Robot
/
lnav
mirror of https://github.com/tstack/lnav.git
2
0
Fork 0
Code Issues Releases Wiki Activity

[sql] fix some issues found while trying to query some logs

This commit is contained in:
Timothy Stack 2019-03-21 07:46:51 -07:00
parent 2fa603d07e
commit 062d480fea
15 changed files with 323 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/json-extension-functions.cc
View File

@ -473,9 +473,12 @@ int json_extension_functions(struct FuncDef **basic_funcs,
.sql_function()
.with_parameter({"json", "The JSON object to query."})
.with_parameter({"ptr", "The JSON-Pointer to lookup in the object."})
.with_parameter(help_text("default", "The default value if the value was not found")
.optional())
.with_tags({"json"})
.with_example({"SELECT jget('1', '')"})
.with_example({"SELECT jget('{ \"a\": 1, \"b\": 2 }', '/b')"})
.with_example({"SELECT jget(null, '/msg', 'Hello')"})
},
{ NULL }

2
src/json_op.cc
View File

@ -221,7 +221,7 @@ int json_op::handle_start_map(void *ctx)
}
}
if (!jo->jo_ptr.expect_map(jo->jo_depth)) {
if (!jo->jo_ptr.expect_map(jo->jo_depth, jo->jo_array_index)) {
retval = 0;
}

255
src/json_ptr.cc
View File

@ -173,3 +173,258 @@ const yajl_callbacks json_ptr_walk::callbacks = {
handle_start_array,
handle_end_array
};
size_t
json_ptr::encode(char *dst, size_t dst_len, const char *src, size_t src_len)
{
size_t retval = 0;
if (src_len == (size_t)-1) {
src_len = strlen(src);
}
for (size_t lpc = 0; lpc < src_len; lpc++) {
switch (src[lpc]) {
case '/':
case '~':
if (retval < dst_len) {
dst[retval] = '~';
retval += 1;
if (src[lpc] == '~') {
dst[retval] = '0';
}
else {
dst[retval] = '1';
}
}
else {
retval += 1;
}
break;
default:
if (retval < dst_len) {
dst[retval] = src[lpc];
}
break;
}
retval += 1;
}
return retval;
}
size_t json_ptr::decode(char *dst, const char *src, ssize_t src_len)
{
size_t retval = 0;
if (src_len == -1) {
src_len = strlen(src);
}
for (int lpc = 0; lpc < src_len; lpc++) {
switch (src[lpc]) {
case '~':
if ((lpc + 1) < src_len) {
switch (src[lpc + 1]) {
case '0':
dst[retval++] = '~';
lpc += 1;
break;
case '1':
dst[retval++] = '/';
lpc += 1;
break;
default:
break;
}
}
break;
default:
dst[retval++] = src[lpc];
break;
}
}
dst[retval] = '\0';
return retval;
}
bool json_ptr::expect_map(int32_t &depth, int32_t &index)
{
bool retval;
if (this->jp_state == MS_DONE) {
retval = true;
}
else if (depth != this->jp_depth) {
retval = true;
}
else if (this->reached_end()) {
retval = true;
}
else if (this->jp_state == MS_VALUE &&
(this->jp_array_index == -1 ||
((index - 1) == this->jp_array_index))) {
if (this->jp_pos[0] == '/') {
this->jp_pos += 1;
this->jp_depth += 1;
this->jp_state = MS_VALUE;
this->jp_array_index = -1;
index = -1;
}
retval = true;
}
else {
retval = true;
}
depth += 1;
return retval;
}
bool json_ptr::at_key(int32_t depth, const char *component, ssize_t len)
{
const char *component_end;
int lpc;
if (this->jp_state == MS_DONE || depth != this->jp_depth) {
return true;
}
if (len == -1) {
len = strlen(component);
}
component_end = component + len;
for (lpc = 0; component < component_end; lpc++, component++) {
char ch = this->jp_pos[lpc];
if (this->jp_pos[lpc] == '~') {
switch (this->jp_pos[lpc + 1]) {
case '0':
ch = '~';
break;
case '1':
ch = '/';
break;
default:
this->jp_state = MS_ERR_INVALID_ESCAPE;
return false;
}
lpc += 1;
}
else if (this->jp_pos[lpc] == '/') {
ch = '\0';
}
if (ch != *component) {
return true;
}
}
this->jp_pos += lpc;
this->jp_state = MS_VALUE;
return true;
}
void json_ptr::exit_container(int32_t &depth, int32_t &index)
{
depth -= 1;
if (this->jp_state == MS_VALUE &&
depth == this->jp_depth &&
(index == -1 || (index - 1 == this->jp_array_index)) &&
this->reached_end()) {
this->jp_state = MS_DONE;
index = -1;
}
}
bool json_ptr::expect_array(int32_t &depth, int32_t &index)
{
bool retval;
if (this->jp_state == MS_DONE) {
retval = true;
}
else if (depth != this->jp_depth) {
retval = true;
}
else if (this->reached_end()) {
retval = true;
}
else if (this->jp_pos[0] == '/') {
int offset;
this->jp_depth += 1;
if (sscanf(this->jp_pos, "/%d%n", &this->jp_array_index, &offset) != 1) {
this->jp_state = MS_ERR_INVALID_INDEX;
retval = true;
}
else if (this->jp_pos[offset] != '\0' && this->jp_pos[offset] != '/') {
this->jp_state = MS_ERR_INVALID_INDEX;
retval = true;
}
else {
index = 0;
this->jp_pos += offset;
this->jp_state = MS_VALUE;
retval = true;
}
}
else {
this->jp_state = MS_ERR_NO_SLASH;
retval = true;
}
depth += 1;
return retval;
}
bool json_ptr::at_index(int32_t &depth, int32_t &index, bool primitive)
{
bool retval;
if (this->jp_state == MS_DONE) {
retval = false;
}
else if (depth < this->jp_depth) {
retval = false;
}
else if (depth == this->jp_depth) {
if (index == -1) {
if (this->jp_array_index == -1) {
retval = this->reached_end();
if (primitive && retval) {
this->jp_state = MS_DONE;
}
}
else {
retval = false;
}
}
else if (index == this->jp_array_index) {
retval = this->reached_end();
this->jp_array_index = -1;
index = -1;
if (primitive && retval) {
this->jp_state = MS_DONE;
}
}
else {
index += 1;
retval = false;
}
}
else if (index == -1) {
retval = this->reached_end();
}
else {
retval = false;
}
return retval;
}

243
src/json_ptr.hh
View File

@ -149,78 +149,9 @@ public:
MS_ERR_INVALID_INDEX,
};
static size_t encode(char *dst, size_t dst_len, const char *src, size_t src_len = -1) {
size_t retval = 0;
static size_t encode(char *dst, size_t dst_len, const char *src, size_t src_len = -1);
if (src_len == (size_t)-1) {
src_len = strlen(src);
}
for (size_t lpc = 0; lpc < src_len; lpc++) {
switch (src[lpc]) {
case '/':
case '~':
if (retval < dst_len) {
dst[retval] = '~';
retval += 1;
if (src[lpc] == '~') {
dst[retval] = '0';
}
else {
dst[retval] = '1';
}
}
else {
retval += 1;
}
break;
default:
if (retval < dst_len) {
dst[retval] = src[lpc];
}
break;
}
retval += 1;
}
return retval;
};
static size_t decode(char *dst, const char *src, ssize_t src_len = -1) {
size_t retval = 0;
if (src_len == -1) {
src_len = strlen(src);
}
for (int lpc = 0; lpc < src_len; lpc++) {
switch (src[lpc]) {
case '~':
if ((lpc + 1) < src_len) {
switch (src[lpc + 1]) {
case '0':
dst[retval++] = '~';
lpc += 1;
break;
case '1':
dst[retval++] = '/';
lpc += 1;
break;
default:
break;
}
}
break;
default:
dst[retval++] = src[lpc];
break;
}
}
dst[retval] = '\0';
return retval;
}
static size_t decode(char *dst, const char *src, ssize_t src_len = -1);
json_ptr(const char *value)
: jp_value(value), jp_pos(value), jp_depth(0), jp_array_index(-1),
@ -228,175 +159,15 @@ public:
};
bool expect_map(int32_t &depth) {
bool retval;
bool expect_map(int32_t &depth, int32_t &index);
if (this->jp_state == MS_DONE) {
retval = true;
}
else if (depth != this->jp_depth) {
retval = true;
}
else if (this->reached_end()) {
retval = true;
}
else if (this->jp_state == MS_VALUE) {
if (this->jp_pos[0] == '/') {
this->jp_pos += 1;
this->jp_depth += 1;
this->jp_state = MS_VALUE;
this->jp_array_index = -1;
}
retval = true;
}
else {
retval = true;
}
depth += 1;
bool at_key(int32_t depth, const char *component, ssize_t len = -1);
return retval;
};
void exit_container(int32_t &depth, int32_t &index);
bool at_key(int32_t depth, const char *component, ssize_t len = -1) {
const char *component_end;
int lpc;
bool expect_array(int32_t &depth, int32_t &index);
if (this->jp_state == MS_DONE || depth != this->jp_depth) {
return true;
}
if (len == -1) {
len = strlen(component);
}
component_end = component + len;
for (lpc = 0; component < component_end; lpc++, component++) {
char ch = this->jp_pos[lpc];
if (this->jp_pos[lpc] == '~') {
switch (this->jp_pos[lpc + 1]) {
case '0':
ch = '~';
break;
case '1':
ch = '/';
break;
default:
this->jp_state = MS_ERR_INVALID_ESCAPE;
return false;
}
lpc += 1;
}
else if (this->jp_pos[lpc] == '/') {
ch = '\0';
}
if (ch != *component) {
return true;
}
}
this->jp_pos += lpc;
this->jp_state = MS_VALUE;
return true;
};
void exit_container(int32_t &depth, int32_t &index) {
depth -= 1;
if (this->jp_state == MS_VALUE &&
depth == this->jp_depth &&
(index == -1 || (index - 1 == this->jp_array_index)) &&
this->reached_end()) {
this->jp_state = MS_DONE;
index = -1;
}
};
bool expect_array(int32_t &depth, int32_t &index) {
bool retval;
if (this->jp_state == MS_DONE) {
retval = true;
}
else if (depth != this->jp_depth) {
retval = true;
}
else if (this->reached_end()) {
retval = true;
}
else if (this->jp_pos[0] == '/') {
int offset;
this->jp_depth += 1;
if (sscanf(this->jp_pos, "/%d%n", &this->jp_array_index, &offset) != 1) {
this->jp_state = MS_ERR_INVALID_INDEX;
retval = true;
}
else if (this->jp_pos[offset] != '\0' && this->jp_pos[offset] != '/') {
this->jp_state = MS_ERR_INVALID_INDEX;
retval = true;
}
else {
index = 0;
this->jp_pos += offset;
this->jp_state = MS_VALUE;
retval = true;
}
}
else {
this->jp_state = MS_ERR_NO_SLASH;
retval = true;
}
depth += 1;
return retval;
};
bool at_index(int32_t &depth, int32_t &index, bool primitive = true) {
bool retval;
if (this->jp_state == MS_DONE) {
retval = false;
}
else if (depth < this->jp_depth) {
retval = false;
}
else if (depth == this->jp_depth) {
if (index == -1) {
if (this->jp_array_index == -1) {
retval = this->reached_end();
if (primitive && retval) {
this->jp_state = MS_DONE;
}
}
else {
retval = false;
}
}
else if (index == this->jp_array_index) {
retval = this->reached_end();
index = -1;
if (primitive && retval) {
this->jp_state = MS_DONE;
}
}
else {
index += 1;
retval = false;
}
}
else if (index == -1) {
retval = this->reached_end();
}
else {
retval = false;
}
return retval;
};
bool at_index(int32_t &depth, int32_t &index, bool primitive = true);
bool reached_end() {
return this->jp_pos[0] == '\0';

9
src/log_format.cc
View File

@ -1004,7 +1004,8 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
return;
}
if (this->jlf_cached_offset != ll.get_offset()) {
if (this->jlf_cached_offset != ll.get_offset() ||
this->jlf_cached_full != full_message) {
yajlpp_parse_context &ypc = *(this->jlf_parse_context);
yajl_handle handle = this->jlf_yajl_handle.in();
json_log_userdata jlu(sbr);
@ -1109,10 +1110,9 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
this->json_append(jfe, str.c_str(), str.size());
}
if (nl_pos == string::npos) {
if (nl_pos == string::npos || full_message) {
lr.lr_end = this->jlf_cached_line.size();
}
else {
} else {
lr.lr_end = lr.lr_start + nl_pos;
}
@ -1243,6 +1243,7 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
}
this->jlf_cached_offset = ll.get_offset();
this->jlf_cached_full = full_message;
}
off_t this_off = 0, next_off = 0;

1
src/log_format.hh
View File

@ -1264,6 +1264,7 @@ public:
std::vector<logline_value> jlf_line_values;
off_t jlf_cached_offset;
bool jlf_cached_full{false};
std::vector<off_t> jlf_line_offsets;
shared_buffer jlf_share_manager;
std::vector<char> jlf_cached_line;

2
src/log_vtab_impl.hh
View File

@ -113,7 +113,7 @@ public:
virtual bool is_valid(log_cursor &lc, logfile_sub_source &lss) {
content_line_t cl(lss.at(lc.lc_curr_line));
std::shared_ptr<logfile> lf = lss.find(cl);
logfile::iterator lf_iter = lf->begin() + cl;
auto lf_iter = lf->begin() + cl;
if (lf_iter->is_continued()) {
return false;

8
src/readline_callbacks.cc
View File

@ -356,7 +356,7 @@ static void rl_search_internal(void *dummy, readline_curses *rc, bool complete =
}
vector<string> kw;
auto iter = rfind_string_attr_if(sa, x, [&al, &name, &kw](auto sa) {
auto iter = rfind_string_attr_if(sa, x, [&al, &name, &kw, x](auto sa) {
if (sa.sa_type != &SQL_FUNCTION_ATTR &&
sa.sa_type != &SQL_KEYWORD_ATTR) {
return false;
@ -366,6 +366,12 @@ static void rl_search_internal(void *dummy, readline_curses *rc, bool complete =
const line_range &lr = sa.sa_range;
int lpc;
if (sa.sa_type == &SQL_FUNCTION_ATTR) {
if (!sa.sa_range.contains(x)) {
return false;
}
}
for (lpc = lr.lr_start; lpc < lr.lr_end; lpc++) {
if (!isalnum(str[lpc]) && str[lpc] != '_') {
break;

2
src/sql_util.cc
View File

@ -925,7 +925,7 @@ void annotate_sql_statement(attr_line_t &al)
if (piter == sa.end()) {
func_range.lr_end = line.length();
} else {
func_range.lr_end = piter->sa_range.lr_end;
func_range.lr_end = piter->sa_range.lr_end - 1;
}
sa.emplace_back(func_range, &SQL_FUNCTION_ATTR);
}

15
test/CMakeLists.txt
View File

@ -27,6 +27,20 @@ add_executable(test_abbrev test_abbrev.cc
../src/pcrepp.cc
../src/lnav_log.cc
../src/spookyhash/SpookyV2.cpp)
add_executable(test_json_ptr test_json_ptr.cc
../src/lnav_util.cc
../../lbuild-debug/src/time_fmts.cc
../src/ptimec_rt.cc
../src/pcrepp.cc
../src/lnav_log.cc
../src/spookyhash/SpookyV2.cpp)
add_executable(drive_json_op drive_json_op.cc
../src/lnav_util.cc
../../lbuild-debug/src/time_fmts.cc
../src/ptimec_rt.cc
../src/pcrepp.cc
../src/lnav_log.cc
../src/spookyhash/SpookyV2.cpp)
add_executable(drive_sql_anno drive_sql_anno.cc ../src/lnav_log.cc ../src/pcrepp.cc)
link_directories(/opt/local/lib)
target_link_libraries(test_pcrepp /usr/local/lib/libpcre.a)
@ -35,3 +49,4 @@ target_link_libraries(lnav_doctests /usr/local/lib/libpcre.a)
target_link_libraries(test_date_time_scanner /usr/local/lib/libpcre.a)
target_link_libraries(test_abbrev /usr/local/lib/libpcre.a)
target_link_libraries(drive_sql_anno /usr/local/lib/libpcre.a)
target_link_libraries(test_json_ptr /usr/local/lib/libpcre.a)

6
test/drive_json_op.cc
View File

@ -152,8 +152,8 @@ int main(int argc, char *argv[])
yajl_gen gen;
ssize_t rc;
gen = yajl_gen_alloc(NULL);
yajl_gen_config(gen, yajl_gen_print_callback, printer, NULL);
gen = yajl_gen_alloc(nullptr);
yajl_gen_config(gen, yajl_gen_print_callback, printer, nullptr);
yajl_gen_config(gen, yajl_gen_beautify, true);
jo.jo_ptr_callbacks.yajl_start_map = handle_start_map;
@ -167,7 +167,7 @@ int main(int argc, char *argv[])
jo.jo_ptr_callbacks.yajl_string = handle_string;
jo.jo_ptr_data = gen;
handle = yajl_alloc(&json_op::ptr_callbacks, NULL, &jo);
handle = yajl_alloc(&json_op::ptr_callbacks, nullptr, &jo);
while ((rc = read(STDIN_FILENO, buffer, sizeof(buffer))) > 0) {
status = yajl_parse(handle, buffer, rc);
if (status == yajl_status_error) {

11
test/expected_help.txt
View File

@ -1305,10 +1305,12 @@ Example
Synopsis
jget(json, ptr) -- Get the value from a JSON object using a JSON-Pointer.
jget(json, ptr, default) -- Get the value from a JSON object using a JSON-
Pointer.
Parameters
json The JSON object to query.
ptr The JSON-Pointer to lookup in the object.
json The JSON object to query.
ptr The JSON-Pointer to lookup in the object.
default The default value if the value was not found
See Also
json_contains()
Examples
@ -1318,6 +1320,9 @@ Examples
#2 ;SELECT jget('{ "a": 1, "b": 2 }', '/b')
#3 ;SELECT jget(null, '/msg', 'Hello')
Synopsis
joinpath(path, ...) -- Join components of a path together.

12
test/test_json_op.sh
View File

@ -100,3 +100,15 @@ EOF
run_test ./drive_json_op get "/ID" <<EOF
{"ID":"P1","ProcessID":"P1","Name":"VxWorks","CanSuspend":true,"CanResume":1,"IsContainer":true,"WordSize":4,"CanTerminate":true,"CanDetach":true,"RCGroup":"P1","SymbolsGroup":"P1","CPUGroup":"P1","DiagnosticTestProcess":true}
EOF
check_output "cannot read key value" <<EOF
"P1"
EOF
run_test ./drive_json_op get "/arr/1/id" <<EOF
{"arr": [{"id": 1}, {"id": 2}]}
EOF
check_output "cannot read key value" <<EOF
2
EOF

6
test/test_json_ptr.cc
View File

@ -55,7 +55,7 @@ int main(int argc, const char *argv[])
depth = 0;
index = -1;
assert(!jptr.at_index(depth, index));
assert(jptr.expect_map(depth));
assert(jptr.expect_map(depth, index));
assert(jptr.at_index(depth, index));
}
@ -64,9 +64,9 @@ int main(int argc, const char *argv[])
depth = 0;
index = -1;
assert(jptr.expect_map(depth));
assert(jptr.expect_map(depth, index));
assert(jptr.at_key(depth, "foo"));
assert(jptr.expect_map(depth));
assert(jptr.expect_map(depth, index));
assert(jptr.at_key(depth, "bar"));
assert(jptr.at_index(depth, index));
}

1
test/test_sessions.sh
View File

@ -6,6 +6,7 @@ export HOME="./sessions"
mkdir -p $HOME
run_test ${lnav_test} -nq \
-c ":reset-session" \
-c ";update access_log set log_mark = 1 where sc_bytes > 60000" \
-c ":goto 1" \
-c ":partition-name middle" \