From a3f5c3239e3d800424ebc7b99fd8edfab8c007f3 Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Mon, 11 Jul 2022 00:09:08 -0700 Subject: [PATCH] [vtab] some more indexing fixes --- src/formats/vmw_log.json | 3 + src/log_format_ext.hh | 3 +- src/log_search_table.cc | 1 + src/log_vtab_impl.cc | 46 ++++-- src/readline_highlighters.cc | 2 + src/sql_help.hh | 1 + src/sql_util.cc | 5 + test/expected/expected.am | 2 + ...a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out | 148 +++++++++--------- ...c7907f907b2335e1328b23fdc46d0968a608d9.err | 2 +- ...4047fb52e4831aabf7d36512247bad6a6a2cf7.err | 2 +- ...9b5940f7533c5fc3d4956a6efce50a9e7132d4.err | 2 +- ...46ee142f80f262c8c14a22751571cc567df525.err | 2 +- ...edc93426e6767aa44ab2356c55327553dcdc8d.err | 2 +- ...4ea85863d4f0ea3b7cb40850ac7c8fde682d57.err | 2 +- ...664c9cda0ae1c48333e21051b5e0eeafd5b4bc.err | 2 +- ...f53d441e22c1d27c27eaa6003c83da1207c063.err | 2 +- ...e121f29bedea0d1a54452cc994b2302ad9dabb.err | 2 +- ...476c76ea51cf479a6a79b037e0cb59871b629c.err | 2 +- ...340cb4c62aabd839ea09235b6ebe41b2bb48f4.err | 2 +- ...814eca259e469b57bf7469787b91e8e8569b17.out | 4 +- ...6839712d088fc7b31618ed90f8ce706c35a9c0.err | 0 ...6839712d088fc7b31618ed90f8ce706c35a9c0.out | 7 + test/test_sql_anno.sh | 2 + 24 files changed, 141 insertions(+), 105 deletions(-) create mode 100644 test/expected/test_sql_anno.sh_be6839712d088fc7b31618ed90f8ce706c35a9c0.err create mode 100644 test/expected/test_sql_anno.sh_be6839712d088fc7b31618ed90f8ce706c35a9c0.out diff --git a/src/formats/vmw_log.json b/src/formats/vmw_log.json index 0a54b3a4..296d7360 100644 --- a/src/formats/vmw_log.json +++ b/src/formats/vmw_log.json @@ -103,6 +103,9 @@ }, "vpx_lro_finish": { "pattern": "\\[VpxLRO\\] -- FINISH (?\\S+)" + }, + "vpx_lro_error": { + "pattern": "\\[VpxLRO\\] -- ERROR (?\\S+) -- (?:(?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:\\((?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\\))?)? -- (?\\S*) -- (?\\S*):\\s*(?.*)" } }, "sample": [ diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index 7286d86f..fc9950ed 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -350,7 +350,8 @@ public: bool elf_builtin_format{false}; struct search_table_def { - std::shared_ptr> + std::shared_ptr< + pcrepp_with_options> std_pattern; std::string std_glob; }; diff --git a/src/log_search_table.cc b/src/log_search_table.cc index 75aa6a44..a73c89e4 100644 --- a/src/log_search_table.cc +++ b/src/log_search_table.cc @@ -206,4 +206,5 @@ log_search_table::filter(log_cursor& lc, logfile_sub_source& lss) lc.lc_log_path.emplace_back(SQLITE_INDEX_CONSTRAINT_GLOB, this->lst_log_path_glob); } + this->lst_match_index = -1; } diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc index 060bcda3..3b9c165c 100644 --- a/src/log_vtab_impl.cc +++ b/src/log_vtab_impl.cc @@ -40,6 +40,8 @@ #include "yajlpp/json_op.hh" #include "yajlpp/yajlpp_def.hh" +// #define DEBUG_INDEXING 1 + using namespace lnav::roles::literals; static auto intern_lifetime = intern_string::get_table_lifetime(); @@ -474,7 +476,7 @@ vt_next_no_rowid(sqlite3_vtab_cursor* cur) if (!vc->log_cursor.lc_indexed_lines.empty()) { vc->log_cursor.lc_curr_line - = vc->log_cursor.lc_indexed_lines.back() - 1_vl; + = vc->log_cursor.lc_indexed_lines.back(); vc->log_cursor.lc_indexed_lines.pop_back(); } @@ -482,6 +484,13 @@ vt_next_no_rowid(sqlite3_vtab_cursor* cur) if (!done) { vc->log_cursor.lc_curr_line += 1_vl; vc->log_cursor.lc_sub_index = 0; + for (auto& col_constraint : vc->log_cursor.lc_indexed_columns) { + vt->vi->vi_column_indexes[col_constraint.cc_column].ci_max_line + = std::max( + vt->vi->vi_column_indexes[col_constraint.cc_column] + .ci_max_line, + vc->log_cursor.lc_curr_line); + } } } while (!done); @@ -494,7 +503,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) auto* vc = (vtab_cursor*) cur; auto* vt = (vtab*) cur->pVtab; -#if 0 +#ifdef DEBUG_INDEXING log_debug("vt_column(%s, %d:%d)", vt->vi->get_name().get(), (int) vc->log_cursor.lc_curr_line, @@ -937,7 +946,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) >= ci.ci_max_line) { std::string value{lv_iter->text_value(), lv_iter->text_length()}; -#if 0 +#ifdef DEBUG_INDEXING log_debug( "updated index for column %d %s -> %d", col, @@ -1186,7 +1195,9 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, index_storage); } - log_info("vt_filter(%s, %d)", vt->vi->get_name().get(), idxNum); +#ifdef DEBUG_INDEXING + log_debug("vt_filter(%s, %d)", vt->vi->get_name().get(), idxNum); +#endif p_cur->log_cursor.lc_format_name.clear(); p_cur->log_cursor.lc_opid = nonstd::nullopt; p_cur->log_cursor.lc_log_path.clear(); @@ -1382,9 +1393,10 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, auto value_len = (size_t) sqlite3_value_bytes(argv[lpc]); -#if 0 - log_debug( - "adding index request for column %d = %s", col, value); +#ifdef DEBUG_INDEXING + log_debug("adding index request for column %d = %s", + col, + value); #endif p_cur->log_cursor.lc_indexed_columns.emplace_back( @@ -1430,7 +1442,9 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, continue; } - // log_debug("adding indexed line %d", (int) vl); +#ifdef DEBUG_INDEXING + log_debug("adding indexed line %d", (int) vl); +#endif p_cur->log_cursor.lc_indexed_lines.push_back(vl); } } @@ -1445,7 +1459,7 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, p_cur->log_cursor.lc_indexed_lines.end(), std::greater<>()); -#if 0 +#ifdef DEBUG_INDEXING log_debug("indexed lines:"); for (auto indline : p_cur->log_cursor.lc_indexed_lines) { log_debug(" %d", (int) indline); @@ -1492,21 +1506,19 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, vt->vi->filter(p_cur->log_cursor, *vt->lss); - while (!p_cur->log_cursor.is_eof() - && (!vt->vi->is_valid(p_cur->log_cursor, *vt->lss) - || !vt->vi->next(p_cur->log_cursor, *vt->lss))) - { - p_cur->log_cursor.lc_curr_line += 1_vl; - p_cur->log_cursor.lc_sub_index = 0; + if (p_cur->log_cursor.lc_indexed_lines.empty()) { + p_cur->log_cursor.lc_indexed_lines.push_back( + p_cur->log_cursor.lc_curr_line); } + auto rc = vt->base.pModule->xNext(p_vtc); -#if 0 +#ifdef DEBUG_INDEXING log_debug("vt_filter() -> cursor_range(%d:%d)", (int) p_cur->log_cursor.lc_curr_line, (int) p_cur->log_cursor.lc_end_line); #endif - return SQLITE_OK; + return rc; } static int diff --git a/src/readline_highlighters.cc b/src/readline_highlighters.cc index 2247a01b..0ca6ed4f 100644 --- a/src/readline_highlighters.cc +++ b/src/readline_highlighters.cc @@ -272,6 +272,8 @@ readline_sqlite_highlighter_int(attr_line_t& al, int x, line_range sub) alb.overlay_attr( line_range{lr.lr_start, (int) line.find('(', lr.lr_start)}, VC_ROLE.value(role_t::VCR_SYMBOL)); + } else if (attr.sa_type == &SQL_NUMBER_ATTR) { + alb.overlay_attr(lr, VC_ROLE.value(role_t::VCR_NUMBER)); } else if (attr.sa_type == &SQL_STRING_ATTR) { if (lr.length() > 1 && al.al_string[lr.lr_end - 1] == '\'') { alb.overlay_attr(lr, VC_ROLE.value(role_t::VCR_STRING)); diff --git a/src/sql_help.hh b/src/sql_help.hh index 6e1d6c22..08bcc519 100644 --- a/src/sql_help.hh +++ b/src/sql_help.hh @@ -42,6 +42,7 @@ extern string_attr_type SQL_KEYWORD_ATTR; extern string_attr_type SQL_IDENTIFIER_ATTR; extern string_attr_type SQL_FUNCTION_ATTR; extern string_attr_type SQL_STRING_ATTR; +extern string_attr_type SQL_NUMBER_ATTR; extern string_attr_type SQL_OPERATOR_ATTR; extern string_attr_type SQL_PAREN_ATTR; extern string_attr_type SQL_GARBAGE_ATTR; diff --git a/src/sql_util.cc b/src/sql_util.cc index 4bf8e63f..bdb71fd5 100644 --- a/src/sql_util.cc +++ b/src/sql_util.cc @@ -965,6 +965,7 @@ string_attr_type SQL_KEYWORD_ATTR("sql_keyword"); string_attr_type SQL_IDENTIFIER_ATTR("sql_ident"); string_attr_type SQL_FUNCTION_ATTR("sql_func"); string_attr_type SQL_STRING_ATTR("sql_string"); +string_attr_type SQL_NUMBER_ATTR("sql_number"); string_attr_type SQL_UNTERMINATED_STRING_ATTR("sql_unstring"); string_attr_type SQL_OPERATOR_ATTR("sql_oper"); string_attr_type SQL_PAREN_ATTR("sql_paren"); @@ -986,6 +987,10 @@ annotate_sql_statement(attr_line_t& al) {pcrepp{R"(\A\(|\A\))"}, &SQL_PAREN_ATTR}, {pcrepp{keyword_re_str, PCRE_CASELESS}, &SQL_KEYWORD_ATTR}, {pcrepp{R"(\A'[^']*('(?:'[^']*')*|$))"}, &SQL_STRING_ATTR}, + { + pcrepp{R"(\A-?\d+(?:\.\d*(?:[eE][\-\+]?\d+)?)?|0x[0-9a-fA-F]+$)"}, + &SQL_NUMBER_ATTR, + }, {pcrepp{R"(\A(((\$|:|@)?\b[a-z_]\w*)|\"([^\"]+)\"|\[([^\]]+)]))", PCRE_CASELESS}, &SQL_IDENTIFIER_ATTR}, diff --git a/test/expected/expected.am b/test/expected/expected.am index ae253e0b..8792e0c7 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -524,6 +524,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql_anno.sh_99da5994c8c90536dbdb1b8ad7dbfb41698a5e8c.out \ $(srcdir)/%reldir%/test_sql_anno.sh_b1a2ddce48beb3e4b1e3ca4b4229a7c21b83b7c4.err \ $(srcdir)/%reldir%/test_sql_anno.sh_b1a2ddce48beb3e4b1e3ca4b4229a7c21b83b7c4.out \ + $(srcdir)/%reldir%/test_sql_anno.sh_be6839712d088fc7b31618ed90f8ce706c35a9c0.err \ + $(srcdir)/%reldir%/test_sql_anno.sh_be6839712d088fc7b31618ed90f8ce706c35a9c0.out \ $(srcdir)/%reldir%/test_sql_anno.sh_c879ba94fdc1a099cf56bd33e5b3e9be65310036.err \ $(srcdir)/%reldir%/test_sql_anno.sh_c879ba94fdc1a099cf56bd33e5b3e9be65310036.out \ $(srcdir)/%reldir%/test_sql_anno.sh_c909647ed0e585002074f55c946f3033df1815b2.err \ diff --git a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out index a8993e7a..b409973a 100644 --- a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out +++ b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out @@ -452,7 +452,7 @@ that is supported by Sqlite3. For example, to get the top ten URLs being accessed in any loaded Apache log files, you can execute: ▌;SELECT cs_uri_stem, count(*) AS total FROM access_log  - ▌ GROUP BY cs_uri_stem ORDER BY total DESC LIMIT 10;  + ▌ GROUP BY cs_uri_stem ORDER BY total DESC LIMIT 10;  The query result view shows the results and graphs any numeric values found in the result, much like the histogram view. @@ -547,7 +547,7 @@ example of a top ten query into the "/tmp/topten.db" file, you can do: ▌;ATTACH DATABASE '/tmp/topten.db' AS topten;  ▌;CREATE TABLE topten.foo AS SELECT cs_uri_stem, count(*) AS total  ▌ FROM access_log GROUP BY cs_uri_stem ORDER BY total DESC  - ▌ LIMIT 10;  + ▌ LIMIT 10;  Dynamic logline Table (experimental) @@ -1716,7 +1716,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] Example #1 To cast the value 1.23 as an integer: - ;SELECT CAST(1.23 AS INTEGER)  + ;SELECT CAST(1.23 AS INTEGER)  @@ -1746,7 +1746,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the absolute value of -1: - ;SELECT abs(-1)  + ;SELECT abs(-1)  @@ -1761,7 +1761,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the arccosine of 0.2: - ;SELECT acos(0.2)  + ;SELECT acos(0.2)  @@ -1776,7 +1776,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the hyperbolic arccosine of 1.2: - ;SELECT acosh(1.2)  + ;SELECT acosh(1.2)  @@ -1791,7 +1791,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the arcsine of 0.2: - ;SELECT asin(0.2)  + ;SELECT asin(0.2)  @@ -1806,7 +1806,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the hyperbolic arcsine of 0.2: - ;SELECT asinh(0.2)  + ;SELECT asinh(0.2)  @@ -1821,7 +1821,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the arctangent of 0.2: - ;SELECT atan(0.2)  + ;SELECT atan(0.2)  @@ -1838,7 +1838,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the angle, in degrees, for the point at (5, 5): - ;SELECT degrees(atan2(5, 5))  + ;SELECT degrees(atan2(5, 5))  @@ -1853,7 +1853,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the hyperbolic arctangent of 0.2: - ;SELECT atanh(0.2)  + ;SELECT atanh(0.2)  @@ -1870,7 +1870,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the angle, in degrees, for the point at (5, 5): - ;SELECT degrees(atn2(5, 5))  + ;SELECT degrees(atn2(5, 5))  @@ -1939,7 +1939,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the ceiling of 1.23: - ;SELECT ceil(1.23)  + ;SELECT ceil(1.23)  @@ -1966,7 +1966,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] upper(), xpath() Example #1 To get a string with the code points 0x48 and 0x49: - ;SELECT char(0x48, 0x49)  + ;SELECT char(0x48, 0x49)  @@ -1990,11 +1990,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] substr(), trim(), unicode(), upper(), xpath() Examples #1 To search for the string 'abc' within 'abcabc' and starting at position 2: - ;SELECT charindex('abc', 'abcabc', 2)  + ;SELECT charindex('abc', 'abcabc', 2)  #2 To search for the string 'abc' within 'abcdef' and starting at position 2: - ;SELECT charindex('abc', 'abcdef', 2)  + ;SELECT charindex('abc', 'abcdef', 2)  @@ -2008,7 +2008,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] Example #1 To get the first non-null value from three parameters: - ;SELECT coalesce(null, 0, null)  + ;SELECT coalesce(null, 0, null)  @@ -2055,7 +2055,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] #3 To get the date portion of the epoch timestamp 1491341842: - ;SELECT date(1491341842, 'unixepoch')  + ;SELECT date(1491341842, 'unixepoch')  @@ -2079,7 +2079,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] #3 To get the date and time portion of the epoch timestamp 1491341842: - ;SELECT datetime(1491341842, 'unixepoch')  + ;SELECT datetime(1491341842, 'unixepoch')  @@ -2171,7 +2171,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] total() Example #1 To raise e to 2: - ;SELECT exp(2)  + ;SELECT exp(2)  @@ -2219,7 +2219,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the floor of 1.23: - ;SELECT floor(1.23)  + ;SELECT floor(1.23)  @@ -2236,15 +2236,15 @@ lnav@googlegroups.com[1] support@lnav.org[2] Examples #1 To generate the numbers in the range [10, 14]: - ;SELECT value FROM generate_series(10, 14)  + ;SELECT value FROM generate_series(10, 14)  #2 To generate every other number in the range [10, 14]: - ;SELECT value FROM generate_series(10, 14, 2)  + ;SELECT value FROM generate_series(10, 14, 2)  #3 To count down from five to 1: - ;SELECT value FROM generate_series(1, 5, -1)  + ;SELECT value FROM generate_series(1, 5, -1)  @@ -2394,7 +2394,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] substr(), trim(), unicode(), upper(), xpath() Example #1 To format an amount: - ;SELECT humanize_file_size(10 * 1024 * 1024)  + ;SELECT humanize_file_size(10 * 1024 * 1024)  @@ -2408,7 +2408,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] Example #1 To get the first non-null value between null and zero: - ;SELECT ifnull(null, 0)  + ;SELECT ifnull(null, 0)  @@ -2501,11 +2501,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] jget(), json_contains(), json_group_array(), json_group_object() Examples #1 To append the number 4 to null: - ;SELECT json_concat(NULL, 4)  + ;SELECT json_concat(NULL, 4)  #2 To append 4 and 5 to the array [1, 2, 3]: - ;SELECT json_concat('[1, 2, 3]', 4, 5)  + ;SELECT json_concat('[1, 2, 3]', 4, 5)  #3 To concatenate two arrays together: @@ -2523,7 +2523,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] jget(), json_concat(), json_group_array(), json_group_object() Examples #1 To test if a JSON array contains the number 4: - ;SELECT json_contains('[1, 2, 3]', 4)  + ;SELECT json_contains('[1, 2, 3]', 4)  #2 To test if a JSON array contains the string 'def': @@ -2540,11 +2540,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] jget(), json_concat(), json_contains(), json_group_object() Examples #1 To create an array from arguments: - ;SELECT json_group_array('one', 2, 3.4)  + ;SELECT json_group_array('one', 2, 3.4)  #2 To create an array from a column of values: - ;SELECT json_group_array(column1) FROM (VALUES (1), (2), (3)) + ;SELECT json_group_array(column1) FROM (VALUES (1), (2), (3)) @@ -2558,11 +2558,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] jget(), json_concat(), json_contains(), json_group_array() Examples #1 To create an object from arguments: - ;SELECT json_group_object('a', 1, 'b', 2)  + ;SELECT json_group_object('a', 1, 'b', 2)  #2 To create an object from a pair of columns: - ;SELECT json_group_object(column1, column2) FROM (VALUES ('a', 1), ('b', 2)) + ;SELECT json_group_object(column1, column2) FROM (VALUES ('a', 1), ('b', 2)) @@ -2586,7 +2586,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] #3 To get the julian day from the timestamp 1491341842: - ;SELECT julianday(1491341842, 'unixepoch')  + ;SELECT julianday(1491341842, 'unixepoch')  @@ -2649,11 +2649,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To get the first character of the string 'abc': - ;SELECT leftstr('abc', 1)  + ;SELECT leftstr('abc', 1)  #2 To get the first ten characters of a string, regardless of size: - ;SELECT leftstr('abc', 10)  + ;SELECT leftstr('abc', 10)  @@ -2748,7 +2748,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] total() Example #1 To get the natual logarithm of 8: - ;SELECT log(8)  + ;SELECT log(8)  @@ -2763,7 +2763,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] pi(), power(), radians(), round(), sign(), square(), sum(), total() Example #1 To get the logarithm of 100: - ;SELECT log10(100)  + ;SELECT log10(100)  @@ -2855,7 +2855,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] total() Examples #1 To get the largest value from the parameters: - ;SELECT max(2, 1, 3)  + ;SELECT max(2, 1, 3)  #2 To get the largest value from an aggregate: @@ -2877,7 +2877,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] total() Examples #1 To get the smallest value from the parameters: - ;SELECT min(2, 1, 3)  + ;SELECT min(2, 1, 3)  #2 To get the smallest value from an aggregate: @@ -2915,11 +2915,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] Examples #1 To test if 1 is different from 1: - ;SELECT nullif(1, 1)  + ;SELECT nullif(1, 1)  #2 To test if 1 is different from 2: - ;SELECT nullif(1, 2)  + ;SELECT nullif(1, 2)  @@ -2940,11 +2940,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To pad the string 'abc' to a length of six characters: - ;SELECT padc('abc', 6) || 'def'  + ;SELECT padc('abc', 6) || 'def'  #2 To pad the string 'abcdef' to a length of eight characters: - ;SELECT padc('abcdef', 8) || 'ghi'  + ;SELECT padc('abcdef', 8) || 'ghi'  @@ -2965,11 +2965,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To pad the string 'abc' to a length of six characters: - ;SELECT padl('abc', 6)  + ;SELECT padl('abc', 6)  #2 To pad the string 'abcdef' to a length of four characters: - ;SELECT padl('abcdef', 4)  + ;SELECT padl('abcdef', 4)  @@ -2990,11 +2990,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To pad the string 'abc' to a length of six characters: - ;SELECT padr('abc', 6) || 'def'  + ;SELECT padr('abc', 6) || 'def'  #2 To pad the string 'abcdef' to a length of four characters: - ;SELECT padr('abcdef', 4) || 'ghi'  + ;SELECT padr('abcdef', 4) || 'ghi'  @@ -3032,7 +3032,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] total() Example #1 To raise two to the power of three: - ;SELECT power(2, 3)  + ;SELECT power(2, 3)  @@ -3063,7 +3063,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] #3 To format 11 with a width of five characters and leading zeroes: - ;SELECT printf('value: %05d', 11)  + ;SELECT printf('value: %05d', 11)  @@ -3114,7 +3114,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] max(), min(), pi(), power(), round(), sign(), square(), sum(), total() Example #1 To convert 180 degrees to radians: - ;SELECT radians(180)  + ;SELECT radians(180)  @@ -3341,7 +3341,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] xpath() Example #1 To repeat the string 'abc' three times: - ;SELECT replicate('abc', 3)  + ;SELECT replicate('abc', 3)  @@ -3383,11 +3383,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] xpath() Examples #1 To get the last character of the string 'abc': - ;SELECT rightstr('abc', 1)  + ;SELECT rightstr('abc', 1)  #2 To get the last ten characters of a string, regardless of size: - ;SELECT rightstr('abc', 10)  + ;SELECT rightstr('abc', 10)  @@ -3406,15 +3406,15 @@ lnav@googlegroups.com[1] support@lnav.org[2] total() Examples #1 To round the number 123.456 to an integer: - ;SELECT round(123.456)  + ;SELECT round(123.456)  #2 To round the number 123.456 to a precision of 1: - ;SELECT round(123.456, 1)  + ;SELECT round(123.456, 1)  #3 To round the number 123.456 to a precision of 5: - ;SELECT round(123.456, 5)  + ;SELECT round(123.456, 5)  @@ -3470,15 +3470,15 @@ lnav@googlegroups.com[1] support@lnav.org[2] total() Examples #1 To get the sign of 10: - ;SELECT sign(10)  + ;SELECT sign(10)  #2 To get the sign of 0: - ;SELECT sign(0)  + ;SELECT sign(0)  #3 To get the sign of -10: - ;SELECT sign(-10)  + ;SELECT sign(-10)  @@ -3504,7 +3504,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] xpath() Examples #1 To get the unicode block element for the value 32 in the range of 0-128: - ;SELECT sparkline(32, 128)  + ;SELECT sparkline(32, 128)  #2 To chart the values in a JSON array: @@ -3540,7 +3540,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] #4 To produce a hash for the parameters where one is a number: - ;SELECT spooky_hash('Hello, World!', 123)  + ;SELECT spooky_hash('Hello, World!', 123)  @@ -3588,7 +3588,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] total() Example #1 To get the square of two: - ;SELECT square(2)  + ;SELECT square(2)  @@ -3662,7 +3662,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] #3 To create a string with the Julian day from the epoch timestamp 1491341842: - ;SELECT strftime('Julian day: %J', 1491341842, 'unixepoch') + ;SELECT strftime('Julian day: %J', 1491341842, 'unixepoch') @@ -3692,22 +3692,22 @@ lnav@googlegroups.com[1] support@lnav.org[2] Examples #1 To get the substring starting at the second character until the end of the string 'abc': - ;SELECT substr('abc', 2)  + ;SELECT substr('abc', 2)  #2 To get the substring of size one starting at the second character of the string 'abc': - ;SELECT substr('abc', 2, 1)  + ;SELECT substr('abc', 2, 1)  #3 To get the substring starting at the last character until the end of the string 'abc': - ;SELECT substr('abc', -1)  + ;SELECT substr('abc', -1)  #4 To get the substring starting at the last character and going backwards one step of the string 'abc': - ;SELECT substr('abc', -1, -1)  + ;SELECT substr('abc', -1, -1)  @@ -3747,7 +3747,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] #3 To get the time portion of the epoch timestamp 1491341842: - ;SELECT time(1491341842, 'unixepoch')  + ;SELECT time(1491341842, 'unixepoch')  @@ -3784,12 +3784,12 @@ lnav@googlegroups.com[1] support@lnav.org[2] #2 To group log messages into five minute buckets and count them: - ;SELECT timeslice(log_time_msecs, '5m') AS slice, count(1) + ;SELECT timeslice(log_time_msecs, '5m') AS slice, count(1)   FROM lnav_example_log GROUP BY slice #3 To group log messages by those before 4:30am and after: - ;SELECT timeslice(log_time_msecs, 'before 4:30am') AS slice, count(1) FROM + ;SELECT timeslice(log_time_msecs, 'before 4:30am') AS slice, count(1) FROM  lnav_example_log GROUP BY slice @@ -3854,7 +3854,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] Examples #1 To get the type of the number 1: - ;SELECT typeof(1)  + ;SELECT typeof(1)  #2 To get the type of the string 'abc': @@ -3983,7 +3983,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] Example #1 To evaluate the number one and return the string 'one': - ;SELECT CASE 1 WHEN 0 THEN 'zero' WHEN 1 THEN 'one' END + ;SELECT CASE 1 WHEN 0 THEN 'zero' WHEN 1 THEN 'one' END @@ -4077,7 +4077,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] Example #1 To mark the syslog message at line 40: - ;UPDATE syslog_log SET log_mark = 1 WHERE log_line = 40 + ;UPDATE syslog_log SET log_mark = 1 WHERE log_line = 40 diff --git a/test/expected/test_events.sh_d9c7907f907b2335e1328b23fdc46d0968a608d9.err b/test/expected/test_events.sh_d9c7907f907b2335e1328b23fdc46d0968a608d9.err index 76cffd5a..0e6fb92a 100644 --- a/test/expected/test_events.sh_d9c7907f907b2335e1328b23fdc46d0968a608d9.err +++ b/test/expected/test_events.sh_d9c7907f907b2335e1328b23fdc46d0968a608d9.err @@ -2,7 +2,7 @@ reason: SQL expression is invalid  |  reason: no such column: sc_status  |   --> /log/watch-expressions/http-errors/expr - |   | sc_status >= 400 AND bad  + |   | sc_status >= 400 AND bad   --> command-option:1  = help: Property Synopsis /log/watch-expressions/http-errors/expr  diff --git a/test/expected/test_meta.sh_154047fb52e4831aabf7d36512247bad6a6a2cf7.err b/test/expected/test_meta.sh_154047fb52e4831aabf7d36512247bad6a6a2cf7.err index 9b6afa8a..1f33d64b 100644 --- a/test/expected/test_meta.sh_154047fb52e4831aabf7d36512247bad6a6a2cf7.err +++ b/test/expected/test_meta.sh_154047fb52e4831aabf7d36512247bad6a6a2cf7.err @@ -4,4 +4,4 @@  |   | 1   |   = help: expecting an array of tag values  --> command-option:1 - | ;UPDATE access_log SET log_tags = 1 WHERE log_line = 1 + | ;UPDATE access_log SET log_tags = 1 WHERE log_line = 1 diff --git a/test/expected/test_meta.sh_3c9b5940f7533c5fc3d4956a6efce50a9e7132d4.err b/test/expected/test_meta.sh_3c9b5940f7533c5fc3d4956a6efce50a9e7132d4.err index f24c28a2..96a52d73 100644 --- a/test/expected/test_meta.sh_3c9b5940f7533c5fc3d4956a6efce50a9e7132d4.err +++ b/test/expected/test_meta.sh_3c9b5940f7533c5fc3d4956a6efce50a9e7132d4.err @@ -8,4 +8,4 @@  |  Description  |  A tag for the log line  --> command-option:1 - | ;UPDATE access_log SET log_tags = json_array('foo') WHERE log_line = 1 + | ;UPDATE access_log SET log_tags = json_array('foo') WHERE log_line = 1 diff --git a/test/expected/test_sql.sh_0d46ee142f80f262c8c14a22751571cc567df525.err b/test/expected/test_sql.sh_0d46ee142f80f262c8c14a22751571cc567df525.err index 0652b4eb..673afd0e 100644 --- a/test/expected/test_sql.sh_0d46ee142f80f262c8c14a22751571cc567df525.err +++ b/test/expected/test_sql.sh_0d46ee142f80f262c8c14a22751571cc567df525.err @@ -1,3 +1,3 @@ ✘ error: the stop parameter is required  --> command-option:1 - | ;SELECT * FROM generate_series(1)  + | ;SELECT * FROM generate_series(1)  diff --git a/test/expected/test_sql.sh_57edc93426e6767aa44ab2356c55327553dcdc8d.err b/test/expected/test_sql.sh_57edc93426e6767aa44ab2356c55327553dcdc8d.err index 490a3a62..979055d2 100644 --- a/test/expected/test_sql.sh_57edc93426e6767aa44ab2356c55327553dcdc8d.err +++ b/test/expected/test_sql.sh_57edc93426e6767aa44ab2356c55327553dcdc8d.err @@ -3,4 +3,4 @@  | |rename-stdin foo   --> ../test/.lnav/formats/default/rename-stdin.lnav:7  | ;SELECT raise_error('no data was redirected to lnav''s standard-input')  - |  WHERE (SELECT count(1) FROM lnav_file WHERE filepath='stdin') = 0 + |  WHERE (SELECT count(1) FROM lnav_file WHERE filepath='stdin') = 0 diff --git a/test/expected/test_sql.sh_764ea85863d4f0ea3b7cb40850ac7c8fde682d57.err b/test/expected/test_sql.sh_764ea85863d4f0ea3b7cb40850ac7c8fde682d57.err index 8432183e..8bdc2a07 100644 --- a/test/expected/test_sql.sh_764ea85863d4f0ea3b7cb40850ac7c8fde682d57.err +++ b/test/expected/test_sql.sh_764ea85863d4f0ea3b7cb40850ac7c8fde682d57.err @@ -1,3 +1,3 @@ ✘ error: Expecting a non-empty pattern value  --> command-option:1 - | ;INSERT INTO lnav_view_filters VALUES ('log', 0, 1, 'out', 'regex', '') + | ;INSERT INTO lnav_view_filters VALUES ('log', 0, 1, 'out', 'regex', '') diff --git a/test/expected/test_sql.sh_7f664c9cda0ae1c48333e21051b5e0eeafd5b4bc.err b/test/expected/test_sql.sh_7f664c9cda0ae1c48333e21051b5e0eeafd5b4bc.err index d2ddc0af..d0680566 100644 --- a/test/expected/test_sql.sh_7f664c9cda0ae1c48333e21051b5e0eeafd5b4bc.err +++ b/test/expected/test_sql.sh_7f664c9cda0ae1c48333e21051b5e0eeafd5b4bc.err @@ -2,4 +2,4 @@  --> command-option:1  | |rename-stdin   --> ../test/.lnav/formats/default/rename-stdin.lnav:6 - | ;SELECT raise_error('expecting the new name for stdin as the first argument') WHERE $1 IS NULL + | ;SELECT raise_error('expecting the new name for stdin as the first argument') WHERE $1 IS NULL diff --git a/test/expected/test_sql.sh_87f53d441e22c1d27c27eaa6003c83da1207c063.err b/test/expected/test_sql.sh_87f53d441e22c1d27c27eaa6003c83da1207c063.err index 5e00096c..0c0a6e87 100644 --- a/test/expected/test_sql.sh_87f53d441e22c1d27c27eaa6003c83da1207c063.err +++ b/test/expected/test_sql.sh_87f53d441e22c1d27c27eaa6003c83da1207c063.err @@ -1,3 +1,3 @@ ✘ error: Expecting an lnav view name for column number 0  --> command-option:1 - | ;INSERT INTO lnav_view_filters VALUES ('bad', 0, 1, 'out', 'regex', 'abc') + | ;INSERT INTO lnav_view_filters VALUES ('bad', 0, 1, 'out', 'regex', 'abc') diff --git a/test/expected/test_sql.sh_ade121f29bedea0d1a54452cc994b2302ad9dabb.err b/test/expected/test_sql.sh_ade121f29bedea0d1a54452cc994b2302ad9dabb.err index c92f8833..82f33db3 100644 --- a/test/expected/test_sql.sh_ade121f29bedea0d1a54452cc994b2302ad9dabb.err +++ b/test/expected/test_sql.sh_ade121f29bedea0d1a54452cc994b2302ad9dabb.err @@ -1,3 +1,3 @@ ✘ error: Invalid regular expression for pattern: missing ) at offset 4  --> command-option:1 - | ;INSERT INTO lnav_view_filters VALUES ('log', 0, 1, 'out', 'regex', 'abc(') + | ;INSERT INTO lnav_view_filters VALUES ('log', 0, 1, 'out', 'regex', 'abc(') diff --git a/test/expected/test_sql.sh_f7476c76ea51cf479a6a79b037e0cb59871b629c.err b/test/expected/test_sql.sh_f7476c76ea51cf479a6a79b037e0cb59871b629c.err index b476e364..bf408af5 100644 --- a/test/expected/test_sql.sh_f7476c76ea51cf479a6a79b037e0cb59871b629c.err +++ b/test/expected/test_sql.sh_f7476c76ea51cf479a6a79b037e0cb59871b629c.err @@ -1,3 +1,3 @@ ✘ error: Expecting an lnav view name for column number 0  --> command-option:1 - | ;INSERT INTO lnav_view_filters VALUES (NULL, 0, 1, 'out', 'regex', 'abc') + | ;INSERT INTO lnav_view_filters VALUES (NULL, 0, 1, 'out', 'regex', 'abc') diff --git a/test/expected/test_sql.sh_f8340cb4c62aabd839ea09235b6ebe41b2bb48f4.err b/test/expected/test_sql.sh_f8340cb4c62aabd839ea09235b6ebe41b2bb48f4.err index e64aa5d8..81f993d0 100644 --- a/test/expected/test_sql.sh_f8340cb4c62aabd839ea09235b6ebe41b2bb48f4.err +++ b/test/expected/test_sql.sh_f8340cb4c62aabd839ea09235b6ebe41b2bb48f4.err @@ -1,3 +1,3 @@ ✘ error: Expecting an value of 'in' or 'out' for column number 3  --> command-option:1 - | ;INSERT INTO lnav_view_filters VALUES ('log', 0 , 1, 'bad', 'regex', 'abc') + | ;INSERT INTO lnav_view_filters VALUES ('log', 0 , 1, 'bad', 'regex', 'abc') diff --git a/test/expected/test_sql_anno.sh_73814eca259e469b57bf7469787b91e8e8569b17.out b/test/expected/test_sql_anno.sh_73814eca259e469b57bf7469787b91e8e8569b17.out index f6f17bfb..84336254 100644 --- a/test/expected/test_sql_anno.sh_73814eca259e469b57bf7469787b91e8e8569b17.out +++ b/test/expected/test_sql_anno.sh_73814eca259e469b57bf7469787b91e8e8569b17.out @@ -1,7 +1,7 @@ SELECT (1 + 2) AS three sql_keyword ------ - sql_garbage - + sql_number - sql_oper - - sql_garbage - + sql_number - sql_keyword -- sql_ident ----- diff --git a/test/expected/test_sql_anno.sh_be6839712d088fc7b31618ed90f8ce706c35a9c0.err b/test/expected/test_sql_anno.sh_be6839712d088fc7b31618ed90f8ce706c35a9c0.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_sql_anno.sh_be6839712d088fc7b31618ed90f8ce706c35a9c0.out b/test/expected/test_sql_anno.sh_be6839712d088fc7b31618ed90f8ce706c35a9c0.out new file mode 100644 index 00000000..902a1db0 --- /dev/null +++ b/test/expected/test_sql_anno.sh_be6839712d088fc7b31618ed90f8ce706c35a9c0.out @@ -0,0 +1,7 @@ + SELECT (1.5 + 2.2) AS decim + sql_keyword ------ + sql_number --- + sql_oper - + sql_number --- + sql_keyword -- + sql_ident ----- diff --git a/test/test_sql_anno.sh b/test/test_sql_anno.sh index 71b50f0f..9aa41cc1 100644 --- a/test/test_sql_anno.sh +++ b/test/test_sql_anno.sh @@ -39,6 +39,8 @@ run_cap_test ./drive_sql_anno "SELECT 'hello, world!' FROM \"my table\"" # math run_cap_test ./drive_sql_anno "SELECT (1 + 2) AS three" +run_cap_test ./drive_sql_anno "SELECT (1.5 + 2.2) AS decim" + # subqueries run_cap_test ./drive_sql_anno "SELECT * FROM (SELECT foo, bar FROM baz)"