From 72c1c48e234b5627a8f7b443a50fa835f3903103 Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Fri, 1 Jul 2022 16:02:14 -0700 Subject: [PATCH] [perf] fixing slow queries --- NEWS | 20 + docs/schemas/format-v1.schema.json | 5 + src/CMakeLists.txt | 2 + src/all_logs_vtab.cc | 9 +- src/column_namer.cc | 52 +- src/column_namer.hh | 30 +- src/field_overlay_source.cc | 7 +- src/formats/formats.am | 1 + src/formats/procstate_log.json | 22 + src/formats/vmw_log.json | 6 + src/internals/sql-ref.rst | 101 ++- src/lnav_commands.cc | 8 +- src/log_data_helper.cc | 3 +- src/log_data_table.cc | 28 +- src/log_data_table.hh | 2 - src/log_format.cc | 50 +- src/log_format_ext.hh | 4 +- src/log_format_impls.cc | 2 +- src/log_format_loader.cc | 5 + src/log_search_table.cc | 78 +- src/log_search_table.hh | 13 +- src/log_vtab_impl.cc | 767 ++++++++++++------ src/log_vtab_impl.hh | 25 +- src/logfile.hh | 2 + src/regexp_vtab.cc | 291 ++++++- src/sql_util.cc | 31 +- src/sql_util.hh | 18 +- src/string-extension-functions.cc | 20 +- src/time_formats.am | 1 + test/CMakeLists.txt | 5 + test/Makefile.am | 9 +- test/expected/expected.am | 16 + ...b4cb9bd6586f9694100db76734b19a75158eab.err | 2 +- ...800217930a6a30e68c4efb20f6959c4f71aeb0.err | 4 +- ...a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out | 354 ++++---- ...5b73be0b991e0e8d6735e31df5b37c4286321b.err | 4 +- ...8e14a26d8261c9f72747118a469266121d5459.out | 6 +- ...32c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.err | 0 ...32c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.out | 2 + ...4306f0e5f610ba71f521ba3d19fe158ece0ba5.err | 0 ...4306f0e5f610ba71f521ba3d19fe158ece0ba5.out | 2 + ...b8da04734fadf3b9eea80e0af997e38e0fb811.err | 0 ...b8da04734fadf3b9eea80e0af997e38e0fb811.out | 3 + ...f137b4b2e1dcc15efa8c0164a13e5db4e8856b.err | 0 ...f137b4b2e1dcc15efa8c0164a13e5db4e8856b.out | 11 + ...a024607c9339423b749ac1a2eb3e49fe9776e5.err | 0 ...a024607c9339423b749ac1a2eb3e49fe9776e5.out | 2 + ...15cff21c4a1b7c4a976b5574eb930b2605cd2f.err | 0 ...15cff21c4a1b7c4a976b5574eb930b2605cd2f.out | 12 + ...f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.err | 0 ...f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.out | 6 + test/logfile_procstate.0 | 43 + test/test_column_namer.cc | 68 ++ test/test_sql.sh | 28 +- test/test_sql_anno.sh | 3 + test/test_sql_indexes.sh | 12 + test/test_sql_search_table.sh | 7 + test/test_sql_str_func.sh | 2 + 58 files changed, 1584 insertions(+), 620 deletions(-) create mode 100644 src/formats/procstate_log.json create mode 100644 test/expected/test_sql.sh_5532c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.err create mode 100644 test/expected/test_sql.sh_5532c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.out create mode 100644 test/expected/test_sql.sh_764306f0e5f610ba71f521ba3d19fe158ece0ba5.err create mode 100644 test/expected/test_sql.sh_764306f0e5f610ba71f521ba3d19fe158ece0ba5.out create mode 100644 test/expected/test_sql.sh_c5b8da04734fadf3b9eea80e0af997e38e0fb811.err create mode 100644 test/expected/test_sql.sh_c5b8da04734fadf3b9eea80e0af997e38e0fb811.out create mode 100644 test/expected/test_sql_anno.sh_08f137b4b2e1dcc15efa8c0164a13e5db4e8856b.err create mode 100644 test/expected/test_sql_anno.sh_08f137b4b2e1dcc15efa8c0164a13e5db4e8856b.out create mode 100644 test/expected/test_sql_indexes.sh_52a024607c9339423b749ac1a2eb3e49fe9776e5.err create mode 100644 test/expected/test_sql_indexes.sh_52a024607c9339423b749ac1a2eb3e49fe9776e5.out create mode 100644 test/expected/test_sql_indexes.sh_5815cff21c4a1b7c4a976b5574eb930b2605cd2f.err create mode 100644 test/expected/test_sql_indexes.sh_5815cff21c4a1b7c4a976b5574eb930b2605cd2f.out create mode 100644 test/expected/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.err create mode 100644 test/expected/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.out create mode 100644 test/logfile_procstate.0 create mode 100644 test/test_column_namer.cc create mode 100644 test/test_sql_indexes.sh create mode 100644 test/test_sql_search_table.sh diff --git a/NEWS b/NEWS index 25cf9079..5a41be6c 100644 --- a/NEWS +++ b/NEWS @@ -29,6 +29,9 @@ lnav v0.10.2: each log message. If the expressions evaluates to true, an event is published to the "lnav_events" table that includes the message contents. + * Added the "regexp_capture_into_json()" table-valued-function that + is similar to "regexp_capture()", but returns a single row with a + JSON value for each match instead of a row for each capture. * Added a "top_meta" column to the lnav_views table that contains metadata related to the top line in the view. * Added a "log_opid" hidden column to all log tables that contains @@ -36,6 +39,9 @@ lnav v0.10.2: * Moved the "log_format" column from the all_logs table to a hidden column on all tables. * Add format for UniFi gateway. + * Added a "glob" property to search tables defined in log formats + to constrain searches to log messages from files that have a + matching log_path value. Breaking Changes: * Added a 'language' column to the lnav_view_filters table that @@ -46,11 +52,25 @@ lnav v0.10.2: * Removed the summary overlay at the bottom of the log view that displayed things like "Error rate" and the time span. It doesn't seem like anyone used it. + * Removed the "log_msg_instance" column from the logline and search + tables since it causes problems with performance. + * Search tables now search for multiple matches within a message + instead of stopping at the first hit. Each additional match is + returned as a separate row. A "match_index" column has been + added to capture the index of the match within the message. + The table regex is also compiled with the "multiline" flag enabled + so the meaning of the '^' and '$' metacharacters are changed + to match the start/end of a line instead of the start/end of + the entire message string. + * Search tables defined in formats are now constrained to only + match log messages that are in that log format instead of all + log messages. Fixes: * Toggling enabled/disabled filters when there is a SQL expression no longer causes a crash. * Fix a crash related to long lines that are word wrapped. + * lnav v0.10.1: Features: diff --git a/docs/schemas/format-v1.schema.json b/docs/schemas/format-v1.schema.json index e46d8a92..687d7236 100644 --- a/docs/schemas/format-v1.schema.json +++ b/docs/schemas/format-v1.schema.json @@ -441,6 +441,11 @@ "title": "//search-table//pattern", "description": "The regular expression for this search table.", "type": "string" + }, + "glob": { + "title": "//search-table//glob", + "description": "Glob pattern used to constrain hits to messages that match the given pattern.", + "type": "string" } }, "additionalProperties": false diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81210d66..540eba3e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -132,6 +132,8 @@ set(FORMAT_FILES formats/openstack_log.json formats/page_log.json formats/papertrail_log.json + formats/pcap_log.json + formats/procstate_log.json formats/snaplogic_log.json formats/sssd_log.json formats/strace_log.json diff --git a/src/all_logs_vtab.cc b/src/all_logs_vtab.cc index b5f09e64..ada48c86 100644 --- a/src/all_logs_vtab.cc +++ b/src/all_logs_vtab.cc @@ -102,12 +102,5 @@ all_logs_vtab::extract(logfile* lf, bool all_logs_vtab::next(log_cursor& lc, logfile_sub_source& lss) { - lc.lc_curr_line = lc.lc_curr_line + 1_vl; - lc.lc_sub_index = 0; - - if (lc.is_eof()) { - return true; - } - - return this->is_valid(lc, lss); + return true; } diff --git a/src/column_namer.cc b/src/column_namer.cc index e7c12fa8..84945dcb 100644 --- a/src/column_namer.cc +++ b/src/column_namer.cc @@ -35,17 +35,28 @@ #include "base/itertools.hh" #include "base/lnav_log.hh" -#include "base/string_util.hh" #include "config.h" #include "sql_util.hh" +const char* column_namer::BUILTIN_COL = "col"; + +column_namer::column_namer(language lang) : cn_language(lang) {} + bool -column_namer::existing_name(const std::string& in_name) const +column_namer::existing_name(const string_fragment& in_name) const { - if (std::binary_search( - std::begin(sql_keywords), std::end(sql_keywords), toupper(in_name))) - { - return true; + switch (this->cn_language) { + case language::SQL: { + auto upped = toupper(in_name.to_string()); + + if (std::binary_search( + std::begin(sql_keywords), std::end(sql_keywords), upped)) { + return true; + } + break; + } + case language::JSON: + break; } if (this->cn_builtin_names | lnav::itertools::find(in_name)) { @@ -59,35 +70,44 @@ column_namer::existing_name(const std::string& in_name) const return false; } -std::string -column_namer::add_column(const std::string& in_name) +string_fragment +column_namer::add_column(const string_fragment& in_name) { - auto base_name = in_name; - std::string retval; + string_fragment base_name; + string_fragment retval; + fmt::memory_buffer buf; int num = 0; if (in_name.empty()) { - base_name = "col"; + base_name = string_fragment{BUILTIN_COL}; + } else { + base_name = in_name; } retval = base_name; - auto counter_iter = this->cn_name_counters.find(retval); if (counter_iter != this->cn_name_counters.end()) { num = ++counter_iter->second; - retval = fmt::format(FMT_STRING("{}_{}"), base_name, num); + fmt::format_to(buf, FMT_STRING("{}_{}"), base_name, num); + retval = string_fragment{buf.data(), 0, (int) buf.size()}; } while (this->existing_name(retval)) { if (num == 0) { - this->cn_name_counters[retval] = num; + this->cn_name_counters[base_name] = num; } - log_debug("column name already exists: %s", retval.c_str()); - retval = fmt::format(FMT_STRING("{}_{}"), base_name, num); + log_debug( + "column name already exists: %.*s", retval.length(), retval.data()); + fmt::format_to(buf, FMT_STRING("{}_{}"), base_name, num); + retval = string_fragment{buf.data(), 0, (int) buf.size()}; num += 1; } + auto* mem = this->cn_alloc.allocate(retval.length() + 1); + memcpy(mem, retval.data(), retval.length()); + mem[retval.length()] = '\0'; + retval = string_fragment{mem, 0, retval.length()}; this->cn_names.emplace_back(retval); return retval; diff --git a/src/column_namer.hh b/src/column_namer.hh index 53e48078..120db55e 100644 --- a/src/column_namer.hh +++ b/src/column_namer.hh @@ -36,15 +36,35 @@ #include #include +#include "ArenaAlloc/arenaalloc.h" +#include "base/intern_string.hh" + class column_namer { public: - bool existing_name(const std::string& in_name) const; + enum class language { + SQL, + JSON, + }; - std::string add_column(const std::string& in_name); + column_namer(language lang); - std::vector cn_builtin_names{"col"}; - std::vector cn_names{}; - std::unordered_map cn_name_counters{}; + bool existing_name(const string_fragment& in_name) const; + + string_fragment add_column(const string_fragment& in_name); + + static const char* BUILTIN_COL; + + ArenaAlloc::Alloc cn_alloc; + language cn_language; + std::vector cn_builtin_names{string_fragment(BUILTIN_COL)}; + std::vector cn_names; + std::unordered_map< + string_fragment, + size_t, + frag_hasher, + std::equal_to, + ArenaAlloc::Alloc>> + cn_name_counters; }; #endif diff --git a/src/field_overlay_source.cc b/src/field_overlay_source.cc index 80f0316f..c793d2bc 100644 --- a/src/field_overlay_source.cc +++ b/src/field_overlay_source.cc @@ -239,7 +239,8 @@ field_overlay_source::build_field_lines(const listview_curses& lv) = this->fos_log_helper.ldh_parser->get_element_string( iter->e_sub_elements->front()); - colname = this->fos_log_helper.ldh_namer->add_column(colname); + colname + = this->fos_log_helper.ldh_namer->add_column(colname).to_string(); this->fos_unknown_key_size = std::max(this->fos_unknown_key_size, (int) colname.length()); } @@ -410,14 +411,14 @@ field_overlay_source::build_field_lines(const listview_curses& lv) for (size_t lpc = 0; lpc < this->fos_log_helper.ldh_parser->dp_pairs.size(); lpc++, ++iter) { - auto& name = this->fos_log_helper.ldh_namer->cn_names[lpc]; + auto name = this->fos_log_helper.ldh_namer->cn_names[lpc]; auto val = this->fos_log_helper.ldh_parser->get_element_string( iter->e_sub_elements->back()); attr_line_t al(fmt::format(FMT_STRING(" {} = {}"), name, val)); al.with_attr( string_attr(line_range(3, 3 + name.length()), - VC_STYLE.value(vc.attrs_for_ident(name)))); + VC_STYLE.value(vc.attrs_for_ident(name.to_string())))); this->fos_lines.emplace_back(al); this->add_key_line_attrs( diff --git a/src/formats/formats.am b/src/formats/formats.am index 69e82911..bca37ec1 100644 --- a/src/formats/formats.am +++ b/src/formats/formats.am @@ -23,6 +23,7 @@ FORMAT_FILES = \ $(srcdir)/%reldir%/page_log.json \ $(srcdir)/%reldir%/papertrail_log.json \ $(srcdir)/%reldir%/pcap_log.json \ + $(srcdir)/%reldir%/procstate_log.json \ $(srcdir)/%reldir%/snaplogic_log.json \ $(srcdir)/%reldir%/sssd_log.json \ $(srcdir)/%reldir%/strace_log.json \ diff --git a/src/formats/procstate_log.json b/src/formats/procstate_log.json new file mode 100644 index 00000000..493839b1 --- /dev/null +++ b/src/formats/procstate_log.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "procstate_log": { + "title": "Process State", + "description": "Periodic dumps of process state", + "regex": { + "std": { + "pattern": "========== Start of system state dump at (?.*) ==========" + } + }, + "sample": [ + { + "line": "========== Start of system state dump at Thu Jun 2 00:01:01 UTC 2022 ==========" + } + ], + "search-table": { + "procstate_procs": { + "pattern": "^(?\\S+)\\s+(?\\d+)\\s+(?\\d+\\.\\d+)\\s+(?\\d+\\.\\d+)\\s+(?\\d+)\\s+(?\\d+)\\s(?\\S+)\\s+(?\\S+)\\s+(?\\S+)\\s+(?\\S+)\\s+(?(?[^ \\n]+)(?: (?[^\\n]+))?)$" + } + } + } +} \ No newline at end of file diff --git a/src/formats/vmw_log.json b/src/formats/vmw_log.json index 5393616d..4059eb33 100644 --- a/src/formats/vmw_log.json +++ b/src/formats/vmw_log.json @@ -93,6 +93,12 @@ "identifier": true } }, + "search-table": { + "vpxd_session_stats": { + "pattern": "/SessionStats/SessionPool/Session/Id='(?[^']+)'/Username='(?[^']+)'/ClientIP='(?[^']+)'(?[^ ]+) (?[^\\n]+)", + "glob": "*/vpxd-profile*" + } + }, "sample": [ { "line": "2021-05-24T20:31:05.671Z - last log rotation time, 2021-05-24T09:30:02.683Z - time the service was last started, Section for VMware ESX, pid=1000080910, version=7.0.3, build=0, option=DEBUG" diff --git a/src/internals/sql-ref.rst b/src/internals/sql-ref.rst index ee3d375f..ba76c024 100644 --- a/src/internals/sql-ref.rst +++ b/src/internals/sql-ref.rst @@ -772,7 +772,7 @@ char(*X*) HI **See Also** - :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -805,7 +805,7 @@ charindex(*needle*, *haystack*, *\[start\]*) 0 **See Also** - :ref:`char`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1069,7 +1069,7 @@ endswith(*str*, *suffix*) 0 **See Also** - :ref:`char`, :ref:`charindex`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1124,7 +1124,7 @@ extract(*str*) {"col_0":1.0,"col_1":2.0} **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1325,7 +1325,7 @@ group_concat(*X*, *\[sep\]*) hw,gw **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1349,7 +1349,7 @@ group_spooky_hash(*str*) 4e7a190aead058cb123c94290f29c34a **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1365,7 +1365,7 @@ gunzip(*b*) * **b** --- The blob to decompress **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1381,7 +1381,7 @@ gzip(*value*) * **value** --- The value to compress **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1427,7 +1427,7 @@ humanize_file_size(*value*) 10.0MB **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1475,7 +1475,7 @@ instr(*haystack*, *needle*) 2 **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1828,7 +1828,7 @@ leftstr(*str*, *N*) abc **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -1852,7 +1852,7 @@ length(*str*) 3 **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2042,7 +2042,7 @@ logfmt2json(*str*) {"foo":1,"bar":2,"name":"Rolo Tomassi"} **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2066,7 +2066,7 @@ lower(*str*) abc **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2098,7 +2098,7 @@ ltrim(*str*, *\[chars\]*) c **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2255,7 +2255,7 @@ padc(*str*, *len*) abcdef ghi **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2287,7 +2287,7 @@ padl(*str*, *len*) abcdef **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2319,7 +2319,7 @@ padr(*str*, *len*) abcdefghi **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2417,7 +2417,7 @@ printf(*format*, *X*) value: 00011 **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2441,7 +2441,7 @@ proper(*str*) Hello, World! **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2624,7 +2624,34 @@ regexp_capture(*string*, *pattern*) 1 2 3 8 9 2 **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + +---- + + +.. _regexp_capture_into_json: + +regexp_capture_into_json(*string*, *pattern*) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + A table-valued function that executes a regular-expression over a string and returns the captured values as a JSON object. If the regex only matches a subset of the input string, it will be rerun on the remaining parts of the string until no more matches are found. + + **Parameters** + * **string\*** --- The string to match against the given pattern. + * **pattern\*** --- The regular expression to match. + + **Examples** + To extract the key/value pairs 'a'/1 and 'b'/2 from the string 'a=1; b=2': + + .. code-block:: custsqlite + + ;SELECT * FROM regexp_capture_into_json('a=1; b=2', '(\w+)=(\d+)') + match_index content + 0 {"col_0":"a","col_1":1} + 1 {"col_0":"b","col_1":2} + + **See Also** + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2663,7 +2690,7 @@ regexp_match(*re*, *str*) {"num":123,"str":"four"} **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_replace`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_replace`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2696,7 +2723,7 @@ regexp_replace(*str*, *re*, *repl*) <123> **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_match`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_match`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2729,7 +2756,7 @@ replace(*str*, *old*, *replacement*) zbc **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2754,7 +2781,7 @@ replicate(*str*, *N*) abcabcabc **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2778,7 +2805,7 @@ reverse(*str*) cba **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2810,7 +2837,7 @@ rightstr(*str*, *N*) abc **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2906,7 +2933,7 @@ rtrim(*str*, *\[chars\]*) a **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -2976,7 +3003,7 @@ sparkline(*value*, *\[upper\]*) ▁▂▃▄▅▆▇█ **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -3021,7 +3048,7 @@ spooky_hash(*str*) f96b3d9c1a19f4394c97a1b79b1880df **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -3135,7 +3162,7 @@ startswith(*str*, *prefix*) 0 **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -3160,7 +3187,7 @@ strfilter(*source*, *include*) bcbc **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -3247,7 +3274,7 @@ substr(*str*, *start*, *\[size\]*) b **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`trim`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -3454,7 +3481,7 @@ trim(*str*, *\[chars\]*) abc **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`unicode`, :ref:`upper`, :ref:`xpath` ---- @@ -3507,7 +3534,7 @@ unicode(*X*) 97 **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`upper`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`upper`, :ref:`xpath` ---- @@ -3545,7 +3572,7 @@ upper(*str*) ABC **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`xpath` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`xpath` ---- @@ -3588,7 +3615,7 @@ xpath(*xpath*, *xmldoc*) Hello ★ /abc/def/text() {} Hello ★ **See Also** - :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper` + :ref:`char`, :ref:`charindex`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`upper` ---- diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index 8c706245..f7a5fb37 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -1460,7 +1460,7 @@ com_pipe_to(exec_context& ec, lpc++, ++iter) { std::string colname = ldh.ldh_parser->get_element_string( iter->e_sub_elements->front()); - colname = ldh.ldh_namer->add_column(colname); + colname = ldh.ldh_namer->add_column(colname).to_string(); std::string val = ldh.ldh_parser->get_element_string( iter->e_sub_elements->back()); setenv(colname.c_str(), val.c_str(), 1); @@ -4832,14 +4832,14 @@ command_prompt(std::vector& args) for (auto& cn_name : ldh.ldh_namer->cn_names) { lnav_data.ld_rl_view->add_possibility( - ln_mode_t::COMMAND, "colname", cn_name); + ln_mode_t::COMMAND, "colname", cn_name.to_string()); } for (const auto& iter : ldh.ldh_namer->cn_builtin_names) { - if (iter == "col") { + if (iter == column_namer::BUILTIN_COL) { continue; } lnav_data.ld_rl_view->add_possibility( - ln_mode_t::COMMAND, "colname", iter); + ln_mode_t::COMMAND, "colname", iter.to_string()); } ldh.clear(); diff --git a/src/log_data_helper.cc b/src/log_data_helper.cc index 0b2ae2d6..96f53ebc 100644 --- a/src/log_data_helper.cc +++ b/src/log_data_helper.cc @@ -92,7 +92,8 @@ log_data_helper::parse_line(content_line_t line, bool allow_middle) this->ldh_msg_format.clear(); this->ldh_parser->dp_msg_format = &this->ldh_msg_format; this->ldh_parser->parse(); - this->ldh_namer = std::make_unique(); + this->ldh_namer + = std::make_unique(column_namer::language::SQL); this->ldh_json_pairs.clear(); this->ldh_xml_pairs.clear(); diff --git a/src/log_data_table.cc b/src/log_data_table.cc index 846c282e..2e598cc5 100644 --- a/src/log_data_table.cc +++ b/src/log_data_table.cc @@ -36,7 +36,7 @@ log_data_table::log_data_table(logfile_sub_source& lss, content_line_t template_line, intern_string_t table_name) : log_vtab_impl(table_name), ldt_log_source(lss), - ldt_template_line(template_line), ldt_instance(-1) + ldt_template_line(template_line) { std::shared_ptr lf = lss.find(template_line); auto format = lf->get_format(); @@ -49,9 +49,6 @@ log_data_table::log_data_table(logfile_sub_source& lss, void log_data_table::get_columns_int() { - static intern_string_t instance_name - = intern_string::lookup("log_msg_instance"); - auto& cols = this->ldt_cols; auto& metas = this->ldt_value_metas; content_line_t cl_copy = this->ldt_template_line; @@ -75,19 +72,16 @@ log_data_table::get_columns_int() data_scanner ds(line, body.lr_start, body.lr_end); data_parser dp(&ds); - column_namer cn; + column_namer cn{column_namer::language::SQL}; dp.parse(); - metas.emplace_back( - instance_name, value_kind_t::VALUE_INTEGER, cols.size(), format.get()); - cols.emplace_back("log_msg_instance", SQLITE_INTEGER); for (auto pair_iter = dp.dp_pairs.begin(); pair_iter != dp.dp_pairs.end(); ++pair_iter) { std::string key_str = dp.get_element_string(pair_iter->e_sub_elements->front()); - std::string colname = cn.add_column(key_str); + auto colname = cn.add_column(key_str).to_string(); int sql_type = SQLITE3_TEXT; value_kind_t kind = value_kind_t::VALUE_TEXT; std::string collator; @@ -117,14 +111,7 @@ log_data_table::get_columns_int() bool log_data_table::next(log_cursor& lc, logfile_sub_source& lss) { - if (lc.lc_curr_line == -1_vl) { - this->ldt_instance = -1; - } - - lc.lc_curr_line = lc.lc_curr_line + 1_vl; - lc.lc_sub_index = 0; - - if (lc.lc_curr_line == (int) lss.text_line_count()) { + if (lc.is_eof()) { return true; } @@ -168,7 +155,6 @@ log_data_table::next(log_cursor& lc, logfile_sub_source& lss) this->ldt_pairs.clear(); this->ldt_pairs.swap(dp.dp_pairs, __FILE__, __LINE__); - this->ldt_instance += 1; return true; } @@ -182,10 +168,8 @@ log_data_table::extract(logfile* lf, auto meta_iter = this->ldt_value_metas.begin(); this->ldt_format_impl->extract(lf, line_number, line, values); - values.emplace_back(*meta_iter, this->ldt_instance); - ++meta_iter; - for (auto& ldt_pair : this->ldt_pairs) { - const data_parser::element& pvalue = ldt_pair.get_pair_value(); + for (const auto& ldt_pair : this->ldt_pairs) { + const auto& pvalue = ldt_pair.get_pair_value(); switch (pvalue.value_token()) { case DT_NUMBER: { diff --git a/src/log_data_table.hh b/src/log_data_table.hh index c9ea2ecb..7a6c5ea3 100644 --- a/src/log_data_table.hh +++ b/src/log_data_table.hh @@ -58,7 +58,6 @@ public: void get_foreign_keys(std::vector& keys_inout) const override { log_vtab_impl::get_foreign_keys(keys_inout); - keys_inout.emplace_back("log_msg_instance"); } bool next(log_cursor& lc, logfile_sub_source& lss) override; @@ -75,7 +74,6 @@ private: shared_buffer_ref ldt_current_line; data_parser::element_list_t ldt_pairs; std::shared_ptr ldt_format_impl; - int64_t ldt_instance; std::vector ldt_cols; std::vector ldt_value_metas; }; diff --git a/src/log_format.cc b/src/log_format.cc index 9ce4f5b4..8f05f658 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -556,13 +556,14 @@ json_array_end(void* ctx) return 1; } -static struct json_path_container json_log_handlers - = {json_path_handler(pcrepp("\\w+")) - .add_cb(read_json_null) - .add_cb(read_json_bool) - .add_cb(read_json_int) - .add_cb(read_json_double) - .add_cb(read_json_field)}; +static struct json_path_container json_log_handlers = { + json_path_handler(pcrepp("\\w+")) + .add_cb(read_json_null) + .add_cb(read_json_bool) + .add_cb(read_json_int) + .add_cb(read_json_double) + .add_cb(read_json_field), +}; static int rewrite_json_field(yajlpp_parse_context* ypc, const unsigned char* str, @@ -2307,6 +2308,8 @@ external_log_format::register_vtabs( auto lst = std::make_shared( *elf_search_table.second.std_pattern, elf_search_table.first); + lst->lst_format = this; + lst->lst_log_path_glob = elf_search_table.second.std_glob; auto errmsg = vtab_manager->register_vtab(lst); if (!errmsg.empty()) { #if 0 @@ -2343,10 +2346,12 @@ external_log_format::match_samples(const std::vector& samples) const class external_log_table : public log_format_vtab_impl { public: - external_log_table(const external_log_format& elf) - : log_format_vtab_impl(elf), elt_format(elf){}; + explicit external_log_table(const external_log_format& elf) + : log_format_vtab_impl(elf), elt_format(elf) + { + } - void get_columns(std::vector& cols) const + void get_columns(std::vector& cols) const override { const external_log_format& elf = this->elt_format; @@ -2368,9 +2373,9 @@ public: cols[vd->vd_meta.lvm_column].vc_collator = vd->vd_collate; cols[vd->vd_meta.lvm_column].vc_comment = vd->vd_description; } - }; + } - void get_foreign_keys(std::vector& keys_inout) const + void get_foreign_keys(std::vector& keys_inout) const override { log_vtab_impl::get_foreign_keys(keys_inout); @@ -2379,19 +2384,16 @@ public: keys_inout.emplace_back(elf_value_def.first.to_string()); } } - }; + } - virtual bool next(log_cursor& lc, logfile_sub_source& lss) + bool next(log_cursor& lc, logfile_sub_source& lss) override { - lc.lc_curr_line = lc.lc_curr_line + 1_vl; - lc.lc_sub_index = 0; - if (lc.is_eof()) { return true; } content_line_t cl(lss.at(lc.lc_curr_line)); - auto lf = lss.find_file_ptr(cl); + auto* lf = lss.find_file_ptr(cl); auto lf_iter = lf->begin() + cl; uint8_t mod_id = lf_iter->get_module_id(); @@ -2444,12 +2446,12 @@ public: } return false; - }; + } - virtual void extract(logfile* lf, - uint64_t line_number, - shared_buffer_ref& line, - std::vector& values) + void extract(logfile* lf, + uint64_t line_number, + shared_buffer_ref& line, + std::vector& values) override { auto format = lf->get_format(); @@ -2467,7 +2469,7 @@ public: this->vi_attrs.clear(); format->annotate(line_number, line, this->vi_attrs, values, false); } - }; + } const external_log_format& elt_format; module_format elt_module_format; diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index 797441eb..cab4567a 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -347,7 +347,9 @@ public: bool elf_builtin_format{false}; struct search_table_def { - std::shared_ptr std_pattern; + std::shared_ptr> + std_pattern; + std::string std_glob; }; std::map elf_search_tables; diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index 972ca54d..893c89de 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -745,7 +745,7 @@ public: class bro_log_table : public log_format_vtab_impl { public: - bro_log_table(const bro_log_format& format) + explicit bro_log_table(const bro_log_format& format) : log_format_vtab_impl(format), blt_format(format) { } diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index 752386b4..4f6b62df 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -731,6 +731,11 @@ static struct json_path_container search_table_def_handlers = { .with_synopsis("") .with_description("The regular expression for this search table.") .for_field(&external_log_format::search_table_def::std_pattern), + json_path_handler("glob") + .with_synopsis("") + .with_description("Glob pattern used to constrain hits to messages " + "that match the given pattern.") + .for_field(&external_log_format::search_table_def::std_glob), }; static struct json_path_container search_table_handlers = { diff --git a/src/log_search_table.cc b/src/log_search_table.cc index 7ce34b52..49fcf7e7 100644 --- a/src/log_search_table.cc +++ b/src/log_search_table.cc @@ -33,30 +33,31 @@ #include "config.h" #include "sql_util.hh" -const static std::string LOG_MSG_INSTANCE = "log_msg_instance"; -static auto instance_name = intern_string::lookup("log_msg_instance"); -static auto instance_meta - = logline_value_meta(instance_name, value_kind_t::VALUE_INTEGER, 0); +const static std::string MATCH_INDEX = "match_index"; +static auto match_index_name = intern_string::lookup("match_index"); +static auto match_index_meta + = logline_value_meta(match_index_name, value_kind_t::VALUE_INTEGER, 0); log_search_table::log_search_table(pcrepp pattern, intern_string_t table_name) - : log_vtab_impl(table_name), lst_regex(std::move(pattern)), lst_instance(-1) + : log_vtab_impl(table_name), lst_regex(std::move(pattern)) { - this->vi_supports_indexes = false; this->get_columns_int(this->lst_cols); } void log_search_table::get_columns_int(std::vector& cols) { - column_namer cn; + column_namer cn{column_namer::language::SQL}; - cols.emplace_back(LOG_MSG_INSTANCE, SQLITE_INTEGER); + cols.emplace_back(MATCH_INDEX, SQLITE_INTEGER); for (int lpc = 0; lpc < this->lst_regex.get_capture_count(); lpc++) { std::string collator; std::string colname; int sqlite_type = SQLITE3_TEXT; - colname = cn.add_column(this->lst_regex.name_for_capture(lpc)); + colname = cn.add_column( + string_fragment{this->lst_regex.name_for_capture(lpc)}) + .to_string(); if (this->lst_regex.captures().size() == (size_t) this->lst_regex.get_capture_count()) { @@ -93,25 +94,39 @@ void log_search_table::get_foreign_keys(std::vector& keys_inout) const { log_vtab_impl::get_foreign_keys(keys_inout); - keys_inout.emplace_back("log_msg_instance"); + keys_inout.emplace_back(MATCH_INDEX); } bool log_search_table::next(log_cursor& lc, logfile_sub_source& lss) { - if (lc.lc_curr_line == -1_vl) { - this->lst_instance = -1; + if (this->lst_match_index >= 0) { + this->lst_input.pi_offset = this->lst_input.pi_next_offset; + if (this->lst_regex.match( + this->lst_match_context, this->lst_input, PCRE_NO_UTF8_CHECK)) + { + this->lst_match_index += 1; + return true; + } + + lc.lc_curr_line += 1_vl; + lc.lc_sub_index = 0; } - lc.lc_curr_line = lc.lc_curr_line + 1_vl; - lc.lc_sub_index = 0; + this->lst_match_index = -1; - if (lc.lc_curr_line == (int) lss.text_line_count()) { + while (!lc.is_eof() && !this->is_valid(lc, lss)) { + lc.lc_curr_line += 1_vl; + lc.lc_sub_index = 0; + } + + if (lc.is_eof()) { return true; } auto cl = lss.at(lc.lc_curr_line); - auto lf = lss.find(cl); + auto* lf = lss.find_file_ptr(cl); + auto lf_iter = lf->begin() + cl; if (!lf_iter->is_message()) { @@ -124,14 +139,16 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss) lf->read_full_message(lf_iter, this->lst_current_line); lf->get_format()->annotate( cl, this->lst_current_line, sa, line_values, false); - pcre_input pi( + this->lst_input.reset( this->lst_current_line.get_data(), 0, this->lst_current_line.length()); - if (!this->lst_regex.match(this->lst_match_context, pi)) { + if (!this->lst_regex.match( + this->lst_match_context, this->lst_input, PCRE_NO_UTF8_CHECK)) + { return false; } - this->lst_instance += 1; + this->lst_match_index = 0; return true; } @@ -142,11 +159,30 @@ log_search_table::extract(logfile* lf, shared_buffer_ref& line, std::vector& values) { - values.emplace_back(instance_meta, this->lst_instance); + values.emplace_back(match_index_meta, this->lst_match_index); for (int lpc = 0; lpc < this->lst_regex.get_capture_count(); lpc++) { - auto cap = this->lst_match_context[lpc]; + const auto* cap = this->lst_match_context[lpc]; values.emplace_back(this->lst_column_metas[lpc], line, line_range{cap->c_begin, cap->c_end}); } } + +void +log_search_table::get_primary_keys(std::vector& keys_out) const +{ + keys_out.emplace_back("log_line"); + keys_out.emplace_back("match_index"); +} + +void +log_search_table::filter(log_cursor& lc, logfile_sub_source& lss) +{ + if (this->lst_format != nullptr) { + lc.lc_format_name = this->lst_format->get_name(); + } + if (!this->lst_log_path_glob.empty()) { + lc.lc_log_path.emplace_back(SQLITE_INDEX_CONSTRAINT_GLOB, + this->lst_log_path_glob); + } +} diff --git a/src/log_search_table.hh b/src/log_search_table.hh index 731a8afd..43c4feb1 100644 --- a/src/log_search_table.hh +++ b/src/log_search_table.hh @@ -41,19 +41,19 @@ class log_search_table : public log_vtab_impl { public: - static int pattern_options() - { - return PCRE_CASELESS; - } + static int pattern_options() { return PCRE_CASELESS | PCRE_MULTILINE; } log_search_table(pcrepp pattern, intern_string_t table_name); + void get_primary_keys(std::vector& keys_out) const override; + void get_columns_int(std::vector& cols); void get_columns(std::vector& cols) const override { cols = this->lst_cols; } + void filter(log_cursor& lc, logfile_sub_source& lss) override; void get_foreign_keys(std::vector& keys_inout) const override; @@ -65,10 +65,13 @@ public: std::vector& values) override; pcrepp lst_regex; + log_format* lst_format{nullptr}; + std::string lst_log_path_glob; shared_buffer_ref lst_current_line; + pcre_input lst_input{""}; pcre_context_static<128> lst_match_context; std::vector lst_column_metas; - int64_t lst_instance; + int64_t lst_match_index{-1}; std::vector lst_cols; }; diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc index 64bffe49..7b5122fd 100644 --- a/src/log_vtab_impl.cc +++ b/src/log_vtab_impl.cc @@ -48,7 +48,7 @@ static struct log_cursor log_cursor_latest; thread_local _log_vtab_data log_vtab_data; static const char* LOG_COLUMNS = R"( ( - log_line INTEGER PRIMARY KEY, -- The line number for the log message + log_line INTEGER, -- The line number for the log message log_part TEXT COLLATE naturalnocase, -- The partition the message is in log_time DATETIME, -- The adjusted timestamp for the log message log_actual_time DATETIME HIDDEN, -- The timestamp from the original log file for this message @@ -72,9 +72,20 @@ static const char* LOG_FOOTER_COLUMNS = R"( log_text TEXT HIDDEN, -- The full text of the log message log_body TEXT HIDDEN, -- The body of the log message log_raw_text TEXT HIDDEN -- The raw text from the log file -); )"; +enum class log_footer_columns : uint32_t { + opid, + format, + format_regex, + time_msecs, + path, + unique_path, + text, + body, + raw_text, +}; + static const char* type_to_string(int type) { @@ -132,6 +143,29 @@ log_vtab_impl::get_table_statement() } oss << LOG_FOOTER_COLUMNS; + { + std::vector primary_keys; + + this->get_primary_keys(primary_keys); + if (!primary_keys.empty()) { + auto first = true; + + oss << ", PRIMARY KEY ("; + for (const auto& pkey : primary_keys) { + if (!first) { + oss << ", "; + } + oss << pkey; + first = false; + } + oss << ")\n"; + } else { + oss << ", PRIMARY KEY (log_line)\n"; + } + } + + oss << ");\n"; + log_debug("log_vtab_impl.get_table_statement() -> %s", oss.str().c_str()); return oss.str(); @@ -146,7 +180,7 @@ log_vtab_impl::logline_value_to_sqlite_type(value_kind_t kind) switch (kind) { case value_kind_t::VALUE_JSON: type = SQLITE3_TEXT; - subtype = 74; + subtype = JSON_SUBTYPE; break; case value_kind_t::VALUE_NULL: case value_kind_t::VALUE_TEXT: @@ -201,17 +235,43 @@ log_vtab_impl::is_valid(log_cursor& lc, logfile_sub_source& lss) auto* lf = lss.find_file_ptr(cl); auto lf_iter = lf->begin() + cl; + if (!lc.lc_format_name.empty() + && lc.lc_format_name != lf->get_format_name()) { + return false; + } + if (!lf_iter->is_message()) { return false; } - if (lc.lc_log_path && lf->get_filename() != lc.lc_log_path.value()) { - return false; + if (!lc.lc_log_path.empty()) { + if (lf == lc.lc_last_log_path_match) { + } else if (lf == lc.lc_last_log_path_mismatch) { + return false; + } else { + for (const auto& path_cons : lc.lc_log_path) { + if (!path_cons.matches(lf->get_filename())) { + lc.lc_last_log_path_mismatch = lf; + return false; + } + } + lc.lc_last_log_path_match = lf; + } } - if (lc.lc_unique_path && lf->get_unique_path() != lc.lc_unique_path.value()) - { - return false; + if (!lc.lc_unique_path.empty()) { + if (lf == lc.lc_last_unique_path_match) { + } else if (lf == lc.lc_last_unique_path_mismatch) { + return false; + } else { + for (const auto& path_cons : lc.lc_unique_path) { + if (!path_cons.matches(lf->get_unique_path())) { + lc.lc_last_unique_path_mismatch = lf; + return false; + } + } + lc.lc_last_unique_path_match = lf; + } } if (lc.lc_opid && lf_iter->get_opid() != lc.lc_opid.value().value) { @@ -311,7 +371,7 @@ vt_open(sqlite3_vtab* p_svt, sqlite3_vtab_cursor** pp_cursor) { vtab* p_vt = (vtab*) p_svt; - p_vt->base.zErrMsg = NULL; + p_vt->base.zErrMsg = nullptr; vtab_cursor* p_cur = new vtab_cursor(); @@ -319,10 +379,9 @@ vt_open(sqlite3_vtab* p_svt, sqlite3_vtab_cursor** pp_cursor) p_cur->base.pVtab = p_svt; p_cur->log_cursor.lc_opid = nonstd::nullopt; - p_cur->log_cursor.lc_curr_line = -1_vl; + p_cur->log_cursor.lc_curr_line = 0_vl; p_cur->log_cursor.lc_end_line = vis_line_t(p_vt->lss->text_line_count()); p_cur->log_cursor.lc_sub_index = 0; - vt_next((sqlite3_vtab_cursor*) p_cur); return SQLITE_OK; } @@ -330,7 +389,7 @@ vt_open(sqlite3_vtab* p_svt, sqlite3_vtab_cursor** pp_cursor) static int vt_close(sqlite3_vtab_cursor* cur) { - vtab_cursor* p_cur = (vtab_cursor*) cur; + auto* p_cur = (vtab_cursor*) cur; /* Free cursor struct. */ delete p_cur; @@ -341,7 +400,7 @@ vt_close(sqlite3_vtab_cursor* cur) static int vt_eof(sqlite3_vtab_cursor* cur) { - vtab_cursor* vc = (vtab_cursor*) cur; + auto* vc = (vtab_cursor*) cur; return vc->log_cursor.is_eof(); } @@ -349,20 +408,62 @@ vt_eof(sqlite3_vtab_cursor* cur) static int vt_next(sqlite3_vtab_cursor* cur) { - vtab_cursor* vc = (vtab_cursor*) cur; - vtab* vt = (vtab*) cur->pVtab; - bool done = false; + auto* vc = (vtab_cursor*) cur; + auto* vt = (vtab*) cur->pVtab; + auto done = false; + + vc->line_values.clear(); + vc->log_cursor.lc_curr_line += 1_vl; + vc->log_cursor.lc_sub_index = 0; + do { + log_cursor_latest = vc->log_cursor; + if (((log_cursor_latest.lc_curr_line % 1024) == 0) + && (log_vtab_data.lvd_progress != nullptr + && log_vtab_data.lvd_progress(log_cursor_latest))) + { + break; + } + while (vc->log_cursor.lc_curr_line != -1_vl && !vc->log_cursor.is_eof() + && !vt->vi->is_valid(vc->log_cursor, *vt->lss)) + { + vc->log_cursor.lc_curr_line += 1_vl; + vc->log_cursor.lc_sub_index = 0; + } + if (vc->log_cursor.is_eof()) { + done = true; + } else { + done = vt->vi->next(vc->log_cursor, *vt->lss); + if (!done) { + vc->log_cursor.lc_curr_line += 1_vl; + vc->log_cursor.lc_sub_index = 0; + } + } + } while (!done); + + return SQLITE_OK; +} + +static int +vt_next_no_rowid(sqlite3_vtab_cursor* cur) +{ + auto* vc = (vtab_cursor*) cur; + auto* vt = (vtab*) cur->pVtab; + auto done = false; vc->line_values.clear(); do { log_cursor_latest = vc->log_cursor; if (((log_cursor_latest.lc_curr_line % 1024) == 0) - && (log_vtab_data.lvd_progress != NULL + && (log_vtab_data.lvd_progress != nullptr && log_vtab_data.lvd_progress(log_cursor_latest))) { break; } done = vt->vi->next(vc->log_cursor, *vt->lss); + if (!done) { + vc->log_cursor.lc_curr_line += 1_vl; + vc->log_cursor.lc_sub_index = 0; + } } while (!done); return SQLITE_OK; @@ -371,13 +472,13 @@ vt_next(sqlite3_vtab_cursor* cur) static int vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) { - vtab_cursor* vc = (vtab_cursor*) cur; - vtab* vt = (vtab*) cur->pVtab; + auto* vc = (vtab_cursor*) cur; + auto* vt = (vtab*) cur->pVtab; content_line_t cl(vt->lss->at(vc->log_cursor.lc_curr_line)); uint64_t line_number; auto ld = vt->lss->find_data(cl, line_number); - auto lf = (*ld)->get_file_ptr(); + auto* lf = (*ld)->get_file_ptr(); auto ll = lf->begin() + line_number; require(col >= 0); @@ -386,11 +487,12 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) switch (col) { case VT_COL_LINE_NUMBER: { sqlite3_result_int64(ctx, vc->log_cursor.lc_curr_line); - } break; + break; + } case VT_COL_PARTITION: { - vis_bookmarks& vb = vt->tc->get_bookmarks(); - bookmark_vector& bv = vb[&textview_curses::BM_META]; + auto& vb = vt->tc->get_bookmarks(); + const auto& bv = vb[&textview_curses::BM_META]; if (bv.empty()) { sqlite3_result_null(ctx); @@ -401,12 +503,8 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) if (iter != bv.begin()) { --iter; content_line_t part_line = vt->lss->at(*iter); - std::map& bm_meta - = vt->lss->get_user_bookmark_metadata(); - std::map::iterator - meta_iter; - - meta_iter = bm_meta.find(part_line); + auto& bm_meta = vt->lss->get_user_bookmark_metadata(); + auto meta_iter = bm_meta.find(part_line); if (meta_iter != bm_meta.end() && !meta_iter->second.bm_name.empty()) { sqlite3_result_text(ctx, @@ -420,7 +518,8 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) sqlite3_result_null(ctx); } } - } break; + break; + } case VT_COL_LOG_TIME: { char buffer[64]; @@ -428,7 +527,8 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) sql_strftime( buffer, sizeof(buffer), ll->get_time(), ll->get_millis()); sqlite3_result_text(ctx, buffer, strlen(buffer), SQLITE_TRANSIENT); - } break; + break; + } case VT_COL_LOG_ACTUAL_TIME: { char buffer[64]; @@ -445,7 +545,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) time_range = find_string_attr_range(vt->vi->vi_attrs, &logline::L_TIMESTAMP); - const char* time_src + const auto* time_src = vc->log_msg.get_data() + time_range.lr_start; struct timeval actual_tv; struct exttm tm; @@ -507,7 +607,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) if (bm_iter == bm.end() || bm_iter->second.bm_comment.empty()) { sqlite3_result_null(ctx); } else { - const bookmark_metadata& meta = bm_iter->second; + const auto& meta = bm_iter->second; sqlite3_result_text(ctx, meta.bm_comment.c_str(), meta.bm_comment.length(), @@ -523,7 +623,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) if (bm_iter == bm.end() || bm_iter->second.bm_tags.empty()) { sqlite3_result_null(ctx); } else { - const bookmark_metadata& meta = bm_iter->second; + const auto& meta = bm_iter->second; yajlpp_gen gen; @@ -541,19 +641,19 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) sqlite3_result_text( ctx, sf.data(), sf.length(), SQLITE_TRANSIENT); - sqlite3_result_subtype(ctx, 'J'); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); } break; } case VT_COL_FILTERS: { - auto& filter_mask + const auto& filter_mask = (*ld)->ld_filter_state.lfo_filter_state.tfs_mask; if (!filter_mask[line_number]) { sqlite3_result_null(ctx); } else { - auto& filters = vt->lss->get_filters(); + const auto& filters = vt->lss->get_filters(); yajlpp_gen gen; yajl_gen_config(gen, yajl_gen_beautify, false); @@ -561,7 +661,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) { yajlpp_array arr(gen); - for (auto& filter : filters) { + for (const auto& filter : filters) { if (filter->lf_deleted) { continue; } @@ -575,18 +675,18 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) } to_sqlite(ctx, gen.to_string_fragment()); - sqlite3_result_subtype(ctx, 'J'); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); } break; } default: if (col > (VT_COL_MAX + vt->vi->vi_column_count - 1)) { - int post_col_number - = col - (VT_COL_MAX + vt->vi->vi_column_count - 1) - 1; + auto footer_column = static_cast( + col - (VT_COL_MAX + vt->vi->vi_column_count - 1) - 1); - switch (post_col_number) { - case 0: { + switch (footer_column) { + case log_footer_columns::opid: { if (vc->line_values.empty()) { lf->read_full_message(ll, vc->log_msg); vt->vi->extract( @@ -608,7 +708,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) } break; } - case 1: { + case log_footer_columns::format: { auto format_name = lf->get_format_name(); sqlite3_result_text(ctx, format_name.get(), @@ -616,7 +716,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) SQLITE_STATIC); break; } - case 2: { + case log_footer_columns::format_regex: { auto pat_name = lf->get_format()->get_pattern_name(line_number); sqlite3_result_text(ctx, @@ -625,25 +725,25 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) SQLITE_STATIC); break; } - case 3: { + case log_footer_columns::time_msecs: { sqlite3_result_int64(ctx, ll->get_time_in_millis()); break; } - case 4: { + case log_footer_columns::path: { const auto& fn = lf->get_filename(); sqlite3_result_text( ctx, fn.c_str(), fn.length(), SQLITE_STATIC); break; } - case 5: { + case log_footer_columns::unique_path: { const auto& fn = lf->get_unique_path(); sqlite3_result_text( ctx, fn.c_str(), fn.length(), SQLITE_STATIC); break; } - case 6: { + case log_footer_columns::text: { shared_buffer_ref line; lf->read_full_message(ll, line); @@ -653,7 +753,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) SQLITE_TRANSIENT); break; } - case 7: { + case log_footer_columns::body: { if (vc->line_values.empty()) { lf->read_full_message(ll, vc->log_msg); vt->vi->extract( @@ -676,7 +776,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) } break; } - case 8: { + case log_footer_columns::raw_text: { auto read_res = lf->read_raw_message(ll); if (read_res.isErr()) { @@ -704,11 +804,9 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) } int sub_col = col - VT_COL_MAX; - std::vector::iterator lv_iter; - - lv_iter = find_if(vc->line_values.begin(), - vc->line_values.end(), - logline_value_cmp(NULL, sub_col)); + auto lv_iter = find_if(vc->line_values.begin(), + vc->line_values.end(), + logline_value_cmp(nullptr, sub_col)); if (lv_iter != vc->line_values.end()) { if (!lv_iter->lv_meta.lvm_struct_name.empty()) { @@ -718,7 +816,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) { yajlpp_map root(gen); - for (auto& lv_struct : vc->line_values) { + for (const auto& lv_struct : vc->line_values) { if (lv_struct.lv_meta.lvm_column != sub_col) { continue; } @@ -751,7 +849,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) nullptr, &jo)); - auto json_in + const auto* json_in = (const unsigned char*) lv_struct.text_value(); auto json_len = lv_struct.text_length(); @@ -783,7 +881,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) auto sf = gen.to_string_fragment(); sqlite3_result_text( ctx, sf.data(), sf.length(), SQLITE_TRANSIENT); - sqlite3_result_subtype(ctx, 74); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); } else { switch (lv_iter->lv_meta.lvm_kind) { case value_kind_t::VALUE_NULL: @@ -794,7 +892,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) lv_iter->text_value(), lv_iter->text_length(), SQLITE_TRANSIENT); - sqlite3_result_subtype(ctx, 74); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); break; } case value_kind_t::VALUE_STRUCT: @@ -896,36 +994,121 @@ vt_rowid(sqlite3_vtab_cursor* cur, sqlite_int64* p_rowid) void log_cursor::update(unsigned char op, vis_line_t vl, constraint_t cons) { - if (vl < 0) { - vl = -1_vl; - } - this->lc_opid = nonstd::nullopt; switch (op) { case SQLITE_INDEX_CONSTRAINT_EQ: - if (vl < this->lc_end_line) { + if (vl < 0_vl) { + this->lc_curr_line = this->lc_end_line; + } else if (vl < this->lc_end_line) { this->lc_curr_line = vl; if (cons == constraint_t::unique) { - this->lc_end_line = vis_line_t(this->lc_curr_line + 1); + this->lc_end_line = this->lc_curr_line + 1_vl; } } break; case SQLITE_INDEX_CONSTRAINT_GE: + if (vl < 0_vl) { + vl = 0_vl; + } this->lc_curr_line = vl; break; case SQLITE_INDEX_CONSTRAINT_GT: - this->lc_curr_line - = vis_line_t(vl + (cons == constraint_t::unique ? 1 : 0)); + if (vl < 0_vl) { + this->lc_curr_line = 0_vl; + } else { + this->lc_curr_line + = vl + (cons == constraint_t::unique ? 1_vl : 0_vl); + } break; case SQLITE_INDEX_CONSTRAINT_LE: - this->lc_end_line - = vis_line_t(vl + (cons == constraint_t::unique ? 1 : 0)); + if (vl < 0_vl) { + this->lc_curr_line = this->lc_end_line; + } else { + this->lc_end_line + = vl + (cons == constraint_t::unique ? 1_vl : 0_vl); + } break; case SQLITE_INDEX_CONSTRAINT_LT: - this->lc_end_line = vl; + if (vl <= 0_vl) { + this->lc_curr_line = this->lc_end_line; + } else { + this->lc_end_line = vl; + } break; } } +log_cursor::string_constraint::string_constraint(unsigned char op, + std::string value) + : sc_op(op), sc_value(std::move(value)) +{ + if (op == SQLITE_INDEX_CONSTRAINT_REGEXP) { + try { + this->sc_pattern + = std::make_shared(this->sc_value, PCRE_UTF8); + } catch (const pcrepp::error& err) { + log_error("unable to compile regexp constraint: %s -- %s", + this->sc_value.c_str(), + err.e_msg.c_str()); + } + } +} + +bool +log_cursor::string_constraint::matches(const std::string& sf) const +{ + switch (this->sc_op) { + case SQLITE_INDEX_CONSTRAINT_EQ: + case SQLITE_INDEX_CONSTRAINT_IS: + return sf == this->sc_value; + case SQLITE_INDEX_CONSTRAINT_NE: + case SQLITE_INDEX_CONSTRAINT_ISNOT: + return sf != this->sc_value; + case SQLITE_INDEX_CONSTRAINT_GT: + return sf > this->sc_value; + case SQLITE_INDEX_CONSTRAINT_LE: + return sf <= this->sc_value; + case SQLITE_INDEX_CONSTRAINT_LT: + return sf < this->sc_value; + case SQLITE_INDEX_CONSTRAINT_GE: + return sf >= this->sc_value; + case SQLITE_INDEX_CONSTRAINT_LIKE: + return sqlite3_strlike(this->sc_value.c_str(), sf.data(), 0) == 0; + case SQLITE_INDEX_CONSTRAINT_GLOB: + return sqlite3_strglob(this->sc_value.c_str(), sf.data()) == 0; + case SQLITE_INDEX_CONSTRAINT_REGEXP: { + if (this->sc_pattern != nullptr) { + pcre_context_static<30> pc; + pcre_input pi(sf); + + return this->sc_pattern->match(pc, pi, PCRE_NO_UTF8_CHECK); + } + // return true here so that the regexp is actually run and fails + return true; + } + case SQLITE_INDEX_CONSTRAINT_ISNOTNULL: + return true; + default: + return false; + } +} + +struct time_range { + nonstd::optional tr_begin; + nonstd::optional tr_end; + + bool empty() const { return !this->tr_begin && !this->tr_end; } + + void add(const timeval& tv) + { + if (!this->tr_begin || tv < this->tr_begin) { + this->tr_begin = tv; + } + if (!this->tr_end || this->tr_end < tv) { + this->tr_end = tv; + } + } +}; + static int vt_filter(sqlite3_vtab_cursor* p_vtc, int idxNum, @@ -933,68 +1116,103 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, int argc, sqlite3_value** argv) { - vtab_cursor* p_cur = (vtab_cursor*) p_vtc; - vtab* vt = (vtab*) p_vtc->pVtab; - sqlite3_index_info::sqlite3_index_constraint* index - = (sqlite3_index_info::sqlite3_index_constraint*) idxStr; + auto* p_cur = (vtab_cursor*) p_vtc; + auto* vt = (vtab*) p_vtc->pVtab; + sqlite3_index_info::sqlite3_index_constraint* index = nullptr; - log_info("(%p) filter called: %d", vt, idxNum); - p_cur->log_cursor.lc_opid = nonstd::nullopt; - p_cur->log_cursor.lc_curr_line = -1_vl; - p_cur->log_cursor.lc_end_line = vis_line_t(vt->lss->text_line_count()); - vt_next(p_vtc); - - if (!idxNum) { - return SQLITE_OK; + if (idxStr) { + auto desc_len = strlen(idxStr); + auto index_len = idxNum * sizeof(*index); + auto storage_len = desc_len + 128 + index_len; + auto* remaining_storage = const_cast( + static_cast(idxStr + desc_len + 1)); + auto* index_storage + = std::align(alignof(sqlite3_index_info::sqlite3_index_constraint), + index_len, + remaining_storage, + storage_len); + index = reinterpret_cast( + index_storage); } + log_info("vt_filter(%s, %d)", vt->vi->get_name().get(), idxNum); + p_cur->log_cursor.lc_opid = nonstd::nullopt; + p_cur->log_cursor.lc_curr_line = 0_vl; + p_cur->log_cursor.lc_end_line = vis_line_t(vt->lss->text_line_count()); + + nonstd::optional log_time_range; + nonstd::optional opid_val; + std::vector log_path_constraints; + std::vector log_unique_path_constraints; + for (int lpc = 0; lpc < idxNum; lpc++) { auto col = index[lpc].iColumn; + auto op = index[lpc].op; switch (col) { - case VT_COL_LINE_NUMBER: + case VT_COL_LINE_NUMBER: { + auto vl = vis_line_t(sqlite3_value_int64(argv[lpc])); + p_cur->log_cursor.update( - index[lpc].op, - vis_line_t(sqlite3_value_int64(argv[lpc])), - log_cursor::constraint_t::unique); + op, vl, log_cursor::constraint_t::unique); break; + } case VT_COL_LOG_TIME: if (sqlite3_value_type(argv[lpc]) == SQLITE3_TEXT) { - const unsigned char* datestr - = sqlite3_value_text(argv[lpc]); + const auto* datestr + = (const char*) sqlite3_value_text(argv[lpc]); + auto datelen = sqlite3_value_bytes(argv[lpc]); date_time_scanner dts; struct timeval tv; struct exttm mytm; - dts.scan((const char*) datestr, - strlen((const char*) datestr), - nullptr, - &mytm, - tv); - auto vl_opt = vt->lss->find_from_time(tv); - if (!vl_opt) { - p_cur->log_cursor.lc_curr_line - = p_cur->log_cursor.lc_end_line; + const auto* date_end + = dts.scan(datestr, datelen, nullptr, &mytm, tv); + if (date_end != (datestr + datelen)) { + log_warning( + " log_time constraint is not a valid datetime, " + "index will not be applied: %s", + datestr); } else { - p_cur->log_cursor.update( - index[lpc].op, - vl_opt.value(), - log_cursor::constraint_t::none); + switch (op) { + case SQLITE_INDEX_CONSTRAINT_EQ: + case SQLITE_INDEX_CONSTRAINT_IS: + if (!log_time_range) { + log_time_range = time_range{}; + } + log_time_range->add(tv); + break; + case SQLITE_INDEX_CONSTRAINT_GT: + case SQLITE_INDEX_CONSTRAINT_GE: + if (!log_time_range) { + log_time_range = time_range{}; + } + log_time_range->tr_begin = tv; + break; + case SQLITE_INDEX_CONSTRAINT_LT: + case SQLITE_INDEX_CONSTRAINT_LE: + if (!log_time_range) { + log_time_range = time_range{}; + } + log_time_range->tr_end = tv; + break; + } } + } else { + log_warning( + " log_time constraint is not text, index will not be " + "applied: value_type(%d)=%d", + lpc, + sqlite3_value_type(argv[lpc])); } break; default: { if (col > (VT_COL_MAX + vt->vi->vi_column_count - 1)) { - int post_col_number - = col - (VT_COL_MAX + vt->vi->vi_column_count - 1) - 1; - nonstd::optional min_time; - nonstd::optional max_time; - nonstd::optional opid_val; - nonstd::optional log_path; - nonstd::optional unique_path; + auto footer_column = static_cast( + col - (VT_COL_MAX + vt->vi->vi_column_count - 1) - 1); - switch (post_col_number) { - case 0: { + switch (footer_column) { + case log_footer_columns::opid: { if (sqlite3_value_type(argv[lpc]) != SQLITE3_TEXT) { continue; } @@ -1002,6 +1220,9 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, = (const char*) sqlite3_value_text(argv[lpc]); auto opid_len = sqlite3_value_bytes(argv[lpc]); auto opid = string_fragment{opid_str, 0, opid_len}; + if (!log_time_range) { + log_time_range = time_range{}; + } for (const auto& file_data : *vt->lss) { if (file_data->get_file_ptr() == nullptr) { continue; @@ -1013,25 +1234,16 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, if (iter == r_opid_map->end()) { continue; } - if (!min_time - || iter->second.otr_begin - < min_time.value()) { - min_time = iter->second.otr_begin; - } - if (!max_time - || max_time.value() < iter->second.otr_end) - { - max_time = iter->second.otr_end; - } + log_time_range->add(iter->second.otr_begin); + log_time_range->add(iter->second.otr_end); } opid_val = log_cursor::opid_hash{ static_cast( hash_str(opid_str, opid_len))}; - log_debug("filter opid %d", opid_val.value().value); break; } - case 3: { + case log_footer_columns::path: { if (sqlite3_value_type(argv[lpc]) != SQLITE3_TEXT) { continue; } @@ -1039,24 +1251,34 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, const auto* filename = (const char*) sqlite3_value_text(argv[lpc]); auto fn_len = sqlite3_value_bytes(argv[lpc]); - const auto fn_str = std::string(filename, fn_len); + const auto fn_constraint + = log_cursor::string_constraint{ + op, std::string(filename, fn_len)}; + auto found = false; + if (!log_time_range) { + log_time_range = time_range{}; + } for (const auto& file_data : *vt->lss) { auto* lf = file_data->get_file_ptr(); if (lf == nullptr) { continue; } - if (fn_str == lf->get_filename()) { - min_time = lf->front().get_timeval(); - max_time = lf->back().get_timeval(); + if (fn_constraint.matches(lf->get_filename())) { + found = true; + log_time_range->add( + lf->front().get_timeval()); + log_time_range->add( + lf->back().get_timeval()); } } - if (min_time) { - log_path = std::move(fn_str); + if (found) { + log_path_constraints.emplace_back( + fn_constraint); } break; } - case 4: { + case log_footer_columns::unique_path: { if (sqlite3_value_type(argv[lpc]) != SQLITE3_TEXT) { continue; } @@ -1064,69 +1286,87 @@ vt_filter(sqlite3_vtab_cursor* p_vtc, const auto* filename = (const char*) sqlite3_value_text(argv[lpc]); auto fn_len = sqlite3_value_bytes(argv[lpc]); - const auto fn_str = std::string(filename, fn_len); + const auto fn_constraint + = log_cursor::string_constraint{ + op, std::string(filename, fn_len)}; + auto found = false; + if (!log_time_range) { + log_time_range = time_range{}; + } for (const auto& file_data : *vt->lss) { auto* lf = file_data->get_file_ptr(); if (lf == nullptr) { continue; } - if (fn_str == lf->get_unique_path()) { - min_time = lf->front().get_timeval(); - max_time = lf->back().get_timeval(); + if (fn_constraint.matches( + lf->get_unique_path())) { + found = true; + log_time_range->add( + lf->front().get_timeval()); + log_time_range->add( + lf->back().get_timeval()); } } - if (min_time) { - unique_path = std::move(fn_str); + if (found) { + log_unique_path_constraints.emplace_back( + fn_constraint); } break; } } - - if (!min_time) { - log_debug("no min time"); - p_cur->log_cursor.lc_curr_line - = p_cur->log_cursor.lc_end_line; - continue; - } - - log_debug("found min time: %d.%06d", - min_time.value().tv_sec, - min_time.value().tv_usec); - auto vl_opt = vt->lss->row_for_time(min_time.value()); - if (!vl_opt) { - log_debug("time not found"); - p_cur->log_cursor.lc_curr_line - = p_cur->log_cursor.lc_end_line; - continue; - } - auto vl_max_opt = vt->lss->row_for_time(max_time.value()); - - log_debug("got row %d", (int) vl_opt.value()); - p_cur->log_cursor.update(index[lpc].op, - vl_opt.value(), - log_cursor::constraint_t::none); - if (vl_max_opt) { - log_debug("got max row %d", (int) vl_max_opt.value()); - p_cur->log_cursor.lc_end_line - = vl_max_opt.value() + 1_vl; - } - p_cur->log_cursor.lc_opid = opid_val; - p_cur->log_cursor.lc_log_path = log_path; - p_cur->log_cursor.lc_unique_path = unique_path; } break; } } } - while (!p_cur->log_cursor.is_eof() - && !vt->vi->is_valid(p_cur->log_cursor, *vt->lss)) - { - p_cur->log_cursor.lc_curr_line += 1_vl; + if (!log_time_range) { + } else if (log_time_range->empty()) { + p_cur->log_cursor.lc_curr_line = p_cur->log_cursor.lc_end_line; + } else { + if (log_time_range->tr_begin) { + auto vl_opt + = vt->lss->row_for_time(log_time_range->tr_begin.value()); + if (!vl_opt) { + p_cur->log_cursor.lc_curr_line = p_cur->log_cursor.lc_end_line; + } else { + p_cur->log_cursor.lc_curr_line = vl_opt.value(); + } + } + if (log_time_range->tr_end) { + auto vl_max_opt + = vt->lss->row_for_time(log_time_range->tr_end.value()); + if (vl_max_opt) { + p_cur->log_cursor.lc_end_line = vl_max_opt.value(); + for (const auto& msg_info : + vt->lss->window_at(vl_max_opt.value())) { + if (log_time_range->tr_end.value() + < msg_info.get_logline().get_timeval()) { + break; + } + p_cur->log_cursor.lc_end_line + = msg_info.get_vis_line() + 1_vl; + } + } + } } - log_debug("cursor %d %d", + p_cur->log_cursor.lc_opid = opid_val; + p_cur->log_cursor.lc_log_path = std::move(log_path_constraints); + p_cur->log_cursor.lc_unique_path = std::move(log_unique_path_constraints); + + vt->vi->filter(p_cur->log_cursor, *vt->lss); + + while (!p_cur->log_cursor.is_eof() + && (!vt->vi->is_valid(p_cur->log_cursor, *vt->lss) + || !vt->vi->next(p_cur->log_cursor, *vt->lss))) + { + p_cur->log_cursor.lc_curr_line += 1_vl; + p_cur->log_cursor.lc_sub_index = 0; + } + + log_debug("vt_filter() -> cursor_range(%d:%d)", (int) p_cur->log_cursor.lc_curr_line, (int) p_cur->log_cursor.lc_end_line); @@ -1137,63 +1377,84 @@ static int vt_best_index(sqlite3_vtab* tab, sqlite3_index_info* p_info) { std::vector indexes; + std::vector index_desc; int argvInUse = 0; - vtab* vt = (vtab*) tab; + auto* vt = (vtab*) tab; - log_info( - "(%p) best index called: nConstraint=%d", tab, p_info->nConstraint); + log_info("vt_best_index(%s, nConstraint=%d)", + vt->vi->get_name().get(), + p_info->nConstraint); if (!vt->vi->vi_supports_indexes) { return SQLITE_OK; } for (int lpc = 0; lpc < p_info->nConstraint; lpc++) { - if (!p_info->aConstraint[lpc].usable - || p_info->aConstraint[lpc].op == SQLITE_INDEX_CONSTRAINT_MATCH + const auto& constraint = p_info->aConstraint[lpc]; + if (!constraint.usable || constraint.op == SQLITE_INDEX_CONSTRAINT_MATCH #ifdef SQLITE_INDEX_CONSTRAINT_OFFSET - || p_info->aConstraint[lpc].op == SQLITE_INDEX_CONSTRAINT_OFFSET - || p_info->aConstraint[lpc].op == SQLITE_INDEX_CONSTRAINT_LIMIT + || constraint.op == SQLITE_INDEX_CONSTRAINT_OFFSET + || constraint.op == SQLITE_INDEX_CONSTRAINT_LIMIT #endif ) { - log_debug(" [%d] not usable", lpc); + log_debug(" column %d: is not usable (usable=%d, op: %s)", + lpc, + constraint.usable, + sql_constraint_op_name(constraint.op)); continue; } - auto col = p_info->aConstraint[lpc].iColumn; - log_debug("column number %d", col); + auto col = constraint.iColumn; + auto op = constraint.op; + log_debug(" column %d: op: %s", col, sql_constraint_op_name(op)); switch (col) { case VT_COL_LINE_NUMBER: { - log_debug("line number index %d", p_info->aConstraint[lpc].op); + argvInUse += 1; + indexes.push_back(constraint); + p_info->aConstraintUsage[lpc].argvIndex = argvInUse; + index_desc.emplace_back(fmt::format( + FMT_STRING("log_line {} ?"), sql_constraint_op_name(op))); + break; + } + case VT_COL_LOG_TIME: { argvInUse += 1; indexes.push_back(p_info->aConstraint[lpc]); p_info->aConstraintUsage[lpc].argvIndex = argvInUse; + index_desc.emplace_back(fmt::format( + FMT_STRING("log_time {} ?"), sql_constraint_op_name(op))); break; } default: { if (col > (VT_COL_MAX + vt->vi->vi_column_count - 1)) { - int post_col_number - = col - (VT_COL_MAX + vt->vi->vi_column_count - 1) - 1; + auto footer_column = static_cast( + col - (VT_COL_MAX + vt->vi->vi_column_count - 1) - 1); - log_debug("post column numer %d", post_col_number); - switch (post_col_number) { - case 0: { - log_debug("opid index"); - argvInUse += 1; - indexes.push_back(p_info->aConstraint[lpc]); - p_info->aConstraintUsage[lpc].argvIndex = argvInUse; + switch (footer_column) { + case log_footer_columns::opid: { + if (op == SQLITE_INDEX_CONSTRAINT_EQ) { + argvInUse += 1; + indexes.push_back(constraint); + p_info->aConstraintUsage[lpc].argvIndex + = argvInUse; + index_desc.emplace_back("log_opid = ?"); + } break; } - case 3: { - log_debug("log_path index"); + case log_footer_columns::path: { argvInUse += 1; - indexes.push_back(p_info->aConstraint[lpc]); + indexes.push_back(constraint); p_info->aConstraintUsage[lpc].argvIndex = argvInUse; + index_desc.emplace_back( + fmt::format(FMT_STRING("log_path {} ?"), + sql_constraint_op_name(op))); break; } - case 4: { - log_debug("log_unique_path index"); + case log_footer_columns::unique_path: { argvInUse += 1; - indexes.push_back(p_info->aConstraint[lpc]); + indexes.push_back(constraint); p_info->aConstraintUsage[lpc].argvIndex = argvInUse; + index_desc.emplace_back( + fmt::format(FMT_STRING("log_unique_path {} ?"), + sql_constraint_op_name(op))); break; } } @@ -1203,45 +1464,35 @@ vt_best_index(sqlite3_vtab* tab, sqlite3_index_info* p_info) } } - if (!argvInUse) { - log_debug("fall back to log_time"); - for (int lpc = 0; lpc < p_info->nConstraint; lpc++) { - if (!p_info->aConstraint[lpc].usable - || p_info->aConstraint[lpc].op == SQLITE_INDEX_CONSTRAINT_MATCH -#ifdef SQLITE_INDEX_CONSTRAINT_OFFSET - || p_info->aConstraint[lpc].op == SQLITE_INDEX_CONSTRAINT_OFFSET - || p_info->aConstraint[lpc].op == SQLITE_INDEX_CONSTRAINT_LIMIT -#endif - ) - { - continue; - } - - switch (p_info->aConstraint[lpc].iColumn) { - case VT_COL_LOG_TIME: - argvInUse += 1; - indexes.push_back(p_info->aConstraint[lpc]); - p_info->aConstraintUsage[lpc].argvIndex = argvInUse; - break; - } - } - } - if (argvInUse) { + auto full_desc = fmt::format(FMT_STRING("SEARCH {} USING {}"), + vt->vi->get_name().get(), + fmt::join(index_desc, " AND ")); + log_info("found index: %s", full_desc.c_str()); + sqlite3_index_info::sqlite3_index_constraint* index_copy; - size_t len = indexes.size() * sizeof(*index_copy); - - log_info("found index, passing %d args", argvInUse); - - index_copy - = (sqlite3_index_info::sqlite3_index_constraint*) sqlite3_malloc( - len); - if (!index_copy) { + auto index_len = indexes.size() * sizeof(*index_copy); + size_t len = full_desc.size() + 128 + index_len; + auto* storage = sqlite3_malloc(len); + if (!storage) { return SQLITE_NOMEM; } - memcpy(index_copy, &indexes[0], len); + auto* desc_storage = static_cast(storage); + memcpy(desc_storage, full_desc.c_str(), full_desc.size() + 1); + auto* remaining_storage + = static_cast(desc_storage + full_desc.size() + 1); + len -= full_desc.size() - 1; + auto* index_storage + = std::align(alignof(sqlite3_index_info::sqlite3_index_constraint), + index_len, + remaining_storage, + len); + index_copy + = reinterpret_cast( + index_storage); + memcpy(index_copy, &indexes[0], index_len); p_info->idxNum = argvInUse; - p_info->idxStr = (char*) index_copy; + p_info->idxStr = static_cast(storage); p_info->needToFreeIdxStr = 1; p_info->estimatedCost = 10.0; } @@ -1312,13 +1563,11 @@ vt_update(sqlite3_vtab* tab, auto json_error = lnav::to_json(top_error); tab->zErrMsg = sqlite3_mprintf("lnav-error:%s", json_error.c_str()); - log_debug("dump %s", json_error.c_str()); return SQLITE_ERROR; } } - bookmark_vector& bv - = vt->tc->get_bookmarks()[&textview_curses::BM_META]; + auto& bv = vt->tc->get_bookmarks()[&textview_curses::BM_META]; bool has_meta = part_name != nullptr || log_comment != nullptr || log_tags != nullptr; @@ -1394,11 +1643,33 @@ static sqlite3_module generic_vtab_module = { vt_column, /* xColumn - read data */ vt_rowid, /* xRowid - read data */ vt_update, /* xUpdate - write data */ - NULL, /* xBegin - begin transaction */ - NULL, /* xSync - sync transaction */ - NULL, /* xCommit - commit transaction */ - NULL, /* xRollback - rollback transaction */ - NULL, /* xFindFunction - function overloading */ + nullptr, /* xBegin - begin transaction */ + nullptr, /* xSync - sync transaction */ + nullptr, /* xCommit - commit transaction */ + nullptr, /* xRollback - rollback transaction */ + nullptr, /* xFindFunction - function overloading */ +}; + +static sqlite3_module no_rowid_vtab_module = { + 0, /* iVersion */ + vt_create, /* xCreate - create a vtable */ + vt_connect, /* xConnect - associate a vtable with a connection */ + vt_best_index, /* xBestIndex - best index */ + vt_disconnect, /* xDisconnect - disassociate a vtable with a connection */ + vt_destroy, /* xDestroy - destroy a vtable */ + vt_open, /* xOpen - open a cursor */ + vt_close, /* xClose - close a cursor */ + vt_filter, /* xFilter - configure scan constraints */ + vt_next_no_rowid, /* xNext - advance a cursor */ + vt_eof, /* xEof - inidicate end of result set*/ + vt_column, /* xColumn - read data */ + nullptr, /* xRowid - read data */ + nullptr, /* xUpdate - write data */ + nullptr, /* xBegin - begin transaction */ + nullptr, /* xSync - sync transaction */ + nullptr, /* xCommit - commit transaction */ + nullptr, /* xRollback - rollback transaction */ + nullptr, /* xFindFunction - function overloading */ }; static int @@ -1420,6 +1691,8 @@ log_vtab_manager::log_vtab_manager(sqlite3* memdb, { sqlite3_create_module( this->vm_db, "log_vtab_impl", &generic_vtab_module, this); + sqlite3_create_module( + this->vm_db, "log_vtab_no_rowid_impl", &no_rowid_vtab_module, this); sqlite3_progress_handler(memdb, 32, progress_callback, nullptr); } @@ -1438,16 +1711,20 @@ log_vtab_manager::register_vtab(std::shared_ptr vi) std::string retval; if (this->vm_impls.find(vi->get_name()) == this->vm_impls.end()) { + std::vector primary_keys; auto_mem errmsg; auto_mem sql; int rc; this->vm_impls[vi->get_name()] = vi; + vi->get_primary_keys(primary_keys); + sql = sqlite3_mprintf( "CREATE VIRTUAL TABLE %s " - "USING log_vtab_impl(%s)", + "USING %s(%s)", vi->get_name().get(), + primary_keys.empty() ? "log_vtab_impl" : "log_vtab_no_rowid_impl", vi->get_name().get()); rc = sqlite3_exec(this->vm_db, sql, nullptr, nullptr, errmsg.out()); if (rc != SQLITE_OK) { @@ -1483,9 +1760,6 @@ log_vtab_manager::unregister_vtab(intern_string_t name) bool log_format_vtab_impl::next(log_cursor& lc, logfile_sub_source& lss) { - lc.lc_curr_line = lc.lc_curr_line + 1_vl; - lc.lc_sub_index = 0; - if (lc.is_eof()) { return true; } @@ -1502,7 +1776,8 @@ log_format_vtab_impl::next(log_cursor& lc, logfile_sub_source& lss) auto format = lf->get_format(); if (format->get_name() == this->lfvi_format.get_name()) { return true; - } else if (mod_id && mod_id == this->lfvi_format.lf_mod_index) { + } + if (mod_id && mod_id == this->lfvi_format.lf_mod_index) { // XXX return true; } diff --git a/src/log_vtab_impl.hh b/src/log_vtab_impl.hh index 516cfffc..5a52ed91 100644 --- a/src/log_vtab_impl.hh +++ b/src/log_vtab_impl.hh @@ -61,13 +61,28 @@ struct log_cursor { unsigned int value : 6; }; + struct string_constraint { + unsigned char sc_op; + std::string sc_value; + std::shared_ptr sc_pattern; + + string_constraint(unsigned char op, std::string value); + + bool matches(const std::string& sf) const; + }; + vis_line_t lc_curr_line; int lc_sub_index; vis_line_t lc_end_line; + intern_string_t lc_format_name; nonstd::optional lc_opid; - nonstd::optional lc_log_path; - nonstd::optional lc_unique_path; + std::vector lc_log_path; + logfile* lc_last_log_path_match{nullptr}; + logfile* lc_last_log_path_mismatch{nullptr}; + std::vector lc_unique_path; + logfile* lc_last_unique_path_match{nullptr}; + logfile* lc_last_unique_path_mismatch{nullptr}; enum class constraint_t { none, @@ -124,7 +139,7 @@ public: virtual ~log_vtab_impl() = default; - const intern_string_t get_name() const { return this->vi_name; } + intern_string_t get_name() const { return this->vi_name; } intern_string_t get_tags_name() const { return this->vi_tags_name; } @@ -132,12 +147,16 @@ public: virtual bool is_valid(log_cursor& lc, logfile_sub_source& lss); + virtual void filter(log_cursor& lc, logfile_sub_source& lss) {} + virtual bool next(log_cursor& lc, logfile_sub_source& lss) = 0; virtual void get_columns(std::vector& cols) const {} virtual void get_foreign_keys(std::vector& keys_inout) const; + virtual void get_primary_keys(std::vector& keys_out) const {} + virtual void extract(logfile* lf, uint64_t line_number, shared_buffer_ref& line, diff --git a/src/logfile.hh b/src/logfile.hh index 2fad39c0..217edbbe 100644 --- a/src/logfile.hh +++ b/src/logfile.hh @@ -153,6 +153,8 @@ public: */ std::shared_ptr get_format() const { return this->lf_format; } + log_format* get_format_ptr() const { return this->lf_format.get(); } + intern_string_t get_format_name() const; text_format_t get_text_format() const { return this->lf_text_format; } diff --git a/src/regexp_vtab.cc b/src/regexp_vtab.cc index ca444abb..1787dd7d 100644 --- a/src/regexp_vtab.cc +++ b/src/regexp_vtab.cc @@ -28,11 +28,13 @@ */ #include "base/lnav_log.hh" +#include "column_namer.hh" #include "config.h" #include "pcrepp/pcrepp.hh" #include "sql_help.hh" #include "sql_util.hh" #include "vtab_module.hh" +#include "yajlpp/yajlpp.hh" enum { RC_COL_MATCH_INDEX, @@ -71,30 +73,21 @@ CREATE TABLE regexp_capture ( std::unique_ptr c_input; std::string c_content; bool c_content_as_blob{false}; - int c_index; - int c_start_index; + int c_index{0}; bool c_matched{false}; - int c_match_index; - sqlite3_int64 c_rowid; + int c_match_index{0}; + sqlite3_int64 c_rowid{0}; - cursor(sqlite3_vtab* vt) - : base({vt}), c_index(0), c_start_index(0), c_match_index(0), - c_rowid(0) - { - this->c_context.set_count(0); - }; + cursor(sqlite3_vtab* vt) : base({vt}) { this->c_context.set_count(0); } - int reset() - { - return SQLITE_OK; - }; + int reset() { return SQLITE_OK; } int next() { if (this->c_index >= (this->c_context.get_count() - 1)) { this->c_input->pi_offset = this->c_input->pi_next_offset; - this->c_matched - = this->c_pattern.match(this->c_context, *(this->c_input)); + this->c_matched = this->c_pattern.match( + this->c_context, *(this->c_input), PCRE_NO_UTF8_CHECK); this->c_index = -1; this->c_match_index += 1; } @@ -106,24 +99,21 @@ CREATE TABLE regexp_capture ( this->c_index += 1; return SQLITE_OK; - }; + } - int eof() - { - return this->c_pattern.empty() || !this->c_matched; - }; + int eof() { return this->c_pattern.empty() || !this->c_matched; } int get_rowid(sqlite3_int64& rowid_out) { rowid_out = this->c_rowid; return SQLITE_OK; - }; + } }; int get_column(const cursor& vc, sqlite3_context* ctx, int col) { - pcre_context::capture_t& cap = vc.c_context.all()[vc.c_index]; + auto& cap = vc.c_context.all()[vc.c_index]; switch (col) { case RC_COL_MATCH_INDEX: @@ -246,9 +236,222 @@ rcFilter(sqlite3_vtab_cursor* pVtabCursor, pCur->c_context.set_count(0); pCur->c_input = std::make_unique(pCur->c_content); - pCur->c_matched = pCur->c_pattern.match(pCur->c_context, *(pCur->c_input)); + pCur->c_matched = pCur->c_pattern.match( + pCur->c_context, *(pCur->c_input), PCRE_NO_UTF8_CHECK); - log_debug("matched %d", pCur->c_matched); + return SQLITE_OK; +} + +enum { + RCJ_COL_MATCH_INDEX, + RCJ_COL_CONTENT, + RCJ_COL_VALUE, + RCJ_COL_PATTERN, +}; + +struct regexp_capture_into_json { + static constexpr const char* NAME = "regexp_capture_into_json"; + static constexpr const char* CREATE_STMT = R"( +-- The regexp_capture_into_json() table-valued function allows you to execute a +-- regular-expression over a given string and get the captured data as rows in +-- a table. +CREATE TABLE regexp_capture_into_json ( + match_index INTEGER, + content TEXT, + value TEXT HIDDEN, + pattern TEXT HIDDEN +); +)"; + + struct cursor { + sqlite3_vtab_cursor base; + pcrepp c_pattern; + pcre_context_static<30> c_context; + std::unique_ptr c_input; + std::unique_ptr c_namer; + std::string c_content; + bool c_content_as_blob{false}; + bool c_matched{false}; + size_t c_match_index{0}; + sqlite3_int64 c_rowid{0}; + + cursor(sqlite3_vtab* vt) : base({vt}) { this->c_context.set_count(0); } + + int reset() { return SQLITE_OK; } + + int next() + { + this->c_input->pi_offset = this->c_input->pi_next_offset; + this->c_matched = this->c_pattern.match( + this->c_context, *(this->c_input), PCRE_NO_UTF8_CHECK); + this->c_match_index += 1; + + if (this->c_pattern.empty() || !this->c_matched) { + return SQLITE_OK; + } + + return SQLITE_OK; + } + + int eof() { return this->c_pattern.empty() || !this->c_matched; } + + int get_rowid(sqlite3_int64& rowid_out) + { + rowid_out = this->c_rowid; + + return SQLITE_OK; + } + }; + + int get_column(const cursor& vc, sqlite3_context* ctx, int col) + { + switch (col) { + case RCJ_COL_MATCH_INDEX: + sqlite3_result_int64(ctx, vc.c_match_index); + break; + case RCJ_COL_CONTENT: { + yajlpp_gen gen; + yajl_gen_config(gen, yajl_gen_beautify, false); + + { + yajlpp_map root_map(gen); + + for (int lpc = 0; lpc < vc.c_pattern.get_capture_count(); + lpc++) { + const auto& colname = vc.c_namer->cn_names[lpc]; + const auto* cap = vc.c_context[lpc]; + + yajl_gen_pstring(gen, colname.data(), colname.length()); + + if (!cap->is_valid()) { + yajl_gen_null(gen); + } else { + auto* cap_start = vc.c_input->get_substr_start(cap); + char* cap_copy = (char*) alloca(cap->length() + 1); + long long int i_value; + double d_value; + int end_index; + + memcpy(cap_copy, cap_start, cap->length()); + cap_copy[cap->length()] = '\0'; + + if (sscanf(cap_copy, "%lld%n", &i_value, &end_index) + == 1 + && (end_index == cap->length())) + { + yajl_gen_integer(gen, i_value); + } else if (sscanf(cap_copy, + "%lf%n", + &d_value, + &end_index) + == 1 + && (end_index == cap->length())) + { + yajl_gen_number(gen, cap_start, cap->length()); + } else { + yajl_gen_pstring(gen, cap_start, cap->length()); + } + } + } + } + + auto sf = gen.to_string_fragment(); + sqlite3_result_text( + ctx, sf.data(), sf.length(), SQLITE_TRANSIENT); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + break; + } + case RCJ_COL_VALUE: + if (vc.c_content_as_blob) { + sqlite3_result_blob64(ctx, + vc.c_content.c_str(), + vc.c_content.length(), + SQLITE_STATIC); + } else { + sqlite3_result_text(ctx, + vc.c_content.c_str(), + vc.c_content.length(), + SQLITE_STATIC); + } + break; + case RCJ_COL_PATTERN: { + auto str = vc.c_pattern.get_pattern(); + + sqlite3_result_text( + ctx, str.c_str(), str.length(), SQLITE_TRANSIENT); + break; + } + } + + return SQLITE_OK; + } +}; + +static int +rcjBestIndex(sqlite3_vtab* tab, sqlite3_index_info* pIdxInfo) +{ + vtab_index_constraints vic(pIdxInfo); + vtab_index_usage viu(pIdxInfo); + + for (auto iter = vic.begin(); iter != vic.end(); ++iter) { + if (iter->op != SQLITE_INDEX_CONSTRAINT_EQ) { + continue; + } + + switch (iter->iColumn) { + case RCJ_COL_VALUE: + case RCJ_COL_PATTERN: + viu.column_used(iter); + break; + } + } + + viu.allocate_args(2); + return SQLITE_OK; +} + +static int +rcjFilter(sqlite3_vtab_cursor* pVtabCursor, + int idxNum, + const char* idxStr, + int argc, + sqlite3_value** argv) +{ + auto* pCur = (regexp_capture_into_json::cursor*) pVtabCursor; + + if (argc != 2) { + pCur->c_content.clear(); + pCur->c_pattern.clear(); + return SQLITE_OK; + } + + auto byte_count = sqlite3_value_bytes(argv[0]); + auto blob = (const char*) sqlite3_value_blob(argv[0]); + + pCur->c_content_as_blob = (sqlite3_value_type(argv[0]) == SQLITE_BLOB); + pCur->c_content.assign(blob, byte_count); + + const char* pattern = (const char*) sqlite3_value_text(argv[1]); + auto re_res = pcrepp::from_str(pattern); + if (re_res.isErr()) { + pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( + "Invalid regular expression: %s", re_res.unwrapErr().ce_msg); + return SQLITE_ERROR; + } + + pCur->c_pattern = re_res.unwrap(); + pCur->c_namer + = std::make_unique(column_namer::language::JSON); + for (int lpc = 0; lpc < pCur->c_pattern.get_capture_count(); lpc++) { + pCur->c_namer->add_column( + string_fragment{pCur->c_pattern.name_for_capture(lpc)}); + } + + pCur->c_context.set_count(0); + + pCur->c_input = std::make_unique(pCur->c_content); + pCur->c_matched = pCur->c_pattern.match( + pCur->c_context, *(pCur->c_input), PCRE_NO_UTF8_CHECK); return SQLITE_OK; } @@ -306,5 +509,43 @@ register_regexp_vtab(sqlite3* db) ensure(rc == SQLITE_OK); + static vtab_module> + REGEXP_CAPTURE_INTO_JSON_MODULE; + static help_text regexp_capture_into_json_help + = help_text( + "regexp_capture_into_json", + "A table-valued function that executes a " + "regular-expression over a string and returns the captured " + "values as a JSON object. If the regex only matches a " + "subset of the input string, it will be rerun on the " + "remaining parts of the string until no more matches are found.") + .sql_table_valued_function() + .with_parameter( + {"string", "The string to match against the given pattern."}) + .with_parameter({"pattern", "The regular expression to match."}) + .with_result({ + "match_index", + "The match iteration. This value will increase " + "each time a new match is found in the input string.", + }) + .with_result({"content", "The captured values from the string."}) + .with_tags({"string"}) + .with_example({ + "To extract the key/value pairs 'a'/1 and 'b'/2 " + "from the string 'a=1; b=2'", + "SELECT * FROM regexp_capture_into_json('a=1; b=2', " + "'(\\w+)=(\\d+)')", + }); + + REGEXP_CAPTURE_INTO_JSON_MODULE.vm_module.xBestIndex = rcjBestIndex; + REGEXP_CAPTURE_INTO_JSON_MODULE.vm_module.xFilter = rcjFilter; + + rc = REGEXP_CAPTURE_INTO_JSON_MODULE.create(db, "regexp_capture_into_json"); + sqlite_function_help.insert(std::make_pair("regexp_capture_into_json", + ®exp_capture_into_json_help)); + regexp_capture_into_json_help.index_tags(); + + ensure(rc == SQLITE_OK); + return rc; } diff --git a/src/sql_util.cc b/src/sql_util.cc index 024f1568..3c17fb3a 100644 --- a/src/sql_util.cc +++ b/src/sql_util.cc @@ -256,7 +256,28 @@ const char* sql_function_names[] = { "julianday(", "strftime(", - nullptr}; + nullptr, +}; + +const std::unordered_map sql_constraint_names = { + {SQLITE_INDEX_CONSTRAINT_EQ, "="}, + {SQLITE_INDEX_CONSTRAINT_GT, ">"}, + {SQLITE_INDEX_CONSTRAINT_LE, "<="}, + {SQLITE_INDEX_CONSTRAINT_LT, "<"}, + {SQLITE_INDEX_CONSTRAINT_GE, ">="}, + {SQLITE_INDEX_CONSTRAINT_MATCH, "MATCH"}, + {SQLITE_INDEX_CONSTRAINT_LIKE, "LIKE"}, + {SQLITE_INDEX_CONSTRAINT_GLOB, "GLOB"}, + {SQLITE_INDEX_CONSTRAINT_REGEXP, "REGEXP"}, + {SQLITE_INDEX_CONSTRAINT_NE, "!="}, + {SQLITE_INDEX_CONSTRAINT_ISNOT, "IS NOT"}, + {SQLITE_INDEX_CONSTRAINT_ISNOTNULL, "IS NOT NULL"}, + {SQLITE_INDEX_CONSTRAINT_ISNULL, "IS NULL"}, + {SQLITE_INDEX_CONSTRAINT_IS, "IS"}, + {SQLITE_INDEX_CONSTRAINT_LIMIT, "LIMIT"}, + {SQLITE_INDEX_CONSTRAINT_OFFSET, "OFFSET"}, + {SQLITE_INDEX_CONSTRAINT_FUNCTION, "function"}, +}; std::multimap sqlite_function_help; @@ -917,7 +938,7 @@ annotate_sql_statement(attr_line_t& al) { static const std::string keyword_re_str = R"(\A)" + sql_keyword_re(); - static struct { + static const struct { pcrepp re; string_attr_type* type; } PATTERNS[] = { @@ -926,7 +947,7 @@ annotate_sql_statement(attr_line_t& al) {pcrepp{R"(\A\(|\A\))"}, &SQL_PAREN_ATTR}, {pcrepp{keyword_re_str, PCRE_CASELESS}, &SQL_KEYWORD_ATTR}, {pcrepp{R"(\A'[^']*('(?:'[^']*')*|$))"}, &SQL_STRING_ATTR}, - {pcrepp{R"(\A(\$?\b[a-z_]\w*)|\"([^\"]+)\"|\[([^\]]+)])", + {pcrepp{R"(\A(((\$|:|@)?\b[a-z_]\w*)|\"([^\"]+)\"|\[([^\]]+)]))", PCRE_CASELESS}, &SQL_IDENTIFIER_ATTR}, {pcrepp{R"(\A--.*)"}, &SQL_COMMENT_ATTR}, @@ -934,7 +955,7 @@ annotate_sql_statement(attr_line_t& al) {pcrepp{R"(\A.)"}, &SQL_GARBAGE_ATTR}, }; - static pcrepp ws_pattern(R"(\A\s+)"); + static const pcrepp ws_pattern(R"(\A\s+)"); pcre_context_static<30> pc; pcre_input pi(al.get_string()); @@ -947,7 +968,7 @@ annotate_sql_statement(attr_line_t& al) } for (const auto& pat : PATTERNS) { if (pat.re.match(pc, pi, PCRE_ANCHORED)) { - pcre_context::capture_t* cap = pc.all(); + auto* cap = pc.all(); struct line_range lr(cap->c_begin, cap->c_end); sa.emplace_back(lr, pat.type->value()); diff --git a/src/sql_util.hh b/src/sql_util.hh index c7ba9cd9..5de614a4 100644 --- a/src/sql_util.hh +++ b/src/sql_util.hh @@ -34,6 +34,7 @@ #include #include +#include #include #include @@ -47,10 +48,23 @@ extern const char* sql_keywords[145]; extern const char* sql_function_names[]; +extern const std::unordered_map + sql_constraint_names; -typedef int (*sqlite_exec_callback)(void*, int, char**, char**); +inline const char* +sql_constraint_op_name(unsigned char op) +{ + auto iter = sql_constraint_names.find(op); + if (iter == sql_constraint_names.end()) { + return "??"; + } + + return iter->second; +} + +using sqlite_exec_callback = int (*)(void*, int, char**, char**); typedef std::vector db_table_list_t; -typedef std::map db_table_map_t; +using db_table_map_t = std::map; struct sqlite_metadata_callbacks { sqlite_exec_callback smc_collation_list; diff --git a/src/string-extension-functions.cc b/src/string-extension-functions.cc index cee16dcf..8774cac0 100644 --- a/src/string-extension-functions.cc +++ b/src/string-extension-functions.cc @@ -40,9 +40,11 @@ using namespace mapbox; -typedef struct { +struct cache_entry { std::shared_ptr re2; -} cache_entry; + std::shared_ptr cn{ + std::make_shared(column_namer::language::JSON)}; +}; static cache_entry* find_re(string_fragment re) @@ -59,6 +61,10 @@ find_re(string_fragment re) auto pair = cache.insert( std::make_pair(string_fragment{c.re2->get_pattern()}, c)); + for (int lpc = 0; lpc < c.re2->get_capture_count(); lpc++) { + c.cn->add_column(string_fragment{c.re2->name_for_capture(lpc)}); + } + iter = pair.first; } @@ -87,7 +93,7 @@ regexp_match(string_fragment re, const char* str) throw pcrepp::error("regular expression does not have any captures"); } - if (!extractor.match(pc, pi)) { + if (!extractor.match(pc, pi, PCRE_NO_UTF8_CHECK)) { return static_cast(nullptr); } @@ -123,14 +129,12 @@ regexp_match(string_fragment re, const char* str) return string_fragment(str, cap->c_begin, cap->c_end); } else { yajlpp_map root_map(gen); - column_namer cn; for (int lpc = 0; lpc < extractor.get_capture_count(); lpc++) { - std::string colname - = cn.add_column(extractor.name_for_capture(lpc)); - pcre_context::capture_t* cap = pc[lpc]; + const auto& colname = reobj->cn->cn_names[lpc]; + const auto* cap = pc[lpc]; - yajl_gen_string(gen, colname); + yajl_gen_pstring(gen, colname.data(), colname.length()); if (!cap->is_valid()) { yajl_gen_null(gen); diff --git a/src/time_formats.am b/src/time_formats.am index e73db950..a91b7749 100644 --- a/src/time_formats.am +++ b/src/time_formats.am @@ -25,6 +25,7 @@ TIME_FORMATS = \ "%a %b %d %H:%M:%S " \ "%a %b %d %H:%M:%S.%L " \ "%a %b %d %H:%M " \ + "%a %b %e %H:%M:%S %Z %Y" \ "%d/%b/%Y:%H:%M:%S +0000" \ "%d/%b/%Y:%H:%M:%S %z" \ "%d-%b-%Y %H:%M:%S %z" \ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d5f5df26..75f12018 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,6 +20,11 @@ add_executable(test_auto_mem test_auto_mem.cc test_stubs.cc) target_link_libraries(test_auto_mem diag) add_test(NAME test_auto_mem COMMAND test_auto_mem) +add_executable(test_column_namer test_column_namer.cc test_stubs.cc) +target_include_directories(test_column_namer PUBLIC ../src/third-party/doctest-root) +target_link_libraries(test_column_namer diag) +add_test(NAME test_column_namer COMMAND test_column_namer) + add_executable(document.sections.tests document.sections.tests.cc test_stubs.cc) target_include_directories(document.sections.tests PUBLIC ../src/third-party/doctest-root) target_link_libraries(document.sections.tests diag) diff --git a/test/Makefile.am b/test/Makefile.am index ff9f60b2..b4fe7818 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -197,11 +197,13 @@ dist_noinst_SCRIPTS = \ test_sql.sh \ test_sql_anno.sh \ test_sql_coll_func.sh \ + test_sql_fs_func.sh \ + test_sql_indexes.sh \ test_sql_json_func.sh \ + test_sql_search_table.sh \ test_sql_str_func.sh \ test_sql_time_func.sh \ test_sql_xml_func.sh \ - test_sql_fs_func.sh \ test_tui.sh \ test_view_colors.sh \ test_vt52_curses.sh \ @@ -297,6 +299,7 @@ dist_noinst_DATA = \ logfile_openam.0 \ logfile_plain.0 \ logfile_pretty.0 \ + logfile_procstate.0 \ logfile_rollover.0 \ logfile_rollover.1 \ logfile_strace_log.0 \ @@ -389,8 +392,10 @@ TESTS = \ test_sql.sh \ test_sql_anno.sh \ test_sql_coll_func.sh \ - test_sql_json_func.sh \ test_sql_fs_func.sh \ + test_sql_indexes.sh \ + test_sql_json_func.sh \ + test_sql_search_table.sh \ test_sql_str_func.sh \ test_sql_time_func.sh \ test_sql_xml_func.sh \ diff --git a/test/expected/expected.am b/test/expected/expected.am index 304067f2..0f861f83 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -420,6 +420,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql.sh_50c0b2c93b646b848a017764bde8a4282c556e2d.out \ $(srcdir)/%reldir%/test_sql.sh_528e48a03cdfa7cfbe263a6e22a65606247a8a95.err \ $(srcdir)/%reldir%/test_sql.sh_528e48a03cdfa7cfbe263a6e22a65606247a8a95.out \ + $(srcdir)/%reldir%/test_sql.sh_5532c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.err \ + $(srcdir)/%reldir%/test_sql.sh_5532c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.out \ $(srcdir)/%reldir%/test_sql.sh_56047c9470e515bc3e3709354c01e5d50462cde7.err \ $(srcdir)/%reldir%/test_sql.sh_56047c9470e515bc3e3709354c01e5d50462cde7.out \ $(srcdir)/%reldir%/test_sql.sh_57427f3c4b4ec785ffff7c5802c10db0d3e547cf.err \ @@ -440,6 +442,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql.sh_6ffd89498b9a7758ded6717148fc2ce77a12621b.out \ $(srcdir)/%reldir%/test_sql.sh_753c343a256d1286750314957d1b4e155464e03e.err \ $(srcdir)/%reldir%/test_sql.sh_753c343a256d1286750314957d1b4e155464e03e.out \ + $(srcdir)/%reldir%/test_sql.sh_764306f0e5f610ba71f521ba3d19fe158ece0ba5.err \ + $(srcdir)/%reldir%/test_sql.sh_764306f0e5f610ba71f521ba3d19fe158ece0ba5.out \ $(srcdir)/%reldir%/test_sql.sh_764ea85863d4f0ea3b7cb40850ac7c8fde682d57.err \ $(srcdir)/%reldir%/test_sql.sh_764ea85863d4f0ea3b7cb40850ac7c8fde682d57.out \ $(srcdir)/%reldir%/test_sql.sh_7f664c9cda0ae1c48333e21051b5e0eeafd5b4bc.err \ @@ -484,6 +488,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql.sh_c20b0320096342c180146a5d18a6de82319d70b2.out \ $(srcdir)/%reldir%/test_sql.sh_c353ef036c505b75996252138fbd4c8d22e8149c.err \ $(srcdir)/%reldir%/test_sql.sh_c353ef036c505b75996252138fbd4c8d22e8149c.out \ + $(srcdir)/%reldir%/test_sql.sh_c5b8da04734fadf3b9eea80e0af997e38e0fb811.err \ + $(srcdir)/%reldir%/test_sql.sh_c5b8da04734fadf3b9eea80e0af997e38e0fb811.out \ $(srcdir)/%reldir%/test_sql.sh_c73dec2706fc0b9a124f5da3a83f40d8d3255beb.err \ $(srcdir)/%reldir%/test_sql.sh_c73dec2706fc0b9a124f5da3a83f40d8d3255beb.out \ $(srcdir)/%reldir%/test_sql.sh_c7e1dbf4605914720b55787785abfafdf2c4178a.err \ @@ -506,6 +512,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql.sh_f8340cb4c62aabd839ea09235b6ebe41b2bb48f4.out \ $(srcdir)/%reldir%/test_sql_anno.sh_028d5d5af2f3519b59d349d41cb7ecf385253b51.err \ $(srcdir)/%reldir%/test_sql_anno.sh_028d5d5af2f3519b59d349d41cb7ecf385253b51.out \ + $(srcdir)/%reldir%/test_sql_anno.sh_08f137b4b2e1dcc15efa8c0164a13e5db4e8856b.err \ + $(srcdir)/%reldir%/test_sql_anno.sh_08f137b4b2e1dcc15efa8c0164a13e5db4e8856b.out \ $(srcdir)/%reldir%/test_sql_anno.sh_0a37c43350ddd7a2d0d75695be32fac083ad04a4.err \ $(srcdir)/%reldir%/test_sql_anno.sh_0a37c43350ddd7a2d0d75695be32fac083ad04a4.out \ $(srcdir)/%reldir%/test_sql_anno.sh_1151e5b727f6b57070bf2c8f047f1d7e02b803a6.err \ @@ -606,6 +614,10 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql_fs_func.sh_e24cf3f35643f945392e7d7a4ca82fea98b4519e.out \ $(srcdir)/%reldir%/test_sql_fs_func.sh_f31f240313ddec806aa6f353ceed707dfd9aaf16.err \ $(srcdir)/%reldir%/test_sql_fs_func.sh_f31f240313ddec806aa6f353ceed707dfd9aaf16.out \ + $(srcdir)/%reldir%/test_sql_indexes.sh_52a024607c9339423b749ac1a2eb3e49fe9776e5.err \ + $(srcdir)/%reldir%/test_sql_indexes.sh_52a024607c9339423b749ac1a2eb3e49fe9776e5.out \ + $(srcdir)/%reldir%/test_sql_indexes.sh_5815cff21c4a1b7c4a976b5574eb930b2605cd2f.err \ + $(srcdir)/%reldir%/test_sql_indexes.sh_5815cff21c4a1b7c4a976b5574eb930b2605cd2f.out \ $(srcdir)/%reldir%/test_sql_json_func.sh_017d24148f3e28f719429b709f4aa5478f458443.err \ $(srcdir)/%reldir%/test_sql_json_func.sh_017d24148f3e28f719429b709f4aa5478f458443.out \ $(srcdir)/%reldir%/test_sql_json_func.sh_191436b38db80b1dd9e7e0814c31c5fa7239dc51.err \ @@ -674,6 +686,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql_json_func.sh_f34205b59e04f261897ad89f659595c743a18ca9.out \ $(srcdir)/%reldir%/test_sql_json_func.sh_f34f5dfa938a1ac7721f924beb16bbceec127a1b.err \ $(srcdir)/%reldir%/test_sql_json_func.sh_f34f5dfa938a1ac7721f924beb16bbceec127a1b.out \ + $(srcdir)/%reldir%/test_sql_search_table.sh_df0fd242f57a96d40f466493938cda0789a094fa.err \ + $(srcdir)/%reldir%/test_sql_search_table.sh_df0fd242f57a96d40f466493938cda0789a094fa.out \ $(srcdir)/%reldir%/test_sql_str_func.sh_005b9365ac99596e539f47c9fe432668c209b21f.err \ $(srcdir)/%reldir%/test_sql_str_func.sh_005b9365ac99596e539f47c9fe432668c209b21f.out \ $(srcdir)/%reldir%/test_sql_str_func.sh_04712488fe50554eb36d3ced80f9a033602f3daa.err \ @@ -714,6 +728,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql_str_func.sh_660288b48d9b30244621d873944938f7ef043976.out \ $(srcdir)/%reldir%/test_sql_str_func.sh_6607c0dd8baff16930eb3e0daf6354af5b50052b.err \ $(srcdir)/%reldir%/test_sql_str_func.sh_6607c0dd8baff16930eb3e0daf6354af5b50052b.out \ + $(srcdir)/%reldir%/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.err \ + $(srcdir)/%reldir%/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.out \ $(srcdir)/%reldir%/test_sql_str_func.sh_71f37db33504b2c08a7a3323c482556f53d88100.err \ $(srcdir)/%reldir%/test_sql_str_func.sh_71f37db33504b2c08a7a3323c482556f53d88100.out \ $(srcdir)/%reldir%/test_sql_str_func.sh_77fc174faeec1eda687a9373dbdbdd1aaef56e20.err \ diff --git a/test/expected/test_cmds.sh_12b4cb9bd6586f9694100db76734b19a75158eab.err b/test/expected/test_cmds.sh_12b4cb9bd6586f9694100db76734b19a75158eab.err index 8619928d..7623418c 100644 --- a/test/expected/test_cmds.sh_12b4cb9bd6586f9694100db76734b19a75158eab.err +++ b/test/expected/test_cmds.sh_12b4cb9bd6586f9694100db76734b19a75158eab.err @@ -1,6 +1,6 @@ ✘ error: filter expression failed with: unable to parse time slice value: bad -- Unrecognized input  --> command-option:1 - | :filter-expr timeslice(:log_time_msecs, 'bad') is not null + | :filter-expr timeslice(:log_time_msecs, 'bad') is not null  = help: :filter-expr expr ══════════════════════════════════════════════════════════════════════ Set the filter expression diff --git a/test/expected/test_cmds.sh_36800217930a6a30e68c4efb20f6959c4f71aeb0.err b/test/expected/test_cmds.sh_36800217930a6a30e68c4efb20f6959c4f71aeb0.err index 404c4cc5..e5fda030 100644 --- a/test/expected/test_cmds.sh_36800217930a6a30e68c4efb20f6959c4f71aeb0.err +++ b/test/expected/test_cmds.sh_36800217930a6a30e68c4efb20f6959c4f71aeb0.err @@ -1,4 +1,4 @@ -✘ error: invalid filter expression: :sc_bytes # ff +✘ error: invalid filter expression: :sc_bytes # ff reason: unrecognized token: "#"  --> command-option:1 - | :filter-expr :sc_bytes # ff  + | :filter-expr :sc_bytes # ff  diff --git a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out index ab23160a..1c6abf70 100644 --- a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out +++ b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out @@ -966,12 +966,12 @@ lnav@googlegroups.com[1] support@lnav.org[2] :hide-lines-before, :hide-unmarked-lines, :toggle-filtering Examples #1 To set a filter expression that matched syslog messages from 'syslogd': - :filter-expr :log_procname = 'syslogd'  + :filter-expr :log_procname = 'syslogd'  #2 To set a filter expression that matches log messages where 'id' is followed by a number and contains the string 'foo': - :filter-expr :log_body REGEXP 'id\d+' AND :log_body REGEXP 'foo' + :filter-expr :log_body REGEXP 'id\d+' AND :log_body REGEXP 'foo' @@ -1140,7 +1140,7 @@ lnav@googlegroups.com[1] support@lnav.org[2] :clear-mark-expr, :hide-unmarked-lines, :mark, :next-mark, :prev-mark Example #1 To mark lines from 'dhclient' that mention 'eth0': - :mark-expr :log_procname = 'dhclient' AND :log_body LIKE '%eth0%' + :mark-expr :log_procname = 'dhclient' AND :log_body LIKE '%eth0%' @@ -1958,10 +1958,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), - substr(), trim(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), + upper(), xpath() Example #1 To get a string with the code points 0x48 and 0x49: ;SELECT char(0x48, 0x49)  @@ -1982,10 +1983,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), printf(), - proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + proper(), regexp_capture(), regexp_capture_into_json(), regexp_match(), + regexp_replace(), replace(), replicate(), reverse(), rightstr(), + rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), + substr(), trim(), unicode(), upper(), xpath() Examples #1 To search for the string 'abc' within 'abcabc' and starting at position 2: ;SELECT charindex('abc', 'abcabc', 2)  @@ -2143,10 +2144,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), printf(), - proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + proper(), regexp_capture(), regexp_capture_into_json(), regexp_match(), + regexp_replace(), replace(), replicate(), reverse(), rightstr(), + rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), + substr(), trim(), unicode(), upper(), xpath() Examples #1 To test if the string 'notbad.jpg' ends with '.jpg': ;SELECT endswith('notbad.jpg', '.jpg')  @@ -2182,10 +2183,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), printf(), - proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + proper(), regexp_capture(), regexp_capture_into_json(), regexp_match(), + regexp_replace(), replace(), replicate(), reverse(), rightstr(), + rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), + substr(), trim(), unicode(), upper(), xpath() Examples #1 To extract key/value pairs from a string: ;SELECT extract('foo=1 bar=2 name="Rolo Tomassi"')  @@ -2296,10 +2297,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), printf(), - proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + proper(), regexp_capture(), regexp_capture_into_json(), regexp_match(), + regexp_replace(), replace(), replicate(), reverse(), rightstr(), + rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), + substr(), trim(), unicode(), upper(), xpath() Examples #1 To concatenate the values of the column 'ex_procname' from the table 'lnav_example_log': @@ -2325,10 +2326,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), printf(), - proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + proper(), regexp_capture(), regexp_capture_into_json(), regexp_match(), + regexp_replace(), replace(), replicate(), reverse(), rightstr(), + rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), + substr(), trim(), unicode(), upper(), xpath() Example #1 To produce a hash of all of the values of 'column1': ;SELECT group_spooky_hash(column1) FROM (VALUES ('abc'), ('123')) @@ -2344,10 +2345,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() gzip(value, ...) ══════════════════════════════════════════════════════════════════════ @@ -2358,10 +2359,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), - substr(), trim(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), + upper(), xpath() hex(X) ══════════════════════════════════════════════════════════════════════ @@ -2385,10 +2387,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), printf(), - proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + proper(), regexp_capture(), regexp_capture_into_json(), regexp_match(), + regexp_replace(), replace(), replicate(), reverse(), rightstr(), + rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), + substr(), trim(), unicode(), upper(), xpath() Example #1 To format an amount: ;SELECT humanize_file_size(10 * 1024 * 1024)  @@ -2421,10 +2423,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Example #1 To test get the position of 'b' in the string 'abc': ;SELECT instr('abc', 'b')  @@ -2640,10 +2642,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To get the first character of the string 'abc': ;SELECT leftstr('abc', 1)  @@ -2664,10 +2666,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), logfmt2json(), lower(), ltrim(), padc(), padl(), padr(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Example #1 To get the length of the string 'abc': ;SELECT length('abc')  @@ -2783,10 +2785,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), lower(), ltrim(), padc(), padl(), padr(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Example #1 To extract key/value pairs from a log message: ;SELECT logfmt2json('foo=1 bar=2 name="Rolo Tomassi"') @@ -2803,10 +2805,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), ltrim(), padc(), padl(), padr(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Example #1 To lowercase the string 'AbC': ;SELECT lower('AbC')  @@ -2824,10 +2826,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), padc(), padl(), padr(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To trim the leading whitespace from the string ' abc': ;SELECT ltrim(' abc')  @@ -2931,10 +2933,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padl(), padr(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To pad the string 'abc' to a length of six characters: ;SELECT padc('abc', 6) || 'def'  @@ -2956,10 +2958,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padr(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To pad the string 'abc' to a length of six characters: ;SELECT padl('abc', 6)  @@ -2981,10 +2983,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - printf(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + printf(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To pad the string 'abc' to a length of six characters: ;SELECT padr('abc', 6) || 'def'  @@ -3046,10 +3048,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), proper(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + padr(), proper(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Examples #1 To substitute 'World' into the string 'Hello, %s!': ;SELECT printf('Hello, %s!', 'World')  @@ -3073,10 +3075,10 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), regexp_capture(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + padr(), printf(), regexp_capture(), regexp_capture_into_json(), + regexp_match(), regexp_replace(), replace(), replicate(), reverse(), + rightstr(), rtrim(), sparkline(), spooky_hash(), startswith(), + strfilter(), substr(), trim(), unicode(), upper(), xpath() Example #1 To capitalize the words in the string 'hello, world!': ;SELECT proper('hello, world!')  @@ -3195,16 +3197,44 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_match(), regexp_replace(), - replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), - spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), - upper(), xpath() + padr(), printf(), proper(), regexp_capture_into_json(), regexp_match(), + regexp_replace(), replace(), replicate(), reverse(), rightstr(), + rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), + substr(), trim(), unicode(), upper(), xpath() Example #1 To extract the key/value pairs 'a'/1 and 'b'/2 from the string 'a=1; b=2': ;SELECT * FROM regexp_capture('a=1; b=2', '(\w+)=(\d+)') +regexp_capture_into_json(string, pattern) +══════════════════════════════════════════════════════════════════════ + A table-valued function that executes a regular-expression over a + string and returns the captured values as a JSON object. If the + regex only matches a subset of the input string, it will be rerun on + the remaining parts of the string until no more matches are found. +Parameters + string The string to match against the given pattern. + pattern The regular expression to match. +Results + match_index The match iteration. This value will + increase each time a new match is found in the input + string. + content The captured values from the string. +See Also + char(), charindex(), endswith(), extract(), group_concat(), + group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), + leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), + padr(), printf(), proper(), regexp_capture(), regexp_match(), + regexp_replace(), replace(), replicate(), reverse(), rightstr(), + rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), + substr(), trim(), unicode(), upper(), xpath() +Example +#1 To extract the key/value pairs 'a'/1 and 'b'/2 from the string 'a=1; b=2': + ;SELECT * FROM regexp_capture_into_json('a=1; b=2', '(\w+)=(\d+)') + + + regexp_match(re, str) ══════════════════════════════════════════════════════════════════════ Match a string against a regular expression and return the capture @@ -3216,10 +3246,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_replace(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), - substr(), trim(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_replace(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), + upper(), xpath() Examples #1 To capture the digits from the string '123': ;SELECT regexp_match('(\d+)', '123')  @@ -3249,10 +3280,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_match(), replace(), replicate(), reverse(), rightstr(), rtrim(), - sparkline(), spooky_hash(), startswith(), strfilter(), substr(), - trim(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_match(), replace(), + replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), + upper(), xpath() Examples #1 To replace the word at the start of the string 'Hello, World!' with 'Goodbye': ;SELECT regexp_replace('Hello, World!', '^(\w+)', 'Goodbye') @@ -3276,10 +3308,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replicate(), reverse(), rightstr(), rtrim(), - sparkline(), spooky_hash(), startswith(), strfilter(), substr(), - trim(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), + upper(), xpath() Examples #1 To replace the string 'x' with 'z' in 'abc': ;SELECT replace('abc', 'x', 'z')  @@ -3300,10 +3333,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), reverse(), rightstr(), rtrim(), - sparkline(), spooky_hash(), startswith(), strfilter(), substr(), - trim(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), reverse(), rightstr(), rtrim(), sparkline(), spooky_hash(), + startswith(), strfilter(), substr(), trim(), unicode(), upper(), + xpath() Example #1 To repeat the string 'abc' three times: ;SELECT replicate('abc', 3)  @@ -3319,10 +3353,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), rightstr(), rtrim(), - sparkline(), spooky_hash(), startswith(), strfilter(), substr(), - trim(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), + upper(), xpath() Example #1 To reverse the string 'abc': ;SELECT reverse('abc')  @@ -3340,10 +3375,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rtrim(), - sparkline(), spooky_hash(), startswith(), strfilter(), substr(), - trim(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rtrim(), sparkline(), spooky_hash(), + startswith(), strfilter(), substr(), trim(), unicode(), upper(), + xpath() Examples #1 To get the last character of the string 'abc': ;SELECT rightstr('abc', 1)  @@ -3406,10 +3442,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - sparkline(), spooky_hash(), startswith(), strfilter(), substr(), - trim(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), + upper(), xpath() Examples #1 To trim the whitespace from the end of the string 'abc ': ;SELECT rtrim('abc ')  @@ -3459,10 +3496,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), spooky_hash(), startswith(), strfilter(), substr(), trim(), - unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), + startswith(), strfilter(), substr(), trim(), unicode(), upper(), + xpath() Examples #1 To get the unicode block element for the value 32 in the range of 0-128: ;SELECT sparkline(32, 128)  @@ -3482,10 +3520,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), startswith(), strfilter(), substr(), trim(), - unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + startswith(), strfilter(), substr(), trim(), unicode(), upper(), + xpath() Examples #1 To produce a hash for the string 'Hello, World!': ;SELECT spooky_hash('Hello, World!')  @@ -3562,10 +3601,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), strfilter(), substr(), trim(), - unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), strfilter(), substr(), trim(), unicode(), upper(), + xpath() Examples #1 To test if the string 'foobar' starts with 'foo': ;SELECT startswith('foobar', 'foo')  @@ -3587,10 +3627,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), startswith(), substr(), trim(), - unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), substr(), trim(), unicode(), upper(), + xpath() Example #1 To get the 'b', 'c', and 'd' characters from the string 'abcabc': ;SELECT strfilter('abcabc', 'bcd')  @@ -3642,10 +3683,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), trim(), - unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), trim(), unicode(), upper(), + xpath() Examples #1 To get the substring starting at the second character until the end of the string 'abc': @@ -3787,10 +3829,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), - substr(), unicode(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), unicode(), upper(), + xpath() Examples #1 To trim whitespace from the start and end of the string ' abc ': ;SELECT trim(' abc ')  @@ -3828,10 +3871,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), - substr(), trim(), upper(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), upper(), + xpath() Example #1 To get the unicode code point for the first character of 'abc': ;SELECT unicode('abc')  @@ -3855,10 +3899,11 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), - substr(), trim(), unicode(), xpath() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), + xpath() Example #1 To uppercase the string 'aBc': ;SELECT upper('aBc')  @@ -3883,21 +3928,22 @@ lnav@googlegroups.com[1] support@lnav.org[2] char(), charindex(), endswith(), extract(), group_concat(), group_spooky_hash(), gunzip(), gzip(), humanize_file_size(), instr(), leftstr(), length(), logfmt2json(), lower(), ltrim(), padc(), padl(), - padr(), printf(), proper(), regexp_capture(), regexp_match(), - regexp_replace(), replace(), replicate(), reverse(), rightstr(), - rtrim(), sparkline(), spooky_hash(), startswith(), strfilter(), - substr(), trim(), unicode(), upper() + padr(), printf(), proper(), regexp_capture(), + regexp_capture_into_json(), regexp_match(), regexp_replace(), + replace(), replicate(), reverse(), rightstr(), rtrim(), sparkline(), + spooky_hash(), startswith(), strfilter(), substr(), trim(), unicode(), + upper() Examples #1 To select the XML nodes on the path '/abc/def': - ;SELECT * FROM xpath('/abc/def', '<abc><def a="b">Hello</def><def>Bye</def></abc>') + ;SELECT * FROM xpath('/abc/def', '<abc><def a="b">Hello<def>Bye') #2 To select all 'a' attributes on the path '/abc/def': - ;SELECT * FROM xpath('/abc/def/@a', '<abc><def a="b">Hello</def><def>Bye</def></abc>') + ;SELECT * FROM xpath('/abc/def/@a', '<abc><def a="b">Hello<def>Bye') #3 To select the text nodes on the path '/abc/def': - ;SELECT * FROM xpath('/abc/def/text()', '<abc><def a="b">Hello &#x2605;</def></abc>') + ;SELECT * FROM xpath('/abc/def/text()', '<abc><def a="b">Hello ★') diff --git a/test/expected/test_cmds.sh_ed5b73be0b991e0e8d6735e31df5b37c4286321b.err b/test/expected/test_cmds.sh_ed5b73be0b991e0e8d6735e31df5b37c4286321b.err index a775a892..6e228f27 100644 --- a/test/expected/test_cmds.sh_ed5b73be0b991e0e8d6735e31df5b37c4286321b.err +++ b/test/expected/test_cmds.sh_ed5b73be0b991e0e8d6735e31df5b37c4286321b.err @@ -1,4 +1,4 @@ -✘ error: invalid mark expression: :log_procname lik +✘ error: invalid mark expression: :log_procname lik reason: near "lik": syntax error  --> command-option:1 - | :mark-expr :log_procname lik  + | :mark-expr :log_procname lik  diff --git a/test/expected/test_logfile.sh_c18e14a26d8261c9f72747118a469266121d5459.out b/test/expected/test_logfile.sh_c18e14a26d8261c9f72747118a469266121d5459.out index ec7fc974..dd50fb46 100644 --- a/test/expected/test_logfile.sh_c18e14a26d8261c9f72747118a469266121d5459.out +++ b/test/expected/test_logfile.sh_c18e14a26d8261c9f72747118a469266121d5459.out @@ -1,3 +1,3 @@ -log_line log_part log_time log_idle_msecs log_level log_mark log_comment log_tags log_filters log_msg_instance col_0 col_1 - 0 2021-05-19 08:00:01.000 0 info 0 0 1.0 /abc/def - 2 2021-05-19 08:00:03.000 2000 info 0 1 3.0 /ghi/jkl +log_line log_part log_time log_idle_msecs log_level log_mark log_comment log_tags log_filters col_0 col_1 + 0 2021-05-19 08:00:01.000 0 info 0 1.0 /abc/def + 2 2021-05-19 08:00:03.000 2000 info 0 3.0 /ghi/jkl diff --git a/test/expected/test_sql.sh_5532c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.err b/test/expected/test_sql.sh_5532c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_sql.sh_5532c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.out b/test/expected/test_sql.sh_5532c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.out new file mode 100644 index 00000000..dbb2a2f6 --- /dev/null +++ b/test/expected/test_sql.sh_5532c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.out @@ -0,0 +1,2 @@ +log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,log_hostname,log_msgid,log_pid,log_pri,log_procname,log_struct,log_syslog_tag,syslog_version,col_0,TTY,PWD,USER,COMMAND +0,,2007-11-03 09:47:02.000,0,info,0,,,[1],veridian,,,,sudo,,sudo,,timstack,pts/6,/auto/wstimstack/rpms/lbuild/test,root,/usr/bin/tail /var/log/messages diff --git a/test/expected/test_sql.sh_764306f0e5f610ba71f521ba3d19fe158ece0ba5.err b/test/expected/test_sql.sh_764306f0e5f610ba71f521ba3d19fe158ece0ba5.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_sql.sh_764306f0e5f610ba71f521ba3d19fe158ece0ba5.out b/test/expected/test_sql.sh_764306f0e5f610ba71f521ba3d19fe158ece0ba5.out new file mode 100644 index 00000000..c5c885d4 --- /dev/null +++ b/test/expected/test_sql.sh_764306f0e5f610ba71f521ba3d19fe158ece0ba5.out @@ -0,0 +1,2 @@ + col_0  +eth0.IPv4 diff --git a/test/expected/test_sql.sh_c5b8da04734fadf3b9eea80e0af997e38e0fb811.err b/test/expected/test_sql.sh_c5b8da04734fadf3b9eea80e0af997e38e0fb811.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_sql.sh_c5b8da04734fadf3b9eea80e0af997e38e0fb811.out b/test/expected/test_sql.sh_c5b8da04734fadf3b9eea80e0af997e38e0fb811.out new file mode 100644 index 00000000..590760c7 --- /dev/null +++ b/test/expected/test_sql.sh_c5b8da04734fadf3b9eea80e0af997e38e0fb811.out @@ -0,0 +1,3 @@ +log_line  col_0  + 0 eth0.IPv4  + 7 eth0.IPv4 diff --git a/test/expected/test_sql_anno.sh_08f137b4b2e1dcc15efa8c0164a13e5db4e8856b.err b/test/expected/test_sql_anno.sh_08f137b4b2e1dcc15efa8c0164a13e5db4e8856b.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_sql_anno.sh_08f137b4b2e1dcc15efa8c0164a13e5db4e8856b.out b/test/expected/test_sql_anno.sh_08f137b4b2e1dcc15efa8c0164a13e5db4e8856b.out new file mode 100644 index 00000000..0472df67 --- /dev/null +++ b/test/expected/test_sql_anno.sh_08f137b4b2e1dcc15efa8c0164a13e5db4e8856b.out @@ -0,0 +1,11 @@ + SELECT * from vmw_log, regexp_capture(log_body, '--> /SessionStats/SessionPool/Session/(?[^\n]+)') + sql_keyword ------ + sql_oper - + sql_keyword ---- + sql_ident ------- + sql_comma - + sql_ident -------------- + sql_func -------------------------------------------------------------------------------- + sql_ident -------- + sql_comma - + sql_string ------------------------------------------------------- diff --git a/test/expected/test_sql_indexes.sh_52a024607c9339423b749ac1a2eb3e49fe9776e5.err b/test/expected/test_sql_indexes.sh_52a024607c9339423b749ac1a2eb3e49fe9776e5.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_sql_indexes.sh_52a024607c9339423b749ac1a2eb3e49fe9776e5.out b/test/expected/test_sql_indexes.sh_52a024607c9339423b749ac1a2eb3e49fe9776e5.out new file mode 100644 index 00000000..a35aee7a --- /dev/null +++ b/test/expected/test_sql_indexes.sh_52a024607c9339423b749ac1a2eb3e49fe9776e5.out @@ -0,0 +1,2 @@ +id parent notused  detail  + 2  0  0 SCAN syslog_log VIRTUAL TABLE INDEX 1:SEARCH syslog_log USING log_path GLOB ?  diff --git a/test/expected/test_sql_indexes.sh_5815cff21c4a1b7c4a976b5574eb930b2605cd2f.err b/test/expected/test_sql_indexes.sh_5815cff21c4a1b7c4a976b5574eb930b2605cd2f.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_sql_indexes.sh_5815cff21c4a1b7c4a976b5574eb930b2605cd2f.out b/test/expected/test_sql_indexes.sh_5815cff21c4a1b7c4a976b5574eb930b2605cd2f.out new file mode 100644 index 00000000..d1ec5f31 --- /dev/null +++ b/test/expected/test_sql_indexes.sh_5815cff21c4a1b7c4a976b5574eb930b2605cd2f.out @@ -0,0 +1,12 @@ +log_line log_part  log_time log_idle_msecs log_level log_mark log_comment log_tags log_filters log_hostname log_msgid log_pid log_pri log_procname log_struct  log_syslog_tag syslog_version  + 0  <NULL> 2006-12-03 09:23:38.000  0 error   0  <NULL>  <NULL>  <NULL> veridian   <NULL> 7998   <NULL> automount   <NULL> automount[7998]   <NULL>  + 1 2006-12-03 09:23:38.000 0 info 0 veridian 16442 automount automount[16442] + 2  <NULL> 2006-12-03 09:23:38.000  0 error   0  <NULL>  <NULL>  <NULL> veridian   <NULL> 7999   <NULL> automount   <NULL> automount[7999]   <NULL>  +  3 2007-01-03 09:47:02.000 2679804000 info 0 veridian sudo sudo + 4  <NULL> 2007-11-03 09:23:38.000  26264196000 error   0  <NULL>  <NULL>  <NULL> veridian   <NULL> 7998   <NULL> automount   <NULL> automount[7998]   <NULL>  + 5 2007-11-03 09:23:38.000 0 info 0 veridian 16442 automount automount[16442] + 6  <NULL> 2007-11-03 09:23:38.000  0 error   0  <NULL>  <NULL>  <NULL> veridian   <NULL> 7999   <NULL> automount   <NULL> automount[7999]   <NULL>  +  7 2007-11-03 09:47:02.000 1404000 info 0 veridian sudo sudo + 8  <NULL> 2021-11-03 09:23:38.000  441848196000 info   0  <NULL>  <NULL>  <NULL> veridian   <NULL> 7998   <NULL> foo   <NULL> foo[7998]   <NULL>  + 9 2021-11-03 09:23:38.000 0 info 0 veridian 16442 foo foo[16442] + 10  <NULL> 2021-11-03 09:23:38.000  0 info   0  <NULL>  <NULL>  <NULL> veridian   <NULL> 7999   <NULL> foo   <NULL> foo[7999]   <NULL>  diff --git a/test/expected/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.err b/test/expected/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.out b/test/expected/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.out new file mode 100644 index 00000000..913b8ce1 --- /dev/null +++ b/test/expected/test_sql_str_func.sh_69f5d49e62da48e188bd9d6af4bd3adeb21eb7d1.out @@ -0,0 +1,6 @@ +Row 0: + Column match_index: 0 + Column content: {"col_0":1,"col_1":2} +Row 1: + Column match_index: 1 + Column content: {"col_0":3,"col_1":4} diff --git a/test/logfile_procstate.0 b/test/logfile_procstate.0 new file mode 100644 index 00000000..43878aa8 --- /dev/null +++ b/test/logfile_procstate.0 @@ -0,0 +1,43 @@ +========== Start of system state dump at Thu Jun 2 00:01:01 UTC 2022 ========== + +/bin/ps auxww +USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND +root 1 0.0 0.0 158392 7792 ? Ss Jun01 0:14 /lib/systemd/systemd --switched-root --system --deserialize 16 +root 2 0.0 0.0 0 0 ? S Jun01 0:00 [kthreadd] +root 3 0.0 0.0 0 0 ? I< Jun01 0:00 [rcu_gp] +root 4 0.0 0.0 0 0 ? I< Jun01 0:00 [rcu_par_gp] +root 6 0.0 0.0 0 0 ? I< Jun01 0:00 [kworker/0:0H-kblockd] +========== End of system state dump ========== + + +========== Start of system state dump at Thu Jun 2 00:02:01 UTC 2022 ========== + +/bin/ps auxww +USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND +root 1 0.0 0.0 158392 7792 ? Ss Jun01 0:14 /lib/systemd/systemd --switched-root --system --deserialize 16 +root 2 0.0 0.0 0 0 ? S Jun01 0:00 [kthreadd] +root 3 0.0 0.0 0 0 ? I< Jun01 0:00 [rcu_gp] +root 4 0.0 0.0 0 0 ? I< Jun01 0:00 [rcu_par_gp] +root 6 0.0 0.0 0 0 ? I< Jun01 0:00 [kworker/0:0H-kblockd] +root 8 0.0 0.0 0 0 ? I< Jun01 0:00 [mm_percpu_wq] +root 9 0.0 0.0 0 0 ? S Jun01 0:00 [ksoftirqd/0] +root 10 0.0 0.0 0 0 ? I Jun01 0:23 [rcu_sched] +root 11 0.0 0.0 0 0 ? I Jun01 0:00 [rcu_bh] +root 12 0.0 0.0 0 0 ? S Jun01 0:00 [migration/0] +root 14 0.0 0.0 0 0 ? S Jun01 0:00 [cpuhp/0] +========== End of system state dump ========== + + +========== Start of system state dump at Thu Jun 2 00:03:01 UTC 2022 ========== + +/bin/ps auxww +USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND +root 1 0.0 0.0 158392 7792 ? Ss Jun01 0:14 /lib/systemd/systemd --switched-root --system --deserialize 16 +root 2 0.0 0.0 0 0 ? S Jun01 0:00 [kthreadd] +root 3 0.0 0.0 0 0 ? I< Jun01 0:00 [rcu_gp] +root 4 0.0 0.0 0 0 ? I< Jun01 0:00 [rcu_par_gp] +root 6 0.0 0.0 0 0 ? I< Jun01 0:00 [kworker/0:0H-kblockd] +root 8 0.0 0.0 0 0 ? I< Jun01 0:00 [mm_percpu_wq] +root 9 0.0 0.0 0 0 ? S Jun01 0:00 [ksoftirqd/0] +========== End of system state dump ========== + diff --git a/test/test_column_namer.cc b/test/test_column_namer.cc new file mode 100644 index 00000000..7eaece7c --- /dev/null +++ b/test/test_column_namer.cc @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "config.h" + +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include "column_namer.hh" +#include "doctest/doctest.h" + +TEST_CASE("column_namer::default") +{ + column_namer cn{column_namer::language::SQL}; + + auto def_name0 = cn.add_column(string_fragment{}); + CHECK(def_name0 == "col_0"); + auto def_name1 = cn.add_column(string_fragment{}); + CHECK(def_name1 == "col_1"); +} + +TEST_CASE("column_namer::no-collision") +{ + column_namer cn{column_namer::language::SQL}; + + auto name0 = cn.add_column(string_fragment{"abc"}); + CHECK(name0 == "abc"); + auto name1 = cn.add_column(string_fragment{"def"}); + CHECK(name1 == "def"); +} + +TEST_CASE("column_namer::collisions") +{ + column_namer cn{column_namer::language::SQL}; + + auto name0 = cn.add_column(string_fragment{"abc"}); + CHECK(name0 == "abc"); + auto name1 = cn.add_column(string_fragment{"abc"}); + CHECK(name1 == "abc_0"); + auto name2 = cn.add_column(string_fragment{"abc"}); + CHECK(name2 == "abc_1"); +} diff --git a/test/test_sql.sh b/test/test_sql.sh index efdb3deb..5c295b1e 100644 --- a/test/test_sql.sh +++ b/test/test_sql.sh @@ -645,18 +645,12 @@ log_line EOF -run_test ${lnav_test} -n \ +run_cap_test ${lnav_test} -n \ -c ':filter-in sudo' \ -c ";select * from logline" \ -c ':write-csv-to -' \ ${test_dir}/logfile_syslog.0 -check_output "logline table is not working" <,2007-11-03 09:47:02.000,0,info,0,,,[1],veridian,,,,sudo,,sudo,,0,timstack,pts/6,/auto/wstimstack/rpms/lbuild/test,root,/usr/bin/tail /var/log/messages -EOF - - run_test ${lnav_test} -n \ -c ':goto 1' \ -c ";select log_line, log_pid, col_0 from logline" \ @@ -884,6 +878,7 @@ CREATE VIRTUAL TABLE lnav_file USING lnav_file_impl(); CREATE VIEW lnav_view_filters_and_stats AS SELECT * FROM lnav_view_filters LEFT NATURAL JOIN lnav_view_filter_stats; CREATE VIRTUAL TABLE regexp_capture USING regexp_capture_impl(); +CREATE VIRTUAL TABLE regexp_capture_into_json USING regexp_capture_into_json_impl(); CREATE VIRTUAL TABLE xpath USING xpath_impl(); CREATE VIRTUAL TABLE fstat USING fstat_impl(); CREATE TABLE lnav_events ( @@ -891,7 +886,6 @@ CREATE TABLE lnav_events ( content TEXT ); CREATE TABLE http_status_codes ( - status integer PRIMARY KEY, EOF @@ -1008,11 +1002,11 @@ check_output "write-json-to isn't working?" < 4" \ + -c ";select col_0 from logline where log_line > 4" \ ${test_dir}/logfile_for_join.0 run_test ${lnav_test} -d "/tmp/lnav.err" -n \ @@ -1129,25 +1123,25 @@ EOF run_test ${lnav_test} -n \ -c ":create-search-table search_test1 (\w+), world!" \ - -c ";select log_msg_instance, col_0 from search_test1" \ + -c ";select col_0 from search_test1" \ -c ":write-csv-to -" \ ${test_dir}/logfile_multiline.0 check_output "create-search-table is not working?" < 0" \ + -c ";select col_0 from search_test1 where log_line > 0" \ -c ":write-csv-to -" \ ${test_dir}/logfile_multiline.0 check_output "create-search-table is not working with where clause?" < /SessionStats/SessionPool/Session/(?[^\n]+)')" diff --git a/test/test_sql_indexes.sh b/test/test_sql_indexes.sh new file mode 100644 index 00000000..e0f4ba55 --- /dev/null +++ b/test/test_sql_indexes.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +export YES_COLOR=1 + +run_cap_test ${lnav_test} -n \ + -c ";EXPLAIN QUERY PLAN SELECT * FROM syslog_log WHERE log_path GLOB '*/logfile_syslog.*'" \ + ${test_dir}/logfile_syslog.* + +run_cap_test ${lnav_test} -n \ + -c ";SELECT * FROM syslog_log WHERE log_path GLOB '*/logfile_syslog.*'" \ + ${test_dir}/logfile_syslog.* + diff --git a/test/test_sql_search_table.sh b/test/test_sql_search_table.sh new file mode 100644 index 00000000..939472df --- /dev/null +++ b/test/test_sql_search_table.sh @@ -0,0 +1,7 @@ +#! /bin/bash + +export YES_COLOR=1 + +run_cap_test ${lnav_test} -n \ + -c ';SELECT * FROM procstate_procs' \ + ${test_dir}/logfile_procstate.0 diff --git a/test/test_sql_str_func.sh b/test/test_sql_str_func.sh index fa6c25f6..52ef3eed 100644 --- a/test/test_sql_str_func.sh +++ b/test/test_sql_str_func.sh @@ -97,3 +97,5 @@ run_cap_test ./drive_sql "SELECT * FROM regexp_capture('foo bar', '(')" run_cap_test ./drive_sql "SELECT * FROM regexp_capture('1 2 3 45', '(\d+)')" run_cap_test ./drive_sql "SELECT * FROM regexp_capture('foo foo', '^foo')" + +run_cap_test ./drive_sql "SELECT * FROM regexp_capture_into_json('foo=1 bar=2; foo=3 bar=4', 'foo=(\d+) bar=(\d+)')"