diff --git a/NEWS b/NEWS index cb50d0d6..edd32a71 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,13 @@ lnav v0.11.1: + Features: + * Additional validation checks for log formats have been + added and will result in warnings. Pass `-W` on the + command-line to view the warnings. The following new + check have been added: + - Each regex must have a corresponding sample log message + that it matches. + - Each sample must be matched by only one regex. + Breaking changes: * The regexp_capture() table-valued-function now returns NULL instead of an empty string for the `capture_name` column if diff --git a/README b/README index 498c09c4..c89669ba 100644 --- a/README +++ b/README @@ -18,7 +18,7 @@ PREREQUISITES The following software packages are required to build/run lnav: gcc/clang - A C++14-compatible compiler. - libpcre - The Perl Compatible Regular Expression (PCRE) library. + libpcre2 - The Perl Compatible Regular Expression v2 (PCRE2) library. sqlite - The SQLite database engine. Version 3.9.0 or higher is required. ncurses - The ncurses text UI library. readline - The readline line editing library. diff --git a/README.md b/README.md index 372d25ac..5f1185a1 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ The following alternatives are also available: The following software packages are required to build lnav: - gcc/clang - A C++14-compatible compiler. -- libpcre - The Perl Compatible Regular Expression (PCRE) library. +- libpcre2 - The Perl Compatible Regular Expression v2 (PCRE2) library. - sqlite - The SQLite database engine. Version 3.9.0 or higher is required. - ncurses - The ncurses text UI library. - readline - The readline line editing library. diff --git a/docs/source/commands.rst b/docs/source/commands.rst index 18fb5b22..e786c585 100644 --- a/docs/source/commands.rst +++ b/docs/source/commands.rst @@ -35,7 +35,7 @@ first few lines of the file given as its argument: Screenshot of the preview shown for the :code:`:open` command. The :lnavcmd:`:filter-out pattern` command is another instance where the preview behavior -can help you craft the correct command-line. This command takes a PCRE regular +can help you craft the correct command-line. This command takes a PCRE2 regular expression that specifies the log messages that should be filtered out of the view. The preview for this command will highlight the portion of the log messages that match the expression in red. Thus, you can be certain that the diff --git a/docs/source/formats.rst b/docs/source/formats.rst index 3191f81a..49c6ef25 100644 --- a/docs/source/formats.rst +++ b/docs/source/formats.rst @@ -94,7 +94,7 @@ like so: "$schema": "https://lnav.org/schemas/format-v1.schema.json" } -Each format to be defined in the file should a separate field in the top-level +Each format to be defined in the file should be a separate field in the top-level object. The field name should be the symbolic name of the format. This value will also be used as the SQL table name for the log. The value for each field should be another object with the following fields: @@ -111,11 +111,19 @@ should be another object with the following fields: .. _format_regex: :regex: This object contains sub-objects that describe the message formats - to match in a plain log file. Log files that contain JSON messages should - not specify this field. + to match in a plain-text log file. Each :code:`regex` MUST only match one + type of log message. It must not match log messages that are matched by + other regexes in this format. This uniqueness requirement is necessary + because **lnav** will "lock-on" to a regex and use it to match against + the next line in a file. So, if the regexes do not uniquely match each + type of log message, messages can be matched by the wrong regex. The + "lock-on" behavior is needed to avoid the performance hit of having to + try too many different regexes. + + .. note:: Log files that contain JSON messages should not specify this field. :pattern: The regular expression that should be used to match log messages. - The `PCRE `_ library is used by **lnav** to do all + The `PCRE2 `_ library is used by **lnav** to do all regular expression matching. :module-format: If true, this regex will only be used to parse message diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 2379ecb1..b095c14a 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -1,4 +1,3 @@ - Introduction ============ @@ -18,8 +17,7 @@ Dependencies When compiling from source, the following dependencies are required: * `NCurses `_ -* `PCRE `_ -- Versions greater than 8.20 give better - performance since the PCRE JIT will be leveraged. +* `PCRE2 `_ * `SQLite `_ * `ZLib `_ * `Bzip2 `_ diff --git a/docs/source/ui.rst b/docs/source/ui.rst index 654efa9e..c7d7ffe8 100644 --- a/docs/source/ui.rst +++ b/docs/source/ui.rst @@ -140,7 +140,7 @@ patterns and execute internal commands, such as converting a unix-timestamp into a human-readable date. The following key-presses will activate a corresponding prompt: -* :kbd:`/` - The search prompt. You can enter a PCRE-flavored regular +* :kbd:`/` - The search prompt. You can enter a PCRE2-flavored regular expression to search for in the current view. * :kbd:`:` - The command prompt. Commands are used to perform common operations. diff --git a/src/formats/access_log.json b/src/formats/access_log.json index 911b8087..7130103b 100644 --- a/src/formats/access_log.json +++ b/src/formats/access_log.json @@ -7,7 +7,7 @@ "multiline": false, "regex": { "ts-first-noquotes": { - "pattern": "^(?\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?[^ ]+) (?[^ ]+) (?[A-Z]+) (?[^ \\?]+)(?:\\?(?[^ ]*))? (?:-1|\\d+) (?\\d+) \\d+\\s*(?.*)" + "pattern": "^(?\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?[^ ]+) (?[^ ]+) (?[A-Z]+) (?!\")(?[^ \\?]+)(?:\\?(?[^ ]*))? (?:-1|\\d+) (?\\d+) \\d+\\s*(?.*)" }, "ts-first": { "pattern": "^(?\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?[^ ]+) (?[^ ]+) (?[A-Z]+) \"(?[^ \\?]+)(?:\\?(?[^ ]*))?\" (?:-1|\\d+) (?\\d+) \\d+\\s*(?.*)" @@ -94,6 +94,14 @@ "line": "10.112.72.172 - - [11/Feb/2013:06:43:36 +0000] \"GET /client/ HTTP/1.1\" 404 5778 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17\"", "level": "error" }, + { + "line": "2013-02-11T06:43:36 10.112.72.172 - GET \"/client/\" -1 200 5778 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17\"", + "level": "info" + }, + { + "line": "2013-02-11T06:43:36 10.112.72.172 - GET /client/ -1 200 5778 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17\"", + "level": "info" + }, { "line": "10.1.10.51 - - [23/Dec/2014:21:20:35 +0000] \"POST /api/1/rest/foo/bar HTTP/1.1\" 200 - \"-\" \"-\" 293" }, diff --git a/src/formats/cups_log.json b/src/formats/cups_log.json index 9f869c83..b79f6363 100644 --- a/src/formats/cups_log.json +++ b/src/formats/cups_log.json @@ -8,7 +8,7 @@ "pattern": "^(?[IEW]) \\[(?\\d{2}/\\S{3,8}/\\d{4}:\\d{2}:\\d{2}:\\d{2} [+-]\\d{2,4})\\] (?
\\w+): (?.*)$" }, "default": { - "pattern": "^(?[IEW]) \\[(?\\d{2}/\\S{3,8}/\\d{4}:\\d{2}:\\d{2}:\\d{2} [+-]\\d{2,4})\\] (?.*)$" + "pattern": "^(?[IEW]) \\[(?\\d{2}/\\S{3,8}/\\d{4}:\\d{2}:\\d{2}:\\d{2} [+-]\\d{2,4})\\] (?!\\w+:)(?.*)$" } }, "level": { diff --git a/src/formats/esx_syslog_log.json b/src/formats/esx_syslog_log.json index 1bd06eab..85fa8815 100644 --- a/src/formats/esx_syslog_log.json +++ b/src/formats/esx_syslog_log.json @@ -55,6 +55,9 @@ { "line": "2022-06-02T05:34:23Z In(14)[+] hostprofile[1001430319]: {'mode': 'Disabled', 'exceptionUsers': []}" }, + { + "line": "2022-06-02 In(14) hostprofile[1001430319]: 05:34:23.666 {'mode': 'Disabled', 'exceptionUsers': []}" + }, { "line": "2022-06-01T13:42:40.681Z In(05) host-16250 Skip service health check. State STOPPED, Curr request 0" } diff --git a/src/formats/java_log.json b/src/formats/java_log.json index ecfd9afd..0d6297ac 100644 --- a/src/formats/java_log.json +++ b/src/formats/java_log.json @@ -30,7 +30,7 @@ "pattern": "^\\[(?\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\]\\s*(?ERROR|WARN|INFO|DEBUG)\\s*\\d+\\[(?[^\\]]+)\\]\\s+-\\s+(?[^\\(]+)\\.(?\\w+)\\((?[^:]+):(?\\d+)\\)\\s+-\\s+(?.*)$" }, "vmw3": { - "pattern": "^(?\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s*\\|\\s*(?ERROR|WARN|INFO|DEBUG)\\s*\\|\\s*(?[^\\|]+)\\s*\\|\\s*(?[^\\|]+)\\s*\\|\\s*(?.*)$" + "pattern": "^(?\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s*\\|\\s*(?ERROR|WARN|INFO|DEBUG)\\s*\\|\\s*(?[^\\|]+)\\s*\\|\\s*(?[^\\|]+)\\s*\\|\\s+(?!\\d+\\s*\\|)(?.*)$" }, "vmw-sso": { "pattern": "^(?\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}Z?)\\s+(?ERROR|WARN|INFO|DEBUG)\\s+[\\w\\-]+\\[\\d+:(?[^\\]]+)\\]\\s+\\[CorId=(?[^\\s\\]]*)(?:\\s+OpId=(?[^\\]]*))?\\]\\s+\\[(?[^\\]]+)\\]\\s+(?.*)$" diff --git a/src/formats/openstack_log.json b/src/formats/openstack_log.json index 23104ea4..4dc280fe 100644 --- a/src/formats/openstack_log.json +++ b/src/formats/openstack_log.json @@ -13,7 +13,7 @@ "pattern": "^(?\\w+) (?\\S+) \\[(?[^\\]]+)\\] (?.*)" }, "keystone": { - "pattern": "^[(](?[^)]+)[)]: (?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?\\w+) (?.*)" + "pattern": "^[(](?[^)]+)[)]: (?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?\\w+) (?!\\()(?.*)" }, "keystone-debug": { "pattern": "^[(](?[^)]+)[)]: (?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?\\w+) [(](?[^)]+)[)] (?.*)" diff --git a/src/formats/page_log.json b/src/formats/page_log.json index c5129766..0758b132 100644 --- a/src/formats/page_log.json +++ b/src/formats/page_log.json @@ -7,7 +7,7 @@ "multiline": false, "regex": { "pre-1.7": { - "pattern": "^(?[\\w_\\-\\.]+) (?[\\w\\.\\-]+) (?\\d+) \\[(?[^\\]]+)\\] (?total|\\d+) (?\\d+) (?[^ ]+) (?[\\w\\.:\\-]+)(?.*)$" + "pattern": "^(?[\\w_\\-\\.]+) (?[\\w\\.\\-]+) (?\\d+) \\[(?[^\\]]+)\\] (?total|\\d+) (?\\d+) (?[^ ]+) (?[\\w\\.:\\-]+)$" }, "1.7": { "pattern": "^(?[\\w_\\-\\.]+) (?[\\w\\.\\-]+) (?\\d+) \\[(?[^\\]]+)\\] (?total|\\d+) (?\\d+) (?[^ ]+) (?[\\w\\.:\\-]+) (?.+) (?[^ ]+) (?.+)(?.*)$" diff --git a/src/formats/unifi_log.json b/src/formats/unifi_log.json index f4aa4bdb..369e7532 100644 --- a/src/formats/unifi_log.json +++ b/src/formats/unifi_log.json @@ -15,7 +15,7 @@ "pattern": "^(?[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?[^\\s]+) (?\\w+)\\.(?\\w+) (?kernel): \\[(?:\\s*(?\\d+\\.\\d+))\\]\\s(?:\\[(?[^\\]]+)\\]\\s*)?(?:ALIEN BLOCK: )?IN=(?(?:\\d|\\w)*) OUT=(?(?:\\d|\\w)*) MAC=(?:(?(?:[0-9a-f]{2}:){5}[0-9a-f]{2})(?::(?[^\\s]+)))? SRC=(?(?:[\\d\\.])+) DST=(?(?:[\\d\\.])+) LEN=(?(?:\\d+)) TOS=(?0x(?:[0-9A-F])+) PREC=(?0x(?:[0-9A-F])+) TTL=(?\\d+) ID=(?\\d+) (?(?:DF) )?PROTO=(?(?!TCP|UDP)(?:\\w+))(?.*)$" }, "kernel-other": { - "pattern": "^(?[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?[^\\s]+) (?\\w+)\\.(?\\w+) (?kernel): (?:\\[(?:\\s*(?\\d+\\.\\d+))\\]\\s)?(?[^\\[].*)$" + "pattern": "^(?[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?[^\\s]+) (?\\w+)\\.(?\\w+) (?kernel): (?:\\[(?:\\s*(?\\d+\\.\\d+))\\]\\s)?(?!IN|ALIEN BLOCK)(?[^\\[].*)$" }, "dnsmasq-dhcp": { "pattern": "^(?[A-Z][a-z]{2}\\s+\\d+\\s+\\d+:\\d+:\\d+) (?[^\\s]+) (?\\w+)\\.(?\\w+) (?dnsmasq-dhcp[A-Za-z0-9\\.\\-]*)(?:\\[(?\\d+)\\])?: (?DHCP[^(]+)(?:\\((?[^)]*)\\)) (?:(?(?:\\d{1,3}\\.){3}\\d{1,3}) )?(?(?:[0-9a-f]{2}:)+[0-9a-f]{2})(?: (?.*))?$" diff --git a/src/lnav.cc b/src/lnav.cc index 5afc2fc6..4ac728c9 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -596,6 +596,10 @@ make it easier to navigate through files quickly. .append(" ") .append("An additional configuration directory.\n") .append(" ") + .append("-W"_symbol) + .append(" ") + .append("Print warnings related to lnav's configuration.\n") + .append(" ") .append("-u"_symbol) .append(" ") .append("Update formats installed from git repositories.\n") @@ -1946,9 +1950,19 @@ wait_for_children() } while (lnav_data.ld_looping); } +struct mode_flags_t { + bool mf_check_configs{false}; + bool mf_install{false}; + bool mf_update_formats{false}; + bool mf_no_default{false}; + bool mf_print_warnings{false}; +}; + static int -print_user_msgs(std::vector error_list) +print_user_msgs(std::vector error_list, + mode_flags_t mf) { + size_t warning_count = 0; int retval = EXIT_SUCCESS; for (auto& iter : error_list) { @@ -1959,6 +1973,13 @@ print_user_msgs(std::vector error_list) case lnav::console::user_message::level::ok: out_file = stdout; break; + case lnav::console::user_message::level::warning: + warning_count += 1; + if (!mf.mf_print_warnings) { + continue; + } + out_file = stderr; + break; default: out_file = stderr; break; @@ -1970,16 +1991,29 @@ print_user_msgs(std::vector error_list) } } + if (warning_count > 0 && !mf.mf_print_warnings + && !(lnav_data.ld_flags & LNF_HEADLESS) + && (std::chrono::system_clock::now() - lnav_data.ld_last_dot_lnav_time + > 24h)) + { + lnav::console::print( + stderr, + lnav::console::user_message::warning( + attr_line_t() + .append(lnav::roles::number(fmt::to_string(warning_count))) + .append(" issues were detected when checking lnav's " + "configuration")) + .with_help( + attr_line_t("pass ") + .append(lnav::roles::symbol("-W")) + .append(" on the command line to display the issues\n") + .append("(this message will only be displayed once a " + "day)"))); + } + return retval; } -struct mode_flags_t { - bool mf_check_configs{false}; - bool mf_install{false}; - bool mf_update_formats{false}; - bool mf_no_default{false}; -}; - enum class verbosity_t : int { quiet, standard, @@ -2040,6 +2074,11 @@ main(int argc, char* argv[]) stable_sort(lnav_data.ld_db_key_names.begin(), lnav_data.ld_db_key_names.end()); + auto dot_lnav_path = lnav::paths::dotlnav(); + std::error_code last_write_ec; + lnav_data.ld_last_dot_lnav_time + = ghc::filesystem::last_write_time(dot_lnav_path, last_write_ec); + ensure_dotlnav(); log_install_handlers(); @@ -2176,6 +2215,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' "-R", lnav_data.ld_active_files.fc_rotated, "rotated"); auto* recurse_flag = app.add_flag( "-r", lnav_data.ld_active_files.fc_recursive, "recurse"); + app.add_flag("-W", mode_flags.mf_print_warnings); auto* headless_flag = app.add_flag( "-n", [](size_t count) { lnav_data.ld_flags |= LNF_HEADLESS; }, @@ -2188,7 +2228,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' perror("Read key from STDIN"); } }; - app.add_flag("-W", wait_cb); + app.add_flag("-S", wait_cb); auto cmd_appender = [](std::string cmd) { lnav_data.ld_commands.emplace_back(cmd); }; @@ -2273,7 +2313,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' return EXIT_SUCCESS; } catch (const CLI::ParseError& e) { if (!arg_errors.empty()) { - print_user_msgs(arg_errors); + print_user_msgs(arg_errors, mode_flags); return e.get_exit_code(); } @@ -2312,7 +2352,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' load_config(lnav_data.ld_config_paths, config_errors); if (!config_errors.empty()) { - if (print_user_msgs(config_errors) != EXIT_SUCCESS) { + if (print_user_msgs(config_errors, mode_flags) != EXIT_SUCCESS) { return EXIT_FAILURE; } } @@ -2421,7 +2461,9 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' auto format_list = load_format_file(src_path, loader_errors); if (!loader_errors.empty()) { - if (print_user_msgs(loader_errors) != EXIT_SUCCESS) { + if (print_user_msgs(loader_errors, mode_flags) + != EXIT_SUCCESS) + { return EXIT_FAILURE; } } @@ -2613,7 +2655,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' load_format_vtabs(lnav_data.ld_vtab_manager.get(), loader_errors); if (!loader_errors.empty()) { - if (print_user_msgs(loader_errors) != EXIT_SUCCESS) { + if (print_user_msgs(loader_errors, mode_flags) != EXIT_SUCCESS) { if (mmode_ops == nullptr) { return EXIT_FAILURE; } @@ -2623,7 +2665,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' if (mmode_ops) { auto perform_res = lnav::management::perform(mmode_ops); - return print_user_msgs(perform_res); + return print_user_msgs(perform_res, mode_flags); } if (!mode_flags.mf_check_configs && !lnav_data.ld_show_help_view) { diff --git a/src/lnav.hh b/src/lnav.hh index 9ed478e3..8460cd78 100644 --- a/src/lnav.hh +++ b/src/lnav.hh @@ -258,6 +258,8 @@ struct lnav_data_t { bool ld_initial_build{false}; bool ld_show_help_view{false}; + + ghc::filesystem::file_time_type ld_last_dot_lnav_time; }; struct static_service {}; diff --git a/src/log_format.cc b/src/log_format.cc index d575ce38..a4b667f8 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -672,7 +672,7 @@ external_log_format::scan_for_partial(shared_buffer_ref& sbr, const auto& pat = this->elf_pattern_order[this->last_pattern_index()]; if (!this->lf_multiline) { - len_out = pat->p_pcre.value->match_partial(sbr.to_string_fragment()); + len_out = pat->p_pcre.pp_value->match_partial(sbr.to_string_fragment()); return true; } @@ -682,7 +682,7 @@ external_log_format::scan_for_partial(shared_buffer_ref& sbr, return false; } - len_out = pat->p_pcre.value->match_partial(sbr.to_string_fragment()); + len_out = pat->p_pcre.pp_value->match_partial(sbr.to_string_fragment()); return (int) len_out > pat->p_timestamp_end; } @@ -803,7 +803,7 @@ external_log_format::scan(logfile& lf, while (::next_format(this->elf_pattern_order, curr_fmt, pat_index)) { auto* fpat = this->elf_pattern_order[curr_fmt].get(); - auto* pat = fpat->p_pcre.value.get(); + auto* pat = fpat->p_pcre.pp_value.get(); if (fpat->p_module_format) { continue; @@ -913,12 +913,12 @@ external_log_format::scan(logfile& lf, int mod_pat_index = mod_elf->last_pattern_index(); auto& mod_pat = *mod_elf->elf_pattern_order[mod_pat_index]; - auto mod_md = mod_pat.p_pcre.value->create_match_data(); - auto match_res - = mod_pat.p_pcre.value->capture_from(body_cap.value()) - .into(mod_md) - .matches(PCRE2_NO_UTF_CHECK) - .ignore_error(); + auto mod_md = mod_pat.p_pcre.pp_value->create_match_data(); + auto match_res = mod_pat.p_pcre.pp_value + ->capture_from(body_cap.value()) + .into(mod_md) + .matches(PCRE2_NO_UTF_CHECK) + .ignore_error(); if (match_res) { auto mod_level_cap = mod_md[mod_pat.p_level_field_index]; @@ -1021,8 +1021,8 @@ external_log_format::module_scan(string_fragment body_cap, continue; } - auto md = pat.value->create_match_data(); - auto match_res = pat.value->capture_from(body_cap) + auto md = pat.pp_value->create_match_data(); + auto match_res = pat.pp_value->capture_from(body_cap) .into(md) .matches(PCRE2_NO_UTF_CHECK) .ignore_error(); @@ -1072,12 +1072,13 @@ external_log_format::annotate(uint64_t line_number, int pat_index = this->pattern_index_for_line(line_number); auto& pat = *this->elf_pattern_order[pat_index]; - sa.reserve(pat.p_pcre.value->get_capture_count()); - auto md = pat.p_pcre.value->create_match_data(); - auto match_res = pat.p_pcre.value->capture_from(line.to_string_fragment()) - .into(md) - .matches(PCRE2_NO_UTF_CHECK) - .ignore_error(); + sa.reserve(pat.p_pcre.pp_value->get_capture_count()); + auto md = pat.p_pcre.pp_value->create_match_data(); + auto match_res + = pat.p_pcre.pp_value->capture_from(line.to_string_fragment()) + .into(md) + .matches(PCRE2_NO_UTF_CHECK) + .ignore_error(); if (!match_res) { // A continued line still needs a body. lr.lr_start = 0; @@ -1085,7 +1086,7 @@ external_log_format::annotate(uint64_t line_number, sa.emplace_back(lr, SA_BODY.value()); if (!this->lf_multiline) { auto len - = pat.p_pcre.value->match_partial(line.to_string_fragment()); + = pat.p_pcre.pp_value->match_partial(line.to_string_fragment()); sa.emplace_back( line_range{(int) len, -1}, SA_INVALID.value("Log line does not match any pattern")); @@ -1271,8 +1272,8 @@ read_json_field(yajlpp_parse_context* ypc, const unsigned char* str, size_t len) jlu->jlu_format->lf_timestamp_flags = tm_out.et_flags & ~ETF_MACHINE_ORIENTED; jlu->jlu_base_line->set_time(tv_out); - } else if (jlu->jlu_format->elf_level_pointer.value != nullptr) { - if (jlu->jlu_format->elf_level_pointer.value + } else if (jlu->jlu_format->elf_level_pointer.pp_value != nullptr) { + if (jlu->jlu_format->elf_level_pointer.pp_value ->find_in(field_name.to_string_fragment(), PCRE2_NO_UTF_CHECK) .ignore_error() .has_value()) @@ -1764,7 +1765,7 @@ external_log_format::build(std::vector& errors) { pattern& pat = *iter->second; - if (pat.p_pcre.value == nullptr) { + if (pat.p_pcre.pp_value == nullptr) { continue; } @@ -1772,7 +1773,7 @@ external_log_format::build(std::vector& errors) this->elf_has_module_format = true; } - for (auto named_cap : pat.p_pcre.value->get_named_captures()) { + for (auto named_cap : pat.p_pcre.pp_value->get_named_captures()) { const intern_string_t name = intern_string::lookup(named_cap.get_name()); @@ -1802,8 +1803,8 @@ external_log_format::build(std::vector& errors) ivd.ivd_index = named_cap.get_index(); if (!vd->vd_unit_field.empty()) { - ivd.ivd_unit_field_index - = pat.p_pcre.value->name_index(vd->vd_unit_field.get()); + ivd.ivd_unit_field_index = pat.p_pcre.pp_value->name_index( + vd->vd_unit_field.get()); } else { ivd.ivd_unit_field_index = -1; } @@ -1908,11 +1909,11 @@ external_log_format::build(std::vector& errors) bool found_in_pattern = false; for (const auto& pat : this->elf_patterns) { - if (pat.second->p_pcre.value == nullptr) { + if (pat.second->p_pcre.pp_value == nullptr) { continue; } - auto cap_index = pat.second->p_pcre.value->name_index( + auto cap_index = pat.second->p_pcre.pp_value->name_index( vd->vd_meta.lvm_name.get()); if (cap_index >= 0) { found_in_pattern = true; @@ -1920,7 +1921,7 @@ external_log_format::build(std::vector& errors) } for (auto named_cap : - pat.second->p_pcre.value->get_named_captures()) + pat.second->p_pcre.pp_value->get_named_captures()) { available_captures.insert(named_cap.get_name().to_string()); } @@ -1970,8 +1971,8 @@ external_log_format::build(std::vector& errors) for (const auto& td_pair : this->lf_tag_defs) { const auto& td = td_pair.second; - if (td->ftd_pattern.value == nullptr - || td->ftd_pattern.value->get_pattern().empty()) + if (td->ftd_pattern.pp_value == nullptr + || td->ftd_pattern.pp_value->get_pattern().empty()) { errors.emplace_back( lnav::console::user_message::error( @@ -2000,23 +2001,26 @@ external_log_format::build(std::vector& errors) .with_snippets(this->get_snippets())); } - for (auto& elf_sample : this->elf_samples) { + for (size_t sample_index = 0; sample_index < this->elf_samples.size(); + sample_index += 1) + { + auto& elf_sample = this->elf_samples[sample_index]; auto sample_lines = string_fragment(elf_sample.s_line.pp_value).split_lines(); bool found = false; for (auto pat_iter = this->elf_pattern_order.begin(); - pat_iter != this->elf_pattern_order.end() && !found; + pat_iter != this->elf_pattern_order.end(); ++pat_iter) { auto& pat = *(*pat_iter); - if (!pat.p_pcre.value) { + if (!pat.p_pcre.pp_value) { continue; } - auto md = pat.p_pcre.value->create_match_data(); - auto match_res = pat.p_pcre.value->capture_from(sample_lines[0]) + auto md = pat.p_pcre.pp_value->create_match_data(); + auto match_res = pat.p_pcre.pp_value->capture_from(sample_lines[0]) .into(md) .matches() .ignore_error(); @@ -2029,16 +2033,20 @@ external_log_format::build(std::vector& errors) continue; } - if (pat.p_pcre.value->name_index(this->lf_timestamp_field.get()) + elf_sample.s_matched_regexes.insert(pat.p_name.to_string()); + pat.p_matched_samples.insert(sample_index); + + if (pat.p_pcre.pp_value->name_index(this->lf_timestamp_field.get()) < 0) { attr_line_t notes; bool first_note = true; - if (pat.p_pcre.value->get_capture_count() > 0) { + if (pat.p_pcre.pp_value->get_capture_count() > 0) { notes.append("the following captures are available:\n "); } - for (auto named_cap : pat.p_pcre.value->get_named_captures()) { + for (auto named_cap : pat.p_pcre.pp_value->get_named_captures()) + { if (!first_note) { notes.append(", "); } @@ -2136,7 +2144,7 @@ external_log_format::build(std::vector& errors) .append(" property"))); } - log_level_t level = this->convert_level( + auto level = this->convert_level( level_cap.value_or(string_fragment::invalid()), nullptr); if (elf_sample.s_level != LEVEL_UNKNOWN @@ -2166,12 +2174,13 @@ external_log_format::build(std::vector& errors) { auto full_match_res - = pat.p_pcre.value->capture_from(elf_sample.s_line.pp_value) + = pat.p_pcre.pp_value + ->capture_from(elf_sample.s_line.pp_value) .into(md) .matches() .ignore_error(); if (!full_match_res) { - attr_line_t regex_al = pat.p_pcre.value->get_pattern(); + attr_line_t regex_al = pat.p_pcre.pp_value->get_pattern(); lnav::snippets::regex_highlighter( regex_al, -1, line_range{0, (int) regex_al.length()}); errors.emplace_back( @@ -2193,7 +2202,7 @@ external_log_format::build(std::vector& errors) } else if (static_cast(full_match_res->f_all.length()) != elf_sample.s_line.pp_value.length()) { - attr_line_t regex_al = pat.p_pcre.value->get_pattern(); + attr_line_t regex_al = pat.p_pcre.pp_value->get_pattern(); lnav::snippets::regex_highlighter( regex_al, -1, line_range{0, (int) regex_al.length()}); auto match_length @@ -2233,12 +2242,12 @@ external_log_format::build(std::vector& errors) for (const auto& pat_iter : this->elf_pattern_order) { auto& pat = *pat_iter; - if (!pat.p_pcre.value) { + if (!pat.p_pcre.pp_value) { continue; } partial_indexes.emplace_back( - pat.p_pcre.value->match_partial(sample_lines[0]), + pat.p_pcre.pp_value->match_partial(sample_lines[0]), pat.p_name); max_name_width = std::max(max_name_width, pat.p_name.size()); } @@ -2270,7 +2279,7 @@ external_log_format::build(std::vector& errors) attr_line_t regex_note; for (const auto& pat_iter : this->elf_pattern_order) { - if (!pat_iter->p_pcre.value) { + if (!pat_iter->p_pcre.pp_value) { regex_note .append( lnav::roles::symbol(fmt::format(FMT_STRING("{:{}}"), @@ -2280,7 +2289,7 @@ external_log_format::build(std::vector& errors) continue; } - attr_line_t regex_al = pat_iter->p_pcre.value->get_pattern(); + attr_line_t regex_al = pat_iter->p_pcre.pp_value->get_pattern(); lnav::snippets::regex_highlighter( regex_al, -1, line_range{0, (int) regex_al.length()}); @@ -2303,6 +2312,52 @@ external_log_format::build(std::vector& errors) } } + if (!this->elf_samples.empty()) { + for (const auto& elf_sample : this->elf_samples) { + if (elf_sample.s_matched_regexes.size() <= 1) { + continue; + } + + errors.emplace_back( + lnav::console::user_message::warning( + attr_line_t("invalid log format: ") + .append_quoted( + lnav::roles::symbol(this->elf_name.to_string()))) + .with_reason( + attr_line_t( + "sample is matched by more than one regex: ") + .join(elf_sample.s_matched_regexes, + VC_ROLE.value(role_t::VCR_SYMBOL), + ", ")) + .with_snippet(lnav::console::snippet::from( + elf_sample.s_line.pp_location, + attr_line_t().append(lnav::roles::quoted_code( + elf_sample.s_line.pp_value)))) + .with_help("log format regexes must match a single type " + "of log message")); + } + + for (const auto& pat : this->elf_pattern_order) { + if (pat->p_module_format) { + continue; + } + + if (pat->p_matched_samples.empty()) { + errors.emplace_back( + lnav::console::user_message::warning( + attr_line_t("invalid pattern: ") + .append_quoted( + lnav::roles::symbol(pat->p_config_path))) + .with_reason("pattern does not match any samples") + .with_snippet(lnav::console::snippet::from( + pat->p_pcre.pp_location, "")) + .with_help( + "every pattern should have at least one sample " + "that it matches")); + } + } + } + for (auto& elf_value_def : this->elf_value_defs) { if (elf_value_def.second->vd_foreign_key || elf_value_def.second->vd_meta.lvm_identifier) @@ -2441,8 +2496,8 @@ external_log_format::build(std::vector& errors) attrs.ta_attrs |= A_BLINK; } - if (hd.hd_pattern.value != nullptr) { - this->lf_highlighters.emplace_back(hd.hd_pattern.value); + if (hd.hd_pattern.pp_value != nullptr) { + this->lf_highlighters.emplace_back(hd.hd_pattern.pp_value); this->lf_highlighters.back() .with_name(hd_pair.first.to_string()) .with_format_name(this->elf_name) @@ -2458,12 +2513,13 @@ external_log_format::register_vtabs( std::vector& errors) { for (auto& elf_search_table : this->elf_search_tables) { - if (elf_search_table.second.std_pattern.value == nullptr) { + if (elf_search_table.second.std_pattern.pp_value == nullptr) { continue; } auto lst = std::make_shared( - elf_search_table.second.std_pattern.value, elf_search_table.first); + elf_search_table.second.std_pattern.pp_value, + elf_search_table.first); lst->lst_format = this; lst->lst_log_path_glob = elf_search_table.second.std_glob; if (elf_search_table.second.std_level != LEVEL_UNKNOWN) { @@ -2487,11 +2543,11 @@ external_log_format::match_samples(const std::vector& samples) const for (const auto& pat_iter : this->elf_pattern_order) { auto& pat = *pat_iter; - if (!pat.p_pcre.value) { + if (!pat.p_pcre.pp_value) { continue; } - if (pat.p_pcre.value->find_in(sample_iter.s_line.pp_value) + if (pat.p_pcre.pp_value->find_in(sample_iter.s_line.pp_value) .ignore_error()) { return true; @@ -2673,11 +2729,11 @@ external_log_format::specialized(int fmt_lock) bool external_log_format::match_name(const std::string& filename) { - if (this->elf_filename_pcre.value == nullptr) { + if (this->elf_filename_pcre.pp_value == nullptr) { return true; } - return this->elf_filename_pcre.value->find_in(filename) + return this->elf_filename_pcre.pp_value->find_in(filename) .ignore_error() .has_value(); } @@ -2755,7 +2811,7 @@ external_log_format::convert_level(string_fragment sf, retval = string2level(sf.data(), sf.length()); } else { for (const auto& elf_level_pattern : this->elf_level_patterns) { - if (elf_level_pattern.second.lp_pcre.value + if (elf_level_pattern.second.lp_pcre.pp_value ->find_in(sf, PCRE2_NO_UTF_CHECK) .ignore_error() .has_value()) diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index 7aa2d9f8..e966b279 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -46,6 +46,7 @@ public: positioned_property s_line; std::string s_description; log_level_t s_level{LEVEL_UNKNOWN}; + std::set s_matched_regexes; }; struct value_def { @@ -113,6 +114,7 @@ public: int p_body_field_index{-1}; int p_timestamp_end{-1}; bool p_module_format{false}; + std::set p_matched_samples; }; struct level_pattern { @@ -311,7 +313,8 @@ public: return ""; } int pat_index = this->pattern_index_for_line(line_number); - return this->elf_pattern_order[pat_index]->p_pcre.value->get_pattern(); + return this->elf_pattern_order[pat_index] + ->p_pcre.pp_value->get_pattern(); } log_level_t convert_level(string_fragment str, diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index d009aaa0..e110cc20 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -308,7 +308,7 @@ read_levels(yajlpp_parse_context* ypc, const unsigned char* str, size_t len) value_frag.to_string(), lnav::console::to_user_message(PATTERN_SRC, ce)); } else { - elf->elf_level_patterns[level].lp_pcre.value + elf->elf_level_patterns[level].lp_pcre.pp_value = compile_res.unwrap().to_shared(); } diff --git a/src/logfile.cc b/src/logfile.cc index 6fe7ebcf..cf0c6fa6 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -640,7 +640,7 @@ logfile::rebuild_index(nonstd::optional deadline) continue; } - if (td->ftd_pattern.value + if (td->ftd_pattern.pp_value ->find_in(sf, PCRE2_NO_UTF_CHECK) .ignore_error() .has_value()) diff --git a/src/regex101.import.cc b/src/regex101.import.cc index d0f7d345..e955aa05 100644 --- a/src/regex101.import.cc +++ b/src/regex101.import.cc @@ -372,7 +372,7 @@ regex101::convert_format_pattern( { regex101::client::entry en; - en.e_regex = pattern->p_pcre.value->get_pattern(); + en.e_regex = pattern->p_pcre.pp_value->get_pattern(); for (const auto& sample : format->elf_samples) { if (en.e_test_string.empty()) { en.e_test_string = sample.s_line.pp_value; diff --git a/src/yajlpp/yajlpp.hh b/src/yajlpp/yajlpp.hh index a6aa133b..46fcbf2b 100644 --- a/src/yajlpp/yajlpp.hh +++ b/src/yajlpp/yajlpp.hh @@ -90,41 +90,46 @@ struct positioned_property { }; template -struct factory_container { +struct factory_container : public positioned_property> { template - struct with_default_args { + struct with_default_args : public positioned_property> { template static Result from( - intern_string_t src, Args... args) + intern_string_t src, source_location loc, Args... args) { auto from_res = T::from(args..., DefaultArgs...); if (from_res.isOk()) { - return Ok(with_default_args{from_res.unwrap().to_shared()}); + with_default_args retval; + + retval.pp_path = src; + retval.pp_location = loc; + retval.pp_value = from_res.unwrap().to_shared(); + return Ok(retval); } return Err( lnav::console::to_user_message(src, from_res.unwrapErr())); } - - std::shared_ptr value; }; template static Result from( - intern_string_t src, Args... args) + intern_string_t src, source_location loc, Args... args) { auto from_res = T::from(args...); if (from_res.isOk()) { - return Ok(factory_container{from_res.unwrap().to_shared()}); + factory_container retval; + + retval.pp_path = src; + retval.pp_location = loc; + retval.pp_value = from_res.unwrap().to_shared(); + return Ok(retval); } - return Err( - lnav::console::to_user_message(src, from_res.unwrapErr())); + return Err(lnav::console::to_user_message(src, from_res.unwrapErr())); } - - std::shared_ptr value; }; class yajlpp_gen_context; diff --git a/src/yajlpp/yajlpp_def.hh b/src/yajlpp/yajlpp_def.hh index ea9dfb5a..2e5a4901 100644 --- a/src/yajlpp/yajlpp_def.hh +++ b/src/yajlpp/yajlpp_def.hh @@ -1090,10 +1090,13 @@ struct json_path_handler : public json_path_handler_base { struct int_ { typedef int type; }; - template::type = 0, - typename... Args> + template< + typename C, + typename T, + typename int_::type + = 0, + typename... Args> json_path_handler& for_field(Args... args, T C::*ptr_arg) { this->add_cb(str_field_cb2); @@ -1103,8 +1106,9 @@ struct json_path_handler : public json_path_handler_base { auto* obj = ypc->ypc_obj_stack.top(); auto value_frag = string_fragment::from_bytes(str, len); const auto* jph = ypc->ypc_current_handler; + auto loc = source_location{ypc->ypc_source, ypc->get_line_number()}; - auto from_res = T::from(ypc->get_full_path(), value_frag); + auto from_res = T::from(ypc->get_full_path(), loc, value_frag); if (from_res.isErr()) { jph->report_error( ypc, value_frag.to_string(), from_res.unwrapErr()); diff --git a/test/expected/expected.am b/test/expected/expected.am index 3a7f53dc..89f54e6f 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -214,6 +214,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_config.sh_5105c29004e297521310ca0bd0fd560b01c2c549.out \ $(srcdir)/%reldir%/test_config.sh_5fd9fbccc35e9b06abdd913da0c16bdb306b926e.err \ $(srcdir)/%reldir%/test_config.sh_5fd9fbccc35e9b06abdd913da0c16bdb306b926e.out \ + $(srcdir)/%reldir%/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.err \ + $(srcdir)/%reldir%/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.out \ $(srcdir)/%reldir%/test_config.sh_b08f7523659d1c12f0e59920cd40d17d4a83b72f.err \ $(srcdir)/%reldir%/test_config.sh_b08f7523659d1c12f0e59920cd40d17d4a83b72f.out \ $(srcdir)/%reldir%/test_config.sh_d622658dc98327b1b2fd346802d24bc633e34ac7.err \ @@ -240,8 +242,12 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_format_loader.sh_15e861d2327512a721fd42ae51dc5427689e0bb6.out \ $(srcdir)/%reldir%/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err \ $(srcdir)/%reldir%/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.out \ + $(srcdir)/%reldir%/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err \ + $(srcdir)/%reldir%/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.out \ $(srcdir)/%reldir%/test_format_loader.sh_a47f2b090a5d8a226783835c7ff7d1c8821f11ed.err \ $(srcdir)/%reldir%/test_format_loader.sh_a47f2b090a5d8a226783835c7ff7d1c8821f11ed.out \ + $(srcdir)/%reldir%/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.err \ + $(srcdir)/%reldir%/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.out \ $(srcdir)/%reldir%/test_json_format.sh_168cac40c27f547044c89d39eb0ff2ef81da4b21.err \ $(srcdir)/%reldir%/test_json_format.sh_168cac40c27f547044c89d39eb0ff2ef81da4b21.out \ $(srcdir)/%reldir%/test_json_format.sh_1bb0fd243e916546aea22029245ac590dae17a86.err \ diff --git a/test/expected/test_config.sh_5105c29004e297521310ca0bd0fd560b01c2c549.err b/test/expected/test_config.sh_5105c29004e297521310ca0bd0fd560b01c2c549.err index ec11ba51..092b26a0 100644 --- a/test/expected/test_config.sh_5105c29004e297521310ca0bd0fd560b01c2c549.err +++ b/test/expected/test_config.sh_5105c29004e297521310ca0bd0fd560b01c2c549.err @@ -4,35 +4,17 @@  = note: expecting one of the following $schema values:  https://lnav.org/schemas/config-v1.schema.json  = help: Property Synopsis - /$schema  - Description - The URI that specifies the schema that describes this type of file - Example - https://lnav.org/schemas/config-v1.schema.json -⚠ warning: unexpected value for property “/ui” - --> {test_dir}/bad-config2/formats/invalid-config/config.malformed.json:2 - |  "ui": "theme",  - = help: Available Properties - $schema  - tuning/ - ui/ - log/ - global/ + /$schema  + Description + The URI that specifies the schema that describes this type of file + Example + https://lnav.org/schemas/config-v1.schema.json ✘ error: invalid JSON  --> {test_dir}/bad-config2/formats/invalid-config/config.malformed.json:3  | parse error: object key and value must be separated by a colon (':')  |  "ui": "theme", "abc", "def": "" }  |  (right here) ------^  |  -⚠ warning: unexpected value for property “/ui” - --> {test_dir}/bad-config2/formats/invalid-config/config.truncated.json:2 - |  "ui": "theme"  - = help: Available Properties - $schema  - tuning/ - ui/ - log/ - global/ ✘ error: invalid JSON reason: parse error: premature EOF  --> {test_dir}/bad-config2/formats/invalid-config/config.truncated.json:3 diff --git a/test/expected/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.err b/test/expected/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.err new file mode 100644 index 00000000..ec11ba51 --- /dev/null +++ b/test/expected/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.err @@ -0,0 +1,38 @@ +✘ error: 'bad' is not a supported configuration $schema version + --> {test_dir}/bad-config2/formats/invalid-config/config.bad-schema.json:2 + |  "$schema": "bad"  + = note: expecting one of the following $schema values: +  https://lnav.org/schemas/config-v1.schema.json + = help: Property Synopsis + /$schema  + Description + The URI that specifies the schema that describes this type of file + Example + https://lnav.org/schemas/config-v1.schema.json +⚠ warning: unexpected value for property “/ui” + --> {test_dir}/bad-config2/formats/invalid-config/config.malformed.json:2 + |  "ui": "theme",  + = help: Available Properties + $schema  + tuning/ + ui/ + log/ + global/ +✘ error: invalid JSON + --> {test_dir}/bad-config2/formats/invalid-config/config.malformed.json:3 + | parse error: object key and value must be separated by a colon (':') + |  "ui": "theme", "abc", "def": "" } + |  (right here) ------^ + |  +⚠ warning: unexpected value for property “/ui” + --> {test_dir}/bad-config2/formats/invalid-config/config.truncated.json:2 + |  "ui": "theme"  + = help: Available Properties + $schema  + tuning/ + ui/ + log/ + global/ +✘ error: invalid JSON + reason: parse error: premature EOF + --> {test_dir}/bad-config2/formats/invalid-config/config.truncated.json:3 diff --git a/test/expected/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.out b/test/expected/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.out new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err b/test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err index e8361462..202b4514 100644 --- a/test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err +++ b/test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err @@ -111,6 +111,14 @@ std  = “^(?<timestamp>\d+): (?<pid>\w+) (?<body>.*)$” with-level = “^(?<timestamp>\d+)\| (?<level>\w+) (?<body>\w+)$” +⚠ warning: invalid pattern: “/bad_sample_log/regex/semi” + reason: pattern does not match any samples + --> {test_dir}/bad-config/formats/invalid-sample/format.json:10 + = help: every pattern should have at least one sample that it matches +⚠ warning: invalid pattern: “/bad_sample_log/regex/std” + reason: pattern does not match any samples + --> {test_dir}/bad-config/formats/invalid-sample/format.json:7 + = help: every pattern should have at least one sample that it matches ⚠ warning: invalid value “/invalid_props_log/value/non-existent” reason: no patterns have a capture named “non-existent”  --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 diff --git a/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err b/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err new file mode 100644 index 00000000..202b4514 --- /dev/null +++ b/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err @@ -0,0 +1,171 @@ +✘ error: “invalid(abc” is not a valid regular expression + reason: missing closing parenthesis + --> /invalid_props_log/tags/badtag3/pattern + | invalid(abc  + |  ^ missing closing parenthesis + --> {test_dir}/bad-config/formats/invalid-properties/format.json:35 + |  "pattern": "invalid(abc" + = help: Property Synopsis + /invalid_props_log/tags/badtag3/pattern  + Description + The regular expression to match against the body of the log message + Example + \w+ is down +✘ error: “abc(def” is not a valid regular expression + reason: missing closing parenthesis + --> /invalid_props_log/search-table/bad_table_regex/pattern + | abc(def  + |  ^ missing closing parenthesis  + --> {test_dir}/bad-config/formats/invalid-properties/format.json:40 + |  "pattern": "abc(def"  + = help: Property Synopsis + /invalid_props_log/search-table/bad_table_regex/pattern  + Description + The regular expression for this search table. +✘ error: “^(?\d+: (?.*)$” is not a valid regular expression + reason: missing closing parenthesis + --> /bad_regex_log/regex/std/pattern + | ^(?<timestamp>\d+: (?<body>.*)$  + |  ^ missing closing parenthesis + --> {test_dir}/bad-config/formats/invalid-regex/format.json:6 + |  "pattern": "^(?\\d+: (?.*)$" + = help: Property Synopsis + /bad_regex_log/regex/std/pattern  + Description + The regular expression to match a log message and capture fields. +✘ error: “(foo” is not a valid regular expression + reason: missing closing parenthesis + --> pattern + | (foo  + |  ^ missing closing parenthesis  + --> {test_dir}/bad-config/formats/invalid-regex/format.json:13 + |  "error": "(foo"  + = help: Property Synopsis + /bad_regex_log/level/error  + Description + The regular expression used to match the log text for this level. For JSON logs with numeric levels, this should be the number for the corresponding level. +✘ error: “abc(” is not a valid regular expression + reason: missing closing parenthesis + --> /bad_regex_log/highlights/foobar/pattern + | abc(  + |  ^ missing closing parenthesis  + --> {test_dir}/bad-config/formats/invalid-regex/format.json:25 + |  "pattern": "abc("  + = help: Property Synopsis + /bad_regex_log/highlights/foobar/pattern  + Description + A regular expression to highlight in logs of this format. +✘ error: “foo” is not a valid value for option “/bad_sample_log/value/pid/kind” + --> {test_dir}/bad-config/formats/invalid-sample/format.json:24 + |  "kind": "foo"  + = help: Property Synopsis + /bad_sample_log/value/pid/kind  + Description + The type of data in the field + Allowed Values + string, integer, float, boolean, json, struct, quoted, xml +✘ error: 'bad' is not a supported log format $schema version + --> {test_dir}/bad-config/formats/invalid-schema/format.json:2 + |  "$schema": "bad"  + = note: expecting one of the following $schema values: +  https://lnav.org/schemas/format-v1.schema.json + = help: Property Synopsis + /$schema The URI of the schema for this file + Description + Specifies the type of this file +✘ error: invalid pattern: “incomplete-match” + reason: pattern does not match entire message + --> {test_dir}/bad-config/formats/invalid-regex/format.json:20 + | 1428634687123; foo  + |  ^ matched up to here  + = note: incomplete-match = ^(?<timestamp>\d+); + = help: update the regular expression to fully capture the sample message +✘ error: invalid sample log message: "abc: foo" + reason: unrecognized timestamp -- abc + --> {test_dir}/bad-config/formats/invalid-sample/format.json:30 + = note: the following custom formats were tried: + abc + ^ “%i” matched up to here + = help: If the timestamp format is not supported by default, you can add a custom format with the “timestamp-format” property +✘ error: invalid sample log message: "1428634687123| debug hello" + reason: “debug” does not match the expected level of “info” + --> {test_dir}/bad-config/formats/invalid-sample/format.json:33 + = note: matched regex = with-level + captured level = “debug” +✘ error: invalid pattern: “with-level” + reason: pattern does not match entire multiline sample message + --> {test_dir}/bad-config/formats/invalid-sample/format.json:37 + = note: with-level = ^(?<timestamp>\d+)\| (?<level>\w+) (?<body>\w+)$ + = help: use “.*” to match new-lines +✘ error: invalid sample log message: "1428634687123; foo bar" + reason: sample does not match any patterns + --> {test_dir}/bad-config/formats/invalid-sample/format.json:41 + = note: the following shows how each pattern matched this sample: + 1428634687123; foo bar + ^ bad-time matched up to here + ^ semi matched up to here + ^ std matched up to here + ^ with-level matched up to here + = note: bad-time  = “^(?<timestamp>\w+): (?<body>\w+)$” + semi  = “^(?<timestamp>\d+); (?<body>\w+)$” + std  = “^(?<timestamp>\d+): (?<pid>\w+) (?<body>.*)$” + with-level = “^(?<timestamp>\d+)\| (?<level>\w+) (?<body>\w+)$” + +⚠ warning: invalid pattern: “/bad_sample_log/regex/semi” + reason: pattern does not match any samples + --> {test_dir}/bad-config/formats/invalid-sample/format.json:10 + = help: every pattern should have at least one sample that it matches +⚠ warning: invalid pattern: “/bad_sample_log/regex/std” + reason: pattern does not match any samples + --> {test_dir}/bad-config/formats/invalid-sample/format.json:7 + = help: every pattern should have at least one sample that it matches +⚠ warning: invalid value “/invalid_props_log/value/non-existent” + reason: no patterns have a capture named “non-existent” + --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 + = note: the following captures are available: + body, pid, timestamp + = help: values are populated from captures in patterns, so at least one pattern must have a capture with this value name +✘ error: invalid tag definition “/invalid_props_log/tags/badtag” + reason: tag definitions must have a non-empty pattern + --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 +✘ error: invalid tag definition “/invalid_props_log/tags/badtag2” + reason: tag definitions must have a non-empty pattern + --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 +✘ error: invalid tag definition “/invalid_props_log/tags/badtag3” + reason: tag definitions must have a non-empty pattern + --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 +✘ error: invalid value for property “/invalid_props_log/timestamp-field” + reason: “ts” was not found in the pattern at /invalid_props_log/regex/std + --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 + = note: the following captures are available: + body, pid, timestamp +✘ error: “not a color” is not a valid color value for property “/invalid_props_log/highlights/hl1/color” + reason: Unknown color: 'not a color'. See https://jonasjacek.github.io/colors/ for a list of supported color names + --> {test_dir}/bad-config/formats/invalid-properties/format.json:23 +✘ error: “also not a color” is not a valid color value for property “/invalid_props_log/highlights/hl1/background-color” + reason: Unknown color: 'also not a color'. See https://jonasjacek.github.io/colors/ for a list of supported color names + --> {test_dir}/bad-config/formats/invalid-properties/format.json:24 +✘ error: “no_regexes_log” is not a valid log format + reason: no regexes specified + --> {test_dir}/bad-config/formats/no-regexes/format.json:4 +✘ error: “no_regexes_log” is not a valid log format + reason: log message samples must be included in a format definition + --> {test_dir}/bad-config/formats/no-regexes/format.json:4 +✘ error: “no_sample_log” is not a valid log format + reason: log message samples must be included in a format definition + --> {test_dir}/bad-config/formats/no-samples/format.json:4 +✘ error: failed to compile SQL statement + reason: near "TALE": syntax error + --> {test_dir}/bad-config/formats/invalid-sql/init.sql:4 + | -- comment test  + | CREATE TALE invalid (x y z);  + |  ^ near "TALE": syntax error  +✘ error: failed to execute SQL statement + reason: ✘ error: “abc(” is not a valid regular expression + |  reason: missing closing parenthesis + |   --> arg + |   | abc(  + |   |  ^ missing closing parenthesis + --> {test_dir}/bad-config/formats/invalid-sql/init2.sql + | SELECT regexp_match('abc(', '123')  + | FROM sqlite_master;  diff --git a/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.out b/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.out new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.err b/test/expected/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.err new file mode 100644 index 00000000..f657d874 --- /dev/null +++ b/test/expected/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.err @@ -0,0 +1,61 @@ +✘ error: invalid JSON + --> {test_dir}/bad-config-json/formats/invalid-json/format.json:4 + | parse error: object key and value must be separated by a colon (':') + |  ar_log": { "abc" } } + |  (right here) ------^ + |  +✘ error: “abc(” is not a valid regular expression + reason: missing closing parenthesis + --> /invalid_key_log/level-pointer + | abc(  + |  ^ missing closing parenthesis  + --> {test_dir}/bad-config-json/formats/invalid-key/format.json:4 + |  "level-pointer": "abc(",  + = help: Property Synopsis + /invalid_key_log/level-pointer + Description + A regular-expression that matches the JSON-pointer of the level property +✘ error: “def[ghi” is not a valid regular expression + reason: missing terminating ] for character class + --> /invalid_key_log/file-pattern + | def[ghi  + |  ^ missing terminating ] for character class + --> {test_dir}/bad-config-json/formats/invalid-key/format.json:5 + |  "file-pattern": "def[ghi",  + = help: Property Synopsis + /invalid_key_log/file-pattern + Description + A regular expression that restricts this format to log files with a matching name +⚠ warning: unexpected value for property “/invalid_key_log/value/test/identifiers” + --> {test_dir}/bad-config-json/formats/invalid-key/format.json:14 + |  "identifiers": true  + = help: Available Properties + kind  + collate  + unit/ + identifier  + foreign-key  + hidden  + action-list  + rewriter  + description  +✘ error: “-1.2” is not a valid value for “/invalid_key_log/timestamp-divisor” + reason: value cannot be less than or equal to zero + --> {test_dir}/bad-config-json/formats/invalid-key/format.json:25 + |  "timestamp-divisor": -1.2  + = help: Property Synopsis + /invalid_key_log/timestamp-divisor  + Description + The value to divide a numeric timestamp by in a JSON log. +✘ error: “foobar_log” is not a valid log format + reason: no regexes specified + --> {test_dir}/bad-config-json/formats/invalid-json/format.json:3 +✘ error: “foobar_log” is not a valid log format + reason: log message samples must be included in a format definition + --> {test_dir}/bad-config-json/formats/invalid-json/format.json:3 +✘ error: “invalid_key_log” is not a valid log format + reason: structured logs cannot have regexes + --> {test_dir}/bad-config-json/formats/invalid-key/format.json:4 +✘ error: invalid line format element “/invalid_key_log/line-format/0/field” + reason: “non-existent” is not a defined value + --> {test_dir}/bad-config-json/formats/invalid-key/format.json:22 diff --git a/test/expected/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.out b/test/expected/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.out new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_text_file.sh_5b51b55dff7332c5bee2c9b797c401c5614d574a.out b/test/expected/test_text_file.sh_5b51b55dff7332c5bee2c9b797c401c5614d574a.out index da9fbdfe..b067f564 100644 --- a/test/expected/test_text_file.sh_5b51b55dff7332c5bee2c9b797c401c5614d574a.out +++ b/test/expected/test_text_file.sh_5b51b55dff7332c5bee2c9b797c401c5614d574a.out @@ -142,8 +142,8 @@ The following alternatives are also available: The following software packages are required to build lnav: • gcc/clang - A C++14-compatible compiler. - • libpcre - The Perl Compatible Regular Expression - (PCRE) library. + • libpcre2 - The Perl Compatible Regular Expression v2 + (PCRE2) library. • sqlite - The SQLite database engine. Version 3.9.0 or higher is required. • ncurses - The ncurses text UI library. diff --git a/test/expected/test_text_file.sh_6a24078983cf1b7a80b6fb65d5186cd125498136.out b/test/expected/test_text_file.sh_6a24078983cf1b7a80b6fb65d5186cd125498136.out index f3f15de0..6180150e 100644 --- a/test/expected/test_text_file.sh_6a24078983cf1b7a80b6fb65d5186cd125498136.out +++ b/test/expected/test_text_file.sh_6a24078983cf1b7a80b6fb65d5186cd125498136.out @@ -113,8 +113,8 @@ The following alternatives are also available: The following software packages are required to build lnav: • gcc/clang - A C++14-compatible compiler. - • libpcre - The Perl Compatible Regular Expression - (PCRE) library. + • libpcre2 - The Perl Compatible Regular Expression v2 + (PCRE2) library. • sqlite - The SQLite database engine. Version 3.9.0 or higher is required. • ncurses - The ncurses text UI library. diff --git a/test/expected/test_text_file.sh_e088ea61a5382458cc48a2607e2639e52b0be1da.out b/test/expected/test_text_file.sh_e088ea61a5382458cc48a2607e2639e52b0be1da.out index f3f15de0..6180150e 100644 --- a/test/expected/test_text_file.sh_e088ea61a5382458cc48a2607e2639e52b0be1da.out +++ b/test/expected/test_text_file.sh_e088ea61a5382458cc48a2607e2639e52b0be1da.out @@ -113,8 +113,8 @@ The following alternatives are also available: The following software packages are required to build lnav: • gcc/clang - A C++14-compatible compiler. - • libpcre - The Perl Compatible Regular Expression - (PCRE) library. + • libpcre2 - The Perl Compatible Regular Expression v2 + (PCRE2) library. • sqlite - The SQLite database engine. Version 3.9.0 or higher is required. • ncurses - The ncurses text UI library. diff --git a/test/formats/timestamp/format.json b/test/formats/timestamp/format.json index f02a1166..2a3740ad 100644 --- a/test/formats/timestamp/format.json +++ b/test/formats/timestamp/format.json @@ -7,16 +7,19 @@ "pattern": "^(?\\d+) (?.*)$" }, "non_epoch": { - "pattern": "^(?\\d+-\\d+-\\d+ \\d+:\\d+:\\d+\\.\\d+) (?.*)$" + "pattern": "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d+) (?.*)$" } }, - "timestamp-format" : [ + "timestamp-format": [ "%i", "%Y-%m-%d %H:%M:%S.%f" ], "sample": [ { "line": "1428634687123 Hello, World!" + }, + { + "line": "2022-09-10 19:57:36.123456 Hello, World" } ] } diff --git a/test/test_config.sh b/test/test_config.sh index e6d60fd6..4722abba 100755 --- a/test/test_config.sh +++ b/test/test_config.sh @@ -31,7 +31,7 @@ run_cap_test ${lnav_test} -n \ ${test_dir}/logfile_access_log.0 # config bad theme -run_cap_test ${lnav_test} -n \ +run_cap_test ${lnav_test} -W -n \ -I ${test_dir}/bad-config2 \ ${test_dir}/logfile_access_log.0 diff --git a/test/test_format_loader.sh b/test/test_format_loader.sh index b00c88fb..e5bb4224 100644 --- a/test/test_format_loader.sh +++ b/test/test_format_loader.sh @@ -2,11 +2,11 @@ export YES_COLOR=1 -run_cap_test ${lnav_test} -C \ +run_cap_test ${lnav_test} -W -C \ -I ${test_dir}/bad-config-json if test x"$HAVE_SQLITE3_ERROR_OFFSET" != x""; then - run_cap_test env LC_ALL=C ${lnav_test} -C \ + run_cap_test env LC_ALL=C ${lnav_test} -W -C \ -I ${test_dir}/bad-config fi