[logfile] allow max unrecognized lines to be tuned

docs/source/config.rst

@@ -198,6 +198,8 @@ Reference
 .. jsonschema:: ../../src/internals/config-v1.schema.json#/properties/ui/properties/keymap-defs/patternProperties/([\w\-]+)
 
 
+.. _tuning:
+
 Tuning
 ------
 

docs/source/formats.rst

@@ -425,8 +425,8 @@ Executing the format file should then install it automatically:
 Format Order When Scanning a File
 ---------------------------------
 
-When **lnav** loads a file, it tries each log format against the first ~1000
-lines of the file trying to find a match.  When a match is found, that log
+When **lnav** loads a file, it tries each log format against the first 15,000
+lines [#]_ of the file trying to find a match.  When a match is found, that log
 format will be locked in and used for the rest of the lines in that file.
 Since there may be overlap between formats, **lnav** performs a test on
 startup to determine which formats match each others sample lines.  Using
@@ -441,3 +441,6 @@ log file for the "Format order" message:
 .. prompt:: bash
 
     lnav -d /tmp/lnav.log
+
+.. [#] The maximum number of lines to check can be configured.  See the
+       :ref:`tuning` section for more details.

src/Makefile.am

@@ -216,6 +216,7 @@ noinst_HEADERS = \
 	log_level_re.re \
 	log_search_table.hh \
 	logfile.hh \
+	logfile.cfg.hh \
 	logfile_fwd.hh \
 	logfile_sub_source.hh \
 	mapbox/recursive_wrapper.hpp \

src/internals/config-v1.schema.json

@@ -48,6 +48,20 @@
                         }
                     },
                     "additionalProperties": false
+                },
+                "logfile": {
+                    "description": "Settings related to log files",
+                    "title": "/tuning/logfile",
+                    "type": "object",
+                    "properties": {
+                        "max-unrecognized-lines": {
+                            "title": "/tuning/logfile/max-unrecognized-lines",
+                            "description": "The maximum number of lines in a file to use when detecting the format",
+                            "type": "integer",
+                            "minimum": 1
+                        }
+                    },
+                    "additionalProperties": false
                 }
             },
             "additionalProperties": false

src/lnav_config.cc

@@ -82,6 +82,10 @@ static auto fvc = injector::bind<file_vtab::config>::to_instance(+[]() {
     return &lnav_config.lc_file_vtab;
 });
 
+static auto lc = injector::bind<lnav::logfile::config>::to_instance(+[]() {
+    return &lnav_config.lc_logfile;
+});
+
 ghc::filesystem::path dotlnav_path()
 {
     auto home_env = getenv("HOME");
@@ -905,6 +909,16 @@ static struct json_path_container file_vtab_handlers = {
                    &file_vtab::config::fvc_max_content_size),
 };
 
+static struct json_path_container logfile_handlers = {
+    yajlpp::property_handler("max-unrecognized-lines")
+        .with_synopsis("<lines>")
+        .with_description(
+            "The maximum number of lines in a file to use when detecting the format")
+        .with_min_value(1)
+        .for_field(&_lnav_config::lc_logfile,
+                   &lnav::logfile::config::lc_max_unrecognized_lines),
+};
+
 static struct json_path_container tuning_handlers = {
     yajlpp::property_handler("archive-manager")
         .with_description("Settings related to opening archive files")
@@ -912,6 +926,9 @@ static struct json_path_container tuning_handlers = {
     yajlpp::property_handler("file-vtab")
         .with_description("Settings related to the lnav_file virtual-table")
         .with_children(file_vtab_handlers),
+    yajlpp::property_handler("logfile")
+        .with_description("Settings related to log files")
+        .with_children(logfile_handlers),
 };
 
 static set<string> SUPPORTED_CONFIG_SCHEMAS = {

src/lnav_config.hh

@@ -48,6 +48,7 @@
 #include "lnav_config_fwd.hh"
 #include "archive_manager.cfg.hh"
 #include "file_vtab.cfg.hh"
+#include "logfile.cfg.hh"
 
 /**
  * Compute the path to a file in the user's '.lnav' directory.
@@ -103,6 +104,7 @@ struct _lnav_config {
 
     archive_manager::config lc_archive_manager;
     file_vtab::config lc_file_vtab;
+    lnav::logfile::config lc_logfile;
 };
 
 extern struct _lnav_config lnav_config;

src/logfile.cc

@@ -43,13 +43,14 @@
 #include <time.h>
 
 #include "base/string_util.hh"
+#include "base/injector.hh"
 #include "logfile.hh"
+#include "logfile.cfg.hh"
 #include "log_format.hh"
 #include "lnav_util.hh"
 
 using namespace std;
 
-static const size_t MAX_UNRECOGNIZED_LINES = 1000;
 static const size_t INDEX_RESERVE_INCREMENT = 1024;
 
 logfile::logfile(const string &filename, logfile_open_options &loo)
@@ -165,7 +166,8 @@ bool logfile::process_prefix(shared_buffer_ref &sbr, const line_info &li)
         found = this->lf_format->scan(*this, this->lf_index, li, sbr);
     }
     else if (this->lf_options.loo_detect_format &&
-             this->lf_index.size() < MAX_UNRECOGNIZED_LINES) {
+             this->lf_index.size() <
+             injector::get<const lnav::logfile::config &>().lc_max_unrecognized_lines) {
         auto &root_formats = log_format::get_root_formats();
         vector<std::shared_ptr<log_format>>::iterator iter;
 

src/logfile.cfg.hh

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2021, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * @file logfile.cfg.hh
+ */
+
+#ifndef lnav_logfile_cfg_hh
+#define lnav_logfile_cfg_hh
+
+namespace lnav {
+namespace logfile {
+
+struct config {
+    int64_t lc_max_unrecognized_lines{15000};
+};
+
+}
+}
+
+#endif