Merge pull request #1077 from tstack/dependabot/bundler/docs/nokogiri-1.13.9

Bump nokogiri from 1.13.8 to 1.13.9 in /docs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,7 +8,7 @@ assignees: ''
 ---
 
 **lnav version**
-v0.11.0 is the latest
+v0.11.1 is the latest
 
 **Describe the bug**
 A clear and concise description of what the bug is.

AUTHORS

@@ -43,3 +43,4 @@ Amos Bird
 Cristian Chiru
 Peter Schiffer
 Pedro Pombeiro
+Fredrik Forséll

NEWS.md

@@ -22,6 +22,10 @@ Features:
   that the implementation relies on libcurl which has some
   limitations, like not supporting all types of schemes
   (e.g. `mailto:`).
+* Added the `subsecond-field` and `subsecond-units` log
+  format properties to allow for specifying a separate
+  field for the sub-second portion of a timestamp.
+* Added a keymap for Swedish keyboards.
 
 Breaking changes:
 * The `regexp_capture()` table-valued-function now returns NULL
@@ -31,6 +35,11 @@ Breaking changes:
 Fixes:
 * Reduce the "no patterns have a capture" error to a warning
   so that it doesn't block lnav from starting up.
+* Some ANSI escape sequences will now be removed before testing
+  regexes against a log message.
+* If a line in a JSON-lines log file does not start with a
+  `{`, it will now be shown as-is and will not have the JSON
+  parse error.
 
 Cost of Doing Business:
 * Migrated from pcre to pcre2.

TESTS_ENVIRONMENT.in

@@ -123,7 +123,7 @@ run_cap_test() {
            export has_errors="yes"
        fi
     else
-       echo "FAIL! EXPECTED OUT MISSING"
+       echo "FAIL! EXPECTED OUT MISSING -- ${srcdir}/expected/${test_file_base}_${test_hash}.out"
        export has_errors="yes"
     fi
 

appveyor.yml

@@ -1,4 +1,5 @@
 version: "{build}"
+image: Visual Studio 2022
 
 environment:
   matrix:
@@ -8,7 +9,7 @@ environment:
     cygsetup: setup-x86_64.exe
 
 install:
-  - C:\%cygwin%\%cygsetup% -qnNdO -R C:/%cygwin% -s http://cygwin.mirror.constant.com -l C:/%cygwin%/var/cache/setup -P libpcre-devel -P libncurses-devel -P libreadline-devel -P zlib-devel -P libbz2-devel -P libsqlite3-devel
+  - C:\%cygwin%\%cygsetup% -qnNdOX -R C:/%cygwin% -l C:/%cygwin%/var/cache/setup -P libpcre2-devel -P libncurses-devel -P libreadline-devel -P zlib-devel -P libbz2-devel -P libsqlite3-devel -P libcurl-devel -P libarchive-devel
 
 build_script:
   - C:\%cygwin%\bin\sh -lc "uname -a && gcc --version && cd /cygdrive/c/projects/lnav && ./autogen.sh && ./configure && make && strip src/lnav.exe && ldd src/lnav.exe"

conanfile.py

@@ -4,7 +4,7 @@ from conan.tools.cmake import CMake, CMakeToolchain, CMakeDeps
 
 class LnavConan(ConanFile):
     name = "lnav"
-    version = "0.11.0"
+    version = "0.11.1"
     homepage = "https://lnav.org"
     url = "https://github.com/tstack/lnav.git"
     license = "BSD-2-Clause"

configure.ac

@@ -206,7 +206,7 @@ AS_VAR_SET(ALL_LDFLAGS, "$SQLITE3_LDFLAGS $READLINE_LDFLAGS $LIBARCHIVE_LDFLAGS
 AS_VAR_SET(static_lib_list,
            ["libncurses.a libncursesw.a libreadline.a libsqlite3.a libz.a libtinfo.a libtinfow.a"])
 AS_VAR_SET(static_lib_list,
-           ["$static_lib_list libpcre2.a libncursesw.a libbz2.a"])
+           ["$static_lib_list libpcre2-8.a libncursesw.a libbz2.a"])
 AS_VAR_SET(static_lib_list,
            ["$static_lib_list libgpm.a libcurl.a libcrypto.a libssl.a libssh2.a"])
 AS_VAR_SET(static_lib_list,

docs/02_downloads.md

@@ -11,7 +11,7 @@ The following options are available for installing **lnav**:
 ## Linux
 
 <!-- markdown-link-check-disable-next-line -->
-Download a [statically linked 64-bit binary](https://github.com/tstack/lnav/releases/download/v{{site.version}}/lnav-{{site.version}}-musl-64bit.zip).
+Download a [statically linked 64-bit binary](https://github.com/tstack/lnav/releases/download/v{{site.version}}/lnav-{{site.version}}-x86_64-linux-musl.zip).
 
 Install from the [Snap Store](https://snapcraft.io/lnav):
 
@@ -22,7 +22,7 @@ $ sudo snap install lnav
 ## MacOS
 
 <!-- markdown-link-check-disable-next-line -->
-Download a [statically linked 64-bit binary](https://github.com/tstack/lnav/releases/download/v{{site.version}}/lnav-{{site.version}}-os-x.zip)
+Download a [statically linked 64-bit binary](https://github.com/tstack/lnav/releases/download/v{{site.version}}/lnav-{{site.version}}-x86_64-macos.zip)
 
 Install using [Homebrew](https://formulae.brew.sh/formula/lnav):
 
@@ -49,3 +49,10 @@ $ make install
 
 If you would like to contribute to the development of lnav, visit our page on
 [GitHub](https://github.com/tstack/lnav).
+
+# VSCode Extension
+
+The [lnav VSCode Extension](https://marketplace.visualstudio.com/items?itemName=lnav.lnav)
+can be used to add syntax highlighting to lnav scripts.
+
+![Screenshot of an lnav script](/assets/images/lnav-vscode-extension.png)

docs/Gemfile.lock

@@ -14,7 +14,7 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.23.5)
+    commonmarker (0.23.6)
     concurrent-ruby (1.1.10)
     dnsruby (1.61.9)
       simpleidn (~> 0.1)
@@ -102,8 +102,6 @@ GEM
       pathutil (~> 0.9)
       rouge (>= 1.7, < 4)
       safe_yaml (~> 1.0)
-    jekyll-asciinema (0.2.1)
-      jekyll (>= 2.0)
     jekyll-avatar (0.7.0)
       jekyll (>= 3.0, < 5.0)
     jekyll-coffeescript (1.1.1)
@@ -213,7 +211,9 @@ GEM
       jekyll-feed (~> 0.9)
       jekyll-seo-tag (~> 2.1)
     minitest (5.16.3)
-    nokogiri (1.13.8-arm64-darwin)
+    nokogiri (1.13.9-arm64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.13.9-x86_64-linux)
       racc (~> 1.4)
     octokit (4.25.1)
       faraday (>= 1, < 3)
@@ -256,10 +256,10 @@ GEM
 
 PLATFORMS
   arm64-darwin-21
+  x86_64-linux
 
 DEPENDENCIES
   github-pages
-  jekyll-asciinema (~> 0.2.1)
   minima (~> 2.5)
   tzinfo (~> 1.2)
   tzinfo-data

docs/_config.yml

@@ -19,7 +19,7 @@
 # in the templates via {{ site.myvariable }}.
 
 title: The Logfile Navigator
-version: 0.11.0
+version: 0.11.1
 email: support@lnav.org
 description: >- # this means to ignore newlines until "baseurl:"
   The Logfile Navigator, lnav for short, is an advanced log file viewer

docs/_posts/2022-09-24-vscode-extension.md

@@ -0,0 +1,21 @@
+---
+layout: post
+title: VSCode Extension for lnav
+excerpt: Syntax highlighting for lnav scripts
+---
+
+I've published a simple [Visual Studio Code extension for lnav](
+https://marketplace.visualstudio.com/items?itemName=lnav.lnav)
+that adds syntax highlighting for scripts.  The following is a
+screenshot showing the `dhclient-summary.lnav` script that is
+builtin:
+
+![Screenshot of an lnav script](/assets/images/lnav-vscode-extension.png)
+
+The lnav commands, those prefixed with colons, are marked as
+keywords and the SQL blocks are treated as an embedded language
+and highlighted accordingly.
+
+If people find this useful, we can take it further and add
+support for running the current script/snippet in a new lnav
+process or even talking to an existing one.

docs/schemas/format-v1.schema.json

@@ -101,6 +101,21 @@
                     "description": "The name of the timestamp field in the log message pattern",
                     "type": "string"
                 },
+                "subsecond-field": {
+                    "title": "/<format_name>/subsecond-field",
+                    "description": "The path to the property in a JSON-lines log message that contains the sub-second time value",
+                    "type": "string"
+                },
+                "subsecond-units": {
+                    "title": "/<format_name>/subsecond-units",
+                    "description": "The units of the subsecond-field property value",
+                    "type": "string",
+                    "enum": [
+                        "milli",
+                        "micro",
+                        "nano"
+                    ]
+                },
                 "time-field": {
                     "title": "/<format_name>/time-field",
                     "description": "The name of the time field in the log message pattern.  This field should only be specified if the timestamp field only contains a date.",
@@ -438,8 +453,7 @@
                             "field": {
                                 "title": "/<format_name>/line-format/field",
                                 "description": "The name of the field to substitute at this position",
-                                "type": "string",
-                                "minLength": 1
+                                "type": "string"
                             },
                             "default-value": {
                                 "title": "/<format_name>/line-format/default-value",

docs/source/formats.rst

@@ -202,6 +202,16 @@ should be another object with the following fields:
     to divide the timestamp by to get the number of seconds and fractional
     seconds.
 
+  :subsecond-field: (v0.11.1+) The path to the property in a JSON-lines log
+    message that contains the sub-second time value
+
+  :subsecond-units: (v0.11.1+) The units of the subsecond-field property value.
+    The following values are supported:
+
+    :milli: for milliseconds
+    :micro: for microseconds
+    :nano: for nanoseconds
+
   :ordered-by-time: (v0.8.3+) Indicates that the order of messages in the file
     is time-based.  Files that are not naturally ordered by time will be sorted
     in order to display them in the correct order.  Note that this sorting can
@@ -389,15 +399,15 @@ with the following contents:
 Scripts
 -------
 
-Format directories may also contain '.sql' and '.lnav' files to help automate
+Format directories may also contain :file:`.sql` and :file:`.lnav` files to help automate
 log file analysis.  The SQL files are executed on startup to create any helper
 tables or views and the '.lnav' script files can be executed using the pipe
-hotkey (|).  For example, **lnav** includes a "partition-by-boot" script that
+hotkey :kbd:`|`.  For example, **lnav** includes a "partition-by-boot" script that
 partitions the log view based on boot messages from the Linux kernel.  A script
 can have a mix of SQL and **lnav** commands, as well as include other scripts.
 The type of statement to execute is determined by the leading character on a
 line: a semi-colon begins a SQL statement; a colon starts an **lnav** command;
-and a pipe (|) denotes another script to be executed.  Lines beginning with a
+and a pipe :code:`|` denotes another script to be executed.  Lines beginning with a
 hash are treated as comments.  The following variables are defined in a script:
 
 .. envvar:: #
@@ -442,6 +452,12 @@ header:
 
       :eval :filter-out ${pattern}
 
+VSCode Extension
+^^^^^^^^^^^^^^^^
+
+The `lnav VSCode Extension <https://marketplace.visualstudio.com/items?itemName=lnav.lnav>`_
+can be installed to add syntax highlighting to lnav scripts.
+
 Installing Formats
 ------------------
 

docs/source/intro.rst

@@ -91,6 +91,7 @@ The builtin keymaps are:
 
   :de: `German <https://github.com/tstack/lnav/blob/master/src/keymaps/de-keymap.json>`_
   :fr: `French <https://github.com/tstack/lnav/blob/master/src/keymaps/fr-keymap.json>`_
+  :sv: `Swedish <https://github.com/tstack/lnav/blob/master/src/keymaps/sv-keymap.json>`_
   :uk: `United Kingdom <https://github.com/tstack/lnav/blob/master/src/keymaps/uk-keymap.json>`_
   :us: `United States <https://github.com/tstack/lnav/blob/master/src/keymaps/us-keymap.json>`_
 

release/Makefile

@@ -1,7 +1,7 @@
 
 VERSION=0.11.1
 
-VERSION_TAG=v$(VERSION)-test
+VERSION_TAG=v$(VERSION)
 
 SRC_VERSION=master
 
@@ -19,7 +19,7 @@ PACKAGE_URLS = \
 	https://ftp.gnu.org/gnu/readline/readline-6.3.tar.gz \
 	https://zlib.net/zlib-1.2.12.tar.gz \
 	https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz \
-	https://www.sqlite.org/2022/sqlite-autoconf-3390200.tar.gz \
+	https://www.sqlite.org/2022/sqlite-autoconf-3390400.tar.gz \
 	https://www.openssl.org/source/openssl-1.0.2n.tar.gz \
 	https://www.libssh2.org/download/libssh2-1.9.0.tar.gz \
 	https://curl.se/download/curl-7.85.0.tar.gz \
@@ -70,6 +70,7 @@ osx-build:
 
 osx-package: clean-outbox osx-build
 	mkdir -p osx-pkg/lnav-${VERSION}
+	git pull --rebase
 	cp ../README ../NEWS.md osx-pkg/lnav-${VERSION}
 	cp osx-build-dir/src/lnav osx-pkg/lnav-${VERSION}
 	cp osx-build-dir/lnav-${VERSION}.tar.gz outbox/

src/Makefile.am

@@ -310,6 +310,7 @@ noinst_HEADERS = \
 	time_T.hh \
 	timer.hh \
 	top_status_source.hh \
+	top_status_source.cfg.hh \
 	unique_path.hh \
 	url_loader.hh \
 	view_curses.hh \

src/archive_manager.cc

@@ -232,7 +232,7 @@ extract(const std::string& filename, const extract_cb& cb)
     }
 
     auto arc_lock = lnav::filesystem::file_lock(tmp_path);
-    auto lock_guard = lnav::filesystem::file_lock::guard(arc_lock);
+    auto lock_guard = lnav::filesystem::file_lock::guard(&arc_lock);
     auto done_path = tmp_path;
 
     done_path += ".done";

src/base/ansi_scrubber.cc

@@ -51,8 +51,11 @@ ansi_regex()
 size_t
 erase_ansi_escapes(string_fragment input)
 {
+    static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
+
     const auto& regex = ansi_regex();
-    auto md = regex.create_match_data();
+    nonstd::optional<int> move_start;
+    size_t fill_index = 0;
 
     auto matcher = regex.capture_from(input).into(md);
     while (true) {
@@ -69,52 +72,55 @@ erase_ansi_escapes(string_fragment input)
         auto sf = md[0].value();
         auto bs_index_res = sf.codepoint_to_byte_index(1);
 
+        if (move_start) {
+            auto move_len = sf.sf_begin - move_start.value();
+            memmove(input.writable_data(fill_index),
+                    input.data() + move_start.value(),
+                    move_len);
+            fill_index += move_len;
+        } else {
+            fill_index = sf.sf_begin;
+        }
+
         if (sf.length() >= 3 && bs_index_res.isOk()
             && sf[bs_index_res.unwrap()] == '\b')
         {
             static const auto OVERSTRIKE_RE
                 = lnav::pcre2pp::code::from_const(R"((\X)\x08(\X))");
 
-            size_t fill_index = 0;
             auto loop_res = OVERSTRIKE_RE.capture_from(sf).for_each(
-                [&fill_index, &sf](lnav::pcre2pp::match_data& over_md) {
+                [&fill_index, &input](lnav::pcre2pp::match_data& over_md) {
                     auto lhs = over_md[1].value();
                     if (lhs == "_") {
                         auto rhs = over_md[2].value();
-                        memmove(sf.writable_data(fill_index),
+                        memmove(input.writable_data(fill_index),
                                 rhs.data(),
                                 rhs.length());
                         fill_index += rhs.length();
                     } else {
-                        memmove(sf.writable_data(fill_index),
+                        memmove(input.writable_data(fill_index),
                                 lhs.data(),
                                 lhs.length());
                         fill_index += lhs.length();
                     }
                 });
-
-            memmove(input.writable_data(sf.sf_begin + fill_index),
-                    md.remaining().data(),
-                    md.remaining().length());
-            input = input.erase(input.sf_string, sf.length() - fill_index);
-        } else {
-            memmove(const_cast<char*>(sf.data()),
-                    md.remaining().data(),
-                    md.remaining().length());
-            input = input.erase(input.sf_string, sf.length());
         }
-
-        matcher.reload_input(input, sf.sf_begin);
+        move_start = md.remaining().sf_begin;
     }
 
-    return input.length();
+    memmove(input.writable_data(fill_index),
+            md.remaining().data(),
+            md.remaining().length());
+    fill_index += md.remaining().length();
+
+    return fill_index;
 }
 
 void
 scrub_ansi_string(std::string& str, string_attrs_t* sa)
 {
+    static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
     const auto& regex = ansi_regex();
-    auto md = regex.create_match_data();
     int64_t origin_offset = 0;
     int last_origin_offset_end = 0;
 

src/base/attr_line.hh

@@ -530,7 +530,7 @@ public:
     attr_line_t& append_quoted(S s)
     {
         this->al_string.append("\u201c");
-        this->al_string.append(std::move(s));
+        this->append(std::move(s));
         this->al_string.append("\u201d");
 
         return *this;

src/base/fs_util.cc

@@ -166,5 +166,18 @@ stat_file(const ghc::filesystem::path& path)
                            strerror(errno)));
 }
 
+file_lock::file_lock(const ghc::filesystem::path& archive_path)
+{
+    auto lock_path = archive_path;
+
+    lock_path += ".lck";
+    auto open_res
+        = lnav::filesystem::create_file(lock_path, O_RDWR | O_CLOEXEC, 0600);
+    if (open_res.isErr()) {
+        throw std::runtime_error(open_res.unwrapErr());
+    }
+    this->lh_fd = open_res.unwrap();
+}
+
 }  // namespace filesystem
 }  // namespace lnav

src/base/fs_util.hh

@@ -82,33 +82,36 @@ class file_lock {
 public:
     class guard {
     public:
-        explicit guard(file_lock& arc_lock) : g_lock(arc_lock)
+        explicit guard(file_lock* arc_lock) : g_lock(arc_lock)
         {
-            this->g_lock.lock();
-        };
+            this->g_lock->lock();
+        }
 
-        ~guard() { this->g_lock.unlock(); };
+        guard(guard&& other) noexcept
+            : g_lock(std::exchange(other.g_lock, nullptr))
+        {
+        }
+
+        ~guard()
+        {
+            if (this->g_lock != nullptr) {
+                this->g_lock->unlock();
+            }
+        }
+
+        guard(const guard&) = delete;
+        guard& operator=(const guard&) = delete;
+        guard& operator=(guard&&) = delete;
 
     private:
-        file_lock& g_lock;
+        file_lock* g_lock;
     };
 
     void lock() const { lockf(this->lh_fd, F_LOCK, 0); }
 
     void unlock() const { lockf(this->lh_fd, F_ULOCK, 0); }
 
-    explicit file_lock(const ghc::filesystem::path& archive_path)
-    {
-        auto lock_path = archive_path;
-
-        lock_path += ".lck";
-        auto open_res = lnav::filesystem::create_file(
-            lock_path, O_RDWR | O_CLOEXEC, 0600);
-        if (open_res.isErr()) {
-            throw std::runtime_error(open_res.unwrapErr());
-        }
-        this->lh_fd = open_res.unwrap();
-    }
+    explicit file_lock(const ghc::filesystem::path& archive_path);
 
     auto_fd lh_fd;
 };

src/base/func_util.hh

@@ -68,6 +68,54 @@ struct noop_func {
 namespace lnav {
 namespace func {
 
+class scoped_cb {
+public:
+    class guard {
+    public:
+        explicit guard(scoped_cb* owner) : g_owner(owner) {}
+
+        guard(const guard&) = delete;
+        guard& operator=(const guard&) = delete;
+
+        guard(guard&& gu) noexcept : g_owner(std::exchange(gu.g_owner, nullptr))
+        {
+        }
+
+        guard& operator=(guard&& gu) noexcept
+        {
+            this->g_owner = std::exchange(gu.g_owner, nullptr);
+            return *this;
+        }
+
+        ~guard()
+        {
+            if (this->g_owner != nullptr) {
+                this->g_owner->s_callback = {};
+            }
+        }
+
+    private:
+        scoped_cb* g_owner;
+    };
+
+    guard install(std::function<void()> cb)
+    {
+        this->s_callback = std::move(cb);
+
+        return guard{this};
+    }
+
+    void operator()()
+    {
+        if (s_callback) {
+            s_callback();
+        }
+    }
+
+private:
+    std::function<void()> s_callback;
+};
+
 template<typename Fn,
          typename... Args,
          std::enable_if_t<std::is_member_pointer<std::decay_t<Fn>>{}, int> = 0>

src/base/injector.bind.hh

@@ -90,6 +90,28 @@ struct bind : singleton_storage<T, Annotations...> {
         return true;
     }
 
+    struct lifetime {
+        ~lifetime()
+        {
+            singleton_storage<T, Annotations...>::ss_owner = nullptr;
+            singleton_storage<T, Annotations...>::ss_data = nullptr;
+        }
+    };
+
+    template<typename I = T,
+             std::enable_if_t<has_injectable<I>::value, bool> = true>
+    static lifetime to_scoped_singleton() noexcept
+    {
+        typename I::injectable* i = nullptr;
+        singleton_storage<T, Annotations...>::ss_owner
+            = create_from_injectable<I>(i)();
+        singleton_storage<T, Annotations...>::ss_data
+            = singleton_storage<T, Annotations...>::ss_owner.get();
+        singleton_storage<T, Annotations...>::ss_scope = scope::singleton;
+
+        return {};
+    }
+
     template<typename... Args>
     static bool to_instance(T* (*f)(Args...)) noexcept
     {

src/base/injector.hh

@@ -55,16 +55,21 @@ void force_linking(Annotation anno);
 template<class...>
 using void_t = void;
 
-template<class, class = void>
-struct has_injectable : std::false_type {
+template<typename T, typename... Annotations>
+struct with_annotations {
+    T value;
 };
 
+template<class, class = void>
+struct has_injectable : std::false_type {};
+
 template<class T>
-struct has_injectable<T, void_t<typename T::injectable>> : std::true_type {
-};
+struct has_injectable<T, void_t<typename T::injectable>> : std::true_type {};
 
 template<typename T, typename... Annotations>
 struct singleton_storage {
+    static scope get_scope() { return ss_scope; }
+
     static T* get()
     {
         static int _[] = {0, (force_linking(Annotations{}), 0)...};
@@ -158,20 +163,16 @@ get()
 }
 
 template<class T>
-struct is_shared_ptr : std::false_type {
-};
+struct is_shared_ptr : std::false_type {};
 
 template<class T>
-struct is_shared_ptr<std::shared_ptr<T>> : std::true_type {
-};
+struct is_shared_ptr<std::shared_ptr<T>> : std::true_type {};
 
 template<class T>
-struct is_vector : std::false_type {
-};
+struct is_vector : std::false_type {};
 
 template<class T>
-struct is_vector<std::vector<T>> : std::true_type {
-};
+struct is_vector<std::vector<T>> : std::true_type {};
 
 template<typename I, typename R, typename... IArgs, typename... Args>
 std::function<std::shared_ptr<I>()> create_from_injectable(R (*)(IArgs...),
@@ -179,22 +180,28 @@ std::function<std::shared_ptr<I>()> create_from_injectable(R (*)(IArgs...),
 
 template<typename T,
          typename... Args,
-         std::enable_if_t<has_injectable<typename T::element_type>::value,
-                          bool> = true,
+         std::enable_if_t<has_injectable<typename T::element_type>::value, bool>
+         = true,
          std::enable_if_t<is_shared_ptr<T>::value, bool> = true>
 T
 get(Args&... args)
 {
     typename T::element_type::injectable* i = nullptr;
 
+    if (singleton_storage<typename T::element_type>::get_scope()
+        == scope::singleton)
+    {
+        return singleton_storage<typename T::element_type>::get_owner();
+    }
     return create_from_injectable<typename T::element_type>(i, args...)();
 }
 
-template<typename T,
-         typename... Annotations,
-         std::enable_if_t<!has_injectable<typename T::element_type>::value,
-                          bool> = true,
-         std::enable_if_t<is_shared_ptr<T>::value, bool> = true>
+template<
+    typename T,
+    typename... Annotations,
+    std::enable_if_t<!has_injectable<typename T::element_type>::value, bool>
+    = true,
+    std::enable_if_t<is_shared_ptr<T>::value, bool> = true>
 T
 get()
 {

src/base/intern_string.hh

@@ -62,6 +62,12 @@ struct string_fragment {
         return string_fragment{str, 0, str != nullptr ? (int) strlen(str) : 0};
     }
 
+    static string_fragment from_c_str(const unsigned char* str)
+    {
+        return string_fragment{
+            str, 0, str != nullptr ? (int) strlen((char*) str) : 0};
+    }
+
     template<typename T, std::size_t N>
     static string_fragment from_const(const T (&str)[N])
     {
@@ -277,6 +283,18 @@ struct string_fragment {
             this->sf_string, this->sf_begin + begin, this->sf_begin + end};
     }
 
+    size_t count(char ch) const {
+        size_t retval = 0;
+
+        for (int lpc = this->sf_begin; lpc < this->sf_end; lpc++) {
+            if (this->sf_string[lpc] == ch) {
+                retval += 1;
+            }
+        }
+
+        return retval;
+    }
+
     nonstd::optional<size_t> find(char ch) const
     {
         for (int lpc = this->sf_begin; lpc < this->sf_end; lpc++) {

src/base/lnav.gzip.cc

@@ -87,6 +87,7 @@ uncompress(const std::string& src, const void* buffer, size_t size)
     int err;
 
     strm.next_in = (Bytef*) buffer;
+    strm.msg = Z_NULL;
     strm.avail_in = size;
     strm.total_in = 0;
     strm.total_out = 0;

src/base/result.h

@@ -859,11 +859,11 @@ struct Result {
     template<typename U = T>
     typename std::enable_if<
         !std::is_same<U, void>::value,
-        U
+        T
     >::type
     unwrapOr(const U& defaultValue) const {
         if (isOk()) {
-            return storage().template get<U>();
+            return storage().template get<T>();
         }
         return defaultValue;
     }

src/base/snippet_highlighters.hh

@@ -37,7 +37,7 @@ namespace snippets {
 
 void regex_highlighter(attr_line_t& al, int x, line_range sub);
 
-}
+}  // namespace snippets
 }  // namespace lnav
 
 #endif

src/base/time_util.cc

@@ -35,6 +35,48 @@
 
 #include "config.h"
 
+namespace lnav {
+
+ssize_t
+strftime_rfc3339(
+    char* buffer, size_t buffer_size, lnav::time64_t tim, int millis, char sep)
+{
+    struct tm gmtm;
+    int year, month, index = 0;
+
+    secs2tm(tim, &gmtm);
+    year = gmtm.tm_year + 1900;
+    month = gmtm.tm_mon + 1;
+    buffer[index++] = '0' + ((year / 1000) % 10);
+    buffer[index++] = '0' + ((year / 100) % 10);
+    buffer[index++] = '0' + ((year / 10) % 10);
+    buffer[index++] = '0' + ((year / 1) % 10);
+    buffer[index++] = '-';
+    buffer[index++] = '0' + ((month / 10) % 10);
+    buffer[index++] = '0' + ((month / 1) % 10);
+    buffer[index++] = '-';
+    buffer[index++] = '0' + ((gmtm.tm_mday / 10) % 10);
+    buffer[index++] = '0' + ((gmtm.tm_mday / 1) % 10);
+    buffer[index++] = sep;
+    buffer[index++] = '0' + ((gmtm.tm_hour / 10) % 10);
+    buffer[index++] = '0' + ((gmtm.tm_hour / 1) % 10);
+    buffer[index++] = ':';
+    buffer[index++] = '0' + ((gmtm.tm_min / 10) % 10);
+    buffer[index++] = '0' + ((gmtm.tm_min / 1) % 10);
+    buffer[index++] = ':';
+    buffer[index++] = '0' + ((gmtm.tm_sec / 10) % 10);
+    buffer[index++] = '0' + ((gmtm.tm_sec / 1) % 10);
+    buffer[index++] = '.';
+    buffer[index++] = '0' + ((millis / 100) % 10);
+    buffer[index++] = '0' + ((millis / 10) % 10);
+    buffer[index++] = '0' + ((millis / 1) % 10);
+    buffer[index] = '\0';
+
+    return index;
+}
+
+}
+
 static time_t BAD_DATE = -1;
 
 time_t

src/base/time_util.hh

@@ -42,7 +42,13 @@ namespace lnav {
 
 using time64_t = uint64_t;
 
-}
+ssize_t strftime_rfc3339(char* buffer,
+                         size_t buffer_size,
+                         lnav::time64_t tim,
+                         int millis,
+                         char sep = ' ');
+
+}  // namespace lnav
 
 struct tm* secs2tm(lnav::time64_t tim, struct tm* res);
 /**
@@ -103,10 +109,7 @@ struct exttm {
     unsigned int et_flags{0};
     long et_gmtoff{0};
 
-    exttm()
-    {
-        memset(&this->et_tm, 0, sizeof(this->et_tm));
-    }
+    exttm() { memset(&this->et_tm, 0, sizeof(this->et_tm)); }
 
     bool operator==(const exttm& other) const
     {
@@ -141,6 +144,12 @@ operator!=(const struct timeval& left, const struct timeval& right)
     return left.tv_sec != right.tv_sec || left.tv_usec != right.tv_usec;
 }
 
+inline bool
+operator==(const struct timeval& left, const struct timeval& right)
+{
+    return left.tv_sec == right.tv_sec || left.tv_usec == right.tv_usec;
+}
+
 inline struct timeval
 operator-(const struct timeval& lhs, const struct timeval& rhs)
 {

src/bound_tags.hh

@@ -34,8 +34,6 @@
 
 struct last_relative_time_tag {};
 
-struct sqlite_db_tag {};
-
 struct sql_cmd_map_tag {};
 
 enum {

src/command_executor.cc

@@ -66,8 +66,6 @@ SELECT count(*) AS total, min(log_line) AS log_line, log_msg_format
 int
 sql_progress(const struct log_cursor& lc)
 {
-    static sig_atomic_t sql_counter = 0;
-
     ssize_t total = lnav_data.ld_log_source.text_line_count();
     off_t off = lc.lc_curr_line;
 
@@ -83,42 +81,11 @@ sql_progress(const struct log_cursor& lc)
         return 1;
     }
 
+    static sig_atomic_t sql_counter = 0;
+
     if (ui_periodic_timer::singleton().time_to_update(sql_counter)) {
-        struct timeval current_time = {0, 0};
-        int ch;
-
-        while ((ch = getch()) != ERR) {
-            if (current_time.tv_sec == 0) {
-                gettimeofday(&current_time, nullptr);
-            }
-            lnav_data.ld_user_message_source.clear();
-
-            alerter::singleton().new_input(ch);
-
-            lnav_data.ld_input_dispatcher.new_input(current_time, ch);
-
-            lnav_data.ld_view_stack.top() | [ch](auto tc) {
-                lnav_data.ld_key_repeat_history.update(ch, tc->get_top());
-            };
-
-            if (!lnav_data.ld_looping) {
-                // No reason to keep processing input after the
-                // user has quit.  The view stack will also be
-                // empty, which will cause issues.
-                break;
-            }
-        }
-
         lnav_data.ld_bottom_source.update_loading(off, total);
-        lnav_data.ld_top_source.update_time();
-        lnav_data.ld_status[LNS_TOP].do_update();
-        lnav_data.ld_status[LNS_BOTTOM].do_update();
-        lnav_data.ld_rl_view->do_update();
-        if (handle_winch()) {
-            layout_views();
-            lnav_data.ld_view_stack.do_update();
-        }
-        refresh();
+        lnav_data.ld_status_refresher();
     }
 
     return 0;
@@ -132,9 +99,7 @@ sql_progress_finished()
     }
 
     lnav_data.ld_bottom_source.update_loading(0, 0);
-    lnav_data.ld_top_source.update_time();
-    lnav_data.ld_status[LNS_TOP].do_update();
-    lnav_data.ld_status[LNS_BOTTOM].do_update();
+    lnav_data.ld_status_refresher();
     lnav_data.ld_views[LNV_DB].redo_search();
 }
 

src/document.sections.cc

@@ -251,7 +251,6 @@ public:
     metadata walk()
     {
         metadata_builder mb;
-        data_token_t dt = DT_INVALID;
         size_t garbage_count = 0;
 
         while (garbage_count < 1000) {
@@ -260,6 +259,7 @@ public:
                 break;
             }
 
+            auto dt = tokenize_res->tr_token;
             element el(tokenize_res->tr_token, tokenize_res->tr_capture);
 
             switch (dt) {

src/file_vtab.cc

@@ -210,6 +210,7 @@ CREATE TABLE lnav_file (
                 loo.loo_include_in_session = true;
                 this->lf_collection.fc_file_names[path] = std::move(loo);
                 lf->set_filename(path);
+                this->lf_collection.regenerate_unique_file_names();
 
                 init_session();
                 load_session();

src/filter_status_source.cc

@@ -116,13 +116,12 @@ filter_status_source::statusview_fields()
             role_t::VCR_STATUS_STITCH_NORMAL_TO_TITLE);
         this->tss_fields[TSF_TITLE].set_value(" " ANSI_ROLE("T") "ext Filters ",
                                               role_t::VCR_STATUS_HOTKEY);
-        this->tss_fields[TSF_TITLE].set_role(
-            role_t::VCR_STATUS_DISABLED_TITLE);
-        this->tss_fields[TSF_STITCH_TITLE].set_stitch_value(
-            role_t::VCR_STATUS, role_t::VCR_STATUS);
+        this->tss_fields[TSF_TITLE].set_role(role_t::VCR_STATUS_DISABLED_TITLE);
+        this->tss_fields[TSF_STITCH_TITLE].set_stitch_value(role_t::VCR_STATUS,
+                                                            role_t::VCR_STATUS);
     } else {
-        this->tss_fields[TSF_FILES_TITLE].set_value(
-            " " ANSI_ROLE("F") "iles ", role_t::VCR_STATUS_HOTKEY);
+        this->tss_fields[TSF_FILES_TITLE].set_value(" " ANSI_ROLE("F") "iles ",
+                                                    role_t::VCR_STATUS_HOTKEY);
         if (lnav_data.ld_active_files.fc_name_to_errors.empty()) {
             this->tss_fields[TSF_FILES_TITLE].set_role(
                 role_t::VCR_STATUS_DISABLED_TITLE);
@@ -142,9 +141,8 @@ filter_status_source::statusview_fields()
         this->tss_fields[TSF_FILES_RIGHT_STITCH].set_stitch_value(
             role_t::VCR_STATUS_STITCH_NORMAL_TO_TITLE,
             role_t::VCR_STATUS_STITCH_TITLE_TO_NORMAL);
-        this->tss_fields[TSF_TITLE].set_value(
-            " " ANSI_ROLE("T") "ext Filters ",
-            role_t::VCR_STATUS_TITLE_HOTKEY);
+        this->tss_fields[TSF_TITLE].set_value(" " ANSI_ROLE("T") "ext Filters ",
+                                              role_t::VCR_STATUS_TITLE_HOTKEY);
         this->tss_fields[TSF_TITLE].set_role(role_t::VCR_STATUS_TITLE);
         this->tss_fields[TSF_STITCH_TITLE].set_stitch_value(
             role_t::VCR_STATUS_STITCH_TITLE_TO_NORMAL,
@@ -194,9 +192,13 @@ filter_status_source::statusview_value_for_field(int field)
 void
 filter_status_source::update_filtered(text_sub_source* tss)
 {
-    status_field& sf = this->tss_fields[TSF_FILTERED];
+    if (tss == nullptr) {
+        return;
+    }
 
-    if (tss == nullptr || tss->get_filtered_count() == 0) {
+    auto& sf = this->tss_fields[TSF_FILTERED];
+
+    if (tss->get_filtered_count() == 0) {
         if (tss->tss_apply_filters) {
             sf.clear();
         } else {
@@ -205,8 +207,8 @@ filter_status_source::update_filtered(text_sub_source* tss)
                 ":toggle-filtering" ANSI_NORM);
         }
     } else {
-        ui_periodic_timer& timer = ui_periodic_timer::singleton();
-        attr_line_t& al = sf.get_value();
+        auto& timer = ui_periodic_timer::singleton();
+        auto& al = sf.get_value();
 
         if (tss->get_filtered_count() == this->bss_last_filtered_count) {
             if (timer.fade_diff(this->bss_filter_counter) == 0) {
@@ -215,8 +217,7 @@ filter_status_source::update_filtered(text_sub_source* tss)
                                          VC_STYLE.value(text_attrs{A_BOLD})));
             }
         } else {
-            this->tss_fields[TSF_FILTERED].set_role(
-                role_t::VCR_ALERT_STATUS);
+            this->tss_fields[TSF_FILTERED].set_role(role_t::VCR_ALERT_STATUS);
             this->bss_last_filtered_count = tss->get_filtered_count();
             timer.start_fade(this->bss_filter_counter, 3);
         }
@@ -286,7 +287,8 @@ filter_help_status_source::statusview_fields()
                                            : "Enable Filtering");
             }
         } else if (lnav_data.ld_mode == ln_mode_t::FILES
-                   && lnav_data.ld_session_loaded) {
+                   && lnav_data.ld_session_loaded)
+        {
             auto& lv = lnav_data.ld_files_view;
             auto sel = files_model::from_selection(lv.get_selection());
 

src/formats/access_log.json

@@ -13,7 +13,7 @@
                 "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?<c_ip>[^ ]+) (?<cs_username>[^ ]+) (?<cs_method>[A-Z]+) \"(?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))?\" (?:-1|\\d+) (?<sc_status>\\d+) \\d+\\s*(?<body>.*)"
             },
             "std": {
-                "pattern": "^(?<c_ip>[\\w\\.:\\-]+)\\s+[\\w\\.\\-]+\\s+(?<cs_username>\\S+)\\s+\\[(?<timestamp>[^\\]]+)\\] \"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]+)\" \"(?<cs_user_agent>[^\"]+)\")?\\s*(?<body>.*)"
+                "pattern": "^(?<c_ip>[\\w\\.:\\-]+)\\s+[\\w\\.\\-]+\\s+(?<cs_username>\\S+)\\s+\\[(?<timestamp>[^\\]]+)\\] \"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]*)\" \"(?<cs_user_agent>[^\"]+)\")?\\s*(?<body>.*)"
             },
             "std-vhost": {
                 "pattern": "^(?<cs_host>[\\w\\-\\.]*)(?::\\d+)?\\s+(?<c_ip>[\\w\\.:\\-]+)\\s+[\\w\\.\\-]+\\s+(?<cs_username>\\S+)\\s+\\[(?<timestamp>[^\\]]+)\\] \"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]+)\" \"(?<cs_user_agent>[^\"]+)\")?\\s*(?<body>.*)"
@@ -107,7 +107,11 @@
             },
             {
                 "line": "www.example.com 1.2.3.4 - theuser [10/Feb/2012:16:41:07 -0500] \"GET / HTTP/1.0\" 200 368 \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11\""
+            },
+            {
+                "line": "10.112.2.3 - - [16/Sep/2022:00:53:14 +0200] \"POST /api/v4/jobs/request HTTP/1.1\" 204 0 \"\" \"gitlab-runner 15.3.0 (15-3-stable; go1.19; linux/amd64)\" -",
+                "level": "info"
             }
         ]
     }
-}
+}

src/fs-extension-functions.cc

@@ -79,7 +79,8 @@ sql_dirname(const char* path_in)
 
     text_end = strlen(path_in) - 1;
     while (text_end >= 0
-           && (path_in[text_end] == '/' || path_in[text_end] == '\\')) {
+           && (path_in[text_end] == '/' || path_in[text_end] == '\\'))
+    {
         text_end -= 1;
     }
 
@@ -249,7 +250,8 @@ fs_extension_functions(struct FuncDef** basic_funcs,
          * TODO: add other functions like normpath, ...
          */
 
-        {nullptr}};
+        {nullptr},
+    };
 
     *basic_funcs = fs_funcs;
     *agg_funcs = nullptr;

src/keymaps/keymaps.am

@@ -3,6 +3,7 @@ KEYMAP_FILES = \
     $(srcdir)/%reldir%/de-keymap.json \
     $(srcdir)/%reldir%/default-keymap.json \
     $(srcdir)/%reldir%/fr-keymap.json \
+    $(srcdir)/%reldir%/sv-keymap.json \
     $(srcdir)/%reldir%/uk-keymap.json \
     $(srcdir)/%reldir%/us-keymap.json \
     $()

src/keymaps/sv-keymap.json

@@ -0,0 +1,27 @@
+{
+    "$schema": "https://lnav.org/schemas/config-v1.schema.json",
+    "ui": {
+        "keymap-defs": {
+            "sv": {
+                "x22": {
+                    "command": ":goto last 20 minutes after the hour"
+                },
+                "xc2xa4": {
+                    "command": ":goto last 40 minutes after the hour"
+                },
+                "xe2x82xac": {
+                    "command": ":goto last 40 minutes after the hour"
+                },
+                "x26": {
+                    "command": ":goto last hour"
+                },
+                "x3d": {
+                    "command": ":goto last day"
+                },
+                "x2b": {
+                    "command": ";UPDATE lnav_views SET paused = 1 - paused"
+                }
+            }
+        }
+    }
+}

src/line_buffer.cc

@@ -131,13 +131,6 @@ private:
 };
 /* XXX END */
 
-static int32_t
-read_le32(const unsigned char* data)
-{
-    return ((data[0] << 0) | (data[1] << 8) | (data[2] << 16)
-            | (data[3] << 24));
-}
-
 #define Z_BUFSIZE      65536U
 #define SYNCPOINT_SIZE (1024 * 1024)
 line_buffer::gz_indexed::gz_indexed()
@@ -198,11 +191,69 @@ line_buffer::gz_indexed::continue_stream()
 }
 
 void
-line_buffer::gz_indexed::open(int fd)
+line_buffer::gz_indexed::open(int fd, header_data& hd)
 {
     this->close();
     this->init_stream();
     this->gz_fd = fd;
+
+    unsigned char name[1024];
+    unsigned char comment[4096];
+
+    name[0] = '\0';
+    comment[0] = '\0';
+
+    gz_header gz_hd;
+    memset(&gz_hd, 0, sizeof(gz_hd));
+    gz_hd.name = name;
+    gz_hd.name_max = sizeof(name);
+    gz_hd.comment = comment;
+    gz_hd.comm_max = sizeof(comment);
+
+    Bytef inbuf[8192];
+    Bytef outbuf[8192];
+    this->strm.next_out = outbuf;
+    this->strm.total_out = 0;
+    this->strm.avail_out = sizeof(outbuf);
+    this->strm.next_in = inbuf;
+    this->strm.total_in = 0;
+
+    if (inflateGetHeader(&this->strm, &gz_hd) == Z_OK) {
+        auto rc = pread(fd, inbuf, sizeof(inbuf), 0);
+        if (rc >= 0) {
+            this->strm.avail_in = rc;
+
+            inflate(&this->strm, Z_BLOCK);
+            inflateEnd(&this->strm);
+
+            this->strm.next_out = Z_NULL;
+            this->strm.next_in = Z_NULL;
+            this->strm.next_in = Z_NULL;
+            this->strm.total_in = 0;
+            this->strm.avail_in = 0;
+            this->init_stream();
+
+            switch (gz_hd.done) {
+                case 0:
+                    log_debug("%d: no gzip header data", fd);
+                    break;
+                case 1:
+                    hd.hd_mtime.tv_sec = gz_hd.time;
+                    hd.hd_name = std::string((char*) name);
+                    hd.hd_comment = std::string((char*) comment);
+                    break;
+                default:
+                    log_error("%d: failed to read gzip header data", fd);
+                    break;
+            }
+        } else {
+            log_error("%d: failed to read gzip header from file: %s",
+                      fd,
+                      strerror(errno));
+        }
+    } else {
+        log_error("%d: unable to get gzip header", fd);
+    }
 }
 
 int
@@ -365,10 +416,9 @@ line_buffer::set_fd(auto_fd& fd)
                         close(gzfd);
                         throw error(errno);
                     }
-                    this->lb_gz_file.writeAccess()->open(gzfd);
+                    this->lb_gz_file.writeAccess()->open(gzfd, this->lb_header);
                     this->lb_compressed = true;
-                    this->lb_file_time
-                        = read_le32((const unsigned char*) &gz_id[4]);
+                    this->lb_file_time = this->lb_header.hd_mtime.tv_sec;
                     if (this->lb_file_time < 0) {
                         this->lb_file_time = 0;
                     }
@@ -1309,7 +1359,7 @@ line_buffer::enable_cache()
         "%d:cache file path: %s", this->lb_fd.get(), cached_file_path.c_str());
 
     auto fl = lnav::filesystem::file_lock(cached_file_path);
-    auto guard = lnav::filesystem::file_lock::guard(fl);
+    auto guard = lnav::filesystem::file_lock::guard(&fl);
 
     if (ghc::filesystem::exists(cached_done_path)) {
         log_info("%d:using existing cache file");

src/line_buffer.hh

@@ -72,11 +72,24 @@ public:
     static const ssize_t MAX_LINE_BUFFER_SIZE;
     class error : public std::exception {
     public:
-        error(int err) : e_err(err){};
+        explicit error(int err) : e_err(err) {}
 
         int e_err;
     };
 
+    struct header_data {
+        timeval hd_mtime{};
+        auto_buffer hd_extra{auto_buffer::alloc(0)};
+        std::string hd_name;
+        std::string hd_comment;
+
+        bool empty() const
+        {
+            return this->hd_mtime.tv_sec == 0 && this->hd_extra.empty()
+                && this->hd_name.empty() && this->hd_comment.empty();
+        }
+    };
+
 #define GZ_WINSIZE           32768U /*> gzip's max supported dictionary is 15-bits */
 #define GZ_RAW_MODE          (-15) /*> Raw inflate data mode */
 #define GZ_HEADER_MODE       (15 + 32) /*> Automatic zstd or gzip decoding */
@@ -104,7 +117,7 @@ public:
         void close();
         void init_stream();
         void continue_stream();
-        void open(int fd);
+        void open(int fd, header_data& hd);
         int stream_data(void* buf, size_t size);
         void seek(off_t offset);
 
@@ -242,6 +255,8 @@ public:
 
     size_t get_buffer_size() const { return this->lb_buffer.size(); }
 
+    const header_data& get_header_data() const { return this->lb_header; }
+
     void enable_cache();
 
     static void cleanup_cache();
@@ -348,6 +363,8 @@ private:
     stats lb_stats;
 
     nonstd::optional<auto_fd> lb_cached_fd;
+
+    header_data lb_header;
 };
 
 #endif

src/lnav.cc

@@ -230,8 +230,7 @@ static auto bound_active_files = injector::bind<file_collection>::to_instance(
     +[]() { return &lnav_data.ld_active_files; });
 
 static auto bound_sqlite_db
-    = injector::bind<auto_mem<sqlite3, sqlite_close_wrapper>,
-                     sqlite_db_tag>::to_instance(&lnav_data.ld_db);
+    = injector::bind<auto_sqlite3>::to_instance(&lnav_data.ld_db);
 
 static auto bound_lnav_flags
     = injector::bind<unsigned long, lnav_flags_tag>::to_instance(
@@ -264,12 +263,6 @@ force_linking(last_relative_time_tag anno)
 {
 }
 
-template<>
-void
-force_linking(sqlite_db_tag anno)
-{
-}
-
 template<>
 void
 force_linking(lnav_flags_tag anno)
@@ -1005,6 +998,55 @@ wait_for_pipers()
     }
 }
 
+struct refresh_status_bars {
+    refresh_status_bars(std::shared_ptr<top_status_source> top_source)
+        : rsb_top_source(std::move(top_source))
+    {
+    }
+
+    using injectable
+        = refresh_status_bars(std::shared_ptr<top_status_source> top_source);
+
+    void doit() const
+    {
+        struct timeval current_time {};
+        int ch;
+
+        gettimeofday(&current_time, nullptr);
+        while ((ch = getch()) != ERR) {
+            lnav_data.ld_user_message_source.clear();
+
+            alerter::singleton().new_input(ch);
+
+            lnav_data.ld_input_dispatcher.new_input(current_time, ch);
+
+            lnav_data.ld_view_stack.top() | [ch](auto tc) {
+                lnav_data.ld_key_repeat_history.update(ch, tc->get_top());
+            };
+
+            if (!lnav_data.ld_looping) {
+                // No reason to keep processing input after the
+                // user has quit.  The view stack will also be
+                // empty, which will cause issues.
+                break;
+            }
+        }
+
+        this->rsb_top_source->update_time(current_time);
+        for (auto& sc : lnav_data.ld_status) {
+            sc.do_update();
+        }
+        lnav_data.ld_rl_view->do_update();
+        if (handle_winch()) {
+            layout_views();
+            lnav_data.ld_view_stack.do_update();
+        }
+        refresh();
+    }
+
+    std::shared_ptr<top_status_source> rsb_top_source;
+};
+
 static void
 looper()
 {
@@ -1340,10 +1382,15 @@ looper()
         lnav_data.ld_spectro_source->ss_exec_context
             = &lnav_data.ld_exec_context;
 
+        auto top_status_lifetime
+            = injector::bind<top_status_source>::to_scoped_singleton();
+
+        auto top_source = injector::get<std::shared_ptr<top_status_source>>();
+
         lnav_data.ld_status[LNS_TOP].set_top(0);
         lnav_data.ld_status[LNS_TOP].set_default_role(
             role_t::VCR_INACTIVE_STATUS);
-        lnav_data.ld_status[LNS_TOP].set_data_source(&lnav_data.ld_top_source);
+        lnav_data.ld_status[LNS_TOP].set_data_source(top_source.get());
         lnav_data.ld_status[LNS_BOTTOM].set_top(-(rlc->get_height() + 1));
         for (auto& stat_bar : lnav_data.ld_status) {
             stat_bar.set_window(lnav_data.ld_window);
@@ -1381,7 +1428,7 @@ looper()
         };
 
         {
-            input_dispatcher& id = lnav_data.ld_input_dispatcher;
+            auto& id = lnav_data.ld_input_dispatcher;
 
             id.id_escape_matcher = match_escape_seq;
             id.id_escape_handler = handle_keyseq;
@@ -1412,7 +1459,15 @@ looper()
             };
         }
 
-        ui_periodic_timer& timer = ui_periodic_timer::singleton();
+        auto refresher_lifetime
+            = injector::bind<refresh_status_bars>::to_scoped_singleton();
+
+        auto refresher = injector::get<std::shared_ptr<refresh_status_bars>>();
+
+        auto refresh_guard = lnav_data.ld_status_refresher.install(
+            [refresher]() { refresher->doit(); });
+
+        auto& timer = ui_periodic_timer::singleton();
         struct timeval current_time;
 
         static sig_atomic_t index_counter;
@@ -1450,7 +1505,7 @@ looper()
 
             gettimeofday(&current_time, nullptr);
 
-            lnav_data.ld_top_source.update_time(current_time);
+            top_source->update_time(current_time);
             lnav_data.ld_preview_view.set_needs_update();
 
             layout_views();
@@ -1562,7 +1617,7 @@ looper()
             lnav_data.ld_user_message_view.do_update();
             if (ui_clock::now() >= next_status_update_time) {
                 echo_views_stmt.execute();
-                lnav_data.ld_top_source.update_user_msg();
+                top_source->update_user_msg();
                 for (auto& sc : lnav_data.ld_status) {
                     sc.do_update();
                 }
@@ -2150,6 +2205,26 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
             } while (!done);
         }
 
+        // XXX
+        lnav_data.ld_log_source.set_preview_sql_filter(nullptr);
+        lnav_data.ld_log_source.set_sql_filter("", nullptr);
+        lnav_data.ld_log_source.set_sql_marker("", nullptr);
+        lnav_config_listener::unload_all();
+
+        {
+            sqlite3_stmt* stmt_iter = nullptr;
+
+            do {
+                stmt_iter = sqlite3_next_stmt(lnav_data.ld_db.in(), stmt_iter);
+                if (stmt_iter != nullptr) {
+                    const auto* stmt_sql = sqlite3_sql(stmt_iter);
+
+                    log_warning("unfinalized SQL statement: %s", stmt_sql);
+                    ensure(false);
+                }
+            } while (stmt_iter != nullptr);
+        }
+
         for (auto& drop_stmt : tables_to_drop) {
             sqlite3_exec(lnav_data.ld_db.in(),
                          drop_stmt.c_str(),

src/lnav.hh

@@ -71,7 +71,6 @@
 #include "sql_util.hh"
 #include "statusview_curses.hh"
 #include "textfile_sub_source.hh"
-#include "top_status_source.hh"
 #include "view_helpers.hh"
 
 class spectrogram_source;
@@ -184,7 +183,6 @@ struct lnav_data_t {
     ln_mode_t ld_last_config_mode{ln_mode_t::FILTER};
 
     statusview_curses ld_status[LNS__MAX];
-    top_status_source ld_top_source;
     bottom_status_source ld_bottom_source;
     filter_status_source ld_filter_status_source;
     filter_help_status_source ld_filter_help_status_source;
@@ -240,7 +238,7 @@ struct lnav_data_t {
     vis_line_t ld_last_pretty_print_top;
 
     std::unique_ptr<log_vtab_manager> ld_vtab_manager;
-    auto_mem<sqlite3, sqlite_close_wrapper> ld_db;
+    auto_sqlite3 ld_db;
 
     std::unordered_map<std::string, std::string> ld_table_ddl;
 
@@ -259,6 +257,8 @@ struct lnav_data_t {
     bool ld_initial_build{false};
     bool ld_show_help_view{false};
 
+    lnav::func::scoped_cb ld_status_refresher;
+
     ghc::filesystem::file_time_type ld_last_dot_lnav_time;
 };
 

src/lnav.indexing.cc

@@ -85,10 +85,7 @@ do_observer_update(const std::shared_ptr<logfile>& lf)
     if (isendwin()) {
         return;
     }
-    lnav_data.ld_top_source.update_time();
-    for (auto& sc : lnav_data.ld_status) {
-        sc.do_update();
-    }
+    lnav_data.ld_status_refresher();
     if (lf && lnav_data.ld_mode == ln_mode_t::FILES
         && !lnav_data.ld_initial_build)
     {
@@ -437,15 +434,7 @@ rescan_files(bool req)
         if (!done && !(lnav_data.ld_flags & LNF_HEADLESS)) {
             lnav_data.ld_files_view.set_needs_update();
             lnav_data.ld_files_view.do_update();
-            lnav_data.ld_top_source.update_time();
-            lnav_data.ld_status[LNS_TOP].do_update();
-            lnav_data.ld_status[LNS_BOTTOM].do_update();
-            lnav_data.ld_rl_view->do_update();
-            if (handle_winch()) {
-                layout_views();
-                lnav_data.ld_view_stack.do_update();
-            }
-            refresh();
+            lnav_data.ld_status_refresher();
         }
     } while (!done && lnav_data.ld_looping);
     return true;

src/lnav_commands.cc

@@ -164,10 +164,10 @@ combined_user_marks(vis_bookmarks& vb)
     const auto& bv_expr = vb[&textview_curses::BM_USER_EXPR];
     bookmark_vector<vis_line_t> retval;
 
-    for (const auto row : bv) {
+    for (const auto& row : bv) {
         retval.insert_once(row);
     }
-    for (const auto row : bv_expr) {
+    for (const auto& row : bv_expr) {
         retval.insert_once(row);
     }
     return retval;

src/lnav_config.cc

@@ -96,6 +96,9 @@ static auto scc = injector::bind<sysclip::config>::to_instance(
 static auto lsc = injector::bind<logfile_sub_source_ns::config>::to_instance(
     +[]() { return &lnav_config.lc_log_source; });
 
+static auto tssc = injector::bind<top_status_source_cfg>::to_instance(
+    +[]() { return &lnav_config.lc_top_status_cfg; });
+
 bool
 check_experimental(const char* feature_name)
 {
@@ -335,11 +338,12 @@ update_installs_from_git()
 
     if (glob(git_formats.c_str(), GLOB_NOCHECK, nullptr, gl.inout()) == 0) {
         for (int lpc = 0; lpc < (int) gl->gl_pathc; lpc++) {
-            char* git_dir = dirname(gl->gl_pathv[lpc]);
+            auto git_dir
+                = ghc::filesystem::path(gl->gl_pathv[lpc]).parent_path();
 
-            printf("Updating formats in %s\n", git_dir);
-            auto pull_cmd
-                = fmt::format(FMT_STRING("cd '{}' && git pull"), git_dir);
+            printf("Updating formats in %s\n", git_dir.c_str());
+            auto pull_cmd = fmt::format(FMT_STRING("cd '{}' && git pull"),
+                                        git_dir.string());
             int ret = system(pull_cmd.c_str());
             if (ret == -1) {
                 std::cerr << "Failed to spawn command "
@@ -516,7 +520,7 @@ static const struct json_path_container global_var_handlers = {
                     paths_out.emplace_back(iter.first);
                 }
             })
-        .FOR_FIELD(_lnav_config, lc_global_vars),
+        .for_field(&_lnav_config::lc_global_vars),
 };
 
 static const struct json_path_container style_config_handlers =
@@ -865,7 +869,7 @@ static const struct json_path_container highlighter_handlers = {
     yajlpp::property_handler("pattern")
         .with_synopsis("regular expression")
         .with_description("The regular expression to highlight")
-        .FOR_FIELD(highlighter_config, hc_regex),
+        .for_field(&highlighter_config::hc_regex),
 
     yajlpp::property_handler("style")
         .with_description(
@@ -903,7 +907,7 @@ static const struct json_path_container theme_vars_handlers = {
                     paths_out.emplace_back(iter.first);
                 }
             })
-        .FOR_FIELD(lnav_theme, lt_vars),
+        .for_field(&lnav_theme::lt_vars),
 };
 
 static const struct json_path_container theme_def_handlers = {
@@ -961,7 +965,8 @@ static const struct json_path_container ui_handlers = {
         .with_description("The format for the clock displayed in "
                           "the top-left corner using strftime(3) conversions")
         .with_example("%a %b %d %H:%M:%S %Z")
-        .for_field(&_lnav_config::lc_ui_clock_format),
+        .for_field(&_lnav_config::lc_top_status_cfg,
+                   &top_status_source_cfg::tssc_clock_format),
     yajlpp::property_handler("dim-text")
         .with_synopsis("bool")
         .with_description("Reduce the brightness of text (useful for xterms). "

src/lnav_config.hh

@@ -52,6 +52,7 @@
 #include "styling.hh"
 #include "sysclip.cfg.hh"
 #include "tailer/tailer.looper.cfg.hh"
+#include "top_status_source.cfg.hh"
 
 /**
  * Check if an experimental feature should be enabled by
@@ -85,7 +86,7 @@ struct key_map {
 };
 
 struct _lnav_config {
-    std::string lc_ui_clock_format;
+    top_status_source_cfg lc_top_status_cfg;
     bool lc_ui_dim_text;
     bool lc_ui_default_colors{true};
     std::string lc_ui_keymap;

src/lnav_config_fwd.hh

@@ -52,6 +52,16 @@ public:
 
     virtual void reload_config(error_reporter& reporter) {}
 
+    virtual void unload_config() {}
+
+    static void unload_all() {
+        auto* lcl = LISTENER_LIST;
+        while (lcl != nullptr) {
+            lcl->unload_config();
+            lcl = lcl->lcl_next;
+        }
+    }
+
     static lnav_config_listener* LISTENER_LIST;
 
     lnav_config_listener* lcl_next;

src/lnav_util.hh

@@ -95,43 +95,42 @@ public:
         return *this;
     }
 
-    template<typename T,
-             typename = std::enable_if<std::is_arithmetic<T>::value>>
-    hasher& update(T value)
+    hasher& update(int64_t value)
     {
+        value = SPOOKYHASH_LITTLE_ENDIAN_64(value);
         this->h_context.Update(&value, sizeof(value));
 
         return *this;
     }
 
-    array_t to_array() {
+    array_t to_array()
+    {
+        uint64_t h1;
+        uint64_t h2;
         array_t retval;
 
-        this->h_context.Final(retval.out(0), retval.out(0));
+        this->h_context.Final(&h1, &h2);
+        *retval.out(0) = SPOOKYHASH_LITTLE_ENDIAN_64(h1);
+        *retval.out(1) = SPOOKYHASH_LITTLE_ENDIAN_64(h2);
         return retval;
     }
 
     void to_string(auto_buffer& buf)
     {
-        array_t bits;
+        array_t bits = this->to_array();
 
-        this->h_context.Final(bits.out(0), bits.out(1));
         bits.to_string(std::back_inserter(buf));
     }
 
     std::string to_string()
     {
-        array_t bits;
-
-        this->h_context.Final(bits.out(0), bits.out(1));
+        array_t bits = this->to_array();
         return bits.to_string();
     }
 
     std::string to_uuid_string()
     {
-        array_t bits;
-
-        this->h_context.Final(bits.out(0), bits.out(1));
+        array_t bits = this->to_array();
         return bits.to_uuid_string();
     }
 
@@ -215,10 +214,7 @@ template<typename A>
 struct final_action {  // slightly simplified
     A act;
     final_action(A a) : act{a} {}
-    ~final_action()
-    {
-        act();
-    }
+    ~final_action() { act(); }
 };
 
 template<typename A>

src/log.watch.cc

@@ -52,8 +52,7 @@ struct compiled_watch_expr {
 struct expressions : public lnav_config_listener {
     void reload_config(error_reporter& reporter) override
     {
-        auto& lnav_db = injector::get<auto_mem<sqlite3, sqlite_close_wrapper>&,
-                                      sqlite_db_tag>();
+        auto& lnav_db = injector::get<auto_sqlite3&>();
 
         if (lnav_db.in() == nullptr) {
             log_warning("db not initialized yet!");
@@ -99,6 +98,10 @@ struct expressions : public lnav_config_listener {
         }
     }
 
+    void unload_config() override {
+        this->e_watch_exprs.clear();
+    }
+
     std::map<std::string, compiled_watch_expr> e_watch_exprs;
 };
 
@@ -114,9 +117,7 @@ eval_with(logfile& lf, logfile::iterator ll)
         return;
     }
 
-    static auto& lnav_db
-        = injector::get<auto_mem<sqlite3, sqlite_close_wrapper>&,
-                        sqlite_db_tag>();
+    static auto& lnav_db = injector::get<auto_sqlite3&>();
 
     char timestamp_buffer[64] = "";
     shared_buffer_ref raw_sbr;

src/log_format.cc

@@ -30,7 +30,6 @@
 #include <memory>
 
 #include <fnmatch.h>
-#include <stdarg.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -312,7 +311,8 @@ log_format::log_scanf(uint32_t line_number,
     int pat_index = this->last_pattern_index();
 
     while (!done && next_format(fmt, curr_fmt, pat_index)) {
-        auto md = fmt[curr_fmt].pcre->create_match_data();
+        static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
+
         auto match_res = fmt[curr_fmt]
                              .pcre->capture_from(line)
                              .into(md)
@@ -480,6 +480,24 @@ read_json_int(yajlpp_parse_context* ypc, long long val)
             tv.tv_sec = tm2sec(&ltm);
         }
         jlu->jlu_base_line->set_time(tv);
+    } else if (jlu->jlu_format->lf_subsecond_field == field_name) {
+        uint64_t millis = 0;
+        switch (jlu->jlu_format->lf_subsecond_unit.value()) {
+            case log_format::subsecond_unit::milli:
+                millis = val;
+                break;
+            case log_format::subsecond_unit::micro:
+                millis = std::chrono::duration_cast<std::chrono::milliseconds>(
+                             std::chrono::microseconds(val))
+                             .count();
+                break;
+            case log_format::subsecond_unit::nano:
+                millis = std::chrono::duration_cast<std::chrono::milliseconds>(
+                             std::chrono::nanoseconds(val))
+                             .count();
+                break;
+        }
+        jlu->jlu_base_line->set_millis(millis);
     } else if (jlu->jlu_format->elf_level_field == field_name) {
         if (jlu->jlu_format->elf_level_pairs.empty()) {
             char level_buf[128];
@@ -577,7 +595,7 @@ json_array_end(void* ctx)
     return 1;
 }
 
-static struct json_path_container json_log_handlers = {
+static const struct json_path_container json_log_handlers = {
     yajlpp::pattern_property_handler("\\w+")
         .add_cb(read_json_null)
         .add_cb(read_json_bool)
@@ -653,7 +671,7 @@ rewrite_json_double(yajlpp_parse_context* ypc, double val)
     return 1;
 }
 
-static struct json_path_container json_log_rewrite_handlers = {
+static const struct json_path_container json_log_rewrite_handlers = {
     yajlpp::pattern_property_handler("\\w+")
         .add_cb(rewrite_json_null)
         .add_cb(rewrite_json_bool)
@@ -707,8 +725,21 @@ external_log_format::scan(logfile& lf,
                           scan_batch_context& sbc)
 {
     if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
-        yajlpp_parse_context& ypc = *(this->jlf_parse_context);
         logline ll(li.li_file_range.fr_offset, 0, 0, LEVEL_INFO);
+        auto line_frag = sbr.to_string_fragment();
+
+        if (!line_frag.startswith("{")) {
+            if (!this->lf_specialized) {
+                return log_format::SCAN_NO_MATCH;
+            }
+
+            ll.set_time(dst.back().get_timeval());
+            ll.set_level(LEVEL_INVALID);
+            dst.emplace_back(ll);
+            return log_format::SCAN_MATCH;
+        }
+
+        auto& ypc = *(this->jlf_parse_context);
         yajl_handle handle = this->jlf_yajl_handle.get();
         json_log_userdata jlu(sbr, &sbc);
 
@@ -749,10 +780,10 @@ external_log_format::scan(logfile& lf,
                     ll.set_ignore(true);
                     dst.emplace_back(ll);
                     return log_format::SCAN_MATCH;
-                } else {
-                    log_debug("no match! %.*s", sbr.length(), line_data);
-                    return log_format::SCAN_NO_MATCH;
                 }
+
+                log_debug("no match! %.*s", sbr.length(), line_data);
+                return log_format::SCAN_NO_MATCH;
             }
 
             jlu.jlu_sub_line_count += this->jlf_line_format_init_count;
@@ -771,11 +802,11 @@ external_log_format::scan(logfile& lf,
             msg = yajl_get_error(
                 handle, 1, (const unsigned char*) sbr.get_data(), sbr.length());
             if (msg != nullptr) {
+                auto msg_frag = string_fragment::from_c_str(msg);
                 log_debug("Unable to parse line at offset %d: %s",
                           li.li_file_range.fr_offset,
                           msg);
-                line_count
-                    = std::count(msg, msg + strlen((char*) msg), '\n') + 1;
+                line_count = msg_frag.count('\n') + 1;
                 yajl_free_error(handle, msg);
             }
             if (!this->lf_specialized) {
@@ -784,7 +815,7 @@ external_log_format::scan(logfile& lf,
             for (int lpc = 0; lpc < line_count; lpc++) {
                 log_level_t level = LEVEL_INVALID;
 
-                ll.set_time(dst.back().get_time());
+                ll.set_time(dst.back().get_timeval());
                 if (lpc > 0) {
                     level = (log_level_t) (level | LEVEL_CONTINUED);
                 }
@@ -802,6 +833,8 @@ external_log_format::scan(logfile& lf,
     auto line_sf = sbr.to_string_fragment();
 
     while (::next_format(this->elf_pattern_order, curr_fmt, pat_index)) {
+        static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
+
         auto* fpat = this->elf_pattern_order[curr_fmt].get();
         auto* pat = fpat->p_pcre.pp_value.get();
 
@@ -809,7 +842,6 @@ external_log_format::scan(logfile& lf,
             continue;
         }
 
-        auto md = pat->create_match_data();
         auto match_res = pat->capture_from(line_sf)
                              .into(md)
                              .matches(PCRE2_NO_UTF_CHECK)
@@ -907,13 +939,15 @@ external_log_format::scan(logfile& lf,
                     mod_iter->second.mf_mod_format);
 
                 if (mod_elf) {
+                    static thread_local auto mod_md
+                        = lnav::pcre2pp::match_data::unitialized();
+
                     shared_buffer_ref body_ref;
 
                     body_cap->trim();
 
                     int mod_pat_index = mod_elf->last_pattern_index();
                     auto& mod_pat = *mod_elf->elf_pattern_order[mod_pat_index];
-                    auto mod_md = mod_pat.p_pcre.pp_value->create_match_data();
                     auto match_res = mod_pat.p_pcre.pp_value
                                          ->capture_from(body_cap.value())
                                          .into(mod_md)
@@ -1014,6 +1048,9 @@ external_log_format::module_scan(string_fragment body_cap,
         int curr_fmt = -1, fmt_lock = -1;
 
         while (::next_format(elf->elf_pattern_order, curr_fmt, fmt_lock)) {
+            static thread_local auto md
+                = lnav::pcre2pp::match_data::unitialized();
+
             auto& fpat = elf->elf_pattern_order[curr_fmt];
             auto& pat = fpat->p_pcre;
 
@@ -1021,7 +1058,6 @@ external_log_format::module_scan(string_fragment body_cap,
                 continue;
             }
 
-            auto md = pat.pp_value->create_match_data();
             auto match_res = pat.pp_value->capture_from(body_cap)
                                  .into(md)
                                  .matches(PCRE2_NO_UTF_CHECK)
@@ -1054,6 +1090,8 @@ external_log_format::annotate(uint64_t line_number,
                               logline_value_vector& values,
                               bool annotate_module) const
 {
+    static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
+
     auto& line = values.lvv_sbr;
     struct line_range lr;
 
@@ -1074,7 +1112,6 @@ external_log_format::annotate(uint64_t line_number,
     auto& pat = *this->elf_pattern_order[pat_index];
 
     sa.reserve(pat.p_pcre.pp_value->get_capture_count());
-    auto md = pat.p_pcre.pp_value->create_match_data();
     auto match_res
         = pat.p_pcre.pp_value->capture_from(line.to_string_fragment())
               .into(md)
@@ -1383,7 +1420,7 @@ external_log_format::get_subline(const logline& ll,
     if (this->jlf_cached_offset != ll.get_offset()
         || this->jlf_cached_full != full_message)
     {
-        yajlpp_parse_context& ypc = *(this->jlf_parse_context);
+        auto& ypc = *(this->jlf_parse_context);
         yajl_handle handle = this->jlf_yajl_handle.get();
         json_log_userdata jlu(sbr, nullptr);
 
@@ -1393,6 +1430,26 @@ external_log_format::get_subline(const logline& ll,
         this->jlf_line_offsets.clear();
         this->jlf_line_attrs.clear();
 
+        auto line_frag = sbr.to_string_fragment();
+
+        if (!line_frag.startswith("{")) {
+            this->jlf_cached_line.resize(line_frag.length());
+            memcpy(this->jlf_cached_line.data(),
+                   line_frag.data(),
+                   line_frag.length());
+            this->jlf_line_values.clear();
+            sbr.share(this->jlf_share_manager,
+                      &this->jlf_cached_line[0],
+                      this->jlf_cached_line.size());
+            this->jlf_line_values.lvv_sbr = sbr;
+            this->jlf_line_attrs.emplace_back(
+                line_range{0, -1},
+                SA_INVALID.value(fmt::format(
+                    FMT_STRING("line at offset {} is not a JSON-line"),
+                    ll.get_offset())));
+            return;
+        }
+
         yajl_reset(handle);
         ypc.set_static_handler(json_log_rewrite_handlers.jpc_children[0]);
         ypc.ypc_userdata = &jlu;
@@ -2002,6 +2059,23 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
                 .with_snippets(this->get_snippets()));
     }
 
+    if (!this->lf_subsecond_field.empty()
+        && !this->lf_subsecond_unit.has_value())
+    {
+        errors.emplace_back(
+            lnav::console::user_message::error(
+                attr_line_t()
+                    .append_quoted(
+                        lnav::roles::symbol(this->elf_name.to_string()))
+                    .append(" is not a valid log format"))
+                .with_reason(attr_line_t()
+                                 .append_quoted("subsecond-unit"_symbol)
+                                 .append(" must be set when ")
+                                 .append_quoted("subsecond-field"_symbol)
+                                 .append(" is used"))
+                .with_snippets(this->get_snippets()));
+    }
+
     for (size_t sample_index = 0; sample_index < this->elf_samples.size();
          sample_index += 1)
     {

src/log_format.hh

@@ -498,6 +498,12 @@ public:
         return intern_string_t::case_lt(lhs->get_name(), rhs->get_name());
     }
 
+    enum class subsecond_unit {
+        milli,
+        micro,
+        nano,
+    };
+
     std::string lf_description;
     uint8_t lf_mod_index{0};
     bool lf_multiline{true};
@@ -505,6 +511,8 @@ public:
     date_time_scanner lf_time_scanner;
     std::vector<pattern_for_lines> lf_pattern_locks;
     intern_string_t lf_timestamp_field{intern_string::lookup("timestamp", -1)};
+    intern_string_t lf_subsecond_field;
+    nonstd::optional<subsecond_unit> lf_subsecond_unit;
     intern_string_t lf_time_field;
     std::vector<const char*> lf_timestamp_format;
     unsigned int lf_timestamp_flags{0};

src/log_format_fwd.hh

@@ -90,23 +90,23 @@ public:
     logline(file_off_t off,
             time_t t,
             uint16_t millis,
-            log_level_t l,
+            log_level_t lev,
             uint8_t mod = 0,
             uint8_t opid = 0)
-        : ll_offset(off), ll_time(t), ll_millis(millis), ll_opid(opid),
-          ll_sub_offset(0), ll_valid_utf(1), ll_level(l), ll_module_id(mod),
-          ll_expr_mark(0)
+        : ll_offset(off), ll_has_ansi(false), ll_time(t), ll_millis(millis),
+          ll_opid(opid), ll_sub_offset(0), ll_valid_utf(1), ll_level(lev),
+          ll_module_id(mod), ll_expr_mark(0)
     {
         memset(this->ll_schema, 0, sizeof(this->ll_schema));
     }
 
     logline(file_off_t off,
             const struct timeval& tv,
-            log_level_t l,
+            log_level_t lev,
             uint8_t mod = 0,
             uint8_t opid = 0)
-        : ll_offset(off), ll_opid(opid), ll_sub_offset(0), ll_valid_utf(1),
-          ll_level(l), ll_module_id(mod), ll_expr_mark(0)
+        : ll_offset(off), ll_has_ansi(false), ll_opid(opid), ll_sub_offset(0),
+          ll_valid_utf(1), ll_level(lev), ll_module_id(mod), ll_expr_mark(0)
     {
         this->set_time(tv);
         memset(this->ll_schema, 0, sizeof(this->ll_schema));

src/log_format_loader.cc

@@ -62,11 +62,14 @@ static void extract_metadata(string_fragment, struct script_metadata& meta_out);
 using log_formats_map_t
     = std::map<intern_string_t, std::shared_ptr<external_log_format>>;
 
+using namespace lnav::roles::literals;
+
 static auto intern_lifetime = intern_string::get_table_lifetime();
 static log_formats_map_t LOG_FORMATS;
 
 struct loader_userdata {
     yajlpp_parse_context* ud_parse_context{nullptr};
+    std::string ud_file_schema;
     ghc::filesystem::path ud_format_path;
     std::vector<intern_string_t>* ud_format_names{nullptr};
     std::vector<lnav::console::user_message>* ud_errors{nullptr};
@@ -76,10 +79,8 @@ static external_log_format*
 ensure_format(const yajlpp_provider_context& ypc, loader_userdata* ud)
 {
     const intern_string_t name = ypc.get_substr_i(0);
-    std::vector<intern_string_t>* formats = ud->ud_format_names;
-    external_log_format* retval;
-
-    retval = LOG_FORMATS[name].get();
+    auto* formats = ud->ud_format_names;
+    auto* retval = LOG_FORMATS[name].get();
     if (retval == nullptr) {
         LOG_FORMATS[name] = std::make_shared<external_log_format>(name);
         retval = LOG_FORMATS[name].get();
@@ -92,7 +93,8 @@ ensure_format(const yajlpp_provider_context& ypc, loader_userdata* ud)
     }
 
     if (!ud->ud_format_path.empty()) {
-        auto i_src_path = intern_string::lookup(ud->ud_format_path.string());
+        const intern_string_t i_src_path
+            = intern_string::lookup(ud->ud_format_path.string());
         auto srcs_iter = retval->elf_format_sources.find(i_src_path);
         if (srcs_iter == retval->elf_format_sources.end()) {
             retval->elf_format_source_order.emplace_back(ud->ud_format_path);
@@ -172,7 +174,7 @@ scaling_factor_provider(const yajlpp_provider_context& ypc,
                         external_log_format::value_def* value_def)
 {
     auto scale_name = ypc.get_substr_i(0);
-    scaling_factor& retval = value_def->vd_unit_scaling[scale_name];
+    auto& retval = value_def->vd_unit_scaling[scale_name];
 
     return &retval;
 }
@@ -380,7 +382,7 @@ ensure_sample(external_log_format* elf, int index)
 static external_log_format::sample*
 sample_provider(const yajlpp_provider_context& ypc, external_log_format* elf)
 {
-    external_log_format::sample& sample = ensure_sample(elf, ypc.ypc_index);
+    auto& sample = ensure_sample(elf, ypc.ypc_index);
 
     return &sample;
 }
@@ -401,7 +403,7 @@ read_json_constant(yajlpp_parse_context* ypc,
     return 1;
 }
 
-static struct json_path_container pattern_handlers = {
+static const struct json_path_container pattern_handlers = {
     yajlpp::property_handler("pattern")
         .with_synopsis("<message-regex>")
         .with_description(
@@ -416,6 +418,14 @@ static struct json_path_container pattern_handlers = {
         .for_field(&external_log_format::pattern::p_module_format),
 };
 
+static const json_path_handler_base::enum_value_t SUBSECOND_UNIT_ENUM[] = {
+    {"milli", log_format::subsecond_unit::milli},
+    {"micro", log_format::subsecond_unit::micro},
+    {"nano", log_format::subsecond_unit::nano},
+
+    json_path_handler_base::ENUM_TERMINATOR,
+};
+
 static const json_path_handler_base::enum_value_t ALIGN_ENUM[] = {
     {"left", external_log_format::json_format_element::align_t::LEFT},
     {"right", external_log_format::json_format_element::align_t::RIGHT},
@@ -444,12 +454,11 @@ static const json_path_handler_base::enum_value_t TRANSFORM_ENUM[] = {
     json_path_handler_base::ENUM_TERMINATOR,
 };
 
-static struct json_path_container line_format_handlers = {
+static const struct json_path_container line_format_handlers = {
     yajlpp::property_handler("field")
         .with_synopsis("<field-name>")
         .with_description(
             "The name of the field to substitute at this position")
-        .with_min_length(1)
         .for_field(&external_log_format::json_format_element::jfe_value),
 
     yajlpp::property_handler("default-value")
@@ -519,22 +528,22 @@ static const json_path_handler_base::enum_value_t SCALE_OP_ENUM[] = {
     json_path_handler_base::ENUM_TERMINATOR,
 };
 
-static struct json_path_container scaling_factor_handlers = {
+static const struct json_path_container scaling_factor_handlers = {
     yajlpp::pattern_property_handler("op")
         .with_enum_values(SCALE_OP_ENUM)
         .for_field(&scaling_factor::sf_op),
 
-    yajlpp::pattern_property_handler("value").FOR_FIELD(scaling_factor,
-                                                        sf_value),
+    yajlpp::pattern_property_handler("value").for_field(
+        &scaling_factor::sf_value),
 };
 
-static struct json_path_container scale_handlers = {
+static const struct json_path_container scale_handlers = {
     yajlpp::pattern_property_handler("(?<scale>[^/]+)")
         .with_obj_provider(scaling_factor_provider)
         .with_children(scaling_factor_handlers),
 };
 
-static struct json_path_container unit_handlers = {
+static const struct json_path_container unit_handlers = {
     yajlpp::property_handler("field")
         .with_synopsis("<field-name>")
         .with_description(
@@ -546,7 +555,7 @@ static struct json_path_container unit_handlers = {
         .with_children(scale_handlers),
 };
 
-static struct json_path_container value_def_handlers = {
+static const struct json_path_container value_def_handlers = {
     yajlpp::property_handler("kind")
         .with_synopsis("<data-type>")
         .with_description("The type of data in the field")
@@ -586,7 +595,7 @@ static struct json_path_container value_def_handlers = {
     yajlpp::property_handler("action-list#")
         .with_synopsis("<string>")
         .with_description("Actions to execute when this field is clicked on")
-        .FOR_FIELD(external_log_format::value_def, vd_action_list),
+        .for_field(&external_log_format::value_def::vd_action_list),
 
     yajlpp::property_handler("rewriter")
         .with_synopsis("<command>")
@@ -602,7 +611,7 @@ static struct json_path_container value_def_handlers = {
         .for_field(&external_log_format::value_def::vd_description),
 };
 
-static struct json_path_container highlighter_def_handlers = {
+static const struct json_path_container highlighter_def_handlers = {
     yajlpp::property_handler("pattern")
         .with_synopsis("<regex>")
         .with_description(
@@ -649,7 +658,7 @@ static const json_path_handler_base::enum_value_t LEVEL_ENUM[] = {
     json_path_handler_base::ENUM_TERMINATOR,
 };
 
-static struct json_path_container sample_handlers = {
+static const struct json_path_container sample_handlers = {
     yajlpp::property_handler("description")
         .with_synopsis("<text>")
         .with_description("A description of this sample.")
@@ -674,14 +683,14 @@ static const json_path_handler_base::enum_value_t TYPE_ENUM[] = {
     json_path_handler_base::ENUM_TERMINATOR,
 };
 
-static struct json_path_container regex_handlers = {
+static const struct json_path_container regex_handlers = {
     yajlpp::pattern_property_handler(R"((?<pattern_name>[^/]+))")
         .with_description("The set of patterns used to match log messages")
         .with_obj_provider(pattern_provider)
         .with_children(pattern_handlers),
 };
 
-static struct json_path_container level_handlers = {
+static const struct json_path_container level_handlers = {
     yajlpp::pattern_property_handler("(?<level>trace|debug[2345]?|info|stats|"
                                      "notice|warning|error|critical|fatal)")
         .add_cb(read_levels)
@@ -693,7 +702,7 @@ static struct json_path_container level_handlers = {
                           "the number for the corresponding level."),
 };
 
-static struct json_path_container value_handlers = {
+static const struct json_path_container value_handlers = {
     yajlpp::pattern_property_handler("(?<value_name>[^/]+)")
         .with_description(
             "The set of values captured by the log message patterns")
@@ -701,7 +710,7 @@ static struct json_path_container value_handlers = {
         .with_children(value_def_handlers),
 };
 
-static struct json_path_container tag_path_handlers = {
+static const struct json_path_container tag_path_handlers = {
     yajlpp::property_handler("glob")
         .with_synopsis("<glob>")
         .with_description("The glob to match against file paths")
@@ -709,7 +718,7 @@ static struct json_path_container tag_path_handlers = {
         .for_field(&format_tag_def::path_restriction::p_glob),
 };
 
-static struct json_path_container format_tag_def_handlers = {
+static const struct json_path_container format_tag_def_handlers = {
     yajlpp::property_handler("paths#")
         .with_description("Restrict tagging to the given paths")
         .for_field(&format_tag_def::ftd_paths)
@@ -731,14 +740,14 @@ static struct json_path_container format_tag_def_handlers = {
         .for_field(&format_tag_def::ftd_level),
 };
 
-static struct json_path_container tag_handlers = {
+static const struct json_path_container tag_handlers = {
     yajlpp::pattern_property_handler(R"((?<tag_name>[\w:;\._\-]+))")
         .with_description("The name of the tag to apply")
         .with_obj_provider(format_tag_def_provider)
         .with_children(format_tag_def_handlers),
 };
 
-static struct json_path_container highlight_handlers = {
+static const struct json_path_container highlight_handlers = {
     yajlpp::pattern_property_handler(R"((?<highlight_name>[^/]+))")
         .with_description("The definition of a highlight")
         .with_obj_provider<external_log_format::highlighter_def,
@@ -752,20 +761,20 @@ static struct json_path_container highlight_handlers = {
         .with_children(highlighter_def_handlers),
 };
 
-static struct json_path_container action_def_handlers = {
+static const struct json_path_container action_def_handlers = {
     json_path_handler("label", read_action_def),
     json_path_handler("capture-output", read_action_bool),
     json_path_handler("cmd#", read_action_cmd),
 };
 
-static struct json_path_container action_handlers = {
+static const struct json_path_container action_handlers = {
     json_path_handler(
         lnav::pcre2pp::code::from_const("(?<action_name>\\w+)").to_shared(),
         read_action_def)
         .with_children(action_def_handlers),
 };
 
-static struct json_path_container search_table_def_handlers = {
+static const struct json_path_container search_table_def_handlers = {
     json_path_handler("pattern")
         .with_synopsis("<regex>")
         .with_description("The regular expression for this search table.")
@@ -782,7 +791,7 @@ static struct json_path_container search_table_def_handlers = {
         .for_field(&external_log_format::search_table_def::std_level),
 };
 
-static struct json_path_container search_table_handlers = {
+static const struct json_path_container search_table_handlers = {
     yajlpp::pattern_property_handler("(?<table_name>\\w+)")
         .with_description(
             "The set of search tables to be automatically defined")
@@ -805,7 +814,7 @@ static const json_path_handler_base::enum_value_t MIME_TYPE_ENUM[] = {
     json_path_handler_base::ENUM_TERMINATOR,
 };
 
-struct json_path_container format_handlers = {
+const struct json_path_container format_handlers = {
     yajlpp::property_handler("regex")
         .with_description(
             "The set of regular expressions used to match log messages")
@@ -836,7 +845,7 @@ struct json_path_container format_handlers = {
     json_path_handler("mime-types#", read_format_field)
         .with_description("A list of mime-types this format should be used for")
         .with_enum_values(MIME_TYPE_ENUM),
-    json_path_handler("level-field", read_format_field)
+    json_path_handler("level-field")
         .with_description(
             "The name of the level field in the log message pattern")
         .for_field(&external_log_format::elf_level_field),
@@ -844,17 +853,25 @@ struct json_path_container format_handlers = {
         .with_description("A regular-expression that matches the JSON-pointer "
                           "of the level property")
         .for_field(&external_log_format::elf_level_pointer),
-    json_path_handler("timestamp-field", read_format_field)
+    json_path_handler("timestamp-field")
         .with_description(
             "The name of the timestamp field in the log message pattern")
         .for_field(&log_format::lf_timestamp_field),
-    json_path_handler("time-field", read_format_field)
+    json_path_handler("subsecond-field")
+        .with_description("The path to the property in a JSON-lines log "
+                          "message that contains the sub-second time value")
+        .for_field(&log_format::lf_subsecond_field),
+    json_path_handler("subsecond-units")
+        .with_description("The units of the subsecond-field property value")
+        .with_enum_values(SUBSECOND_UNIT_ENUM)
+        .for_field(&log_format::lf_subsecond_unit),
+    json_path_handler("time-field")
         .with_description(
             "The name of the time field in the log message pattern.  This "
             "field should only be specified if the timestamp field only "
             "contains a date.")
         .for_field(&log_format::lf_time_field),
-    json_path_handler("body-field", read_format_field)
+    json_path_handler("body-field")
         .with_description(
             "The name of the body field in the log message pattern")
         .for_field(&external_log_format::elf_body_field),
@@ -932,11 +949,11 @@ struct json_path_container format_handlers = {
 static int
 read_id(yajlpp_parse_context* ypc, const unsigned char* str, size_t len)
 {
+    auto* ud = static_cast<loader_userdata*>(ypc->ypc_userdata);
     auto file_id = std::string((const char*) str, len);
 
-    if (find(SUPPORTED_FORMAT_SCHEMAS.begin(),
-             SUPPORTED_FORMAT_SCHEMAS.end(),
-             file_id)
+    ud->ud_file_schema = file_id;
+    if (SUPPORTED_FORMAT_SCHEMAS.find(file_id)
         == SUPPORTED_FORMAT_SCHEMAS.end())
     {
         const auto* handler = ypc->ypc_current_handler;
@@ -959,20 +976,17 @@ read_id(yajlpp_parse_context* ypc, const unsigned char* str, size_t len)
     return 1;
 }
 
-struct json_path_container root_format_handler
-        = json_path_container{
-                json_path_handler("$schema", read_id)
-                        .with_synopsis("The URI of the schema for this file")
-                        .with_description("Specifies the type of this file"),
+const struct json_path_container root_format_handler = json_path_container{
+    json_path_handler("$schema", read_id)
+        .with_synopsis("The URI of the schema for this file")
+        .with_description("Specifies the type of this file"),
 
-                yajlpp::pattern_property_handler(
-                        "(?<format_name>\\w+)")
-                        .with_description(
-                                "The definition of a log file format.")
-                        .with_obj_provider(ensure_format)
-                        .with_children(format_handlers),
-        }
-                .with_schema_id(DEFAULT_FORMAT_SCHEMA);
+    yajlpp::pattern_property_handler("(?<format_name>\\w+)")
+        .with_description("The definition of a log file format.")
+        .with_obj_provider(ensure_format)
+        .with_children(format_handlers),
+}
+    .with_schema_id(DEFAULT_FORMAT_SCHEMA);
 
 static void
 write_sample_file()
@@ -1018,15 +1032,13 @@ write_sample_file()
         struct script_metadata meta;
         auto sf = bsf.to_string_fragment();
         auto_fd script_fd;
-        struct stat st;
 
         extract_metadata(sf, meta);
         auto path
             = fmt::format(FMT_STRING("formats/default/{}.lnav"), meta.sm_name);
         auto script_path = lnav::paths::dotlnav() / path;
-        if (lnav::filesystem::statp(script_path, &st) == 0
-            && st.st_size == sf.length())
-        {
+        auto stat_res = lnav::filesystem::stat_file(script_path);
+        if (stat_res.isOk() && stat_res.unwrap().st_size == sf.length()) {
             // Assume it's the right contents and move on...
             continue;
         }
@@ -1112,6 +1124,30 @@ load_format_file(const ghc::filesystem::path& filename,
         if (rc == 0) {
             ypc.complete_parse();
         }
+
+        if (ud.ud_file_schema.empty()) {
+            static const auto SCHEMA_LINE
+                = attr_line_t()
+                      .append(
+                          fmt::format(FMT_STRING("    \"$schema\": \"{}\","),
+                                      *SUPPORTED_FORMAT_SCHEMAS.begin()))
+                      .with_attr_for_all(
+                          VC_ROLE.value(role_t::VCR_QUOTED_CODE));
+
+            errors.emplace_back(
+                lnav::console::user_message::warning(
+                    attr_line_t("format file is missing ")
+                        .append_quoted("$schema"_symbol)
+                        .append(" property"))
+                    .with_snippet(lnav::console::snippet::from(
+                        intern_string::lookup(filename.string()), ""))
+                    .with_note("the schema specifies the supported format "
+                               "version and can be used with editors to "
+                               "automatically validate the file")
+                    .with_help(attr_line_t("add the following property to the "
+                                           "top-level JSON object:\n")
+                                   .append(SCHEMA_LINE)));
+        }
     }
 
     return retval;
@@ -1127,19 +1163,19 @@ load_from_path(const ghc::filesystem::path& path,
     log_info("loading formats from path: %s", format_path.c_str());
     if (glob(format_path.c_str(), 0, nullptr, gl.inout()) == 0) {
         for (int lpc = 0; lpc < (int) gl->gl_pathc; lpc++) {
-            const char* base = basename(gl->gl_pathv[lpc]);
+            auto filepath = ghc::filesystem::path(gl->gl_pathv[lpc]);
 
-            if (startswith(base, "config.")) {
+            if (startswith(filepath.filename().string(), "config.")) {
+                log_info("  not loading config as format: %s",
+                         filepath.c_str());
                 continue;
             }
 
-            std::string filename(gl->gl_pathv[lpc]);
-            std::vector<intern_string_t> format_list;
-
-            format_list = load_format_file(filename, errors);
+            auto format_list = load_format_file(filepath, errors);
             if (format_list.empty()) {
-                log_warning("Empty format file: %s", filename.c_str());
+                log_warning("Empty format file: %s", filepath.c_str());
             } else {
+                log_info("contents of format file '%s':", filepath.c_str());
                 for (auto iter = format_list.begin(); iter != format_list.end();
                      ++iter)
                 {
@@ -1177,8 +1213,7 @@ load_formats(const std::vector<ghc::filesystem::path>& extra_paths,
         yajl_config(handle, yajl_allow_comments, 1);
         auto sf = bsf.to_string_fragment();
         if (ypc_builtin.parse(sf) != yajl_status_ok) {
-            unsigned char* msg = yajl_get_error(
-                handle, 1, (const unsigned char*) sf.data(), sf.length());
+            auto* msg = yajl_get_error(handle, 1, sf.udata(), sf.length());
 
             errors.emplace_back(
                 lnav::console::user_message::error("invalid json")
@@ -1362,19 +1397,34 @@ extract_metadata(string_fragment contents, struct script_metadata& meta_out)
 void
 extract_metadata_from_file(struct script_metadata& meta_inout)
 {
+    auto stat_res = lnav::filesystem::stat_file(meta_inout.sm_path);
+    if (stat_res.isErr()) {
+        log_warning("unable to open script: %s -- %s",
+                    meta_inout.sm_path.c_str(),
+                    stat_res.unwrapErr().c_str());
+        return;
+    }
+
+    auto st = stat_res.unwrap();
+    if (!S_ISREG(st.st_mode)) {
+        log_warning("script is not a regular file -- %s",
+                    meta_inout.sm_path.c_str());
+        return;
+    }
+
+    auto open_res = lnav::filesystem::open_file(meta_inout.sm_path, O_RDONLY);
+    if (open_res.isErr()) {
+        log_warning("unable to open script file: %s -- %s",
+                    meta_inout.sm_path.c_str(),
+                    open_res.unwrapErr().c_str());
+        return;
+    }
+
+    auto fd = open_res.unwrap();
     char buffer[8 * 1024];
-    auto_mem<FILE> fp(fclose);
-    struct stat st;
-
-    if (lnav::filesystem::statp(meta_inout.sm_path, &st) == -1) {
-        log_warning("unable to open script -- %s", meta_inout.sm_path.c_str());
-    } else if (!S_ISREG(st.st_mode)) {
-        log_warning("not a regular file -- %s", meta_inout.sm_path.c_str());
-    } else if ((fp = fopen(meta_inout.sm_path.c_str(), "r")) != nullptr) {
-        size_t len;
-
-        len = fread(buffer, 1, sizeof(buffer), fp.in());
-        extract_metadata(string_fragment::from_bytes(buffer, len), meta_inout);
+    auto rc = read(fd, buffer, sizeof(buffer));
+    if (rc > 0) {
+        extract_metadata(string_fragment::from_bytes(buffer, rc), meta_inout);
     }
 }
 

src/log_format_loader.hh

@@ -75,7 +75,7 @@ struct available_scripts {
 void find_format_scripts(const std::vector<ghc::filesystem::path>& extra_paths,
                          available_scripts& scripts);
 
-extern struct json_path_container format_handlers;
-extern struct json_path_container root_format_handler;
+extern const struct json_path_container format_handlers;
+extern const struct json_path_container root_format_handler;
 
 #endif

src/log_vtab_impl.cc

@@ -1199,7 +1199,7 @@ log_cursor::string_constraint::string_constraint(unsigned char op,
     : sc_op(op), sc_value(std::move(value))
 {
     if (op == SQLITE_INDEX_CONSTRAINT_REGEXP) {
-        auto compile_res = lnav::pcre2pp::code::from(value);
+        auto compile_res = lnav::pcre2pp::code::from(this->sc_value);
 
         if (compile_res.isErr()) {
             auto ce = compile_res.unwrapErr();
@@ -1890,12 +1890,12 @@ vt_best_index(sqlite3_vtab* tab, sqlite3_index_info* p_info)
     return SQLITE_OK;
 }
 
-static struct json_path_container tags_handler = {
+static const struct json_path_container tags_handler = {
     json_path_handler("#")
         .with_synopsis("tag")
         .with_description("A tag for the log line")
         .with_pattern(R"(^#[^\s]+$)")
-        .FOR_FIELD(bookmark_metadata, bm_tags),
+        .for_field(&bookmark_metadata::bm_tags),
 };
 
 static int

src/logfile.cc

@@ -51,11 +51,22 @@
 #include "log.watch.hh"
 #include "log_format.hh"
 #include "logfile.cfg.hh"
+#include "yajlpp/yajlpp_def.hh"
 
 static auto intern_lifetime = intern_string::get_table_lifetime();
 
 static const size_t INDEX_RESERVE_INCREMENT = 1024;
 
+static const typed_json_path_container<line_buffer::header_data>
+    file_header_handlers = {
+        yajlpp::property_handler("name").for_field(
+            &line_buffer::header_data::hd_name),
+        yajlpp::property_handler("mtime").for_field(
+            &line_buffer::header_data::hd_mtime),
+        yajlpp::property_handler("comment").for_field(
+            &line_buffer::header_data::hd_comment),
+};
+
 Result<std::shared_ptr<logfile>, std::string>
 logfile::open(std::string filename, logfile_open_options& loo)
 {
@@ -120,6 +131,12 @@ logfile::open(std::string filename, logfile_open_options& loo)
 
     lf->lf_indexing = lf->lf_options.loo_is_visible;
 
+    const auto& hdr = lf->lf_line_buffer.get_header_data();
+    if (!hdr.empty()) {
+        lf->lf_embedded_metadata["net.zlib.gzip.header"]
+            = {text_format_t::TF_JSON, file_header_handlers.to_string(hdr)};
+    }
+
     ensure(lf->invariant());
 
     return Ok(lf);
@@ -136,8 +153,6 @@ logfile::~logfile() {}
 bool
 logfile::exists() const
 {
-    struct stat st;
-
     if (!this->lf_actual_path) {
         return true;
     }
@@ -146,13 +161,15 @@ logfile::exists() const
         return true;
     }
 
-    if (lnav::filesystem::statp(this->lf_actual_path.value(), &st) == -1) {
+    auto stat_res = lnav::filesystem::stat_file(this->lf_actual_path.value());
+    if (stat_res.isErr()) {
         log_error("%s: stat failed -- %s",
                   this->lf_actual_path.value().c_str(),
-                  strerror(errno));
+                  stat_res.unwrapErr().c_str());
         return false;
     }
 
+    auto st = stat_res.unwrap();
     return this->lf_stat.st_dev == st.st_dev
         && this->lf_stat.st_ino == st.st_ino
         && this->lf_stat.st_size <= st.st_size;
@@ -589,7 +606,7 @@ logfile::rebuild_index(nonstd::optional<ui_clock::time_point> deadline)
             auto sbr = read_result.unwrap();
             sbr.rtrim(is_line_ending);
 
-            if (li.li_has_ansi) {
+            if (li.li_valid_utf && li.li_has_ansi) {
                 auto tmp_line = sbr.to_string_fragment().to_string();
 
                 scrub_ansi_string(tmp_line, nullptr);

src/logfile_sub_source.cc

@@ -1334,12 +1334,17 @@ logfile_sub_source::set_sql_marker(std::string stmt_str, sqlite3_stmt* stmt)
         }
     }
 
+    this->lss_marker_stmt_text = std::move(stmt_str);
+    this->lss_marker_stmt = stmt;
+
+    if (this->tss_view == nullptr) {
+        return Ok();
+    }
+
     auto& vis_bm = this->tss_view->get_bookmarks();
     auto& expr_marks_bv = vis_bm[&textview_curses::BM_USER_EXPR];
 
     expr_marks_bv.clear();
-    this->lss_marker_stmt_text = std::move(stmt_str);
-    this->lss_marker_stmt = stmt;
     if (this->lss_index_delegate) {
         this->lss_index_delegate->index_start(*this);
     }

src/pcrepp/pcre2pp.cc

@@ -55,6 +55,20 @@ quote(const char* unquoted)
     return retval;
 }
 
+matcher
+capture_builder::into(lnav::pcre2pp::match_data& md) &&
+{
+    if (md.get_capacity() < this->mb_code.get_match_data_capacity()) {
+        md = this->mb_code.create_match_data();
+    }
+
+    return matcher{
+        this->mb_code,
+        this->mb_input,
+        md,
+    };
+}
+
 match_data
 code::create_match_data() const
 {

src/pcrepp/pcre2pp.hh

@@ -106,6 +106,8 @@ public:
 
     int get_count() const { return this->md_capture_end; }
 
+    uint32_t get_capacity() const { return this->md_ovector_count; }
+
 private:
     friend matcher;
     friend code;
@@ -195,14 +197,7 @@ struct capture_builder {
         return *this;
     }
 
-    matcher into(match_data& md) &&
-    {
-        return matcher{
-            this->mb_code,
-            this->mb_input,
-            md,
-        };
-    }
+    matcher into(match_data& md) &&;
 
     template<uint32_t Options = 0, typename F>
     Result<string_fragment, matcher::error> for_each(F func) &&;
@@ -274,6 +269,10 @@ public:
 
     std::vector<string_fragment> get_captures() const;
 
+    uint32_t get_match_data_capacity() const {
+        return this->p_match_proto.md_ovector_count;
+    }
+
     match_data create_match_data() const;
 
     capture_builder capture_from(string_fragment in) const

src/readline_possibilities.cc

@@ -427,7 +427,7 @@ add_config_possibilities()
             if (named_caps.empty()) {
                 rc->add_possibility(ln_mode_t::COMMAND, "config-option", path);
             }
-            for (const auto named_cap : named_caps) {
+            for (const auto& named_cap : named_caps) {
                 if (visited.count(named_cap.get_name().to_string()) == 0) {
                     rc->clear_possibilities(ln_mode_t::COMMAND,
                                             named_cap.get_name().to_string());

src/regex101.import.cc

@@ -54,7 +54,7 @@ regex101::import(const std::string& url,
 {
     static const auto USER_URL = lnav::pcre2pp::code::from_const(
         R"(^https://regex101.com/r/(\w+)(?:/(\d+))?)");
-    static thread_local auto URL_MATCH_DATA = USER_URL.create_match_data();
+    static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
     static const auto NAME_RE = lnav::pcre2pp::code::from_const(R"(^\w+$)");
 
     if (url.empty()) {
@@ -100,10 +100,8 @@ regex101::import(const std::string& url,
                                .append("^ matched up to here"_comment)));
     }
 
-    auto user_find_res = USER_URL.capture_from(url)
-                             .into(URL_MATCH_DATA)
-                             .matches()
-                             .ignore_error();
+    auto user_find_res
+        = USER_URL.capture_from(url).into(md).matches().ignore_error();
     if (!user_find_res) {
         auto partial_len = USER_URL.match_partial(url);
         return Err(lnav::console::user_message::error(
@@ -118,7 +116,7 @@ regex101::import(const std::string& url,
                                       .append("^ matched up to here"_comment)));
     }
 
-    auto permalink = URL_MATCH_DATA[1]->to_string();
+    auto permalink = md[1]->to_string();
 
     auto format_filename = existing_format
         ? fmt::format(FMT_STRING("{}.regex101-{}.json"), name, permalink)

src/regexp_vtab.cc

@@ -106,7 +106,7 @@ CREATE TABLE regexp_capture (
                 this->c_match_index += 1;
             }
 
-            if (this->c_pattern == nullptr || !this->c_matched) {
+            if (!this->c_matched) {
                 return SQLITE_OK;
             }
 
@@ -329,7 +329,7 @@ CREATE TABLE regexp_capture_into_json (
             this->c_matched = match_res.has_value();
             this->c_match_index += 1;
 
-            if (this->c_pattern == nullptr || !this->c_matched) {
+            if (!this->c_matched) {
                 return SQLITE_OK;
             }
 

src/relative_time.cc

@@ -294,8 +294,9 @@ relative_time::from_str(string_fragment str)
 
         bool found = false;
         for (int lpc = 0; lpc < RTT__MAX && !found; lpc++) {
+            static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
+
             token_t token = (token_t) lpc;
-            auto md = MATCHERS[lpc].pcre.create_match_data();
             auto match_res = MATCHERS[lpc]
                                  .pcre.capture_from(remaining)
                                  .into(md)

src/relative_time.hh

@@ -150,7 +150,7 @@ public:
         if (this->rt_previous) {
             return true;
         }
-        for (auto rtf : this->rt_field) {
+        for (const auto& rtf : this->rt_field) {
             if (rtf.value < 0) {
                 return true;
             }
@@ -179,7 +179,7 @@ public:
         if (!this->rt_included_days.empty()) {
             return false;
         }
-        for (auto rtf : this->rt_field) {
+        for (const auto& rtf : this->rt_field) {
             if (rtf.is_set) {
                 return false;
             }

src/session.export.cc

@@ -175,9 +175,7 @@ replace_home_dir(std::string path)
 Result<void, lnav::console::user_message>
 export_to(FILE* file)
 {
-    static auto& lnav_db
-        = injector::get<auto_mem<sqlite3, sqlite_close_wrapper>&,
-                        sqlite_db_tag>();
+    static auto& lnav_db = injector::get<auto_sqlite3&>();
 
     static const char* BOOKMARK_QUERY = R"(
 SELECT log_time_msecs, log_format, log_mark, log_comment, log_tags, log_line_hash

src/session_data.cc

@@ -372,7 +372,7 @@ void
 load_time_bookmarks()
 {
     logfile_sub_source& lss = lnav_data.ld_log_source;
-    auto_mem<sqlite3, sqlite_close_wrapper> db;
+    auto_sqlite3 db;
     auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME;
     auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
     logfile_sub_source::iterator file_iter;
@@ -862,7 +862,7 @@ read_commands(yajlpp_parse_context* ypc, const unsigned char* str, size_t len)
     return 1;
 }
 
-static struct json_path_container view_def_handlers = {
+static const struct json_path_container view_def_handlers = {
     json_path_handler("top_line", read_top_line),
     json_path_handler("search", read_current_search),
     json_path_handler("word_wrap", read_word_wrap),
@@ -870,18 +870,18 @@ static struct json_path_container view_def_handlers = {
     json_path_handler("commands#", read_commands),
 };
 
-static struct json_path_container view_handlers = {
+static const struct json_path_container view_handlers = {
     yajlpp::pattern_property_handler("([^/]+)").with_children(
         view_def_handlers),
 };
 
-static struct json_path_container file_state_handlers = {
+static const struct json_path_container file_state_handlers = {
     yajlpp::property_handler("visible")
         .with_description("Indicates whether the file is visible or not")
         .for_field(&file_state::fs_is_visible),
 };
 
-static struct json_path_container file_states_handlers = {
+static const struct json_path_container file_states_handlers = {
     yajlpp::pattern_property_handler(R"((?<filename>[^/]+))")
         .with_description("Map of file names to file state objects")
         .with_obj_provider<file_state, void>([](const auto& ypc, auto* root) {
@@ -891,7 +891,7 @@ static struct json_path_container file_states_handlers = {
         .with_children(file_state_handlers),
 };
 
-static struct json_path_container view_info_handlers = {
+static const struct json_path_container view_info_handlers = {
     yajlpp::property_handler("save-time")
         .for_field(&session_data_t::sd_save_time),
     yajlpp::property_handler("time-offset")
@@ -1070,7 +1070,7 @@ save_user_bookmarks(sqlite3* db,
 static void
 save_time_bookmarks()
 {
-    auto_mem<sqlite3, sqlite_close_wrapper> db;
+    auto_sqlite3 db;
     auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME;
     auto_mem<char, sqlite3_free> errmsg;
     auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
@@ -1670,7 +1670,7 @@ lnav::session::regex101::insert_entry(const lnav::session::regex101::entry& ei)
 )";
 
     auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME;
-    auto_mem<sqlite3, sqlite_close_wrapper> db;
+    auto_sqlite3 db;
 
     if (sqlite3_open(db_path.c_str(), db.out()) != SQLITE_OK) {
         return;
@@ -1723,7 +1723,7 @@ lnav::session::regex101::get_entries()
 )";
 
     auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME;
-    auto_mem<sqlite3, sqlite_close_wrapper> db;
+    auto_sqlite3 db;
 
     if (sqlite3_open(db_path.c_str(), db.out()) != SQLITE_OK) {
         return Err(std::string());
@@ -1766,7 +1766,7 @@ lnav::session::regex101::delete_entry(const std::string& format_name,
 )";
 
     auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME;
-    auto_mem<sqlite3, sqlite_close_wrapper> db;
+    auto_sqlite3 db;
 
     if (sqlite3_open(db_path.c_str(), db.out()) != SQLITE_OK) {
         return;
@@ -1793,7 +1793,7 @@ lnav::session::regex101::get_entry(const std::string& format_name,
     )";
 
     auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME;
-    auto_mem<sqlite3, sqlite_close_wrapper> db;
+    auto_sqlite3 db;
 
     if (sqlite3_open(db_path.c_str(), db.out()) != SQLITE_OK) {
         return error{std::string()};

src/spookyhash/SpookyV2.cpp

@@ -54,18 +54,18 @@ void SpookyHash::Short(
         // handle all complete sets of 32 bytes
         for (; u.p64 < end; u.p64 += 4)
         {
-            c += u.p64[0];
-            d += u.p64[1];
+            c += SPOOKYHASH_LITTLE_ENDIAN_64(u.p64[0]);
+            d += SPOOKYHASH_LITTLE_ENDIAN_64(u.p64[1]);
             ShortMix(a,b,c,d);
-            a += u.p64[2];
-            b += u.p64[3];
+            a += SPOOKYHASH_LITTLE_ENDIAN_64(u.p64[2]);
+            b += SPOOKYHASH_LITTLE_ENDIAN_64(u.p64[3]);
         }
         
         //Handle the case of 16+ remaining bytes.
         if (remainder >= 16)
         {
-            c += u.p64[0];
-            d += u.p64[1];
+            c += SPOOKYHASH_LITTLE_ENDIAN_64(u.p64[0]);
+            d += SPOOKYHASH_LITTLE_ENDIAN_64(u.p64[1]);
             ShortMix(a,b,c,d);
             u.p64 += 2;
             remainder -= 16;
@@ -83,8 +83,8 @@ void SpookyHash::Short(
     case 13:
         d += ((uint64)u.p8[12]) << 32;
     case 12:
-        d += u.p32[2];
-        c += u.p64[0];
+        d += SPOOKYHASH_LITTLE_ENDIAN_32(u.p32[2]);
+        c += SPOOKYHASH_LITTLE_ENDIAN_64(u.p64[0]);
         break;
     case 11:
         d += ((uint64)u.p8[10]) << 16;
@@ -93,7 +93,7 @@ void SpookyHash::Short(
     case 9:
         d += (uint64)u.p8[8];
     case 8:
-        c += u.p64[0];
+        c += SPOOKYHASH_LITTLE_ENDIAN_64(u.p64[0]);
         break;
     case 7:
         c += ((uint64)u.p8[6]) << 48;
@@ -102,7 +102,7 @@ void SpookyHash::Short(
     case 5:
         c += ((uint64)u.p8[4]) << 32;
     case 4:
-        c += u.p32[0];
+        c += SPOOKYHASH_LITTLE_ENDIAN_32(u.p32[0]);
         break;
     case 3:
         c += ((uint64)u.p8[2]) << 16;

src/spookyhash/SpookyV2.h

@@ -45,6 +45,25 @@
   typedef  uint8_t   uint8;
 #endif
 
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define SPOOKYHASH_LITTLE_ENDIAN_64(b)   ((uint64_t)b)
+#define SPOOKYHASH_LITTLE_ENDIAN_32(b)   ((uint32_t)b)
+#define SPOOKYHASH_LITTLE_ENDIAN_16(b)   ((uint16_t)b)
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#if __GNUC__ * 100 + __GNUC_MINOR__ >= 403 || defined(__clang__)
+#define SPOOKYHASH_LITTLE_ENDIAN_64(b)   __builtin_bswap64(b)
+#define SPOOKYHASH_LITTLE_ENDIAN_32(b)   __builtin_bswap32(b)
+#define SPOOKYHASH_LITTLE_ENDIAN_16(b)   __builtin_bswap16(b)
+#else
+#warning Using bulk byte swap routines. Expect performance issues.
+#define SPOOKYHASH_LITTLE_ENDIAN_64(b)   ((((b) & 0xFF00000000000000ull) >> 56) | (((b) & 0x00FF000000000000ull) >> 40) | (((b) & 0x0000FF0000000000ull) >> 24) | (((b) & 0x000000FF00000000ull) >> 8) | (((b) & 0x00000000FF000000ull) << 8) | (((b) & 0x0000000000FF0000ull) << 24ull) | (((b) & 0x000000000000FF00ull) << 40) | (((b) & 0x00000000000000FFull) << 56))
+#define SPOOKYHASH_LITTLE_ENDIAN_32(b)   ((((b) & 0xFF000000) >> 24) | (((b) & 0x00FF0000) >> 8) | (((b) & 0x0000FF00) << 8) | (((b) & 0x000000FF) << 24))
+#define SPOOKYHASH_LITTLE_ENDIAN_16(b)   ((((b) & 0xFF00) >> 8) | (((b) & 0x00FF) << 8))
+#endif
+#else
+#error Unknow endianness
+#endif
+
 
 class SpookyHash
 {
@@ -136,18 +155,30 @@ public:
         uint64 &s4, uint64 &s5, uint64 &s6, uint64 &s7,
         uint64 &s8, uint64 &s9, uint64 &s10,uint64 &s11)
     {
-      s0 += data[0];    s2 ^= s10;    s11 ^= s0;    s0 = Rot64(s0,11);    s11 += s1;
-      s1 += data[1];    s3 ^= s11;    s0 ^= s1;    s1 = Rot64(s1,32);    s0 += s2;
-      s2 += data[2];    s4 ^= s0;    s1 ^= s2;    s2 = Rot64(s2,43);    s1 += s3;
-      s3 += data[3];    s5 ^= s1;    s2 ^= s3;    s3 = Rot64(s3,31);    s2 += s4;
-      s4 += data[4];    s6 ^= s2;    s3 ^= s4;    s4 = Rot64(s4,17);    s3 += s5;
-      s5 += data[5];    s7 ^= s3;    s4 ^= s5;    s5 = Rot64(s5,28);    s4 += s6;
-      s6 += data[6];    s8 ^= s4;    s5 ^= s6;    s6 = Rot64(s6,39);    s5 += s7;
-      s7 += data[7];    s9 ^= s5;    s6 ^= s7;    s7 = Rot64(s7,57);    s6 += s8;
-      s8 += data[8];    s10 ^= s6;    s7 ^= s8;    s8 = Rot64(s8,55);    s7 += s9;
-      s9 += data[9];    s11 ^= s7;    s8 ^= s9;    s9 = Rot64(s9,54);    s8 += s10;
-      s10 += data[10];    s0 ^= s8;    s9 ^= s10;    s10 = Rot64(s10,22);    s9 += s11;
-      s11 += data[11];    s1 ^= s9;    s10 ^= s11;    s11 = Rot64(s11,46);    s10 += s0;
+      s0 += SPOOKYHASH_LITTLE_ENDIAN_64(data[0]);
+      s2 ^= s10;    s11 ^= s0;    s0 = Rot64(s0,11);    s11 += s1;
+      s1 += SPOOKYHASH_LITTLE_ENDIAN_64(data[1]);
+      s3 ^= s11;    s0 ^= s1;    s1 = Rot64(s1,32);    s0 += s2;
+      s2 += SPOOKYHASH_LITTLE_ENDIAN_64(data[2]);
+      s4 ^= s0;    s1 ^= s2;    s2 = Rot64(s2,43);    s1 += s3;
+      s3 += SPOOKYHASH_LITTLE_ENDIAN_64(data[3]);
+      s5 ^= s1;    s2 ^= s3;    s3 = Rot64(s3,31);    s2 += s4;
+      s4 += SPOOKYHASH_LITTLE_ENDIAN_64(data[4]);
+      s6 ^= s2;    s3 ^= s4;    s4 = Rot64(s4,17);    s3 += s5;
+      s5 += SPOOKYHASH_LITTLE_ENDIAN_64(data[5]);
+      s7 ^= s3;    s4 ^= s5;    s5 = Rot64(s5,28);    s4 += s6;
+      s6 += SPOOKYHASH_LITTLE_ENDIAN_64(data[6]);
+      s8 ^= s4;    s5 ^= s6;    s6 = Rot64(s6,39);    s5 += s7;
+      s7 += SPOOKYHASH_LITTLE_ENDIAN_64(data[7]);
+      s9 ^= s5;    s6 ^= s7;    s7 = Rot64(s7,57);    s6 += s8;
+      s8 += SPOOKYHASH_LITTLE_ENDIAN_64(data[8]);
+      s10 ^= s6;    s7 ^= s8;    s8 = Rot64(s8,55);    s7 += s9;
+      s9 += SPOOKYHASH_LITTLE_ENDIAN_64(data[9]);
+      s11 ^= s7;    s8 ^= s9;    s9 = Rot64(s9,54);    s8 += s10;
+      s10 += SPOOKYHASH_LITTLE_ENDIAN_64(data[10]);
+      s0 ^= s8;    s9 ^= s10;    s10 = Rot64(s10,22);    s9 += s11;
+      s11 += SPOOKYHASH_LITTLE_ENDIAN_64(data[11]);
+      s1 ^= s9;    s10 ^= s11;    s11 = Rot64(s11,46);    s10 += s0;
     }
 
     //
@@ -191,9 +222,18 @@ public:
         uint64 &h4, uint64 &h5, uint64 &h6, uint64 &h7, 
         uint64 &h8, uint64 &h9, uint64 &h10,uint64 &h11)
     {
-        h0 += data[0];   h1 += data[1];   h2 += data[2];   h3 += data[3];
-        h4 += data[4];   h5 += data[5];   h6 += data[6];   h7 += data[7];
-        h8 += data[8];   h9 += data[9];   h10 += data[10]; h11 += data[11];
+        h0 += SPOOKYHASH_LITTLE_ENDIAN_64(data[0]);
+        h1 += SPOOKYHASH_LITTLE_ENDIAN_64(data[1]);
+        h2 += SPOOKYHASH_LITTLE_ENDIAN_64(data[2]);
+        h3 += SPOOKYHASH_LITTLE_ENDIAN_64(data[3]);
+        h4 += SPOOKYHASH_LITTLE_ENDIAN_64(data[4]);
+        h5 += SPOOKYHASH_LITTLE_ENDIAN_64(data[5]);
+        h6 += SPOOKYHASH_LITTLE_ENDIAN_64(data[6]);
+        h7 += SPOOKYHASH_LITTLE_ENDIAN_64(data[7]);
+        h8 += SPOOKYHASH_LITTLE_ENDIAN_64(data[8]);
+        h9 += SPOOKYHASH_LITTLE_ENDIAN_64(data[9]);
+        h10 += SPOOKYHASH_LITTLE_ENDIAN_64(data[10]);
+        h11 += SPOOKYHASH_LITTLE_ENDIAN_64(data[11]);
         EndPartial(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
         EndPartial(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);
         EndPartial(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11);

src/sql_commands.cc

@@ -45,9 +45,7 @@ sql_cmd_dump(exec_context& ec,
              std::string cmdline,
              std::vector<std::string>& args)
 {
-    static auto& lnav_db
-        = injector::get<auto_mem<sqlite3, sqlite_close_wrapper>&,
-                        sqlite_db_tag>();
+    static auto& lnav_db = injector::get<auto_sqlite3&>();
     static auto& lnav_flags = injector::get<unsigned long&, lnav_flags_tag>();
 
     std::string retval;
@@ -90,9 +88,7 @@ sql_cmd_read(exec_context& ec,
              std::string cmdline,
              std::vector<std::string>& args)
 {
-    static auto& lnav_db
-        = injector::get<auto_mem<sqlite3, sqlite_close_wrapper>&,
-                        sqlite_db_tag>();
+    static auto& lnav_db = injector::get<auto_sqlite3&>();
     static auto& lnav_flags = injector::get<unsigned long&, lnav_flags_tag>();
 
     std::string retval;

src/sql_util.cc

@@ -526,44 +526,6 @@ attach_sqlite_db(sqlite3* db, const std::string& filename)
     }
 }
 
-ssize_t
-sql_strftime(
-    char* buffer, size_t buffer_size, lnav::time64_t tim, int millis, char sep)
-{
-    struct tm gmtm;
-    int year, month, index = 0;
-
-    secs2tm(tim, &gmtm);
-    year = gmtm.tm_year + 1900;
-    month = gmtm.tm_mon + 1;
-    buffer[index++] = '0' + ((year / 1000) % 10);
-    buffer[index++] = '0' + ((year / 100) % 10);
-    buffer[index++] = '0' + ((year / 10) % 10);
-    buffer[index++] = '0' + ((year / 1) % 10);
-    buffer[index++] = '-';
-    buffer[index++] = '0' + ((month / 10) % 10);
-    buffer[index++] = '0' + ((month / 1) % 10);
-    buffer[index++] = '-';
-    buffer[index++] = '0' + ((gmtm.tm_mday / 10) % 10);
-    buffer[index++] = '0' + ((gmtm.tm_mday / 1) % 10);
-    buffer[index++] = sep;
-    buffer[index++] = '0' + ((gmtm.tm_hour / 10) % 10);
-    buffer[index++] = '0' + ((gmtm.tm_hour / 1) % 10);
-    buffer[index++] = ':';
-    buffer[index++] = '0' + ((gmtm.tm_min / 10) % 10);
-    buffer[index++] = '0' + ((gmtm.tm_min / 1) % 10);
-    buffer[index++] = ':';
-    buffer[index++] = '0' + ((gmtm.tm_sec / 10) % 10);
-    buffer[index++] = '0' + ((gmtm.tm_sec / 1) % 10);
-    buffer[index++] = '.';
-    buffer[index++] = '0' + ((millis / 100) % 10);
-    buffer[index++] = '0' + ((millis / 10) % 10);
-    buffer[index++] = '0' + ((millis / 1) % 10);
-    buffer[index] = '\0';
-
-    return index;
-}
-
 static void
 sqlite_logger(void* dummy, int code, const char* msg)
 {

src/sql_util.hh

@@ -82,11 +82,15 @@ void dump_sqlite_schema(sqlite3* db, std::string& schema_out);
 
 void attach_sqlite_db(sqlite3* db, const std::string& filename);
 
-ssize_t sql_strftime(char* buffer,
-                     size_t buffer_size,
-                     lnav::time64_t tim,
-                     int millis,
-                     char sep = ' ');
+inline ssize_t
+sql_strftime(char* buffer,
+             size_t buffer_size,
+             lnav::time64_t tim,
+             int millis,
+             char sep = ' ')
+{
+    return lnav::strftime_rfc3339(buffer, buffer_size, tim, millis, sep);
+}
 
 inline ssize_t
 sql_strftime(char* buffer,

src/sqlitepp.hh

@@ -43,6 +43,8 @@
 /* XXX figure out how to do this with the template */
 void sqlite_close_wrapper(void* mem);
 
+using auto_sqlite3 = auto_mem<sqlite3, sqlite_close_wrapper>;
+
 namespace sqlitepp {
 
 inline auto_mem<char>

src/tailer/Makefile.am

@@ -48,7 +48,7 @@ libtailerpp_a_CPPFLAGS = \
 libtailerpp_a_SOURCES = \
     tailerpp.cc
 
-tailerbin.cc: tailer tailer.ape ../../tools/bin2c$(BUILD_EXEEXT)
+tailerbin.cc: tailer.ape ../../tools/bin2c$(BUILD_EXEEXT)
 	../../tools/bin2c$(BUILD_EXEEXT) -n tailer_bin tailerbin $(srcdir)/tailer.ape
 
 libtailerservice_a_CPPFLAGS = \
@@ -92,13 +92,16 @@ TESTS = \
     test_tailer.sh
 
 DISTCLEANFILES = \
+	*.cmd \
 	*.dat \
+	*.out \
 	*.err \
 	*.db \
 	*.dpt \
 	*.diff \
 	*.index \
 	*.tmp \
+	*.outbak \
 	*.errbak \
 	*.tmpbak \
     tailerbin.h \

src/tailer/tailer.looper.cc

@@ -269,7 +269,7 @@ tailer::looper::complete_path(const network::path& path)
 static std::vector<std::string>
 create_ssh_args_from_config(const std::string& dest)
 {
-    auto& cfg = injector::get<const tailer::config&>();
+    const auto& cfg = injector::get<const tailer::config&>();
     std::vector<std::string> retval;
 
     retval.emplace_back(cfg.c_ssh_cmd);

src/tailer/test_tailer.sh

@@ -23,18 +23,7 @@ info: load preview request -- 1234
 info: exiting...
 EOF
 
-run_test ./drive_tailer preview "${test_dir}/remote-log-dir/*"
-
-check_output "preview of file failed?" <<EOF
-preview of file: {test_dir}/remote-log-dir/*
-{test_dir}/remote-log-dir/logfile_access_log.0
-{test_dir}/remote-log-dir/logfile_access_log.1
-
-all done!
-tailer stderr:
-info: load preview request -- 1234
-info: exiting...
-EOF
+run_cap_test ./drive_tailer preview "${test_dir}/remote-log-dir/*"
 
 run_test ./drive_tailer possible "${test_dir}/logfile_access_log.*"
 

src/text_anonymizer.cc

@@ -467,9 +467,11 @@ text_anonymizer::next(string_fragment line)
                 } else {
                     static const auto ATTR_RE
                         = lnav::pcre2pp::code::from_const(R"([\w\-]+=)");
+                    static thread_local auto md
+                        = lnav::pcre2pp::match_data::unitialized();
+
                     auto remaining = string_fragment::from_str_range(
                         open_tag, space_index, open_tag.size());
-                    auto md = ATTR_RE.create_match_data();
 
                     retval += open_tag.substr(0, space_index + 1);
                     while (!remaining.empty()) {

src/text_format.cc

@@ -45,7 +45,7 @@ detect_text_format(string_fragment sf,
     static const auto MARKDOWN_EXT = ghc::filesystem::path(".markdown");
 
     static const auto MAN_MATCHERS = lnav::pcre2pp::code::from_const(
-        R"(^[A-Z]+\(\d\)\s+)", PCRE2_MULTILINE);
+        R"(^[A-Za-z][A-Za-z\-_\+0-9]+\(\d\)\s+)", PCRE2_MULTILINE);
 
     // XXX This is a pretty crude way of detecting format...
     static const auto PYTHON_MATCHERS = lnav::pcre2pp::code::from_const(

src/text_format.hh

@@ -54,6 +54,7 @@ enum class text_format_t {
     TF_SQL,
     TF_XML,
     TF_YAML,
+    TF_TOML,
 };
 
 namespace fmt {
@@ -103,6 +104,9 @@ struct formatter<text_format_t> : formatter<string_view> {
             case text_format_t::TF_YAML:
                 name = "application/yaml";
                 break;
+            case text_format_t::TF_TOML:
+                name = "application/toml";
+                break;
         }
         return formatter<string_view>::format(name, ctx);
     }

src/textfile_highlighters.cc

@@ -429,6 +429,7 @@ setup_highlights(highlight_map_t& hm)
                             "(?<!\\$)\\$\\((\\w+)\\)|"
                             "(?<!\\$)\\$\\{(\\w+)\\}"
                             ")"))
+              .with_nestable(false)
               .with_role(role_t::VCR_VARIABLE);
     hm[{highlight_source_t::INTERNAL, "rust.sym"}]
         = highlighter(xpcre_compile("\\b[A-Z_][A-Z0-9_]+\\b"))

src/textfile_sub_source.cc

@@ -455,9 +455,7 @@ textfile_sub_source::rescan_files(
     textfile_sub_source::scan_callback& callback,
     nonstd::optional<ui_clock::time_point> deadline)
 {
-    static auto& lnav_db
-        = injector::get<auto_mem<sqlite3, sqlite_close_wrapper>&,
-                        sqlite_db_tag>();
+    static auto& lnav_db = injector::get<auto_sqlite3&>();
 
     file_iterator iter;
     bool retval = false;
@@ -520,20 +518,61 @@ textfile_sub_source::rescan_files(
 
                 auto read_res = lf->read_file();
                 if (read_res.isOk()) {
-                    auto content = read_res.unwrap();
-                    auto content_sf = string_fragment{content};
-                    std::string frontmatter;
-                    auto front_matter_terminator = content.length() > 8
-                        ? content.find("\n---\n", 4)
-                        : std::string::npos;
+                    static const auto FRONT_MATTER_RE
+                        = lnav::pcre2pp::code::from_const(
+                            R"((?:^---\n(.*)\n---\n|^\+\+\+\n(.*)\n\+\+\+\n))",
+                            PCRE2_MULTILINE | PCRE2_DOTALL);
+                    static thread_local auto md
+                        = FRONT_MATTER_RE.create_match_data();
 
-                    if (startswith(content, "---\n")
-                        && front_matter_terminator != std::string::npos)
-                    {
-                        frontmatter
-                            = content.substr(4, front_matter_terminator - 3);
-                        content_sf
-                            = content_sf.substr(front_matter_terminator + 4);
+                    auto content = read_res.unwrap();
+                    auto content_sf = string_fragment::from_str(content);
+                    std::string frontmatter;
+                    text_format_t frontmatter_format{text_format_t::TF_UNKNOWN};
+
+                    auto cap_res = FRONT_MATTER_RE.capture_from(content_sf)
+                                       .into(md)
+                                       .matches()
+                                       .ignore_error();
+                    if (cap_res) {
+                        if (md[1]) {
+                            frontmatter_format = text_format_t::TF_YAML;
+                            frontmatter = md[1]->to_string();
+                        } else if (md[2]) {
+                            frontmatter_format = text_format_t::TF_TOML;
+                            frontmatter = md[2]->to_string();
+                        }
+                        content_sf = cap_res->f_remaining;
+                    } else if (content_sf.startswith("{")) {
+                        yajlpp_parse_context ypc(
+                            intern_string::lookup(lf->get_filename()));
+                        auto_mem<yajl_handle_t> handle(yajl_free);
+
+                        handle = yajl_alloc(&ypc.ypc_callbacks, nullptr, &ypc);
+                        yajl_config(
+                            handle.in(), yajl_allow_trailing_garbage, 1);
+                        ypc.with_ignore_unused(true)
+                            .with_handle(handle.in())
+                            .with_error_reporter(
+                                [&lf](const auto& ypc, const auto& um) {
+                                    log_error(
+                                        "%s: failed to parse JSON front matter "
+                                        "-- %s",
+                                        lf->get_filename().c_str(),
+                                        um.um_reason.al_string.c_str());
+                                });
+                        if (ypc.parse_doc(content_sf)) {
+                            auto consumed = ypc.ypc_total_consumed;
+                            if (consumed < content_sf.length()
+                                && content_sf[consumed] == '\n')
+                            {
+                                frontmatter_format = text_format_t::TF_JSON;
+                                frontmatter = string_fragment::from_str_range(
+                                                  content, 0, consumed)
+                                                  .to_string();
+                                content_sf = content_sf.substr(consumed);
+                            }
+                        }
                     }
 
                     md2attr_line mdal;
@@ -553,7 +592,7 @@ textfile_sub_source::rescan_files(
 
                         if (!frontmatter.empty()) {
                             lf_meta["net.daringfireball.markdown.frontmatter"]
-                                = {text_format_t::TF_YAML, frontmatter};
+                                = {frontmatter_format, frontmatter};
                         }
 
                         lnav::events::publish(

src/themes/default-theme.json

@@ -51,6 +51,10 @@
                         "color": "Silver",
                         "background-color": "Teal"
                     },
+                    "scrollbar": {
+                        "color": "Black",
+                        "background-color": "Silver"
+                    },
                     "focused": {
                         "color": "Black",
                         "background-color": "Silver"

src/themes/eldar.json

@@ -60,6 +60,10 @@
                         "color": "$black",
                         "background-color": "Grey37"
                     },
+                    "scrollbar": {
+                        "color": "$black",
+                        "background-color": "$white"
+                    },
                     "h1": {
                         "underline": true
                     },

src/themes/night-owl.json

@@ -68,6 +68,10 @@
                         "color": "$base00",
                         "background-color": "$base3"
                     },
+                    "scrollbar": {
+                        "color": "$black",
+                        "background-color": "$white"
+                    },
                     "h1": {
                         "underline": true
                     },

src/themes/solarized-dark.json

@@ -4,23 +4,23 @@
         "theme-defs": {
             "solarized-dark": {
                 "vars": {
-                    "base03":    "#002b36",
-                    "base02":    "#073642",
-                    "base01":    "#586e75",
-                    "base00":    "#657b83",
-                    "base0":     "#839496",
-                    "base1":     "#93a1a1",
-                    "base2":     "#eee8d5",
-                    "base3":     "#fdf6e3",
-                    "black":     "#002b36",
-                    "yellow":    "#b58900",
-                    "orange":    "#cb4b16",
-                    "red":       "#dc322f",
-                    "magenta":   "#d33682",
-                    "violet":    "#6c71c4",
-                    "blue":      "#268bd2",
-                    "cyan":      "#2aa198",
-                    "green":     "#859900",
+                    "base03": "#002b36",
+                    "base02": "#073642",
+                    "base01": "#586e75",
+                    "base00": "#657b83",
+                    "base0": "#839496",
+                    "base1": "#93a1a1",
+                    "base2": "#eee8d5",
+                    "base3": "#fdf6e3",
+                    "black": "#002b36",
+                    "yellow": "#b58900",
+                    "orange": "#cb4b16",
+                    "red": "#dc322f",
+                    "magenta": "#d33682",
+                    "violet": "#6c71c4",
+                    "blue": "#268bd2",
+                    "cyan": "#2aa198",
+                    "green": "#859900",
                     "semantic_highlight_color": "semantic()"
                 },
                 "styles": {
@@ -69,6 +69,10 @@
                         "color": "$base00",
                         "background-color": "$base3"
                     },
+                    "scrollbar": {
+                        "color": "$base03",
+                        "background-color": "$base0"
+                    },
                     "focused": {
                         "color": "$base03",
                         "background-color": "$base01"

src/themes/solarized-light.json

@@ -4,23 +4,23 @@
         "theme-defs": {
             "solarized-light": {
                 "vars": {
-                    "base03":    "#002b36",
-                    "base02":    "#073642",
-                    "base01":    "#586e75",
-                    "base00":    "#657b83",
-                    "base0":     "#839496",
-                    "base1":     "#93a1a1",
-                    "base2":     "#eee8d5",
-                    "base3":     "#fdf6e3",
-                    "black":     "#002b36",
-                    "yellow":    "#b58900",
-                    "orange":    "#cb4b16",
-                    "red":       "#dc322f",
-                    "magenta":   "#d33682",
-                    "violet":    "#6c71c4",
-                    "blue":      "#268bd2",
-                    "cyan":      "#2aa198",
-                    "green":     "#859900",
+                    "base03": "#002b36",
+                    "base02": "#073642",
+                    "base01": "#586e75",
+                    "base00": "#657b83",
+                    "base0": "#839496",
+                    "base1": "#93a1a1",
+                    "base2": "#eee8d5",
+                    "base3": "#fdf6e3",
+                    "black": "#002b36",
+                    "yellow": "#b58900",
+                    "orange": "#cb4b16",
+                    "red": "#dc322f",
+                    "magenta": "#d33682",
+                    "violet": "#6c71c4",
+                    "blue": "#268bd2",
+                    "cyan": "#2aa198",
+                    "green": "#859900",
                     "semantic_highlight_color": "semantic()"
                 },
                 "styles": {
@@ -69,6 +69,10 @@
                         "color": "$base00",
                         "background-color": "$base3"
                     },
+                    "scrollbar": {
+                        "color": "$base3",
+                        "background-color": "$base00"
+                    },
                     "focused": {
                         "color": "$base03",
                         "background-color": "$base01"

src/third-party/rapidyaml/ryml_all.hpp

@@ -363,7 +363,7 @@ C4_FOR_EACH(PRN_STRUCT_OFFSETS, a, b, c);
 // (end https://github.com/biojppm/c4core/src/c4/platform.hpp)
 
 
-
+#if 0
 //********************************************************************************
 //--------------------------------------------------------------------------------
 // src/c4/cpu.hpp
@@ -505,7 +505,7 @@ C4_FOR_EACH(PRN_STRUCT_OFFSETS, a, b, c);
 
 
 // (end https://github.com/biojppm/c4core/src/c4/cpu.hpp)
-
+#endif
 
 
 //********************************************************************************
@@ -1618,9 +1618,9 @@ using index_sequence_for = make_index_sequence<sizeof...(_Tp)>;
 // amalgamate: removed include of
 // https://github.com/biojppm/c4core/src/c4/cpu.hpp
 //#include "c4/cpu.hpp"
-#if !defined(C4_CPU_HPP_) && !defined(_C4_CPU_HPP_)
-#error "amalgamate: file c4/cpu.hpp must have been included at this point"
-#endif /* C4_CPU_HPP_ */
+//#if !defined(C4_CPU_HPP_) && !defined(_C4_CPU_HPP_)
+//#error "amalgamate: file c4/cpu.hpp must have been included at this point"
+//#endif /* C4_CPU_HPP_ */
 
 // amalgamate: removed include of
 // https://github.com/biojppm/c4core/src/c4/compiler.hpp

src/top_status_source.cc

@@ -29,22 +29,31 @@
 
 #include "top_status_source.hh"
 
-#include <sqlite3.h>
-
 #include "base/injector.hh"
-#include "bound_tags.hh"
 #include "config.h"
 #include "lnav.hh"
-#include "lnav_config.hh"
-#include "logfile_sub_source.hh"
 #include "md2attr_line.hh"
 #include "md4cpp.hh"
 #include "shlex.hh"
 #include "shlex.resolver.hh"
 #include "sql_util.hh"
 #include "sqlitepp.client.hh"
+#include "top_status_source.cfg.hh"
 
-top_status_source::top_status_source()
+static const char* MSG_QUERY = R"(
+SELECT message FROM lnav_user_notifications
+  WHERE message IS NOT NULL AND
+        (expiration IS NULL OR expiration > datetime('now')) AND
+        (views IS NULL OR
+         json_contains(views, (SELECT name FROM lnav_top_view)))
+  ORDER BY priority DESC
+  LIMIT 1
+)";
+
+top_status_source::top_status_source(auto_sqlite3& db,
+                                     const top_status_source_cfg& cfg)
+    : tss_config(cfg),
+      tss_user_msgs_stmt(prepare_stmt(db.in(), MSG_QUERY).unwrap())
 {
     this->tss_fields[TSF_TIME].set_width(28);
     this->tss_fields[TSF_TIME].set_role(role_t::VCR_STATUS_INFO);
@@ -62,7 +71,7 @@ top_status_source::update_time(const timeval& current_time)
     buffer[0] = ' ';
     strftime(&buffer[1],
              sizeof(buffer) - 1,
-             lnav_config.lc_ui_clock_format.c_str(),
+             this->tss_config.tssc_clock_format.c_str(),
              localtime(&current_time.tv_sec));
     sf.set_value(buffer);
 }
@@ -76,40 +85,14 @@ top_status_source::update_time()
     this->update_time(tv);
 }
 
-static const char* MSG_QUERY = R"(
-SELECT message FROM lnav_user_notifications
-  WHERE message IS NOT NULL AND
-        (expiration IS NULL OR expiration > datetime('now')) AND
-        (views IS NULL OR
-         json_contains(views, (SELECT name FROM lnav_top_view)))
-  ORDER BY priority DESC
-  LIMIT 1
-)";
-
-struct user_msg_stmt {
-    user_msg_stmt()
-        : ums_stmt(
-            prepare_stmt(injector::get<auto_mem<sqlite3, sqlite_close_wrapper>&,
-                                       sqlite_db_tag>()
-                             .in(),
-                         MSG_QUERY)
-                .unwrap())
-    {
-    }
-
-    prepared_stmt ums_stmt;
-};
-
 void
 top_status_source::update_user_msg()
 {
-    static thread_local user_msg_stmt um_stmt;
-
     auto& al = this->tss_fields[TSF_USER_MSG].get_value();
     al.clear();
 
-    um_stmt.ums_stmt.reset();
-    auto fetch_res = um_stmt.ums_stmt.fetch_row<std::string>();
+    this->tss_user_msgs_stmt.reset();
+    auto fetch_res = this->tss_user_msgs_stmt.fetch_row<std::string>();
     fetch_res.match(
         [&al](const std::string& value) {
             shlex lexer(value);

src/top_status_source.cfg.hh

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) 2022, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef lnav_top_status_source_cfg_hh
+#define lnav_top_status_source_cfg_hh
+
+struct top_status_source_cfg {
+    std::string tssc_clock_format;
+};
+
+#endif

src/top_status_source.hh

@@ -32,19 +32,30 @@
 
 #include <string>
 
+#include <sqlite3.h>
+
+#include "base/injector.hh"
+#include "bound_tags.hh"
 #include "listview_curses.hh"
+#include "sql_util.hh"
+#include "sqlitepp.client.hh"
 #include "statusview_curses.hh"
+#include "top_status_source.cfg.hh"
 
 class top_status_source : public status_data_source {
 public:
-    typedef enum {
+    enum field_t {
         TSF_TIME,
         TSF_USER_MSG,
 
         TSF__MAX
-    } field_t;
+    };
 
-    top_status_source();
+    explicit top_status_source(auto_sqlite3& db,
+                               const top_status_source_cfg& cfg);
+
+    using injectable
+        = top_status_source(auto_sqlite3& db, const top_status_source_cfg& cfg);
 
     size_t statusview_fields() override { return TSF__MAX; }
 
@@ -60,7 +71,9 @@ public:
     void update_user_msg();
 
 private:
+    const top_status_source_cfg& tss_config;
     status_field tss_fields[TSF__MAX];
+    prepared_stmt tss_user_msgs_stmt;
 };
 
 #endif

src/unique_path.cc

@@ -50,7 +50,7 @@ unique_path_generator::generate()
     while (!this->upg_unique_paths.empty()) {
         std::vector<std::shared_ptr<unique_path_source>> collisions;
 
-        for (auto pair : this->upg_unique_paths) {
+        for (const auto& pair : this->upg_unique_paths) {
             if (pair.second.size() == 1) {
                 if (loop_count > 0) {
                     std::shared_ptr<unique_path_source> src = pair.second[0];

src/view_curses.cc

@@ -588,11 +588,9 @@ attr_for_colors(nonstd::optional<short> fg, nonstd::optional<short> bg)
 view_colors::role_attrs
 view_colors::to_attrs(const lnav_theme& lt,
                       const positioned_property<style_config>& pp_sc,
-                      const positioned_property<style_config>& pp_fallback_sc,
                       lnav_config_listener::error_reporter& reporter)
 {
     const auto& sc = pp_sc.pp_value;
-    const auto& fallback_sc = pp_fallback_sc.pp_value;
     std::string fg1, bg1, fg_color, bg_color;
     intern_string_t role_class;
 
@@ -607,9 +605,6 @@ view_colors::to_attrs(const lnav_theme& lt,
     }
 
     fg1 = sc.sc_color;
-    if (fg1.empty()) {
-        fg1 = fallback_sc.sc_color;
-    }
     bg1 = sc.sc_background_color;
     shlex(fg1).eval(fg_color, lt.lt_vars);
     shlex(bg1).eval(bg_color, lt.lt_vars);
@@ -658,7 +653,7 @@ view_colors::init_roles(const lnav_theme& lt,
 
     /* Setup the mappings from roles to actual colors. */
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_TEXT)]
-        = this->to_attrs(lt, lt.lt_style_text, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_text, reporter);
 
     for (int ansi_fg = 0; ansi_fg < 8; ansi_fg++) {
         for (int ansi_bg = 0; ansi_bg < 8; ansi_bg++) {
@@ -724,44 +719,36 @@ view_colors::init_roles(const lnav_theme& lt,
         .ra_class_name
         = intern_string::lookup("-lnav_styles_search");
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_IDENTIFIER)]
-        = this->to_attrs(
-            lt, lt.lt_style_identifier, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_identifier, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_OK)]
-        = this->to_attrs(lt, lt.lt_style_ok, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_ok, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_INFO)]
-        = this->to_attrs(lt, lt.lt_style_info, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_info, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_ERROR)]
-        = this->to_attrs(lt, lt.lt_style_error, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_error, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_WARNING)]
-        = this->to_attrs(lt, lt.lt_style_warning, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_warning, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_ALT_ROW)]
-        = this->to_attrs(lt, lt.lt_style_alt_text, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_alt_text, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_HIDDEN)]
-        = this->to_attrs(lt, lt.lt_style_hidden, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_hidden, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_ADJUSTED_TIME)]
-        = this->to_attrs(
-            lt, lt.lt_style_adjusted_time, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_adjusted_time, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_SKEWED_TIME)]
-        = this->to_attrs(
-            lt, lt.lt_style_skewed_time, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_skewed_time, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_OFFSET_TIME)]
-        = this->to_attrs(
-            lt, lt.lt_style_offset_time, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_offset_time, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_INVALID_MSG)]
-        = this->to_attrs(
-            lt, lt.lt_style_invalid_msg, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_invalid_msg, reporter);
 
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_STATUS)]
-        = this->to_attrs(lt, lt.lt_style_status, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_status, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_WARN_STATUS)]
-        = this->to_attrs(
-            lt, lt.lt_style_warn_status, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_warn_status, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_ALERT_STATUS)]
-        = this->to_attrs(
-            lt, lt.lt_style_alert_status, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_alert_status, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_ACTIVE_STATUS)]
-        = this->to_attrs(
-            lt, lt.lt_style_active_status, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_active_status, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_ACTIVE_STATUS2)]
         = role_attrs{
             this->vc_role_attrs[lnav::enums::to_underlying(
@@ -778,72 +765,55 @@ view_colors::init_roles(const lnav_theme& lt,
         .ra_reverse.ta_attrs
         |= A_BOLD;
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_STATUS_TITLE)]
-        = this->to_attrs(
-            lt, lt.lt_style_status_title, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_status_title, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_STATUS_SUBTITLE)]
-        = this->to_attrs(
-            lt, lt.lt_style_status_subtitle, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_status_subtitle, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_STATUS_INFO)]
-        = this->to_attrs(
-            lt, lt.lt_style_status_info, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_status_info, reporter);
 
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_STATUS_HOTKEY)]
-        = this->to_attrs(
-            lt, lt.lt_style_status_hotkey, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_status_hotkey, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(
         role_t::VCR_STATUS_TITLE_HOTKEY)]
-        = this->to_attrs(
-            lt, lt.lt_style_status_title_hotkey, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_status_title_hotkey, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(
         role_t::VCR_STATUS_DISABLED_TITLE)]
-        = this->to_attrs(lt,
-                         lt.lt_style_status_disabled_title,
-                         lt.lt_style_status,
-                         reporter);
+        = this->to_attrs(lt, lt.lt_style_status_disabled_title, reporter);
 
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_H1)]
-        = this->to_attrs(lt, lt.lt_style_header[0], lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_header[0], reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_H2)]
-        = this->to_attrs(lt, lt.lt_style_header[1], lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_header[1], reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_H3)]
-        = this->to_attrs(lt, lt.lt_style_header[2], lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_header[2], reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_H4)]
-        = this->to_attrs(lt, lt.lt_style_header[3], lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_header[3], reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_H5)]
-        = this->to_attrs(lt, lt.lt_style_header[4], lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_header[4], reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_H6)]
-        = this->to_attrs(lt, lt.lt_style_header[5], lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_header[5], reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_HR)]
-        = this->to_attrs(lt, lt.lt_style_hr, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_hr, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_HYPERLINK)]
-        = this->to_attrs(lt, lt.lt_style_hyperlink, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_hyperlink, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_LIST_GLYPH)]
-        = this->to_attrs(
-            lt, lt.lt_style_list_glyph, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_list_glyph, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_BREADCRUMB)]
-        = this->to_attrs(
-            lt, lt.lt_style_breadcrumb, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_breadcrumb, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_TABLE_BORDER)]
-        = this->to_attrs(
-            lt, lt.lt_style_table_border, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_table_border, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_TABLE_HEADER)]
-        = this->to_attrs(
-            lt, lt.lt_style_table_header, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_table_header, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_QUOTE_BORDER)]
-        = this->to_attrs(
-            lt, lt.lt_style_quote_border, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_quote_border, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_QUOTED_TEXT)]
-        = this->to_attrs(
-            lt, lt.lt_style_quoted_text, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_quoted_text, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_FOOTNOTE_BORDER)]
-        = this->to_attrs(
-            lt, lt.lt_style_footnote_border, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_footnote_border, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_FOOTNOTE_TEXT)]
-        = this->to_attrs(
-            lt, lt.lt_style_footnote_text, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_footnote_text, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_SNIPPET_BORDER)]
-        = this->to_attrs(
-            lt, lt.lt_style_snippet_border, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_snippet_border, reporter);
 
     {
         positioned_property<style_config> stitch_sc;
@@ -854,7 +824,7 @@ view_colors::init_roles(const lnav_theme& lt,
             = lt.lt_style_status_title.pp_value.sc_background_color;
         this->vc_role_attrs[lnav::enums::to_underlying(
             role_t::VCR_STATUS_STITCH_TITLE_TO_SUB)]
-            = this->to_attrs(lt, stitch_sc, lt.lt_style_status, reporter);
+            = this->to_attrs(lt, stitch_sc, reporter);
     }
     {
         positioned_property<style_config> stitch_sc;
@@ -865,7 +835,7 @@ view_colors::init_roles(const lnav_theme& lt,
             = lt.lt_style_status_subtitle.pp_value.sc_background_color;
         this->vc_role_attrs[lnav::enums::to_underlying(
             role_t::VCR_STATUS_STITCH_SUB_TO_TITLE)]
-            = this->to_attrs(lt, stitch_sc, lt.lt_style_status, reporter);
+            = this->to_attrs(lt, stitch_sc, reporter);
     }
 
     {
@@ -877,7 +847,7 @@ view_colors::init_roles(const lnav_theme& lt,
             = lt.lt_style_status_subtitle.pp_value.sc_background_color;
         this->vc_role_attrs[lnav::enums::to_underlying(
             role_t::VCR_STATUS_STITCH_SUB_TO_NORMAL)]
-            = this->to_attrs(lt, stitch_sc, lt.lt_style_status, reporter);
+            = this->to_attrs(lt, stitch_sc, reporter);
     }
     {
         positioned_property<style_config> stitch_sc;
@@ -888,7 +858,7 @@ view_colors::init_roles(const lnav_theme& lt,
             = lt.lt_style_status.pp_value.sc_background_color;
         this->vc_role_attrs[lnav::enums::to_underlying(
             role_t::VCR_STATUS_STITCH_NORMAL_TO_SUB)]
-            = this->to_attrs(lt, stitch_sc, lt.lt_style_status, reporter);
+            = this->to_attrs(lt, stitch_sc, reporter);
     }
 
     {
@@ -900,7 +870,7 @@ view_colors::init_roles(const lnav_theme& lt,
             = lt.lt_style_status_title.pp_value.sc_background_color;
         this->vc_role_attrs[lnav::enums::to_underlying(
             role_t::VCR_STATUS_STITCH_TITLE_TO_NORMAL)]
-            = this->to_attrs(lt, stitch_sc, lt.lt_style_status, reporter);
+            = this->to_attrs(lt, stitch_sc, reporter);
     }
     {
         positioned_property<style_config> stitch_sc;
@@ -911,33 +881,24 @@ view_colors::init_roles(const lnav_theme& lt,
             = lt.lt_style_status.pp_value.sc_background_color;
         this->vc_role_attrs[lnav::enums::to_underlying(
             role_t::VCR_STATUS_STITCH_NORMAL_TO_TITLE)]
-            = this->to_attrs(lt, stitch_sc, lt.lt_style_status, reporter);
+            = this->to_attrs(lt, stitch_sc, reporter);
     }
 
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_INACTIVE_STATUS)]
-        = this->to_attrs(
-            lt, lt.lt_style_inactive_status, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_inactive_status, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(
         role_t::VCR_INACTIVE_ALERT_STATUS)]
-        = this->to_attrs(lt,
-                         lt.lt_style_inactive_alert_status,
-                         lt.lt_style_alert_status,
-                         reporter);
+        = this->to_attrs(lt, lt.lt_style_inactive_alert_status, reporter);
 
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_POPUP)]
-        = this->to_attrs(lt, lt.lt_style_popup, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_popup, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_FOCUSED)]
-        = this->to_attrs(
-            lt, lt.lt_style_focused, lt.lt_style_focused, reporter);
+        = this->to_attrs(lt, lt.lt_style_focused, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(
         role_t::VCR_DISABLED_FOCUSED)]
-        = this->to_attrs(lt,
-                         lt.lt_style_disabled_focused,
-                         lt.lt_style_disabled_focused,
-                         reporter);
+        = this->to_attrs(lt, lt.lt_style_disabled_focused, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_SCROLLBAR)]
-        = this->to_attrs(
-            lt, lt.lt_style_scrollbar, lt.lt_style_status, reporter);
+        = this->to_attrs(lt, lt.lt_style_scrollbar, reporter);
     {
         positioned_property<style_config> bar_sc;
 
@@ -946,7 +907,7 @@ view_colors::init_roles(const lnav_theme& lt,
             = lt.lt_style_scrollbar.pp_value.sc_background_color;
         this->vc_role_attrs[lnav::enums::to_underlying(
             role_t::VCR_SCROLLBAR_ERROR)]
-            = this->to_attrs(lt, bar_sc, lt.lt_style_alert_status, reporter);
+            = this->to_attrs(lt, bar_sc, reporter);
     }
     {
         positioned_property<style_config> bar_sc;
@@ -956,57 +917,48 @@ view_colors::init_roles(const lnav_theme& lt,
             = lt.lt_style_scrollbar.pp_value.sc_background_color;
         this->vc_role_attrs[lnav::enums::to_underlying(
             role_t::VCR_SCROLLBAR_WARNING)]
-            = this->to_attrs(lt, bar_sc, lt.lt_style_warn_status, reporter);
+            = this->to_attrs(lt, bar_sc, reporter);
     }
 
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_QUOTED_CODE)]
-        = this->to_attrs(
-            lt, lt.lt_style_quoted_code, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_quoted_code, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_CODE_BORDER)]
-        = this->to_attrs(
-            lt, lt.lt_style_code_border, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_code_border, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_KEYWORD)]
-        = this->to_attrs(lt, lt.lt_style_keyword, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_keyword, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_STRING)]
-        = this->to_attrs(lt, lt.lt_style_string, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_string, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_COMMENT)]
-        = this->to_attrs(lt, lt.lt_style_comment, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_comment, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_DOC_DIRECTIVE)]
-        = this->to_attrs(
-            lt, lt.lt_style_doc_directive, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_doc_directive, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_VARIABLE)]
-        = this->to_attrs(lt, lt.lt_style_variable, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_variable, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_SYMBOL)]
-        = this->to_attrs(lt, lt.lt_style_symbol, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_symbol, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_NUMBER)]
-        = this->to_attrs(lt, lt.lt_style_number, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_number, reporter);
 
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_RE_SPECIAL)]
-        = this->to_attrs(
-            lt, lt.lt_style_re_special, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_re_special, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_RE_REPEAT)]
-        = this->to_attrs(lt, lt.lt_style_re_repeat, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_re_repeat, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_FILE)]
-        = this->to_attrs(lt, lt.lt_style_file, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_file, reporter);
 
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_DIFF_DELETE)]
-        = this->to_attrs(
-            lt, lt.lt_style_diff_delete, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_diff_delete, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_DIFF_ADD)]
-        = this->to_attrs(lt, lt.lt_style_diff_add, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_diff_add, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_DIFF_SECTION)]
-        = this->to_attrs(
-            lt, lt.lt_style_diff_section, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_diff_section, reporter);
 
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_LOW_THRESHOLD)]
-        = this->to_attrs(
-            lt, lt.lt_style_low_threshold, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_low_threshold, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_MED_THRESHOLD)]
-        = this->to_attrs(
-            lt, lt.lt_style_med_threshold, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_med_threshold, reporter);
     this->vc_role_attrs[lnav::enums::to_underlying(role_t::VCR_HIGH_THRESHOLD)]
-        = this->to_attrs(
-            lt, lt.lt_style_high_threshold, lt.lt_style_text, reporter);
+        = this->to_attrs(lt, lt.lt_style_high_threshold, reporter);
 
     for (auto level = static_cast<log_level_t>(LEVEL_UNKNOWN + 1);
          level < LEVEL__MAX;
@@ -1018,8 +970,8 @@ view_colors::init_roles(const lnav_theme& lt,
             this->vc_level_attrs[level]
                 = role_attrs{text_attrs{}, text_attrs{}};
         } else {
-            this->vc_level_attrs[level] = this->to_attrs(
-                lt, level_iter->second, lt.lt_style_text, reporter);
+            this->vc_level_attrs[level]
+                = this->to_attrs(lt, level_iter->second, reporter);
         }
     }
 

src/view_curses.hh

@@ -297,7 +297,6 @@ private:
 
     role_attrs to_attrs(const lnav_theme& lt,
                         const positioned_property<style_config>& sc,
-                        const positioned_property<style_config>& fallback_sc,
                         lnav_config_listener::error_reporter& reporter);
 
     role_attrs vc_level_attrs[LEVEL__MAX];

src/yajlpp/json_ptr.cc

@@ -127,10 +127,8 @@ handle_map_key(void* ctx, const unsigned char* key, size_t len)
     } else {
         auto fully_encoded_key = (char*) alloca(required_len);
 
-        json_ptr::encode(fully_encoded_key,
-                         sizeof(fully_encoded_key),
-                         (const char*) key,
-                         len);
+        json_ptr::encode(
+            fully_encoded_key, required_len, (const char*) key, len);
         jpw->jpw_keys.emplace_back(&fully_encoded_key[0], required_len);
     }
 

src/yajlpp/yajlpp.cc

@@ -224,7 +224,9 @@ json_path_handler_base::gen(yajlpp_gen_context& ygc, yajl_gen handle) const
             ygc.ygc_depth += 1;
 
             if (this->jph_obj_provider) {
-                auto md = this->jph_regex->create_match_data();
+                static thread_local auto md
+                    = lnav::pcre2pp::match_data::unitialized();
+
                 auto find_res = this->jph_regex->capture_from(full_path)
                                     .into(md)
                                     .matches();
@@ -381,7 +383,7 @@ json_path_handler_base::gen_schema(yajlpp_gen_context& ygc) const
             schema.gen("examples");
 
             yajlpp_array example_array(ygc.ygc_handle);
-            for (auto& ex : this->jph_examples) {
+            for (const auto& ex : this->jph_examples) {
                 example_array.gen(ex);
             }
         }
@@ -504,7 +506,9 @@ json_path_handler_base::walk(
 
                 ypc.set_path(full_path).with_obj(root).update_callbacks();
                 if (this->jph_obj_provider) {
-                    auto md = this->jph_regex->create_match_data();
+                    static thread_local auto md
+                        = lnav::pcre2pp::match_data::unitialized();
+
                     std::string full_path = lpath + "/";
 
                     if (!this->jph_regex->capture_from(full_path)
@@ -537,7 +541,7 @@ nonstd::optional<int>
 json_path_handler_base::to_enum_value(const string_fragment& sf) const
 {
     for (int lpc = 0; this->jph_enum_values[lpc].first; lpc++) {
-        const enum_value_t& ev = this->jph_enum_values[lpc];
+        const auto& ev = this->jph_enum_values[lpc];
 
         if (sf == ev.first) {
             return ev.second;
@@ -707,7 +711,8 @@ yajlpp_parse_context::update_callbacks(const json_path_container* orig_handlers,
     auto path_frag = string_fragment::from_byte_range(
         this->ypc_path.data(), 1 + child_start, this->ypc_path.size() - 1);
     for (const auto& jph : handlers->jpc_children) {
-        auto md = jph.jph_regex->create_match_data();
+        static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
+
         if (jph.jph_regex->capture_from(path_frag)
                 .into(md)
                 .matches()
@@ -945,8 +950,8 @@ yajlpp_parse_context::handle_unused_or_delete(void* ctx)
     if (!ypc->ypc_handler_stack.empty()
         && ypc->ypc_handler_stack.back()->jph_obj_deleter)
     {
-        auto& jph = ypc->ypc_handler_stack.back();
-        auto md = jph->jph_regex->create_match_data();
+        static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
+
         auto key_start = ypc->ypc_path_index_stack.back();
         auto path_frag = string_fragment::from_byte_range(
             ypc->ypc_path.data(), key_start + 1, ypc->ypc_path.size() - 1);
@@ -1035,6 +1040,7 @@ yajlpp_parse_context::parse_doc(const string_fragment& sf)
 
     auto rc = yajl_parse(this->ypc_handle, this->ypc_json_text, sf.length());
     size_t consumed = yajl_get_bytes_consumed(this->ypc_handle);
+    this->ypc_total_consumed += consumed;
     this->ypc_line_number += std::count(
         &this->ypc_json_text[0], &this->ypc_json_text[consumed], '\n');
 
@@ -1101,7 +1107,7 @@ yajlpp_parse_context::reset(const struct json_path_container* handlers)
 }
 
 void
-yajlpp_parse_context::set_static_handler(json_path_handler_base& jph)
+yajlpp_parse_context::set_static_handler(const json_path_handler_base& jph)
 {
     this->ypc_path.clear();
     this->ypc_path.push_back('/');
@@ -1304,6 +1310,40 @@ yajlpp_parse_context::get_snippet() const
         .with_line(line_number);
 }
 
+void
+json_path_handler_base::validate_string(yajlpp_parse_context& ypc,
+                                        string_fragment sf) const
+{
+    if (this->jph_pattern) {
+        if (!this->jph_pattern->find_in(sf).ignore_error()) {
+            this->report_pattern_error(&ypc, sf.to_string());
+        }
+    }
+    if (sf.empty() && this->jph_min_length > 0) {
+        ypc.report_error(lnav::console::user_message::error(
+                             attr_line_t("invalid value for option ")
+                                 .append_quoted(lnav::roles::symbol(
+                                     ypc.get_full_path().to_string())))
+                             .with_reason("empty values are not allowed")
+                             .with_snippet(ypc.get_snippet())
+                             .with_help(this->get_help_text(&ypc)));
+    } else if (sf.length() < this->jph_min_length) {
+        ypc.report_error(
+            lnav::console::user_message::error(
+                attr_line_t()
+                    .append_quoted(sf)
+                    .append(" is not a valid value for option ")
+                    .append_quoted(
+                        lnav::roles::symbol(ypc.get_full_path().to_string())))
+                .with_reason(attr_line_t("value must be at least ")
+                                 .append(lnav::roles::number(
+                                     fmt::to_string(this->jph_min_length)))
+                                 .append(" characters long"))
+                .with_snippet(ypc.get_snippet())
+                .with_help(this->get_help_text(&ypc)));
+    }
+}
+
 void
 json_path_handler_base::report_pattern_error(yajlpp_parse_context* ypc,
                                              const std::string& value_str) const
@@ -1372,8 +1412,6 @@ void
 json_path_handler_base::report_min_value_error(yajlpp_parse_context* ypc,
                                                long long value) const
 {
-    const auto* jph = ypc->ypc_current_handler;
-
     ypc->report_error(
         lnav::console::user_message::error(
             attr_line_t()
@@ -1383,9 +1421,9 @@ json_path_handler_base::report_min_value_error(yajlpp_parse_context* ypc,
                     lnav::roles::symbol(ypc->get_full_path().to_string())))
             .with_reason(attr_line_t("value must be greater than or equal to ")
                              .append(lnav::roles::number(
-                                 fmt::to_string(jph->jph_min_value))))
+                                 fmt::to_string(this->jph_min_value))))
             .with_snippet(ypc->get_snippet())
-            .with_help(jph->get_help_text(ypc)));
+            .with_help(this->get_help_text(ypc)));
 }
 
 void
@@ -1459,7 +1497,7 @@ json_path_container::gen_properties(yajlpp_gen_context& ygc) const
         {
             yajlpp_map properties(ygc.ygc_handle);
 
-            for (auto& child_handler : this->jpc_children) {
+            for (const auto& child_handler : this->jpc_children) {
                 if (child_handler.jph_is_pattern_property) {
                     continue;
                 }
@@ -1523,5 +1561,5 @@ yajlpp_gen::to_string_fragment()
 
     yajl_gen_get_buf(this->yg_handle.in(), &buf, &len);
 
-    return string_fragment((const char*) buf, 0, len);
+    return string_fragment::from_bytes(buf, len);
 }

src/yajlpp/yajlpp.hh

@@ -50,6 +50,7 @@
 #include "base/lnav.console.hh"
 #include "base/lnav.console.into.hh"
 #include "base/lnav_log.hh"
+#include "base/opt_util.hh"
 #include "json_ptr.hh"
 #include "optional.hpp"
 #include "pcrepp/pcre2pp.hh"
@@ -210,6 +211,20 @@ struct json_path_handler_base {
     nonstd::optional<int> to_enum_value(const string_fragment& sf) const;
     const char* to_enum_string(int value) const;
 
+    template<typename T>
+    std::enable_if_t<!detail::is_optional<T>::value, const char*>
+    to_enum_string(T value) const
+    {
+        return this->to_enum_string((int) value);
+    }
+
+    template<typename T>
+    std::enable_if_t<detail::is_optional<T>::value, const char*> to_enum_string(
+        T value) const
+    {
+        return this->to_enum_string((int) value.value());
+    }
+
     yajl_gen_status gen(yajlpp_gen_context& ygc, yajl_gen handle) const;
     yajl_gen_status gen_schema(yajlpp_gen_context& ygc) const;
     yajl_gen_status gen_schema_type(yajlpp_gen_context& ygc) const;
@@ -267,10 +282,13 @@ struct json_path_handler_base {
     std::function<int(yajlpp_parse_context*)> jph_null_cb;
     std::function<int(yajlpp_parse_context*, int)> jph_bool_cb;
     std::function<int(yajlpp_parse_context*, long long)> jph_integer_cb;
+    std::function<int(yajlpp_parse_context*, double)> jph_double_cb;
     std::function<int(
         yajlpp_parse_context*, const unsigned char* str, size_t len)>
         jph_str_cb;
 
+    void validate_string(yajlpp_parse_context& ypc, string_fragment sf) const;
+
     void report_pattern_error(yajlpp_parse_context* ypc,
                               const std::string& value_str) const;
     void report_min_value_error(yajlpp_parse_context* ypc,
@@ -335,7 +353,7 @@ public:
 
     void reset(const struct json_path_container* handlers);
 
-    void set_static_handler(struct json_path_handler_base& jph);
+    void set_static_handler(const struct json_path_handler_base& jph);
 
     template<typename T>
     yajlpp_parse_context& with_obj(T& obj)
@@ -442,6 +460,7 @@ public:
     yajl_handle ypc_handle{nullptr};
     const unsigned char* ypc_json_text{nullptr};
     size_t ypc_json_text_len{0};
+    size_t ypc_total_consumed{0};
     yajl_callbacks ypc_callbacks;
     yajl_callbacks ypc_alt_callbacks;
     std::vector<char> ypc_path;

src/yajlpp/yajlpp_def.hh

@@ -34,38 +34,13 @@
 
 #include <chrono>
 
+#include "base/date_time_scanner.hh"
+#include "base/time_util.hh"
 #include "config.h"
 #include "mapbox/variant.hpp"
 #include "relative_time.hh"
 #include "yajlpp.hh"
 
-#define FOR_FIELD(T, FIELD) for_field<T, decltype(T ::FIELD), &T ::FIELD>()
-
-inline intern_string_t&
-assign(intern_string_t& lhs, const string_fragment& rhs)
-{
-    lhs = intern_string::lookup(rhs.data(), rhs.length());
-
-    return lhs;
-}
-
-inline std::string&
-assign(std::string& lhs, const string_fragment& rhs)
-{
-    lhs.assign(rhs.data(), rhs.length());
-
-    return lhs;
-}
-
-template<template<typename...> class Container>
-inline Container<std::string>&
-assign(Container<std::string>& lhs, const string_fragment& rhs)
-{
-    lhs.emplace_back(rhs.data(), rhs.length());
-
-    return lhs;
-}
-
 struct json_null_t {
     bool operator==(const json_null_t& other) const { return true; }
 };
@@ -244,54 +219,6 @@ struct json_path_handler : public json_path_handler_base {
         return *this;
     }
 
-    template<typename T, typename MEM_T, MEM_T T::*MEM>
-    static void* get_field_lvalue_cb(void* root,
-                                     nonstd::optional<std::string> name)
-    {
-        auto obj = (T*) root;
-        auto& mem = obj->*MEM;
-
-        return &mem;
-    }
-
-    template<typename T, typename STR_T, STR_T T::*STR>
-    static int string_field_cb(yajlpp_parse_context* ypc,
-                               const unsigned char* str,
-                               size_t len)
-    {
-        auto handler = ypc->ypc_current_handler;
-
-        if (ypc->ypc_locations) {
-            (*ypc->ypc_locations)[ypc->get_full_path()]
-                = source_location{ypc->ypc_source, ypc->get_line_number()};
-        }
-
-        assign(ypc->get_lvalue(ypc->get_obj_member<T, STR_T, STR>()),
-               string_fragment(str, 0, len));
-        handler->jph_validator(*ypc, *handler);
-
-        return 1;
-    }
-
-    template<typename T, typename ENUM_T, ENUM_T T::*ENUM>
-    static int enum_field_cb(yajlpp_parse_context* ypc,
-                             const unsigned char* str,
-                             size_t len)
-    {
-        auto obj = (T*) ypc->ypc_obj_stack.top();
-        auto handler = ypc->ypc_current_handler;
-        auto res = handler->to_enum_value(string_fragment(str, 0, len));
-
-        if (res) {
-            obj->*ENUM = (ENUM_T) res.value();
-        } else {
-            handler->report_enum_error(ypc,
-                                       std::string((const char*) str, len));
-        }
-
-        return 1;
-    }
-
     static int null_field_cb(yajlpp_parse_context* ypc)
     {
         return ypc->ypc_current_handler->jph_null_cb(ypc);
@@ -314,204 +241,9 @@ struct json_path_handler : public json_path_handler_base {
         return ypc->ypc_current_handler->jph_integer_cb(ypc, val);
     }
 
-    template<typename T, typename NUM_T, NUM_T T::*NUM>
-    static int num_field_cb(yajlpp_parse_context* ypc, long long num)
+    static int dbl_field_cb(yajlpp_parse_context* ypc, double val)
     {
-        auto obj = (T*) ypc->ypc_obj_stack.top();
-
-        obj->*NUM = num;
-
-        return 1;
-    }
-
-    template<typename T, typename NUM_T, NUM_T T::*NUM>
-    static int decimal_field_cb(yajlpp_parse_context* ypc, double num)
-    {
-        auto obj = (T*) ypc->ypc_obj_stack.top();
-
-        obj->*NUM = num;
-
-        return 1;
-    }
-
-    template<typename T, typename STR_T, STR_T T::*STR>
-    static void string_field_validator(yajlpp_parse_context& ypc,
-                                       const json_path_handler_base& jph)
-    {
-        auto& field_ptr = ypc.get_rvalue(ypc.get_obj_member<T, STR_T, STR>());
-
-        if (jph.jph_pattern) {
-            auto sf = to_string_fragment(field_ptr);
-
-            if (!jph.jph_pattern->find_in(sf).ignore_error()) {
-                jph.report_pattern_error(&ypc, sf.to_string());
-            }
-        }
-        if (field_ptr.empty() && jph.jph_min_length > 0) {
-            ypc.report_error(
-                lnav::console::user_message::error(
-                    attr_line_t("invalid value for option ")
-                        .template append_quoted(lnav::roles::symbol(
-                            ypc.get_full_path().to_string())))
-                    .with_reason("empty values are not allowed")
-                    .with_snippet(ypc.get_snippet())
-                    .with_help(jph.get_help_text(&ypc)));
-        } else if (field_ptr.size() < jph.jph_min_length) {
-            ypc.report_error(
-                lnav::console::user_message::error(
-                    attr_line_t()
-                        .template append_quoted(field_ptr)
-                        .append(" is not a valid value for option ")
-                        .append_quoted(lnav::roles::symbol(
-                            ypc.get_full_path().to_string())))
-                    .with_reason(attr_line_t("value must be at least ")
-                                     .append(lnav::roles::number(
-                                         fmt::to_string(jph.jph_min_length)))
-                                     .append(" characters long"))
-                    .with_snippet(ypc.get_snippet())
-                    .with_help(jph.get_help_text(&ypc)));
-        }
-    }
-
-    template<typename T, typename NUM_T, NUM_T T::*NUM>
-    static void number_field_validator(yajlpp_parse_context& ypc,
-                                       const json_path_handler_base& jph)
-    {
-        auto& field_ptr = ypc.get_rvalue(ypc.get_obj_member<T, NUM_T, NUM>());
-
-        if (field_ptr < jph.jph_min_value) {
-            jph.report_min_value_error(&ypc, field_ptr);
-        }
-    }
-
-    template<typename T, typename R, R T::*FIELD>
-    static yajl_gen_status field_gen(yajlpp_gen_context& ygc,
-                                     const json_path_handler_base& jph,
-                                     yajl_gen handle)
-    {
-        auto def_obj = (T*) (ygc.ygc_default_stack.empty()
-                                 ? nullptr
-                                 : ygc.ygc_default_stack.top());
-        auto obj = (T*) ygc.ygc_obj_stack.top();
-
-        if (def_obj != nullptr && def_obj->*FIELD == obj->*FIELD) {
-            return yajl_gen_status_ok;
-        }
-
-        if (ygc.ygc_depth) {
-            yajl_gen_string(handle, jph.jph_property);
-        }
-
-        yajlpp_generator gen(handle);
-
-        return gen(obj->*FIELD);
-    }
-
-    template<typename T, typename R, R T::*FIELD>
-    static yajl_gen_status map_field_gen(yajlpp_gen_context& ygc,
-                                         const json_path_handler_base& jph,
-                                         yajl_gen handle)
-    {
-        const auto def_container = (T*) (ygc.ygc_default_stack.empty()
-                                             ? nullptr
-                                             : ygc.ygc_default_stack.top());
-        auto container = (T*) ygc.ygc_obj_stack.top();
-        auto& obj = container->*FIELD;
-        yajl_gen_status rc;
-
-        for (const auto& pair : obj) {
-            if (def_container != nullptr) {
-                auto& def_obj = def_container->*FIELD;
-                auto iter = def_obj.find(pair.first);
-
-                if (iter != def_obj.end() && iter->second == pair.second) {
-                    continue;
-                }
-            }
-
-            if ((rc = yajl_gen_string(handle, pair.first))
-                != yajl_gen_status_ok)
-            {
-                return rc;
-            }
-            if ((rc = yajl_gen_string(handle, pair.second))
-                != yajl_gen_status_ok)
-            {
-                return rc;
-            }
-        }
-
-        return yajl_gen_status_ok;
-    }
-
-    template<typename T, typename STR_T, std::string T::*STR>
-    json_path_handler& for_field()
-    {
-        this->add_cb(string_field_cb<T, STR_T, STR>);
-        this->jph_gen_callback = field_gen<T, STR_T, STR>;
-        this->jph_validator = string_field_validator<T, STR_T, STR>;
-        this->jph_field_getter = get_field_lvalue_cb<T, STR_T, STR>;
-
-        return *this;
-    }
-
-    template<typename T,
-             typename STR_T,
-             std::map<std::string, std::string> T::*STR>
-    json_path_handler& for_field()
-    {
-        this->add_cb(string_field_cb<T, STR_T, STR>);
-        this->jph_gen_callback = map_field_gen<T, STR_T, STR>;
-        this->jph_validator = string_field_validator<T, STR_T, STR>;
-
-        return *this;
-    }
-
-    template<typename T,
-             typename STR_T,
-             std::map<std::string, std::vector<std::string>> T::*STR>
-    json_path_handler& for_field()
-    {
-        this->add_cb(string_field_cb<T, STR_T, STR>);
-        this->jph_validator = string_field_validator<T, STR_T, STR>;
-
-        return *this;
-    }
-
-    template<typename T, typename STR_T, std::vector<std::string> T::*STR>
-    json_path_handler& for_field()
-    {
-        this->add_cb(string_field_cb<T, STR_T, STR>);
-        this->jph_gen_callback = field_gen<T, STR_T, STR>;
-        this->jph_validator = string_field_validator<T, STR_T, STR>;
-
-        return *this;
-    }
-
-    template<typename T, typename STR_T, intern_string_t T::*STR>
-    json_path_handler& for_field()
-    {
-        this->add_cb(string_field_cb<T, intern_string_t, STR>);
-        this->jph_gen_callback = field_gen<T, intern_string_t, STR>;
-        this->jph_validator = string_field_validator<T, intern_string_t, STR>;
-
-        return *this;
-    }
-
-    template<typename T, typename BOOL_T, bool T::*BOOL>
-    json_path_handler& for_field()
-    {
-        this->add_cb(bool_field_cb);
-        this->jph_bool_cb = [&](yajlpp_parse_context* ypc, int val) {
-            auto obj = (T*) ypc->ypc_obj_stack.top();
-
-            obj->*BOOL = static_cast<bool>(val);
-
-            return 1;
-        };
-        this->jph_gen_callback = field_gen<T, bool, BOOL>;
-
-        return *this;
+        return ypc->ypc_current_handler->jph_double_cb(ypc, val);
     }
 
     template<typename T, typename U>
@@ -551,31 +283,56 @@ struct json_path_handler : public json_path_handler_base {
 
     template<typename T, typename... Args>
     struct LastIsEnum {
+        using value_type = typename LastIsEnum<Args...>::value_type;
         static constexpr bool value = LastIsEnum<Args...>::value;
     };
 
     template<typename T, typename U>
     struct LastIsEnum<U T::*> {
+        using value_type = U;
+
+        static constexpr bool value = std::is_enum<U>::value;
+    };
+
+    template<typename T, typename U>
+    struct LastIsEnum<nonstd::optional<U> T::*> {
+        using value_type = U;
+
         static constexpr bool value = std::is_enum<U>::value;
     };
 
     template<typename T, typename... Args>
-    struct LastIsNumber {
-        static constexpr bool value = LastIsNumber<Args...>::value;
+    struct LastIsInteger {
+        static constexpr bool value = LastIsInteger<Args...>::value;
     };
 
     template<typename T, typename U>
-    struct LastIsNumber<U T::*> {
+    struct LastIsInteger<U T::*> {
         static constexpr bool value
             = std::is_integral<U>::value && !std::is_same<U, bool>::value;
     };
 
     template<typename T, typename U>
-    struct LastIsNumber<nonstd::optional<U> T::*> {
+    struct LastIsInteger<nonstd::optional<U> T::*> {
         static constexpr bool value
             = std::is_integral<U>::value && !std::is_same<U, bool>::value;
     };
 
+    template<typename T, typename... Args>
+    struct LastIsFloat {
+        static constexpr bool value = LastIsFloat<Args...>::value;
+    };
+
+    template<typename T, typename U>
+    struct LastIsFloat<U T::*> {
+        static constexpr bool value = std::is_same<U, double>::value;
+    };
+
+    template<typename T, typename U>
+    struct LastIsFloat<nonstd::optional<U> T::*> {
+        static constexpr bool value = std::is_same<U, double>::value;
+    };
+
     template<typename T, typename... Args>
     struct LastIsVector {
         using value_type = typename LastIsVector<Args...>::value_type;
@@ -663,12 +420,7 @@ struct json_path_handler : public json_path_handler_base {
             auto value_str = std::string((const char*) str, len);
             auto jph = ypc->ypc_current_handler;
 
-            if (jph->jph_pattern) {
-                if (!jph->jph_pattern->find_in(value_str).ignore_error()) {
-                    jph->report_pattern_error(ypc, value_str);
-                }
-            }
-
+            jph->validate_string(*ypc, value_str);
             json_path_handler::get_field(obj, args...)
                 .emplace_back(std::move(value_str));
 
@@ -925,12 +677,7 @@ struct json_path_handler : public json_path_handler_base {
             auto value_str = std::string((const char*) str, len);
             auto jph = ypc->ypc_current_handler;
 
-            if (jph->jph_pattern) {
-                if (!jph->jph_pattern->find_in(value_str).ignore_error()) {
-                    jph->report_pattern_error(ypc, value_str);
-                }
-            }
-
+            jph->validate_string(*ypc, value_str);
             json_path_handler::get_field(obj, args...) = std::move(value_str);
 
             return 1;
@@ -965,6 +712,69 @@ struct json_path_handler : public json_path_handler_base {
         return *this;
     }
 
+    template<typename... Args,
+             std::enable_if_t<LastIs<timeval, Args...>::value, bool> = true>
+    json_path_handler& for_field(Args... args)
+    {
+        this->add_cb(str_field_cb2);
+        this->jph_str_cb = [args...](yajlpp_parse_context* ypc,
+                                     const unsigned char* str,
+                                     size_t len) {
+            auto obj = ypc->ypc_obj_stack.top();
+            auto jph = ypc->ypc_current_handler;
+
+            date_time_scanner dts;
+            timeval tv{};
+            exttm tm;
+
+            if (dts.scan((char*) str, len, nullptr, &tm, tv) == nullptr) {
+                ypc->report_error(
+                    lnav::console::user_message::error(
+                        attr_line_t("unrecognized timestamp ")
+                            .append_quoted(
+                                string_fragment::from_bytes(str, len)))
+                        .with_snippet(ypc->get_snippet())
+                        .with_help(jph->get_help_text(ypc)));
+            } else {
+                json_path_handler::get_field(obj, args...) = tv;
+            }
+
+            return 1;
+        };
+        this->jph_gen_callback = [args...](yajlpp_gen_context& ygc,
+                                           const json_path_handler_base& jph,
+                                           yajl_gen handle) {
+            const auto& field = json_path_handler::get_field(
+                ygc.ygc_obj_stack.top(), args...);
+
+            if (!ygc.ygc_default_stack.empty()) {
+                const auto& field_def = json_path_handler::get_field(
+                    ygc.ygc_default_stack.top(), args...);
+
+                if (field == field_def) {
+                    return yajl_gen_status_ok;
+                }
+            }
+
+            if (ygc.ygc_depth) {
+                yajl_gen_string(handle, jph.jph_property);
+            }
+
+            yajlpp_generator gen(handle);
+            char buf[64];
+
+            auto buf_len = lnav::strftime_rfc3339(
+                buf, sizeof(buf), field.tv_sec, field.tv_usec, 'T');
+
+            return gen(string_fragment::from_bytes(buf, buf_len));
+        };
+        this->jph_field_getter
+            = [args...](void* root, nonstd::optional<std::string> name) {
+                  return (void*) &json_path_handler::get_field(root, args...);
+              };
+        return *this;
+    }
+
     template<
         typename... Args,
         std::enable_if_t<LastIs<nonstd::optional<std::string>, Args...>::value,
@@ -980,12 +790,7 @@ struct json_path_handler : public json_path_handler_base {
             auto value_str = std::string((const char*) str, len);
             auto jph = ypc->ypc_current_handler;
 
-            if (jph->jph_pattern) {
-                if (!jph->jph_pattern->find_in(value_str).ignore_error()) {
-                    jph->report_pattern_error(ypc, value_str);
-                }
-            }
-
+            jph->validate_string(*ypc, value_str);
             json_path_handler::get_field(obj, args...) = std::move(value_str);
 
             return 1;
@@ -1039,12 +844,7 @@ struct json_path_handler : public json_path_handler_base {
             auto value_str = std::string((const char*) str, len);
             auto jph = ypc->ypc_current_handler;
 
-            if (jph->jph_pattern) {
-                if (!jph->jph_pattern->find_in(value_str).ignore_error()) {
-                    jph->report_pattern_error(ypc, value_str);
-                }
-            }
-
+            jph->validate_string(*ypc, value_str);
             auto& field = json_path_handler::get_field(obj, args...);
 
             field.pp_path = ypc->get_full_path();
@@ -1093,12 +893,7 @@ struct json_path_handler : public json_path_handler_base {
             auto value_str = std::string((const char*) str, len);
             auto jph = ypc->ypc_current_handler;
 
-            if (jph->jph_pattern) {
-                if (!jph->jph_pattern->find_in(value_str).ignore_error()) {
-                    jph->report_pattern_error(ypc, value_str);
-                }
-            }
-
+            jph->validate_string(*ypc, value_str);
             json_path_handler::get_field(obj, args...)
                 = intern_string::lookup(value_str);
 
@@ -1145,12 +940,7 @@ struct json_path_handler : public json_path_handler_base {
             auto value_str = std::string((const char*) str, len);
             auto jph = ypc->ypc_current_handler;
 
-            if (jph->jph_pattern) {
-                if (!jph->jph_pattern->find_in(value_str).ignore_error()) {
-                    jph->report_pattern_error(ypc, value_str);
-                }
-            }
-
+            jph->validate_string(*ypc, value_str);
             auto& field = json_path_handler::get_field(obj, args...);
             field.pp_path = ypc->get_full_path();
             field.pp_location.sl_source = ypc->ypc_source;
@@ -1222,7 +1012,7 @@ struct json_path_handler : public json_path_handler_base {
     }
 
     template<typename... Args,
-             std::enable_if_t<LastIsNumber<Args...>::value, bool> = true>
+             std::enable_if_t<LastIsInteger<Args...>::value, bool> = true>
     json_path_handler& for_field(Args... args)
     {
         this->add_cb(int_field_cb);
@@ -1275,6 +1065,59 @@ struct json_path_handler : public json_path_handler_base {
         return *this;
     }
 
+    template<typename... Args,
+             std::enable_if_t<LastIsFloat<Args...>::value, bool> = true>
+    json_path_handler& for_field(Args... args)
+    {
+        this->add_cb(dbl_field_cb);
+        this->jph_double_cb = [args...](yajlpp_parse_context* ypc, double val) {
+            auto jph = ypc->ypc_current_handler;
+            auto* obj = ypc->ypc_obj_stack.top();
+
+            if (val < jph->jph_min_value) {
+                jph->report_min_value_error(ypc, val);
+                return 1;
+            }
+
+            json_path_handler::get_field(obj, args...) = val;
+
+            return 1;
+        };
+        this->jph_gen_callback = [args...](yajlpp_gen_context& ygc,
+                                           const json_path_handler_base& jph,
+                                           yajl_gen handle) {
+            const auto& field = json_path_handler::get_field(
+                ygc.ygc_obj_stack.top(), args...);
+
+            if (!ygc.ygc_default_stack.empty()) {
+                const auto& field_def = json_path_handler::get_field(
+                    ygc.ygc_default_stack.top(), args...);
+
+                if (field == field_def) {
+                    return yajl_gen_status_ok;
+                }
+            }
+
+            if (!is_field_set(field)) {
+                return yajl_gen_status_ok;
+            }
+
+            if (ygc.ygc_depth) {
+                yajl_gen_string(handle, jph.jph_property);
+            }
+
+            yajlpp_generator gen(handle);
+
+            return gen(field);
+        };
+        this->jph_field_getter
+            = [args...](void* root, nonstd::optional<std::string> name) {
+                  return (void*) &json_path_handler::get_field(root, args...);
+              };
+
+        return *this;
+    }
+
     template<
         typename... Args,
         std::enable_if_t<LastIs<std::chrono::seconds, Args...>::value, bool>
@@ -1349,8 +1192,7 @@ struct json_path_handler : public json_path_handler_base {
 
             if (res) {
                 json_path_handler::get_field(obj, args...)
-                    = (decltype(json_path_handler::get_field(
-                        obj, args...))) res.value();
+                    = (typename LastIsEnum<Args...>::value_type) res.value();
             } else {
                 handler->report_enum_error(ypc,
                                            std::string((const char*) str, len));
@@ -1373,13 +1215,17 @@ struct json_path_handler : public json_path_handler_base {
                 }
             }
 
+            if (!is_field_set(field)) {
+                return yajl_gen_status_ok;
+            }
+
             if (ygc.ygc_depth) {
                 yajl_gen_string(handle, jph.jph_property);
             }
 
             yajlpp_generator gen(handle);
 
-            return gen(jph.to_enum_string((int) field));
+            return gen(jph.to_enum_string(field));
         };
         this->jph_field_getter
             = [args...](void* root, nonstd::optional<std::string> name) {
@@ -1387,34 +1233,6 @@ struct json_path_handler : public json_path_handler_base {
               };
 
         return *this;
-    };
-
-    template<typename T, typename NUM_T, NUM_T T::*NUM>
-    json_path_handler& for_field(
-        typename std::enable_if<std::is_integral<NUM_T>::value
-                                && !std::is_same<NUM_T, bool>::value>::type*
-            dummy
-        = 0)
-    {
-        this->add_cb(num_field_cb<T, NUM_T, NUM>);
-        this->jph_validator = number_field_validator<T, NUM_T, NUM>;
-        return *this;
-    }
-
-    template<typename T, typename NUM_T, double T::*NUM>
-    json_path_handler& for_field()
-    {
-        this->add_cb(decimal_field_cb<T, NUM_T, NUM>);
-        this->jph_validator = number_field_validator<T, NUM_T, NUM>;
-        return *this;
-    }
-
-    template<typename T, typename ENUM_T, ENUM_T T::*ENUM>
-    json_path_handler& for_field(
-        typename std::enable_if<std::is_enum<ENUM_T>::value>::type* dummy = 0)
-    {
-        this->add_cb(enum_field_cb<T, ENUM_T, ENUM>);
-        return *this;
     }
 
     json_path_handler& with_children(const json_path_container& container);
@@ -1539,7 +1357,7 @@ struct typed_json_path_container : public json_path_container {
         return gen.to_string_fragment().to_string();
     }
 
-    json_string to_json_string(T& obj) const
+    json_string to_json_string(const T& obj) const
     {
         yajlpp_gen gen;
         yajlpp_gen_context ygc(gen, *this);
@@ -1564,6 +1382,7 @@ pattern_property_handler(const T (&path)[N])
 {
     return {lnav::pcre2pp::code::from_const(path).to_shared()};
 }
+
 }  // namespace yajlpp
 
 #endif

test/Makefile.am

@@ -223,7 +223,10 @@ include expected/expected.am
 
 dist_noinst_DATA = \
     $(EXPECTED_FILES) \
+    expected/test_tailer.sh_12f539e535df04364316699f9edeac461aa9f9de.err \
+    expected/test_tailer.sh_12f539e535df04364316699f9edeac461aa9f9de.out \
 	ansi-colors.0.in \
+	bad-config/formats/invalid-json-format/format.json \
 	bad-config/formats/invalid-properties/format.json \
 	bad-config/formats/invalid-regex/format.json \
 	bad-config/formats/invalid-sample/format.json \
@@ -302,10 +305,12 @@ dist_noinst_DATA = \
 	logfile_haproxy.0 \
 	logfile_invalid_json.json \
 	logfile_invalid_json2.json \
+	logfile_mixed_json2.json \
 	logfile_journald.json \
 	logfile_json.json \
 	logfile_json2.json \
 	logfile_json3.json \
+	logfile_json_subsec.json \
 	logfile_leveltest.0 \
 	logfile_logfmt.0 \
 	logfile_multiline.0 \
@@ -347,6 +352,7 @@ dist_noinst_DATA = \
 	multiline.lnav \
 	nested.lnav \
 	mvwattrline_output.0 \
+	textfile_0.md \
 	textfile_ansi.0 \
 	textfile_json_indented.0 \
 	textfile_json_one_line.0 \
@@ -366,6 +372,7 @@ dist_noinst_DATA = \
 	formats/jsontest/rewrite-user.lnav \
 	formats/jsontest2/format.json \
 	formats/jsontest3/format.json \
+	formats/jsontest-subsec/format.json \
 	formats/nestedjson/format.json \
 	formats/scripts/multiline-echo.lnav \
 	formats/scripts/redirecting.lnav \

test/bad-config/formats/invalid-json-format/format.json

@@ -0,0 +1,17 @@
+{
+    "$schema": "https://lnav.org/schemas/format-v1.schema.json",
+    "bad_json_log": {
+        "json": true,
+        "line-format": [
+            {
+                "field": ""
+            },
+            {
+                "field": "__timestamp__",
+                "timestamp-format": ""
+            }
+        ],
+        "value": {
+        }
+    }
+}