[vtab] indexing improvements

src/base/auto_mem.hh

@@ -158,7 +158,13 @@ class auto_buffer {
 public:
     static auto_buffer alloc(size_t capacity)
     {
-        return auto_buffer{(char*) malloc(capacity), capacity};
+        return auto_buffer{capacity == 0 ? nullptr : (char*) malloc(capacity),
+                           capacity};
+    }
+
+    static auto_buffer alloc_bitmap(size_t capacity_in_bits)
+    {
+        return alloc((capacity_in_bits + 7) / 8);
     }
 
     static auto_buffer from(const char* mem, size_t size)
@@ -193,9 +199,10 @@ public:
 
     auto_buffer& operator=(auto_buffer&& other) noexcept
     {
-        this->ab_buffer = other.ab_buffer;
-        this->ab_size = other.ab_size;
-        this->ab_capacity = other.ab_capacity;
+        free(this->ab_buffer);
+        this->ab_buffer = std::exchange(other.ab_buffer, nullptr);
+        this->ab_size = std::exchange(other.ab_size, 0);
+        this->ab_capacity = std::exchange(other.ab_capacity, 0);
         return *this;
     }
 
@@ -216,6 +223,30 @@ public:
 
     const char* begin() const { return this->ab_buffer; }
 
+    bool is_bit_set(size_t bit_offset) const
+    {
+        size_t byte_offset = bit_offset / 8;
+        auto bitmask = 1UL << (bit_offset % 8);
+
+        return this->ab_buffer[byte_offset] & bitmask;
+    }
+
+    void set_bit(size_t bit_offset)
+    {
+        size_t byte_offset = bit_offset / 8;
+        auto bitmask = 1UL << (bit_offset % 8);
+
+        this->ab_buffer[byte_offset] |= bitmask;
+    }
+
+    void clear_bit(size_t bit_offset)
+    {
+        size_t byte_offset = bit_offset / 8;
+        auto bitmask = 1UL << (bit_offset % 8);
+
+        this->ab_buffer[byte_offset] &= ~bitmask;
+    }
+
     std::reverse_iterator<char*> rbegin()
     {
         return std::reverse_iterator<char*>(this->end());
@@ -252,6 +283,8 @@ public:
 
     size_t size() const { return this->ab_size; }
 
+    size_t bitmap_size() const { return this->ab_size * 8; }
+
     bool empty() const { return this->ab_size == 0; }
 
     bool full() const { return this->ab_size == this->ab_capacity; }
@@ -270,6 +303,16 @@ public:
         return *this;
     }
 
+    auto_buffer& resize_bitmap(size_t new_size_in_bits, int fill = 0)
+    {
+        auto new_size = (new_size_in_bits + 7) / 8;
+        assert(new_size <= this->ab_capacity);
+
+        auto old_size = std::exchange(this->ab_size, new_size);
+        memset(this->at(old_size), 0, this->ab_size - old_size);
+        return *this;
+    }
+
     auto_buffer& resize_by(ssize_t amount)
     {
         return this->resize(this->ab_size + amount);
@@ -290,6 +333,11 @@ public:
         this->ab_capacity = new_capacity;
     }
 
+    void expand_bitmap_to(size_t new_capacity_in_bits)
+    {
+        this->expand_to((new_capacity_in_bits + 7) / 8);
+    }
+
     void expand_by(size_t amount)
     {
         if (amount == 0) {

src/base/lnav.gzip.tests.cc

@@ -42,7 +42,7 @@ TEST_CASE("lnav::gzip::uncompress")
 
         CHECK(u_res.isErr());
         CHECK(u_res.unwrapErr()
-              == "unable to uncompress: empty -- buffer error");
+              == "unable to uncompress: empty -- stream error");
     }
 
     {

src/bottom_status_source.cc

@@ -186,6 +186,7 @@ bottom_status_source::update_loading(file_off_t off, file_size_t total)
     auto& sf = this->bss_fields[BSF_LOADING];
 
     require(off >= 0);
+    require(off <= total);
 
     if (total == 0 || (size_t) off == total) {
         sf.set_cylon(false);

src/column_namer.cc

@@ -95,7 +95,11 @@ column_namer::add_column(const string_fragment& in_name)
 
     while (this->existing_name(retval)) {
         if (num == 0) {
-            this->cn_name_counters[base_name] = num;
+            auto* mem = this->cn_alloc.allocate(retval.length() + 1);
+            memcpy(mem, retval.data(), retval.length());
+            mem[retval.length()] = '\0';
+            auto counter_name = string_fragment{mem, 0, retval.length()};
+            this->cn_name_counters[counter_name] = num;
         }
 
         log_debug(

src/formats/procstate_log.json

@@ -5,7 +5,7 @@
         "description": "Periodic dumps of process state",
         "regex": {
             "std": {
-                "pattern": "========== Start of system state dump at (?<timestamp>.*) =========="
+                "pattern": "========== Start of system state dump at (?<timestamp>[^=]+)==========(?<body>.*)"
             }
         },
         "sample": [
@@ -15,7 +15,7 @@
         ],
         "search-table": {
             "procstate_procs": {
-                "pattern": "^(?<user>\\S+)\\s+(?<pid>\\d+)\\s+(?<cpu_pct>\\d+\\.\\d+)\\s+(?<mem_pct>\\d+\\.\\d+)\\s+(?<vsz>\\d+)\\s+(?<rss>\\d+)\\s(?<tty>\\S+)\\s+(?<stat>\\S+)\\s+(?<start_time>\\S+)\\s+(?<cpu_time>\\S+)\\s+(?<cmd>(?<cmd_name>[^ \\n]+)(?: (?<cmd_args>[^\\n]+))?)$"
+                "pattern": "^(?<user>\\S+)\\s+(?<pid>\\d+)\\s+(?<cpu_pct>\\d+(?:\\.\\d+)?)\\s+(?<mem_pct>\\d+(?:\\.\\d+)?)\\s+(?<vsz>\\d+)\\s+(?<rss>\\d+)\\s(?<tty>\\S+)\\s+(?<stat>\\S+)\\s+(?<start_time>\\S+)\\s+(?<cpu_time>\\S+)\\s+(?<cmd>(?<cmd_name>[^ \\n]+)(?: (?<cmd_args>[^\\n]+))?)$"
             }
         }
     }

src/formats/vmw_log.json

@@ -6,10 +6,10 @@
         "url": "http://kb.vmware.com/kb/2004201",
         "regex": {
             "6.0+": {
-                "pattern": "^(?:\\[#\\d+\\] )?(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+)(?:\\(\\d+\\)+)? (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\]:? (?:\\w+ -\\[\\d+\\] )?\\[(?<src>\\w+@\\d+)(?:\\s+sub=(?<sub>Http2(?:Client)?Session #\\d+|HTTP session map|HTTP server|Proxy Req \\d+(?: Tunnel)?|Hostsvc.ResourcePool [\\w+\\-\\.]+|Memory checker|Handle checker|HttpNfcLease-session\\[[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\][0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|Statssvc.StatsCollector.StatsRegistry\\(\\w+\\).Query\\(\\w+\\)|Vimsvc\\.Ticket((?: |\\-)[0-9a-fA-F]{2})+|Req@(?:[\\w\\.\\-@/:]+)(?: M?[\\d\\.]+(?:U[\\d\\.]+)?)?|(?:SSL )?(?:[\\w\\.\\-@/:]+(?:\\[[0-9a-fA-F]+\\])?(?:\\([0-9a-fA-F]+\\))?(?:\\{\\d+\\})?)+)?)?(?:\\s+item=(?<item>[\\w\\.\\-@/:]+))?(?: opI(?:D|d)=(?<opid>(?:req=)?[\\w@ \\-\\.:]+(?!sid|user|reason|update)))?(?: sid=(?<sid>[^ \\]]+))?(?: user=(?<user>[^ \\]<]+(?:<[^>]+>)?))?(?: update=(?<vpxa_update>\\d+))?(?:\\s+reason=(?<reason>[^\\]]+))?\\]\\s*(?<body>.*)(?:\\n.*)?$"
+                "pattern": "^(?:\\[#\\d+\\] )?(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+)(?:\\(\\d+\\)+)? (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\]:? (?:\\w+ -\\[\\d+\\] )?\\[(?<src>\\w+@\\d+)(?:\\s+sub=(?<sub>.*?(?!\\w+=)))?(?:\\s+item=(?<item>[\\w\\.\\-@/:]+))?(?: req=(?<req>[^ \\]]+))?(?: opI(?:D|d)=(?<opid>(?:req=)?[\\w@ \\-\\.:]+?(?!\\w+=)))?(?: sid=(?<sid>[^ \\]]+))?(?: user=(?<user>[^ \\]<]+(?:<[^>]+>)?))?(?: update=(?<vpxa_update>\\d+))?(?:\\s+reason=(?<reason>[^\\]]+))?\\]\\s*(?<body>.*)$"
             },
             "6.0+-nosrc": {
-                "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+)(?:\\(\\d+\\)+)? (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\]:? \\[(?:opI(?:D|d)=(?<opid>[^\\]]+))\\]\\s*(?<body>.*)(?:\\n.*)?$"
+                "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+)(?:\\(\\d+\\)+)? (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\]:? \\[(?:opI(?:D|d)=(?<opid>[^\\]]+))\\]\\s*(?<body>.*)$"
             },
             "section": {
                 "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?:- last log rotation time, \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2}))?\\s*(ESX KMX Agent started.|(?:- time the service was last started(?: \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z)?, )?Section for (?:[^,]+), pid=(?<tid>\\w+).*)"
@@ -86,6 +86,10 @@
                 "kind": "string",
                 "identifier": true
             },
+            "req": {
+                "kind": "string",
+                "identifier": true
+            },
             "sid": {
                 "kind": "string",
                 "identifier": true

src/log_search_table.cc

@@ -127,6 +127,7 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss)
             return true;
         }
 
+        // log_debug("done matching message");
         this->lst_match_index = -1;
         return false;
     }
@@ -150,6 +151,12 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss)
         return false;
     }
 
+    if (this->lst_mismatch_bitmap.is_bit_set(lc.lc_curr_line)) {
+        // log_debug("%d: mismatch, aborting", (int) lc.lc_curr_line);
+        return false;
+    }
+
+    // log_debug("%d: doing message", (int) lc.lc_curr_line);
     lf->read_full_message(lf_iter, this->lst_current_line);
     lf->get_format()->annotate(cl,
                                this->lst_current_line,
@@ -162,6 +169,7 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss)
     if (!this->lst_regex.match(
             this->lst_match_context, this->lst_input, PCRE_NO_UTF8_CHECK))
     {
+        this->lst_mismatch_bitmap.set_bit(lc.lc_curr_line);
         return false;
     }
 
@@ -214,4 +222,29 @@ log_search_table::filter(log_cursor& lc, logfile_sub_source& lss)
         };
     }
     this->lst_match_index = -1;
+
+    if (lss.lss_index_generation != this->lst_index_generation) {
+        log_debug("%s:index generation changed from %d to %d, resetting...",
+                  this->vi_name.c_str(),
+                  this->lst_index_generation,
+                  lss.lss_index_generation);
+        this->lst_mismatch_bitmap
+            = auto_buffer::alloc_bitmap(lss.text_line_count());
+        this->lst_index_generation = lss.lss_index_generation;
+    }
+
+    if (this->lst_mismatch_bitmap.bitmap_size() < lss.text_line_count()) {
+        this->lst_mismatch_bitmap.expand_bitmap_to(lss.text_line_count());
+        this->lst_mismatch_bitmap.resize_bitmap(lss.text_line_count());
+#if 1
+        log_debug("%s:bitmap resize %d:%d",
+                  this->vi_name.c_str(),
+                  this->lst_mismatch_bitmap.size(),
+                  this->lst_mismatch_bitmap.capacity());
+#endif
+    }
+    if (!lc.lc_indexed_lines.empty()) {
+        lc.lc_curr_line = lc.lc_indexed_lines.back();
+        lc.lc_indexed_lines.pop_back();
+    }
 }

src/log_search_table.hh

@@ -78,6 +78,8 @@ public:
     int64_t lst_match_index{-1};
     mutable std::vector<vtab_column> lst_cols;
     std::vector<logline_value> lst_line_values_cache;
+    auto_buffer lst_mismatch_bitmap{auto_buffer::alloc_bitmap(0)};
+    int32_t lst_index_generation{0};
 };
 
 #endif

src/log_vtab_impl.cc

@@ -424,6 +424,61 @@ vt_eof(sqlite3_vtab_cursor* cur)
     return vc->log_cursor.is_eof();
 }
 
+static void
+populate_indexed_columns(vtab_cursor* vc, vtab* vt)
+{
+    if (vc->log_cursor.is_eof() || vc->log_cursor.lc_indexed_columns.empty()) {
+        return;
+    }
+
+    logfile* lf = nullptr;
+
+    for (const auto& ic : vc->log_cursor.lc_indexed_columns) {
+        auto& ci = vt->vi->vi_column_indexes[ic.cc_column];
+
+        if (vc->log_cursor.lc_curr_line < ci.ci_max_line) {
+            continue;
+        }
+
+        if (lf == nullptr) {
+            content_line_t cl(vt->lss->at(vc->log_cursor.lc_curr_line));
+            uint64_t line_number;
+            auto ld = vt->lss->find_data(cl, line_number);
+            lf = (*ld)->get_file_ptr();
+            auto ll = lf->begin() + line_number;
+
+            lf->read_full_message(ll, vc->log_msg);
+            vt->vi->extract(lf, line_number, vc->log_msg, vc->line_values);
+        }
+
+        int sub_col = ic.cc_column - VT_COL_MAX;
+        auto lv_iter = find_if(vc->line_values.begin(),
+                               vc->line_values.end(),
+                               logline_value_cmp(nullptr, sub_col));
+        if (lv_iter == vc->line_values.end()
+            || lv_iter->lv_meta.lvm_kind == value_kind_t::VALUE_NULL)
+        {
+            continue;
+        }
+
+        auto value = lv_iter->to_string();
+
+#ifdef DEBUG_INDEXING
+        log_debug("updated index for column %d %s -> %d",
+                  ic.cc_column,
+                  value.c_str(),
+                  (int) vc->log_cursor.lc_curr_line);
+#endif
+
+        if (ci.ci_value_to_lines[value].empty()
+            || ci.ci_value_to_lines[value].back()
+                != vc->log_cursor.lc_curr_line)
+        {
+            ci.ci_value_to_lines[value].push_back(vc->log_cursor.lc_curr_line);
+        }
+    }
+}
+
 static int
 vt_next(sqlite3_vtab_cursor* cur)
 {
@@ -432,7 +487,12 @@ vt_next(sqlite3_vtab_cursor* cur)
     auto done = false;
 
     vc->line_values.clear();
-    vc->log_cursor.lc_curr_line += 1_vl;
+    if (!vc->log_cursor.lc_indexed_lines.empty()) {
+        vc->log_cursor.lc_curr_line = vc->log_cursor.lc_indexed_lines.back();
+        vc->log_cursor.lc_indexed_lines.pop_back();
+    } else {
+        vc->log_cursor.lc_curr_line += 1_vl;
+    }
     vc->log_cursor.lc_sub_index = 0;
     do {
         log_cursor_latest = vc->log_cursor;
@@ -443,12 +503,6 @@ vt_next(sqlite3_vtab_cursor* cur)
             break;
         }
 
-        if (!vc->log_cursor.lc_indexed_lines.empty()) {
-            vc->log_cursor.lc_curr_line
-                = vc->log_cursor.lc_indexed_lines.back();
-            vc->log_cursor.lc_indexed_lines.pop_back();
-        }
-
         while (vc->log_cursor.lc_curr_line != -1_vl && !vc->log_cursor.is_eof()
                && !vt->vi->is_valid(vc->log_cursor, *vt->lss))
         {
@@ -459,8 +513,16 @@ vt_next(sqlite3_vtab_cursor* cur)
             done = true;
         } else {
             done = vt->vi->next(vc->log_cursor, *vt->lss);
-            if (!done) {
-                vc->log_cursor.lc_curr_line += 1_vl;
+            if (done) {
+                populate_indexed_columns(vc, vt);
+            } else {
+                if (!vc->log_cursor.lc_indexed_lines.empty()) {
+                    vc->log_cursor.lc_curr_line
+                        = vc->log_cursor.lc_indexed_lines.back();
+                    vc->log_cursor.lc_indexed_lines.pop_back();
+                } else {
+                    vc->log_cursor.lc_curr_line += 1_vl;
+                }
                 vc->log_cursor.lc_sub_index = 0;
             }
         }
@@ -486,15 +548,25 @@ vt_next_no_rowid(sqlite3_vtab_cursor* cur)
             break;
         }
 
-        if (!vc->log_cursor.lc_indexed_lines.empty()) {
-            vc->log_cursor.lc_curr_line
-                = vc->log_cursor.lc_indexed_lines.back();
-            vc->log_cursor.lc_indexed_lines.pop_back();
-        }
-
         done = vt->vi->next(vc->log_cursor, *vt->lss);
-        if (!done) {
-            vc->log_cursor.lc_curr_line += 1_vl;
+        if (done) {
+            populate_indexed_columns(vc, vt);
+        } else if (vc->log_cursor.is_eof()) {
+            done = true;
+        } else {
+            require(vc->log_cursor.lc_curr_line < vt->lss->text_line_count());
+
+            if (!vc->log_cursor.lc_indexed_lines.empty()) {
+                vc->log_cursor.lc_curr_line
+                    = vc->log_cursor.lc_indexed_lines.back();
+                vc->log_cursor.lc_indexed_lines.pop_back();
+#ifdef DEBUG_INDEXING
+                log_debug("going to next line from index %d",
+                          (int) vc->log_cursor.lc_curr_line);
+#endif
+            } else {
+                vc->log_cursor.lc_curr_line += 1_vl;
+            }
             vc->log_cursor.lc_sub_index = 0;
             for (auto& col_constraint : vc->log_cursor.lc_indexed_columns) {
                 vt->vi->vi_column_indexes[col_constraint.cc_column].ci_max_line
@@ -946,31 +1018,6 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
                             case value_kind_t::VALUE_TEXT:
                             case value_kind_t::VALUE_XML:
                             case value_kind_t::VALUE_TIMESTAMP: {
-                                auto& ci = vt->vi->vi_column_indexes[col];
-                                auto find_res
-                                    = vc->log_cursor.lc_indexed_columns
-                                    | lnav::itertools::find_if(
-                                          [col](const auto& elem) {
-                                              return elem.cc_column == col;
-                                          });
-                                if (find_res
-                                    && vc->log_cursor.lc_curr_line
-                                        >= ci.ci_max_line) {
-                                    std::string value{lv_iter->text_value(),
-                                                      lv_iter->text_length()};
-#ifdef DEBUG_INDEXING
-                                    log_debug(
-                                        "updated index for column %d %s -> %d",
-                                        col,
-                                        value.c_str(),
-                                        (int) vc->log_cursor.lc_curr_line);
-#endif
-
-                                    ci.ci_value_to_lines[value].push_back(
-                                        vc->log_cursor.lc_curr_line);
-                                    ci.ci_max_line
-                                        = vc->log_cursor.lc_curr_line + 1_vl;
-                                }
                                 sqlite3_result_text(ctx,
                                                     lv_iter->text_value(),
                                                     lv_iter->text_length(),
@@ -1520,6 +1567,10 @@ vt_filter(sqlite3_vtab_cursor* p_vtc,
                         continue;
                     }
 
+                    if (vl < p_cur->log_cursor.lc_curr_line) {
+                        continue;
+                    }
+
 #ifdef DEBUG_INDEXING
                     log_debug("adding indexed line %d", (int) vl);
 #endif
@@ -1537,6 +1588,18 @@ vt_filter(sqlite3_vtab_cursor* p_vtc,
                   p_cur->log_cursor.lc_indexed_lines.end(),
                   std::greater<>());
 
+        if (max_indexed_line
+            && max_indexed_line.value() < vt->lss->text_line_count()) {
+            log_debug("max indexed out of sync, clearing other indexes");
+            p_cur->log_cursor.lc_level_constraint = nonstd::nullopt;
+            p_cur->log_cursor.lc_curr_line = 0_vl;
+            opid_val = nonstd::nullopt;
+            log_time_range = nonstd::nullopt;
+            p_cur->log_cursor.lc_indexed_lines.clear();
+            log_path_constraints.clear();
+            log_unique_path_constraints.clear();
+        }
+
 #ifdef DEBUG_INDEXING
         log_debug("indexed lines:");
         for (auto indline : p_cur->log_cursor.lc_indexed_lines) {
@@ -1582,12 +1645,12 @@ vt_filter(sqlite3_vtab_cursor* p_vtc,
     p_cur->log_cursor.lc_log_path = std::move(log_path_constraints);
     p_cur->log_cursor.lc_unique_path = std::move(log_unique_path_constraints);
 
-    vt->vi->filter(p_cur->log_cursor, *vt->lss);
-
     if (p_cur->log_cursor.lc_indexed_lines.empty()) {
         p_cur->log_cursor.lc_indexed_lines.push_back(
             p_cur->log_cursor.lc_curr_line);
     }
+    vt->vi->filter(p_cur->log_cursor, *vt->lss);
+
     auto rc = vt->base.pModule->xNext(p_vtc);
 
 #ifdef DEBUG_INDEXING

src/log_vtab_impl.hh

@@ -266,7 +266,7 @@ protected:
 };
 
 using sql_progress_callback_t = int (*)(const log_cursor&);
-typedef void (*sql_progress_finished_callback_t)();
+using sql_progress_finished_callback_t = void (*)();
 
 struct _log_vtab_data {
     sql_progress_callback_t lvd_progress;

src/sql_util.cc

@@ -888,6 +888,8 @@ int
 guess_type_from_pcre(const std::string& pattern, std::string& collator)
 {
     try {
+        static const std::vector<int> number_matches = {1, 2};
+
         pcrepp re(pattern);
         std::vector<int> matches;
         int retval = SQLITE3_TEXT;
@@ -910,6 +912,9 @@ guess_type_from_pcre(const std::string& pattern, std::string& collator)
         if (matches.size() == 1) {
             retval = TYPE_TEST_VALUE[matches.front()].sqlite_type;
             collator = TYPE_TEST_VALUE[matches.front()].collator;
+        } else if (matches == number_matches) {
+            retval = SQLITE_FLOAT;
+            collator = "";
         }
 
         return retval;

test/expected/expected.am

@@ -700,6 +700,10 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_sql_json_func.sh_f34205b59e04f261897ad89f659595c743a18ca9.out \
     $(srcdir)/%reldir%/test_sql_json_func.sh_f34f5dfa938a1ac7721f924beb16bbceec127a1b.err \
     $(srcdir)/%reldir%/test_sql_json_func.sh_f34f5dfa938a1ac7721f924beb16bbceec127a1b.out \
+    $(srcdir)/%reldir%/test_sql_search_table.sh_1a0d872ebc492fcecb2e79a0993170d5fc771a5b.err \
+    $(srcdir)/%reldir%/test_sql_search_table.sh_1a0d872ebc492fcecb2e79a0993170d5fc771a5b.out \
+    $(srcdir)/%reldir%/test_sql_search_table.sh_5aaae556ecb1661602f176215e28f661d3404032.err \
+    $(srcdir)/%reldir%/test_sql_search_table.sh_5aaae556ecb1661602f176215e28f661d3404032.out \
     $(srcdir)/%reldir%/test_sql_search_table.sh_df0fd242f57a96d40f466493938cda0789a094fa.err \
     $(srcdir)/%reldir%/test_sql_search_table.sh_df0fd242f57a96d40f466493938cda0789a094fa.out \
     $(srcdir)/%reldir%/test_sql_search_table.sh_ef9373a76853f345d06234f6e0fe11b5d40da27b.err \

test/expected/test_sql_search_table.sh_1a0d872ebc492fcecb2e79a0993170d5fc771a5b.out

@@ -0,0 +1,2 @@
+log_line log_part        log_time         log_idle_msecs log_level log_mark log_comment log_tags log_filters  comp   opid    tid   user   item  prc  reason  req    sid         src        sub   vpxa_update  line   file  match_index   lro_id       entity                 operation                           SessionId                           SessionSubId             
+       2   <NULL> 2022-06-02 11:58:12.376            182 info             0      <NULL>   <NULL>      <NULL> <NULL> e3979f6 45709 <NULL> <NULL> vpxd <NULL> <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846064 SessionManager vim.SessionManager.sessionIsActive 52626140-422b-6287-b4e4-344192c6a01d 523e0a4b-6e83-6bcd-9342-22502dd89866 

test/expected/test_sql_search_table.sh_5aaae556ecb1661602f176215e28f661d3404032.out

@@ -0,0 +1,4 @@
+log_line log_part        log_time         log_idle_msecs log_level log_mark log_comment log_tags log_filters match_index user pid cpu_pct mem_pct vsz rss tty stat start_time cpu_time    cmd      cmd_name  cmd_args 
+       0   <NULL> 2022-06-02 00:01:01.000              0 info             0      <NULL>   <NULL>      <NULL>           1 root   2     0.0     0.0   0   0 ?   S    Jun01      0:00     [kthreadd] [kthreadd]          
+      12   <NULL> 2022-06-02 00:02:01.000          60000 info             0      <NULL>   <NULL>      <NULL>           1 root   2     0.0     0.0   0   0 ?   S    Jun01      0:00     [kthreadd] [kthreadd]          
+      30   <NULL> 2022-06-02 00:03:01.000          60000 info             0      <NULL>   <NULL>      <NULL>           1 root   2     0.0     0.0   0   0 ?   S    Jun01      0:00     [kthreadd] [kthreadd]          

test/expected/test_sql_search_table.sh_ef9373a76853f345d06234f6e0fe11b5d40da27b.out

@@ -1,6 +1,6 @@
-log_line log_part        log_time         log_idle_msecs log_level log_mark log_comment log_tags log_filters  comp             opid              tid   user   item  prc  reason  sid         src        sub   vpxa_update  line   file  match_index   lro_id         entity                       operation                              SessionId                           SessionSubId                                                                     log_body                                                         
-       0   <NULL> 2022-06-02 11:58:12.193              0 info             0      <NULL>   <NULL>      <NULL> <NULL> 7e1280cf                    45715 <NULL> <NULL> vpxd <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846063 SessionManager     vim.SessionManager.sessionIsActive        528e6e0c-246d-58b5-3234-278c6e0c5d0d 52c289ac-2563-48d5-8a8e-f178da022c0d [VpxLRO] -- BEGIN lro-846063 -- SessionManager -- vim.Sessio⋯8b5-3234-278c6e0c5d0d(52c289ac-2563-48d5-8a8e-f178da022c0d) 
-       2   <NULL> 2022-06-02 11:58:12.376            182 info             0      <NULL>   <NULL>      <NULL> <NULL> e3979f6                     45709 <NULL> <NULL> vpxd <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846064 SessionManager     vim.SessionManager.sessionIsActive        52626140-422b-6287-b4e4-344192c6a01d 523e0a4b-6e83-6bcd-9342-22502dd89866 [VpxLRO] -- BEGIN lro-846064 -- SessionManager -- vim.Sessio⋯287-b4e4-344192c6a01d(523e0a4b-6e83-6bcd-9342-22502dd89866) 
-       4   <NULL> 2022-06-02 11:58:12.623            246 info             0      <NULL>   <NULL>      <NULL> <NULL> l3wrhr4o-cbf-h5:70001034-60 47524 <NULL> <NULL> vpxd <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846066 ChangeLogCollector vim.cdc.ChangeLogCollector.waitForChanges 526861fc-0c28-1930-ae5e-d8c2772bf8c2 52a7a308-9646-c054-f1e7-16131c1a7db6 [VpxLRO] -- BEGIN lro-846066 -- ChangeLogCollector -- vim.c⋯1930-ae5e-d8c2772bf8c2(52a7a308-9646-c054-f1e7-16131c1a7db6) 
-       6   <NULL> 2022-06-02 11:58:12.736            113 info             0      <NULL>   <NULL>      <NULL> <NULL> 499b440                     48432 <NULL> <NULL> vpxd <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846067 SessionManager     vim.SessionManager.sessionIsActive        521fe9f6-d061-11a2-ac86-badb3c071373 524cba9b-2cc4-9b70-32e4-421452a404d7 [VpxLRO] -- BEGIN lro-846067 -- SessionManager -- vim.Sessio⋯1a2-ac86-badb3c071373(524cba9b-2cc4-9b70-32e4-421452a404d7) 
-       8   <NULL> 2022-06-02 11:58:12.740              4 info             0      <NULL>   <NULL>      <NULL> <NULL> 55a419df                    48035 <NULL> <NULL> vpxd <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846068 SessionManager     vim.SessionManager.sessionIsActive        52585600-b0bc-76b1-c4d5-4d7708671c5e 523b68ba-e312-9909-a3ca-39cc86aaf206 [VpxLRO] -- BEGIN lro-846068 -- SessionManager -- vim.Sessio⋯6b1-c4d5-4d7708671c5e(523b68ba-e312-9909-a3ca-39cc86aaf206) 
+log_line log_part        log_time         log_idle_msecs log_level log_mark log_comment log_tags log_filters  comp             opid              tid   user   item  prc  reason  req    sid         src        sub   vpxa_update  line   file  match_index   lro_id         entity                       operation                              SessionId                           SessionSubId                                                                     log_body                                                         
+       0   <NULL> 2022-06-02 11:58:12.193              0 info             0      <NULL>   <NULL>      <NULL> <NULL> 7e1280cf                    45715 <NULL> <NULL> vpxd <NULL> <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846063 SessionManager     vim.SessionManager.sessionIsActive        528e6e0c-246d-58b5-3234-278c6e0c5d0d 52c289ac-2563-48d5-8a8e-f178da022c0d [VpxLRO] -- BEGIN lro-846063 -- SessionManager -- vim.Sessio⋯8b5-3234-278c6e0c5d0d(52c289ac-2563-48d5-8a8e-f178da022c0d) 
+       2   <NULL> 2022-06-02 11:58:12.376            182 info             0      <NULL>   <NULL>      <NULL> <NULL> e3979f6                     45709 <NULL> <NULL> vpxd <NULL> <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846064 SessionManager     vim.SessionManager.sessionIsActive        52626140-422b-6287-b4e4-344192c6a01d 523e0a4b-6e83-6bcd-9342-22502dd89866 [VpxLRO] -- BEGIN lro-846064 -- SessionManager -- vim.Sessio⋯287-b4e4-344192c6a01d(523e0a4b-6e83-6bcd-9342-22502dd89866) 
+       4   <NULL> 2022-06-02 11:58:12.623            246 info             0      <NULL>   <NULL>      <NULL> <NULL> l3wrhr4o-cbf-h5:70001034-60 47524 <NULL> <NULL> vpxd <NULL> <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846066 ChangeLogCollector vim.cdc.ChangeLogCollector.waitForChanges 526861fc-0c28-1930-ae5e-d8c2772bf8c2 52a7a308-9646-c054-f1e7-16131c1a7db6 [VpxLRO] -- BEGIN lro-846066 -- ChangeLogCollector -- vim.c⋯1930-ae5e-d8c2772bf8c2(52a7a308-9646-c054-f1e7-16131c1a7db6) 
+       6   <NULL> 2022-06-02 11:58:12.736            113 info             0      <NULL>   <NULL>      <NULL> <NULL> 499b440                     48432 <NULL> <NULL> vpxd <NULL> <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846067 SessionManager     vim.SessionManager.sessionIsActive        521fe9f6-d061-11a2-ac86-badb3c071373 524cba9b-2cc4-9b70-32e4-421452a404d7 [VpxLRO] -- BEGIN lro-846067 -- SessionManager -- vim.Sessio⋯1a2-ac86-badb3c071373(524cba9b-2cc4-9b70-32e4-421452a404d7) 
+       8   <NULL> 2022-06-02 11:58:12.740              4 info             0      <NULL>   <NULL>      <NULL> <NULL> 55a419df                    48035 <NULL> <NULL> vpxd <NULL> <NULL> <NULL> Originator@6876 vpxLro      <NULL> <NULL> <NULL>           0 lro-846068 SessionManager     vim.SessionManager.sessionIsActive        52585600-b0bc-76b1-c4d5-4d7708671c5e 523b68ba-e312-9909-a3ca-39cc86aaf206 [VpxLRO] -- BEGIN lro-846068 -- SessionManager -- vim.Sessio⋯6b1-c4d5-4d7708671c5e(523b68ba-e312-9909-a3ca-39cc86aaf206) 

test/test_auto_mem.cc

@@ -98,5 +98,18 @@ main(int argc, char* argv[])
         assert(last_lf_index == (last_lf_rchr - msg));
     }
 
+    {
+        auto bitmap = auto_buffer::alloc_bitmap(15);
+
+        assert(bitmap.capacity() == 2);
+        bitmap.resize_bitmap(15);
+        assert(bitmap.size() == 2);
+
+        memset(bitmap.in(), 0, bitmap.size());
+        for (size_t lpc = 0; lpc < 15; lpc++) {
+            assert(!bitmap.is_bit_set(lpc));
+        }
+    }
+
     return retval;
 }

test/test_sql_search_table.sh

@@ -9,3 +9,13 @@ run_cap_test ${lnav_test} -n \
 run_cap_test ${lnav_test} -n \
     -c ';SELECT *,log_body FROM vpx_lro_begin' \
     ${test_dir}/logfile_vpxd.0
+
+run_cap_test ${lnav_test} -n \
+    -c ";select * from vpx_lro_begin where log_line > 3 and lro_id = 'lro-846064'" \
+    -c ";select * from vpx_lro_begin where lro_id = 'lro-846064'" \
+    ${test_dir}/logfile_vpxd.0
+
+run_cap_test ${lnav_test} -n \
+    -c ";select * from procstate_procs where cmd_name = '[kthreadd]'" \
+    -c ";select * from procstate_procs where cmd_name = '[kthreadd]'" \
+    ${test_dir}/logfile_procstate.0