[tests] text file handling and make sure the progress status bar is updated during SQL processing

src/bottom_status_source.cc

@@ -188,7 +188,7 @@ bottom_status_source::update_loading(file_off_t off, file_size_t total)
     require(off >= 0);
     require(off <= total);
 
-    if (total == 0 || (size_t) off == total) {
+    if (total == 0) {
         sf.set_cylon(false);
         sf.set_role(role_t::VCR_STATUS);
         if (this->bss_paused) {
@@ -196,6 +196,21 @@ bottom_status_source::update_loading(file_off_t off, file_size_t total)
         } else {
             sf.clear();
         }
+    } else if ((size_t) off == total) {
+        static const std::vector<std::string> DOTS = {
+            "",
+            ".",
+            "..",
+            "...",
+            "..",
+            ".",
+        };
+
+        this->bss_load_percent += 1;
+        sf.set_cylon(true);
+        sf.set_role(role_t::VCR_ACTIVE_STATUS2);
+        sf.set_value(" Working%s ",
+                     DOTS[this->bss_load_percent % DOTS.size()].c_str());
     } else {
         int pct = (int) (((double) off / (double) total) * 100.0);
 

src/lnav.cc

@@ -1041,7 +1041,11 @@ looper()
 
         lnav_data.ld_log_source.lss_sorting_observer
             = [](auto& lss, auto off, auto size) {
-                  lnav_data.ld_bottom_source.update_loading(off, size);
+                  if (off == size) {
+                      lnav_data.ld_bottom_source.update_loading(0, 0);
+                  } else {
+                      lnav_data.ld_bottom_source.update_loading(off, size);
+                  }
                   do_observer_update(nullptr);
               };
 
@@ -2907,6 +2911,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                 text_tc = &lnav_data.ld_views[LNV_TEXT];
                 text_tc->set_top(0_vl);
                 text_tc->set_height(vis_line_t(text_tc->get_inner_height()));
+                setup_highlights(lnav_data.ld_views[LNV_TEXT].get_highlights());
                 if (lnav_data.ld_log_source.text_line_count() == 0
                     && lnav_data.ld_text_source.text_line_count() > 0)
                 {

src/lnav.indexing.cc

@@ -61,7 +61,11 @@ public:
         if ((((size_t) off == total) && (this->lo_last_offset != off))
             || ui_periodic_timer::singleton().time_to_update(index_counter))
         {
-            lnav_data.ld_bottom_source.update_loading(off, total);
+            if (off == total) {
+                lnav_data.ld_bottom_source.update_loading(0, 0);
+            } else {
+                lnav_data.ld_bottom_source.update_loading(off, total);
+            }
             do_observer_update(lf);
             this->lo_last_offset = off;
         }

src/log_level.cc

@@ -33,23 +33,25 @@
 
 #include "config.h"
 
-const char* level_names[LEVEL__MAX + 1] = {"unknown",
-                                           "trace",
-                                           "debug5",
-                                           "debug4",
-                                           "debug3",
-                                           "debug2",
-                                           "debug",
-                                           "info",
-                                           "stats",
-                                           "notice",
-                                           "warning",
-                                           "error",
-                                           "critical",
-                                           "fatal",
-                                           "invalid",
+const char* level_names[LEVEL__MAX + 1] = {
+    "unknown",
+    "trace",
+    "debug5",
+    "debug4",
+    "debug3",
+    "debug2",
+    "debug",
+    "info",
+    "stats",
+    "notice",
+    "warning",
+    "error",
+    "critical",
+    "fatal",
+    "invalid",
 
-                                           nullptr};
+    nullptr,
+};
 
 log_level_t
 abbrev2level(const char* levelstr, ssize_t len)

test/Makefile.am

@@ -204,6 +204,7 @@ dist_noinst_SCRIPTS = \
 	test_sql_time_func.sh \
 	test_sql_views_vtab.sh \
 	test_sql_xml_func.sh \
+	test_text_file.sh \
 	test_tui.sh \
 	test_view_colors.sh \
 	test_vt52_curses.sh \
@@ -401,6 +402,7 @@ TESTS = \
 	test_sql_time_func.sh \
 	test_sql_views_vtab.sh \
 	test_sql_xml_func.sh \
+	test_text_file.sh \
 	test_tui.sh \
 	test_data_parser.sh \
 	test_pretty_print.sh \

test/expected/expected.am

@@ -922,4 +922,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_sql_xml_func.sh_b036c73528a446cba46625767517cdac868aba72.out \
     $(srcdir)/%reldir%/test_sql_xml_func.sh_fefeb387ae14d4171225ea06cbbff3ec43990cf0.err \
     $(srcdir)/%reldir%/test_sql_xml_func.sh_fefeb387ae14d4171225ea06cbbff3ec43990cf0.out \
+    $(srcdir)/%reldir%/test_text_file.sh_801414c6bb6d3f9225973eafa3c6dfa49cd2081d.err \
+    $(srcdir)/%reldir%/test_text_file.sh_801414c6bb6d3f9225973eafa3c6dfa49cd2081d.out \
+    $(srcdir)/%reldir%/test_text_file.sh_c21295f131c221861568bda5014b76ef99bdd11f.err \
+    $(srcdir)/%reldir%/test_text_file.sh_c21295f131c221861568bda5014b76ef99bdd11f.out \
     $()

test/expected/test_cmds.sh_ac45fb0f8f9578c3ded0855f694698ec38ce31ad.out

@@ -1,12 +1,12 @@
 {
-  "foo bar": null,
-  "array": [
+  "foo bar": null,
+  "array": [
     1,
     2,
     3
   ],
-  "obj": {
-    "one": 1,
-    "two": true
+  "obj": {
+    "one": 1,
+    "two": true
   }
 }

test/expected/test_text_file.sh_801414c6bb6d3f9225973eafa3c6dfa49cd2081d.out

@@ -0,0 +1,111 @@
+/**
+ * Copyright (c) 2018, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "log_level.hh"
+
+#include <ctype.h>
+
+#include "config.h"
+
+const char* level_names[LEVEL__MAX + 1] = {
+    "unknown",
+    "trace",
+    "debug5",
+    "debug4",
+    "debug3",
+    "debug2",
+    "debug",
+    "info",
+    "stats",
+    "notice",
+    "warning",
+    "error",
+    "critical",
+    "fatal",
+    "invalid",
+
+    nullptr,
+};
+
+log_level_t
+abbrev2level(const char* levelstr, ssize_t len)
+{
+    if (len == 0 || levelstr[0] == '\0') {
+        return LEVEL_UNKNOWN;
+    }
+
+    switch (toupper(levelstr[0])) {
+        case 'T':
+            return LEVEL_TRACE;
+        case 'D':
+        case 'V':
+            if (len > 1) {
+                switch (levelstr[len - 1]) {
+                    case '2':
+                        return LEVEL_DEBUG2;
+                    case '3':
+                        return LEVEL_DEBUG3;
+                    case '4':
+                        return LEVEL_DEBUG4;
+                    case '5':
+                        return LEVEL_DEBUG5;
+                }
+            }
+            return LEVEL_DEBUG;
+        case 'I':
+            if (len == 7 && toupper(levelstr[1]) == 'N'
+                && toupper(levelstr[2]) == 'V' && toupper(levelstr[3]) == 'A'
+                && toupper(levelstr[4]) == 'L' && toupper(levelstr[5]) == 'I'
+                && toupper(levelstr[6]) == 'D')
+            {
+                return LEVEL_INVALID;
+            }
+            return LEVEL_INFO;
+        case 'S':
+            return LEVEL_STATS;
+        case 'N':
+            return LEVEL_NOTICE;
+        case 'W':
+            return LEVEL_WARNING;
+        case 'E':
+            return LEVEL_ERROR;
+        case 'C':
+            return LEVEL_CRITICAL;
+        case 'F':
+            return LEVEL_FATAL;
+        default:
+            return LEVEL_UNKNOWN;
+    }
+}
+
+int
+levelcmp(const char* l1, ssize_t l1_len, const char* l2, ssize_t l2_len)
+{
+    return abbrev2level(l1, l1_len) - abbrev2level(l2, l2_len);
+}

test/expected/test_text_file.sh_c21295f131c221861568bda5014b76ef99bdd11f.out

@@ -0,0 +1,159 @@
+Build[1][2] Docs[3][4] Coverage Status[5][6] lnav[7][8]
+
+ ▌[1] - https://github.com/tstack/lnav/workflows/ci-build/badge.svg               
+ ▌[2] - https://github.com/tstack/lnav/actions?query=workflow%3Aci-build          
+ ▌[3] - https://readthedocs.org/projects/lnav/badge/?version=latest&style=plastic 
+ ▌[4] - https://docs.lnav.org                                                     
+ ▌[5] - https://coveralls.io/repos/github/tstack/lnav/badge.svg?branch=master     
+ ▌[6] - https://coveralls.io/github/tstack/lnav?branch=master                     
+ ▌[7] - https://snapcraft.io//lnav/badge.svg                                      
+ ▌[8] - https://snapcraft.io/lnav                                                 
+
+This is the source repository for lnav, visit https://lnav.org[1] for
+a high level overview.
+
+ ▌[1] - https://lnav.org 
+
+LNAV – The Logfile Navigator
+
+The Log File Navigator, lnav for short, is an advanced log file viewer
+for the small-scale.  It is a terminal application that can understand
+your log files and make it easy for you to find problems with little
+to no setup.
+
+Screenshot
+
+The following screenshot shows a syslog file. Log lines are displayed
+with highlights. Errors are red and warnings are yellow.
+
+Screenshot[1][2]
+
+ ▌[1] - file://{top_srcdir}/docs/assets/images/lnav-syslog-thumb.png 
+ ▌[2] - file://{top_srcdir}/docs/assets/images/lnav-syslog.png       
+
+Features
+
+ • Log messages from different files are collated together
+   into a single view
+ • Automatic detection of log format
+ • Automatic decompression of GZip and BZip2 files
+ • Filter log messages based on regular expressions
+ • Use SQL to analyze your logs
+ • And more...
+
+Installation
+
+Download a statically-linked binary for Linux/MacOS from the release
+page[1]
+
+ ▌[1] - https://github.com/tstack/lnav/releases/latest#release-artifacts 
+
+Usage
+
+The only file installed is the executable,  lnav .  You can execute it
+with no arguments to view the default set of files:
+
+ ▌$ lnav                                  
+
+You can view all the syslog messages by running:
+
+ ▌$ lnav /var/log/messages*               
+
+Usage with  systemd-journald 
+
+On systems running  systemd-journald , you can use  lnav  as the
+pager:
+
+ ▌$ journalctl | lnav                     
+
+or in follow mode:
+
+ ▌$ journalctl -f | lnav                  
+
+Since  journalctl 's default output format omits the year, if you are
+viewing logs which span multiple years you will need to change the
+output format to include the year, otherwise  lnav  gets confused:
+
+ ▌$ journalctl -o short-iso | lnav        
+
+It is also possible to use  journalctl 's json output format and  lnav
+will make use of additional fields such as PRIORITY and _SYSTEMD_UNIT:
+
+ ▌$ journalctl -o json | lnav             
+
+In case some MESSAGE fields contain special characters such as ANSI
+color codes which are considered as unprintable by journalctl,
+specifying  journalctl 's  -a  option might be preferable in order to
+output those messages still in a non binary representation:
+
+ ▌$ journalctl -a -o json | lnav          
+
+If using systemd v236 or newer, the output fields can be limited to
+the ones actually recognized by  lnav  for increased efficiency:
+
+ ▌$ journalctl -o json --output-fields=MESSAGE,PRIORITY,_PID,SYSLOG_IDENTIFIER,_SYSTEMD_UNIT | lnav    
+
+If your system has been running for a long time, for increased
+efficiency you may want to limit the number of log lines fed into  lnav
+, e.g. via  journalctl 's  -n  or  --since=...  options.
+
+In case of a persistent journal, you may want to limit the number of
+log lines fed into  lnav  via  journalctl 's  -b  option.
+
+Links
+
+ • Main Site[1]
+ • Documentation[2] on Read the Docs
+ • Internal Architecture[3]
+
+ ▌[1] - https://lnav.org                                 
+ ▌[2] - https://docs.lnav.org                            
+ ▌[3] - file://{top_srcdir}/ARCHITECTURE.md 
+
+Contributing
+
+ • Become a Sponsor on GitHub[1]
+
+ ▌[1] - https://github.com/sponsors/tstack 
+
+Building From Source
+
+Prerequisites
+
+The following software packages are required to build lnav:
+
+ • gcc/clang  - A C++14-compatible compiler.
+ • libpcre    - The Perl Compatible Regular Expression
+   (PCRE) library.
+ • sqlite     - The SQLite database engine.  Version 3.9.0
+   or higher is required.
+ • ncurses    - The ncurses text UI library.
+ • readline   - The readline line editing library.
+ • zlib       - The zlib compression library.
+ • bz2        - The bzip2 compression library.
+ • libcurl    - The cURL library for downloading files
+   from URLs.  Version 7.23.0 or higher is required.
+ • libarchive - The libarchive library for opening archive
+   files, like zip/tgz.
+ • wireshark  - The 'tshark' program is used to interpret
+   pcap files.
+
+Build
+
+Lnav follows the usual GNU style for configuring and installing
+software:
+
+Run  ./autogen.sh  if compiling from a cloned repository.
+
+ ▌$ ./configure                           
+ ▌$ make                                  
+ ▌$ sudo make install                     
+
+See Also
+
+Angle-grinder[1] is a tool to slice and dice log files on the
+command-line. If you're familiar with the SumoLogic query language,
+you might find this tool more comfortable to work with.
+
+ ▌[1] - https://github.com/rcoh/angle-grinder 
+

test/test_text_file.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+export YES_COLOR=1
+unset XDG_CONFIG_HOME
+
+run_cap_test ${lnav_test} -n \
+    ${top_srcdir}/README.md
+
+run_cap_test ${lnav_test} -n \
+    ${top_srcdir}/src/log_level.cc