[pretty-print] allow formats to do transforms before pretty-printing

Also started upgrading to C++11.

Fixes #353

CMakeLists.txt

@@ -1,5 +1,6 @@
 
 cmake_minimum_required (VERSION 2.6)
+SET(CMAKE_CXX_STANDARD 11)
 project (lnav)
 add_subdirectory(src)
 add_subdirectory(test)

NEWS

@@ -8,6 +8,10 @@ lnav v0.8.2:
        give you more control over how the displayed line looks.
      * Added a "hidden" option to log format values so that you can hide JSON
        log fields from being displayed if they are not in the line format.
+     * Added a "rewriter" field to log format value definitions that is a
+       command used to rewrite the field in the pretty-printed version of a
+       log message.  For example, the HTTP access log format will rewrite the
+       status code field to include the textual version (e.g. 200 (OK)).
 
      Interface Changes:
      * The color used for text colored via ":highlight" is now based on the

configure.ac

@@ -8,6 +8,8 @@ AC_PREFIX_DEFAULT(/usr)
 
 AC_CANONICAL_HOST
 
+AX_CXX_COMPILE_STDCXX_11([noext], [mandatory])
+
 for defdir in /opt/local /usr/local /usr /; do
     if test -d "$defdir/include"; then
         CPPFLAGS="$CPPFLAGS -I$defdir/include"

docs/source/formats.rst

@@ -158,6 +158,16 @@ fields:
       not be graphed.  This should only need to be set for integer fields.
     :hidden: A boolean for JSON log fields that indicates whether they should
       be displayed if they are not present in the line-format.
+    :rewriter: A command to rewrite this field when pretty-printing log
+      messages containing this value.  The command must start with ':', ';',
+      or '|' to signify whether it is a regular command, SQL query, or a script
+      to be executed.  The other fields in the line are accessible in SQL by
+      using the ':' prefix.  The text value of this field will then be replaced
+      with the result of the command when pretty-printing.  For example, the
+      HTTP access log format will rewrite the status code field to include the
+      textual version (e.g. 200 (OK)) using the following SQL query::
+
+         ;SELECT :sc_status || ' (' || (SELECT message FROM http_status_codes WHERE status = :sc_status) || ') '
 
   :sample: A list of objects that contain sample log messages.  All formats
     must include at least one sample and it must be matched by one of the

m4/ax_cxx_compile_stdcxx.m4

@@ -0,0 +1,562 @@
+# ===========================================================================
+#   http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional])
+#
+# DESCRIPTION
+#
+#   Check for baseline language coverage in the compiler for the specified
+#   version of the C++ standard.  If necessary, add switches to CXX and
+#   CXXCPP to enable support.  VERSION may be '11' (for the C++11 standard)
+#   or '14' (for the C++14 standard).
+#
+#   The second argument, if specified, indicates whether you insist on an
+#   extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g.
+#   -std=c++11).  If neither is specified, you get whatever works, with
+#   preference for an extended mode.
+#
+#   The third argument, if specified 'mandatory' or if left unspecified,
+#   indicates that baseline support for the specified C++ standard is
+#   required and that the macro should error out if no mode with that
+#   support is found.  If specified 'optional', then configuration proceeds
+#   regardless, after defining HAVE_CXX${VERSION} if and only if a
+#   supporting mode is found.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com>
+#   Copyright (c) 2012 Zack Weinberg <zackw@panix.com>
+#   Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu>
+#   Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com>
+#   Copyright (c) 2015 Paul Norman <penorman@mac.com>
+#   Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#serial 4
+
+dnl  This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro
+dnl  (serial version number 13).
+
+AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
+  m4_if([$1], [11], [],
+        [$1], [14], [],
+        [$1], [17], [m4_fatal([support for C++17 not yet implemented in AX_CXX_COMPILE_STDCXX])],
+        [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl
+  m4_if([$2], [], [],
+        [$2], [ext], [],
+        [$2], [noext], [],
+        [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl
+  m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true],
+        [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true],
+        [$3], [optional], [ax_cxx_compile_cxx$1_required=false],
+        [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])])
+  AC_LANG_PUSH([C++])dnl
+  ac_success=no
+  AC_CACHE_CHECK(whether $CXX supports C++$1 features by default,
+  ax_cv_cxx_compile_cxx$1,
+  [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
+    [ax_cv_cxx_compile_cxx$1=yes],
+    [ax_cv_cxx_compile_cxx$1=no])])
+  if test x$ax_cv_cxx_compile_cxx$1 = xyes; then
+    ac_success=yes
+  fi
+
+  m4_if([$2], [noext], [], [dnl
+  if test x$ac_success = xno; then
+    for switch in -std=gnu++$1 -std=gnu++0x; do
+      cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
+      AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
+                     $cachevar,
+        [ac_save_CXX="$CXX"
+         CXX="$CXX $switch"
+         AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
+          [eval $cachevar=yes],
+          [eval $cachevar=no])
+         CXX="$ac_save_CXX"])
+      if eval test x\$$cachevar = xyes; then
+        CXX="$CXX $switch"
+        if test -n "$CXXCPP" ; then
+          CXXCPP="$CXXCPP $switch"
+        fi
+        ac_success=yes
+        break
+      fi
+    done
+  fi])
+
+  m4_if([$2], [ext], [], [dnl
+  if test x$ac_success = xno; then
+    dnl HP's aCC needs +std=c++11 according to:
+    dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf
+    dnl Cray's crayCC needs "-h std=c++11"
+    for switch in -std=c++$1 -std=c++0x +std=c++$1 "-h std=c++$1"; do
+      cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
+      AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
+                     $cachevar,
+        [ac_save_CXX="$CXX"
+         CXX="$CXX $switch"
+         AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
+          [eval $cachevar=yes],
+          [eval $cachevar=no])
+         CXX="$ac_save_CXX"])
+      if eval test x\$$cachevar = xyes; then
+        CXX="$CXX $switch"
+        if test -n "$CXXCPP" ; then
+          CXXCPP="$CXXCPP $switch"
+        fi
+        ac_success=yes
+        break
+      fi
+    done
+  fi])
+  AC_LANG_POP([C++])
+  if test x$ax_cxx_compile_cxx$1_required = xtrue; then
+    if test x$ac_success = xno; then
+      AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.])
+    fi
+  fi
+  if test x$ac_success = xno; then
+    HAVE_CXX$1=0
+    AC_MSG_NOTICE([No compiler with C++$1 support was found])
+  else
+    HAVE_CXX$1=1
+    AC_DEFINE(HAVE_CXX$1,1,
+              [define if the compiler supports basic C++$1 syntax])
+  fi
+  AC_SUBST(HAVE_CXX$1)
+])
+
+
+dnl  Test body for checking C++11 support
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+)
+
+
+dnl  Test body for checking C++14 support
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
+)
+
+
+dnl  Tests for new features in C++11
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[
+
+// If the compiler admits that it is not ready for C++11, why torture it?
+// Hopefully, this will speed up the test.
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 201103L
+
+#error "This is not a C++11 compiler"
+
+#else
+
+namespace cxx11
+{
+
+  namespace test_static_assert
+  {
+
+    template <typename T>
+    struct check
+    {
+      static_assert(sizeof(int) <= sizeof(T), "not big enough");
+    };
+
+  }
+
+  namespace test_final_override
+  {
+
+    struct Base
+    {
+      virtual void f() {}
+    };
+
+    struct Derived : public Base
+    {
+      virtual void f() override {}
+    };
+
+  }
+
+  namespace test_double_right_angle_brackets
+  {
+
+    template < typename T >
+    struct check {};
+
+    typedef check<void> single_type;
+    typedef check<check<void>> double_type;
+    typedef check<check<check<void>>> triple_type;
+    typedef check<check<check<check<void>>>> quadruple_type;
+
+  }
+
+  namespace test_decltype
+  {
+
+    int
+    f()
+    {
+      int a = 1;
+      decltype(a) b = 2;
+      return a + b;
+    }
+
+  }
+
+  namespace test_type_deduction
+  {
+
+    template < typename T1, typename T2 >
+    struct is_same
+    {
+      static const bool value = false;
+    };
+
+    template < typename T >
+    struct is_same<T, T>
+    {
+      static const bool value = true;
+    };
+
+    template < typename T1, typename T2 >
+    auto
+    add(T1 a1, T2 a2) -> decltype(a1 + a2)
+    {
+      return a1 + a2;
+    }
+
+    int
+    test(const int c, volatile int v)
+    {
+      static_assert(is_same<int, decltype(0)>::value == true, "");
+      static_assert(is_same<int, decltype(c)>::value == false, "");
+      static_assert(is_same<int, decltype(v)>::value == false, "");
+      auto ac = c;
+      auto av = v;
+      auto sumi = ac + av + 'x';
+      auto sumf = ac + av + 1.0;
+      static_assert(is_same<int, decltype(ac)>::value == true, "");
+      static_assert(is_same<int, decltype(av)>::value == true, "");
+      static_assert(is_same<int, decltype(sumi)>::value == true, "");
+      static_assert(is_same<int, decltype(sumf)>::value == false, "");
+      static_assert(is_same<int, decltype(add(c, v))>::value == true, "");
+      return (sumf > 0.0) ? sumi : add(c, v);
+    }
+
+  }
+
+  namespace test_noexcept
+  {
+
+    int f() { return 0; }
+    int g() noexcept { return 0; }
+
+    static_assert(noexcept(f()) == false, "");
+    static_assert(noexcept(g()) == true, "");
+
+  }
+
+  namespace test_constexpr
+  {
+
+    template < typename CharT >
+    unsigned long constexpr
+    strlen_c_r(const CharT *const s, const unsigned long acc) noexcept
+    {
+      return *s ? strlen_c_r(s + 1, acc + 1) : acc;
+    }
+
+    template < typename CharT >
+    unsigned long constexpr
+    strlen_c(const CharT *const s) noexcept
+    {
+      return strlen_c_r(s, 0UL);
+    }
+
+    static_assert(strlen_c("") == 0UL, "");
+    static_assert(strlen_c("1") == 1UL, "");
+    static_assert(strlen_c("example") == 7UL, "");
+    static_assert(strlen_c("another\0example") == 7UL, "");
+
+  }
+
+  namespace test_rvalue_references
+  {
+
+    template < int N >
+    struct answer
+    {
+      static constexpr int value = N;
+    };
+
+    answer<1> f(int&)       { return answer<1>(); }
+    answer<2> f(const int&) { return answer<2>(); }
+    answer<3> f(int&&)      { return answer<3>(); }
+
+    void
+    test()
+    {
+      int i = 0;
+      const int c = 0;
+      static_assert(decltype(f(i))::value == 1, "");
+      static_assert(decltype(f(c))::value == 2, "");
+      static_assert(decltype(f(0))::value == 3, "");
+    }
+
+  }
+
+  namespace test_uniform_initialization
+  {
+
+    struct test
+    {
+      static const int zero {};
+      static const int one {1};
+    };
+
+    static_assert(test::zero == 0, "");
+    static_assert(test::one == 1, "");
+
+  }
+
+  namespace test_lambdas
+  {
+
+    void
+    test1()
+    {
+      auto lambda1 = [](){};
+      auto lambda2 = lambda1;
+      lambda1();
+      lambda2();
+    }
+
+    int
+    test2()
+    {
+      auto a = [](int i, int j){ return i + j; }(1, 2);
+      auto b = []() -> int { return '0'; }();
+      auto c = [=](){ return a + b; }();
+      auto d = [&](){ return c; }();
+      auto e = [a, &b](int x) mutable {
+        const auto identity = [](int y){ return y; };
+        for (auto i = 0; i < a; ++i)
+          a += b--;
+        return x + identity(a + b);
+      }(0);
+      return a + b + c + d + e;
+    }
+
+    int
+    test3()
+    {
+      const auto nullary = [](){ return 0; };
+      const auto unary = [](int x){ return x; };
+      using nullary_t = decltype(nullary);
+      using unary_t = decltype(unary);
+      const auto higher1st = [](nullary_t f){ return f(); };
+      const auto higher2nd = [unary](nullary_t f1){
+        return [unary, f1](unary_t f2){ return f2(unary(f1())); };
+      };
+      return higher1st(nullary) + higher2nd(nullary)(unary);
+    }
+
+  }
+
+  namespace test_variadic_templates
+  {
+
+    template <int...>
+    struct sum;
+
+    template <int N0, int... N1toN>
+    struct sum<N0, N1toN...>
+    {
+      static constexpr auto value = N0 + sum<N1toN...>::value;
+    };
+
+    template <>
+    struct sum<>
+    {
+      static constexpr auto value = 0;
+    };
+
+    static_assert(sum<>::value == 0, "");
+    static_assert(sum<1>::value == 1, "");
+    static_assert(sum<23>::value == 23, "");
+    static_assert(sum<1, 2>::value == 3, "");
+    static_assert(sum<5, 5, 11>::value == 21, "");
+    static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, "");
+
+  }
+
+  // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae
+  // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function
+  // because of this.
+  namespace test_template_alias_sfinae
+  {
+
+    struct foo {};
+
+    template<typename T>
+    using member = typename T::member_type;
+
+    template<typename T>
+    void func(...) {}
+
+    template<typename T>
+    void func(member<T>*) {}
+
+    void test();
+
+    void test() { func<foo>(0); }
+
+  }
+
+}  // namespace cxx11
+
+#endif  // __cplusplus >= 201103L
+
+]])
+
+
+dnl  Tests for new features in C++14
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[
+
+// If the compiler admits that it is not ready for C++14, why torture it?
+// Hopefully, this will speed up the test.
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 201402L
+
+#error "This is not a C++14 compiler"
+
+#else
+
+namespace cxx14
+{
+
+  namespace test_polymorphic_lambdas
+  {
+
+    int
+    test()
+    {
+      const auto lambda = [](auto&&... args){
+        const auto istiny = [](auto x){
+          return (sizeof(x) == 1UL) ? 1 : 0;
+        };
+        const int aretiny[] = { istiny(args)... };
+        return aretiny[0];
+      };
+      return lambda(1, 1L, 1.0f, '1');
+    }
+
+  }
+
+  namespace test_binary_literals
+  {
+
+    constexpr auto ivii = 0b0000000000101010;
+    static_assert(ivii == 42, "wrong value");
+
+  }
+
+  namespace test_generalized_constexpr
+  {
+
+    template < typename CharT >
+    constexpr unsigned long
+    strlen_c(const CharT *const s) noexcept
+    {
+      auto length = 0UL;
+      for (auto p = s; *p; ++p)
+        ++length;
+      return length;
+    }
+
+    static_assert(strlen_c("") == 0UL, "");
+    static_assert(strlen_c("x") == 1UL, "");
+    static_assert(strlen_c("test") == 4UL, "");
+    static_assert(strlen_c("another\0test") == 7UL, "");
+
+  }
+
+  namespace test_lambda_init_capture
+  {
+
+    int
+    test()
+    {
+      auto x = 0;
+      const auto lambda1 = [a = x](int b){ return a + b; };
+      const auto lambda2 = [a = lambda1(x)](){ return a; };
+      return lambda2();
+    }
+
+  }
+
+  namespace test_digit_seperators
+  {
+
+    constexpr auto ten_million = 100'000'000;
+    static_assert(ten_million == 100000000, "");
+
+  }
+
+  namespace test_return_type_deduction
+  {
+
+    auto f(int& x) { return x; }
+    decltype(auto) g(int& x) { return x; }
+
+    template < typename T1, typename T2 >
+    struct is_same
+    {
+      static constexpr auto value = false;
+    };
+
+    template < typename T >
+    struct is_same<T, T>
+    {
+      static constexpr auto value = true;
+    };
+
+    int
+    test()
+    {
+      auto x = 0;
+      static_assert(is_same<int, decltype(f(x))>::value, "");
+      static_assert(is_same<int&, decltype(g(x))>::value, "");
+      return x;
+    }
+
+  }
+
+}  // namespace cxx14
+
+#endif  // __cplusplus >= 201402L
+
+]])

m4/ax_cxx_compile_stdcxx_11.m4

@@ -0,0 +1,39 @@
+# ============================================================================
+#  http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_11.html
+# ============================================================================
+#
+# SYNOPSIS
+#
+#   AX_CXX_COMPILE_STDCXX_11([ext|noext], [mandatory|optional])
+#
+# DESCRIPTION
+#
+#   Check for baseline language coverage in the compiler for the C++11
+#   standard; if necessary, add switches to CXX and CXXCPP to enable
+#   support.
+#
+#   This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX
+#   macro with the version set to C++11.  The two optional arguments are
+#   forwarded literally as the second and third argument respectively.
+#   Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for
+#   more information.  If you want to use this macro, you also need to
+#   download the ax_cxx_compile_stdcxx.m4 file.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com>
+#   Copyright (c) 2012 Zack Weinberg <zackw@panix.com>
+#   Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu>
+#   Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com>
+#   Copyright (c) 2015 Paul Norman <penorman@mac.com>
+#   Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 17
+
+AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX])
+AC_DEFUN([AX_CXX_COMPILE_STDCXX_11], [AX_CXX_COMPILE_STDCXX([11], [$1], [$2])])

src/command_executor.cc

@@ -44,6 +44,8 @@
 
 using namespace std;
 
+exec_context INIT_EXEC_CONTEXT;
+
 static const string MSG_FORMAT_STMT =
         "SELECT count(*) as total, min(log_line) as log_line, log_msg_format "
                 "FROM all_logs GROUP BY log_msg_format ORDER BY total desc";
@@ -74,9 +76,9 @@ static int sql_progress(const struct log_cursor &lc)
     return 0;
 }
 
-string execute_from_file(const string &path, int line_number, char mode, const string &cmdline);
+string execute_from_file(exec_context &ec, const string &path, int line_number, char mode, const string &cmdline);
 
-string execute_command(const string &cmdline)
+string execute_command(exec_context &ec, const string &cmdline)
 {
     vector<string> args;
     string         msg;
@@ -93,14 +95,14 @@ string execute_command(const string &cmdline)
             msg = "error: unknown command - " + args[0];
         }
         else {
-            msg = iter->second.c_func(cmdline, args);
+            msg = iter->second.c_func(ec, cmdline, args);
         }
     }
 
     return msg;
 }
 
-string execute_sql(const string &sql, string &alt_msg)
+string execute_sql(exec_context &ec, const string &sql, string &alt_msg)
 {
     db_label_source &dls = lnav_data.ld_db_row_source;
     auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
@@ -147,11 +149,20 @@ string execute_sql(const string &sql, string &alt_msg)
 
         param_count = sqlite3_bind_parameter_count(stmt.in());
         for (int lpc = 0; lpc < param_count; lpc++) {
+            map<string, string>::iterator ov_iter;
             const char *name;
 
             name = sqlite3_bind_parameter_name(stmt.in(), lpc + 1);
-            if (name[0] == '$') {
+            ov_iter = ec.ec_override.find(name);
-                map<string, string> &vars = lnav_data.ld_local_vars.top();
+            if (ov_iter != ec.ec_override.end()) {
+                sqlite3_bind_text(stmt.in(),
+                                  lpc,
+                                  ov_iter->second.c_str(),
+                                  ov_iter->second.length(),
+                                  SQLITE_TRANSIENT);
+            }
+            else if (name[0] == '$') {
+                map<string, string> &vars = ec.ec_local_vars.top();
                 map<string, string>::iterator local_var;
                 const char *env_value;
 
@@ -164,6 +175,40 @@ string execute_sql(const string &sql, string &alt_msg)
                     sqlite3_bind_text(stmt.in(), lpc + 1, env_value, -1, SQLITE_STATIC);
                 }
             }
+            else if (name[0] == ':' && ec.ec_line_values != NULL) {
+                vector<logline_value> &lvalues = *ec.ec_line_values;
+                vector<logline_value>::iterator iter;
+
+                for (iter = lvalues.begin(); iter != lvalues.end(); ++iter) {
+                    if (strcmp(&name[1], iter->lv_name.get()) != 0) {
+                        continue;
+                    }
+                    switch (iter->lv_kind) {
+                        case logline_value::VALUE_BOOLEAN:
+                            sqlite3_bind_int64(stmt.in(), lpc + 1, iter->lv_value.i);
+                            break;
+                        case logline_value::VALUE_FLOAT:
+                            sqlite3_bind_double(stmt.in(), lpc + 1, iter->lv_value.d);
+                            break;
+                        case logline_value::VALUE_INTEGER:
+                            sqlite3_bind_int64(stmt.in(), lpc + 1, iter->lv_value.i);
+                            break;
+                        case logline_value::VALUE_NULL:
+                            sqlite3_bind_null(stmt.in(), lpc + 1);
+                            break;
+                        default:
+                            sqlite3_bind_text(stmt.in(),
+                                              lpc + 1,
+                                              iter->text_value(),
+                                              iter->text_length(),
+                                              SQLITE_TRANSIENT);
+                            break;
+                    }
+                }
+            }
+            else {
+                log_warning("Could not bind variable: %s", name);
+            }
         }
 
         if (lnav_data.ld_rl_view != NULL) {
@@ -182,7 +227,7 @@ string execute_sql(const string &sql, string &alt_msg)
                 break;
 
             case SQLITE_ROW:
-                sql_callback(stmt.in());
+                ec.ec_sql_callback(ec, stmt.in());
                 break;
 
             default: {
@@ -203,29 +248,38 @@ string execute_sql(const string &sql, string &alt_msg)
         lnav_data.ld_views[LNV_DB].reload_data();
         lnav_data.ld_views[LNV_DB].set_left(0);
 
-        if (dls.dls_rows.size() > 0) {
+        if (!ec.ec_accumulator.empty()) {
-            vis_bookmarks &bm =
+            retval = ec.ec_accumulator;
-            lnav_data.ld_views[LNV_LOG].get_bookmarks();
+            ec.ec_accumulator.clear();
+        }
+        else if (dls.dls_rows.size() > 0) {
+            vis_bookmarks &bm = lnav_data.ld_views[LNV_LOG].get_bookmarks();
 
-            if (!(lnav_data.ld_flags & LNF_HEADLESS) &&
+            if (lnav_data.ld_flags & LNF_HEADLESS) {
-                    dls.dls_headers.size() == 1 && !bm[&BM_QUERY].empty()) {
+                if (ec.ec_local_vars.size() == 1) {
+                    ensure_view(&lnav_data.ld_views[LNV_DB]);
+                }
+
+                retval = "";
+                alt_msg = "";
+            }
+            else if (dls.dls_headers.size() == 1 && !bm[&BM_QUERY].empty()) {
                 retval = "";
                 alt_msg = HELP_MSG_2(
                   y, Y,
                   "to move forward/backward through query results "
                   "in the log view");
             }
-            else if (!(lnav_data.ld_flags & LNF_HEADLESS) &&
+            else if (dls.dls_rows.size() == 1) {
-                    dls.dls_rows.size() == 1) {
                 string row;
 
                 dls.text_value_for_line(lnav_data.ld_views[LNV_DB], 0, row, true);
-                retval = "SQL Result: " + row;
+                retval = row;
             }
             else {
                 char row_count[32];
 
-                if (lnav_data.ld_local_vars.size() == 1) {
+                if (ec.ec_local_vars.size() == 1) {
                     ensure_view(&lnav_data.ld_views[LNV_DB]);
                 }
                 snprintf(row_count, sizeof(row_count),
@@ -256,7 +310,7 @@ string execute_sql(const string &sql, string &alt_msg)
     return retval;
 }
 
-static string execute_file_contents(const string &path, bool multiline)
+static string execute_file_contents(exec_context &ec, const string &path, bool multiline)
 {
     string retval;
     FILE *file;
@@ -276,7 +330,7 @@ static string execute_file_contents(const string &path, bool multiline)
     char mode = '\0';
     pair<string, string> dir_and_base = split_path(path);
 
-    lnav_data.ld_path_stack.push(dir_and_base.first);
+    ec.ec_path_stack.push(dir_and_base.first);
     while ((line_size = getline(&line, &line_max_size, file)) != -1) {
         line_number += 1;
 
@@ -293,7 +347,7 @@ static string execute_file_contents(const string &path, bool multiline)
             case ';':
             case '|':
                 if (mode) {
-                    retval = execute_from_file(path, starting_line_number, mode, trim(cmdline));
+                    retval = execute_from_file(ec, path, starting_line_number, mode, trim(cmdline));
                 }
 
                 starting_line_number = line_number;
@@ -305,7 +359,7 @@ static string execute_file_contents(const string &path, bool multiline)
                     cmdline += line;
                 }
                 else {
-                    retval = execute_from_file(path, line_number, ':', line);
+                    retval = execute_from_file(ec, path, line_number, ':', line);
                 }
                 break;
         }
@@ -313,18 +367,18 @@ static string execute_file_contents(const string &path, bool multiline)
     }
 
     if (mode) {
-        retval = execute_from_file(path, starting_line_number, mode, trim(cmdline));
+        retval = execute_from_file(ec, path, starting_line_number, mode, trim(cmdline));
     }
 
     if (file != stdin) {
         fclose(file);
     }
-    lnav_data.ld_path_stack.pop();
+    ec.ec_path_stack.pop();
 
     return retval;
 }
 
-string execute_file(const string &path_and_args, bool multiline)
+string execute_file(exec_context &ec, const string &path_and_args, bool multiline)
 {
     map<string, vector<script_metadata> > scripts;
     map<string, vector<script_metadata> >::iterator iter;
@@ -334,17 +388,17 @@ string execute_file(const string &path_and_args, bool multiline)
 
     log_info("Executing file: %s", path_and_args.c_str());
 
-    if (!lexer.split(split_args, lnav_data.ld_local_vars.top())) {
+    if (!lexer.split(split_args, ec.ec_local_vars.top())) {
         retval = "error: unable to parse path";
     }
     else if (split_args.empty()) {
         retval = "error: no script specified";
     }
     else {
-        lnav_data.ld_local_vars.push(map<string, string>());
+        ec.ec_local_vars.push(map<string, string>());
 
         string script_name = split_args[0];
-        map<string, string> &vars = lnav_data.ld_local_vars.top();
+        map<string, string> &vars = ec.ec_local_vars.top();
         char env_arg_name[32];
         string result, open_error = "file not found";
 
@@ -375,7 +429,7 @@ string execute_file(const string &path_and_args, bool multiline)
             open_error = strerror(errno);
         }
         else {
-            string local_path = lnav_data.ld_path_stack.top() + "/" + script_name;
+            string local_path = ec.ec_path_stack.top() + "/" + script_name;
 
             if (access(local_path.c_str(), R_OK) == 0) {
                 struct script_metadata meta;
@@ -393,37 +447,37 @@ string execute_file(const string &path_and_args, bool multiline)
             for (vector<script_metadata>::iterator path_iter = paths_to_exec.begin();
                  path_iter != paths_to_exec.end();
                  ++path_iter) {
-                result = execute_file_contents(path_iter->sm_path, multiline);
+                result = execute_file_contents(ec, path_iter->sm_path, multiline);
             }
-            retval = "Executed: " + script_name + " -- " + result;
+            retval = result;
         }
         else {
             retval = "error: unknown script -- " + script_name + " -- " + open_error;
         }
-        lnav_data.ld_local_vars.pop();
+        ec.ec_local_vars.pop();
     }
 
     return retval;
 }
 
-string execute_from_file(const string &path, int line_number, char mode, const string &cmdline)
+string execute_from_file(exec_context &ec, const string &path, int line_number, char mode, const string &cmdline)
 {
     string retval, alt_msg;
 
     switch (mode) {
         case ':':
-            retval = execute_command(cmdline);
+            retval = execute_command(ec, cmdline);
             break;
         case '/':
         case ';':
             setup_logline_table();
-            retval = execute_sql(cmdline, alt_msg);
+            retval = execute_sql(ec, cmdline, alt_msg);
             break;
         case '|':
-            retval = execute_file(cmdline);
+            retval = execute_file(ec, cmdline);
             break;
         default:
-            retval = execute_command(cmdline);
+            retval = execute_command(ec, cmdline);
             break;
     }
 
@@ -439,28 +493,55 @@ string execute_from_file(const string &path, int line_number, char mode, const s
     return retval;
 }
 
-void execute_init_commands(vector<pair<string, string> > &msgs)
+string execute_any(exec_context &ec, const string &cmdline_with_mode)
+{
+    string retval, alt_msg, cmdline = cmdline_with_mode.substr(1);
+
+    switch (cmdline_with_mode[0]) {
+        case ':':
+            retval = execute_command(ec, cmdline);
+            break;
+        case '/':
+        case ';':
+            setup_logline_table();
+            retval = execute_sql(ec, cmdline, alt_msg);
+            break;
+        case '|': {
+            retval = execute_file(ec, cmdline);
+            break;
+        }
+        default:
+            retval = execute_command(ec, cmdline);
+            break;
+    }
+
+    if (rescan_files()) {
+        rebuild_indexes(true);
+    }
+
+    return retval;
+}
+
+void execute_init_commands(exec_context &ec, vector<pair<string, string> > &msgs)
 {
     if (lnav_data.ld_cmd_init_done) {
         return;
     }
 
-    for (list<std::string>::iterator iter = lnav_data.ld_commands.begin();
+    for (auto &cmd : lnav_data.ld_commands) {
-         iter != lnav_data.ld_commands.end();
-         ++iter) {
         string msg, alt_msg;
 
-        switch (iter->at(0)) {
+        switch (cmd.at(0)) {
         case ':':
-            msg = execute_command(iter->substr(1));
+            msg = execute_command(ec, cmd.substr(1));
             break;
         case '/':
         case ';':
             setup_logline_table();
-            msg = execute_sql(iter->substr(1), alt_msg);
+            msg = execute_sql(ec, cmd.substr(1), alt_msg);
             break;
         case '|':
-            msg = execute_file(iter->substr(1));
+            msg = execute_file(ec, cmd.substr(1));
             break;
         }
 
@@ -487,7 +568,7 @@ void execute_init_commands(vector<pair<string, string> > &msgs)
     lnav_data.ld_cmd_init_done = true;
 }
 
-int sql_callback(sqlite3_stmt *stmt)
+int sql_callback(exec_context &ec, sqlite3_stmt *stmt)
 {
     logfile_sub_source &lss = lnav_data.ld_log_source;
     db_label_source &dls = lnav_data.ld_db_row_source;
@@ -534,3 +615,30 @@ int sql_callback(sqlite3_stmt *stmt)
 
     return retval;
 }
+
+future<string> pipe_callback(exec_context &ec, const string &cmdline, auto_fd &fd)
+{
+    piper_proc *pp = new piper_proc(fd, false);
+    static int exec_count = 0;
+    char desc[128];
+
+    lnav_data.ld_pipers.push_back(pp);
+    snprintf(desc,
+             sizeof(desc), "[%d] Output of %s",
+             exec_count++,
+             cmdline.c_str());
+    lnav_data.ld_file_names[desc]
+        .with_fd(pp->get_fd())
+        .with_detect_format(false);
+    lnav_data.ld_files_to_front.push_back(make_pair(desc, 0));
+    if (lnav_data.ld_rl_view != NULL) {
+        lnav_data.ld_rl_view->set_alt_value(HELP_MSG_1(
+                                                X, "to close the file"));
+    }
+
+    packaged_task<string()> task([]() { return ""; });
+
+    task();
+
+    return task.get_future();
+}

src/command_executor.hh

@@ -32,14 +32,49 @@
 
 #include <sqlite3.h>
 
+#include <future>
 #include <string>
 
-std::string execute_command(const std::string &cmdline);
+struct exec_context;
 
-std::string execute_sql(const std::string &sql, std::string &alt_msg);
+typedef int (*sql_callback_t)(exec_context &ec, sqlite3_stmt *stmt);
-std::string execute_file(const std::string &path_and_args, bool multiline = true);
-void execute_init_commands(std::vector<std::pair<std::string, std::string> > &msgs);
 
-int sql_callback(sqlite3_stmt *stmt);
+typedef std::future<std::string> (*pipe_callback_t)(
+    exec_context &ec, const std::string &cmdline, auto_fd &fd);
+
+struct exec_context {
+    exec_context(std::vector<logline_value> *line_values = NULL,
+                 sql_callback_t sql_callback = NULL,
+                 pipe_callback_t pipe_callback = NULL)
+        : ec_line_values(line_values),
+          ec_sql_callback(sql_callback),
+          ec_pipe_callback(pipe_callback) {
+        this->ec_local_vars.push(std::map<std::string, std::string>());
+        this->ec_path_stack.push(".");
+    }
+
+    vis_line_t ec_top_line;
+
+    std::map<std::string, std::string> ec_override;
+    std::vector<logline_value> *ec_line_values;
+    std::stack<std::map<std::string, std::string> > ec_local_vars;
+    std::stack<std::string> ec_path_stack;
+
+    std::string ec_accumulator;
+
+    sql_callback_t ec_sql_callback;
+    pipe_callback_t ec_pipe_callback;
+};
+
+std::string execute_command(exec_context &ec, const std::string &cmdline);
+
+std::string execute_sql(exec_context &ec, const std::string &sql, std::string &alt_msg);
+std::string execute_file(exec_context &ec, const std::string &path_and_args, bool multiline = true);
+std::string execute_any(exec_context &ec, const std::string &cmdline);
+void execute_init_commands(exec_context &ec, std::vector<std::pair<std::string, std::string> > &msgs);
+
+int sql_callback(exec_context &ec, sqlite3_stmt *stmt);
+std::future<std::string> pipe_callback(
+    exec_context &ec, const std::string &cmdline, auto_fd &fd);
 
 #endif //LNAV_COMMAND_EXECUTOR_H

src/default-log-formats.json

@@ -51,7 +51,8 @@
             },
             "sc_status" : {
                 "kind" : "integer",
-                "foreign-key" : true
+                "foreign-key" : true,
+                "rewriter" : ";SELECT :sc_status || ' (' || (SELECT message FROM http_status_codes WHERE status = :sc_status) || ') '"
             },
             "sc_bytes" : {
                 "kind" : "integer"

src/hotkeys.cc

@@ -171,6 +171,7 @@ void update_view_name(void)
 void handle_paging_key(int ch)
 {
     textview_curses *   tc  = lnav_data.ld_view_stack.top();
+    exec_context &ec = lnav_data.ld_exec_context;
     logfile_sub_source *lss = NULL;
     bookmarks<vis_line_t>::type &     bm  = tc->get_bookmarks();
 
@@ -431,7 +432,7 @@ void handle_paging_key(int ch)
                 alerter::singleton().chime();
             }
             else {
-                execute_command("zoom-to " + string(lnav_zoom_strings[lnav_data.ld_zoom_level - 1]));
+                execute_command(ec, "zoom-to " + string(lnav_zoom_strings[lnav_data.ld_zoom_level - 1]));
             }
             break;
 
@@ -440,7 +441,7 @@ void handle_paging_key(int ch)
                 alerter::singleton().chime();
             }
             else {
-                execute_command("zoom-to " + string(lnav_zoom_strings[lnav_data.ld_zoom_level + 1]));
+                execute_command(ec, "zoom-to " + string(lnav_zoom_strings[lnav_data.ld_zoom_level + 1]));
             }
             break;
 
@@ -883,6 +884,8 @@ void handle_paging_key(int ch)
                 logfile::iterator ll = lf->begin() + cl;
                 log_data_helper ldh(lss);
 
+                lnav_data.ld_exec_context.ec_top_line = tc->get_top();
+
                 lnav_data.ld_rl_view->clear_possibilities(LNM_COMMAND, "numeric-colname");
                 lnav_data.ld_rl_view->clear_possibilities(LNM_COMMAND, "colname");
 
@@ -985,6 +988,7 @@ void handle_paging_key(int ch)
                 tc == &lnav_data.ld_views[LNV_DB] ||
                 tc == &lnav_data.ld_views[LNV_SCHEMA]) {
                 textview_curses &log_view = lnav_data.ld_views[LNV_LOG];
+                lnav_data.ld_exec_context.ec_top_line = tc->get_top();
 
                 lnav_data.ld_mode = LNM_SQL;
                 setup_logline_table();
@@ -1011,12 +1015,11 @@ void handle_paging_key(int ch)
 
             lnav_data.ld_mode = LNM_EXEC;
 
+            lnav_data.ld_exec_context.ec_top_line = tc->get_top();
             lnav_data.ld_rl_view->clear_possibilities(LNM_EXEC, "__command");
             find_format_scripts(lnav_data.ld_config_paths, scripts);
-            for (map<string, vector<script_metadata> >::iterator iter = scripts.begin();
+            for (const auto &iter : scripts) {
-                 iter != scripts.end();
+                lnav_data.ld_rl_view->add_possibility(LNM_EXEC, "__command", iter.first);
-                 ++iter) {
-                lnav_data.ld_rl_view->add_possibility(LNM_EXEC, "__command", iter->first);
             }
             add_view_text_possibilities(LNM_EXEC, "*", tc);
             add_env_possibilities(LNM_EXEC);
@@ -1204,7 +1207,7 @@ void handle_paging_key(int ch)
             break;
 
         case 'X':
-            lnav_data.ld_rl_view->set_value(execute_command("close"));
+            lnav_data.ld_rl_view->set_value(execute_command(ec, "close"));
             break;
 
         case '\\':
@@ -1269,7 +1272,7 @@ void handle_paging_key(int ch)
             break;
 
         case KEY_CTRL_W:
-            execute_command(lnav_data.ld_views[LNV_LOG].get_word_wrap() ?
+            execute_command(ec, lnav_data.ld_views[LNV_LOG].get_word_wrap() ?
                             "disable-word-wrap" : "enable-word-wrap");
             break;
 

src/json_ptr.hh

@@ -40,8 +40,10 @@
 #include <string>
 #include <vector>
 
-#include "yajlpp.hh"
+#include "auto_mem.hh"
+#include "yajl/api/yajl_parse.h"
 #include "yajl/api/yajl_tree.h"
+#include "yajl/api/yajl_gen.h"
 #include "lnav_log.hh"
 
 class json_ptr_walk {
@@ -183,6 +185,42 @@ public:
         return retval;
     };
 
+    static size_t decode(char *dst, const char *src, ssize_t src_len = -1) {
+        size_t retval = 0;
+
+        if (src_len == -1) {
+            src_len = strlen(src);
+        }
+
+        for (int lpc = 0; lpc < src_len; lpc++) {
+            switch (src[lpc]) {
+                case '~':
+                    if ((lpc + 1) < src_len) {
+                        switch (src[lpc + 1]) {
+                            case '0':
+                                dst[retval++] = '~';
+                                lpc += 1;
+                                break;
+                            case '1':
+                                dst[retval++] = '/';
+                                lpc += 1;
+                                break;
+                            default:
+                                break;
+                        }
+                    }
+                    break;
+                default:
+                    dst[retval++] = src[lpc];
+                    break;
+            }
+        }
+
+        dst[retval] = '\0';
+
+        return retval;
+    }
+
     json_ptr(const char *value)
         : jp_value(value), jp_pos(value), jp_depth(0), jp_array_index(-1),
           jp_state(MS_VALUE) {

src/lnav.cc

@@ -717,6 +717,45 @@ static void open_schema_view(void)
     schema_tc->set_sub_source(new plain_text_source(schema));
 }
 
+static int pretty_sql_callback(exec_context &ec, sqlite3_stmt *stmt)
+{
+    int ncols = sqlite3_column_count(stmt);
+
+    for (int lpc = 0; lpc < ncols; lpc++) {
+        if (lpc > 0) {
+            ec.ec_accumulator += ", ";
+        }
+        ec.ec_accumulator.append((const char *)sqlite3_column_text(stmt, lpc));
+    }
+
+    return 0;
+}
+
+static future<string> pretty_pipe_callback(exec_context &ec,
+                                           const string &cmdline,
+                                           auto_fd &fd)
+{
+    auto retval = std::async(std::launch::async, [&]() {
+        char buffer[1024];
+        ostringstream ss;
+        ssize_t rc;
+
+        while ((rc = read(fd, buffer, sizeof(buffer))) > 0) {
+            ss.write(buffer, rc);
+        }
+
+        string retval = ss.str();
+
+        if (endswith(retval.c_str(), "\n")) {
+            retval.resize(retval.length() - 1);
+        }
+
+        return retval;
+    });
+
+    return retval;
+}
+
 static void open_pretty_view(void)
 {
     static const char *NOTHING_MSG =
@@ -741,6 +780,7 @@ static void open_pretty_view(void)
         for (vis_line_t vl = log_tc->get_top(); vl <= log_tc->get_bottom(); ++vl) {
             content_line_t cl = lss.at(vl);
             logfile *lf = lss.find(cl);
+            log_format *format = lf->get_format();
             logfile::iterator ll = lf->begin() + cl;
             shared_buffer_ref sbr;
 
@@ -750,7 +790,17 @@ static void open_pretty_view(void)
             ll = lf->message_start(ll);
 
             lf->read_full_message(ll, sbr);
-            data_scanner ds(sbr);
+
+            string_attrs_t sa;
+            vector<logline_value> values;
+            string rewritten_line;
+
+            format->annotate(sbr, sa, values);
+            exec_context ec(&values, pretty_sql_callback, pretty_pipe_callback);
+            add_ansi_vars(ec.ec_local_vars.top());
+            format->rewrite(ec, sbr, sa, rewritten_line);
+
+            data_scanner ds(rewritten_line);
             pretty_printer pp(&ds);
 
             // TODO: dump more details of the line in the output.
@@ -1715,6 +1765,8 @@ static void wait_for_pipers(void)
 static void looper(void)
 {
     try {
+        exec_context &ec = lnav_data.ld_exec_context;
+
         readline_context command_context("cmd", &lnav_commands);
 
         readline_context search_context("search");
@@ -1842,7 +1894,7 @@ static void looper(void)
         lnav_data.ld_status[0].window_change();
         lnav_data.ld_status[1].window_change();
 
-        execute_file(dotlnav_path("session"));
+        execute_file(ec, dotlnav_path("session"));
 
         lnav_data.ld_scroll_broadcaster.invoke(lnav_data.ld_view_stack.top());
 
@@ -2058,7 +2110,7 @@ static void looper(void)
 
                     vector<pair<string, string> > msgs;
 
-                    execute_init_commands(msgs);
+                    execute_init_commands(ec, msgs);
 
                     if (!msgs.empty()) {
                         pair<string, string> last_msg = msgs.back();
@@ -2285,6 +2337,7 @@ redraw_listener REDRAW_LISTENER;
 int main(int argc, char *argv[])
 {
     std::vector<std::string> config_errors, loader_errors;
+    exec_context &ec = lnav_data.ld_exec_context;
     int lpc, c, retval = EXIT_SUCCESS;
 
     auto_ptr<piper_proc> stdin_reader;
@@ -2302,10 +2355,11 @@ int main(int argc, char *argv[])
         lnav_data.ld_flags |= LNF_SECURE_MODE;
     }
 
+    lnav_data.ld_exec_context.ec_sql_callback = sql_callback;
+    lnav_data.ld_exec_context.ec_pipe_callback = pipe_callback;
+
     lnav_data.ld_program_name = argv[0];
-    lnav_data.ld_local_vars.push(map<string, string>());
+    add_ansi_vars(ec.ec_local_vars.top());
-    add_ansi_vars(lnav_data.ld_local_vars.top());
-    lnav_data.ld_path_stack.push(".");
 
     rl_readline_name = "lnav";
 
@@ -2913,7 +2967,7 @@ int main(int argc, char *argv[])
                 }
 
                 log_info("Executing initial commands");
-                execute_init_commands(msgs);
+                execute_init_commands(lnav_data.ld_exec_context, msgs);
                 wait_for_pipers();
                 lnav_data.ld_curl_looper.process_all();
                 rebuild_indexes(false);

src/lnav.hh

@@ -63,6 +63,7 @@
 #include "relative_time.hh"
 #include "log_format_loader.hh"
 #include "spectro_source.hh"
+#include "command_executor.hh"
 
 /** The command modes that are available while viewing a file. */
 typedef enum {
@@ -291,12 +292,12 @@ struct _lnav_data {
 
     relative_time ld_last_relative_time;
 
-    std::stack<std::map<std::string, std::string> > ld_local_vars;
-    std::stack<std::string> ld_path_stack;
     std::stack<FILE *> ld_output_stack;
 
     std::map<std::string, std::vector<script_metadata> > ld_scripts;
 
+    exec_context ld_exec_context;
+
     int ld_fifo_counter;
 
     struct key_repeat_history ld_key_repeat_history;

src/lnav_commands.cc

@@ -117,7 +117,7 @@ static string refresh_pt_search()
     return retval;
 }
 
-static string com_adjust_log_time(string cmdline, vector<string> &args)
+static string com_adjust_log_time(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting new time value";
 
@@ -161,7 +161,7 @@ static string com_adjust_log_time(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_unix_time(string cmdline, vector<string> &args)
+static string com_unix_time(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a unix time value";
 
@@ -217,7 +217,7 @@ static string com_unix_time(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_current_time(string cmdline, vector<string> &args)
+static string com_current_time(exec_context &ec, string cmdline, vector<string> &args)
 {
     char      ftime[128];
     struct tm localtm;
@@ -239,7 +239,7 @@ static string com_current_time(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_goto(string cmdline, vector<string> &args)
+static string com_goto(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting line number/percentage, timestamp, or relative time";
 
@@ -319,7 +319,7 @@ static string com_goto(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_relative_goto(string cmdline, vector<string> &args)
+static string com_relative_goto(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting line number/percentage";
 
@@ -348,7 +348,7 @@ static string com_relative_goto(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_goto_mark(string cmdline, vector<string> &args)
+static string com_goto_mark(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "";
 
@@ -442,7 +442,7 @@ static void json_write_row(yajl_gen handle, int row)
     }
 }
 
-static string com_save_to(string cmdline, vector<string> &args)
+static string com_save_to(exec_context &ec, string cmdline, vector<string> &args)
 {
     FILE *outfile = NULL, *toclose = NULL;
     const char *mode    = "";
@@ -467,7 +467,7 @@ static string com_save_to(string cmdline, vector<string> &args)
     vector<string> split_args;
     shlex lexer(fn);
 
-    if (!lexer.split(split_args, lnav_data.ld_local_vars.top())) {
+    if (!lexer.split(split_args, ec.ec_local_vars.top())) {
         return "error: unable to parse arguments";
     }
     if (split_args.size() > 1) {
@@ -643,7 +643,7 @@ static string com_save_to(string cmdline, vector<string> &args)
     return "";
 }
 
-static string com_pipe_to(string cmdline, vector<string> &args)
+static string com_pipe_to(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting command to execute";
 
@@ -697,9 +697,9 @@ static string com_pipe_to(string cmdline, vector<string> &args)
                 log_data_helper ldh(lss);
                 char tmp_str[64];
 
-                ldh.parse_line(tc->get_top(), true);
+                ldh.parse_line(ec.ec_top_line, true);
 
-                snprintf(tmp_str, sizeof(tmp_str), "%d", (int) tc->get_top());
+                snprintf(tmp_str, sizeof(tmp_str), "%d", (int) ec.ec_top_line);
                 setenv("log_line", tmp_str, 1);
                 sql_strftime(tmp_str, sizeof(tmp_str), ldh.ldh_line->get_timeval());
                 setenv("log_time", tmp_str, 1);
@@ -728,30 +728,17 @@ static string com_pipe_to(string cmdline, vector<string> &args)
 
         default:
             bookmark_vector<vis_line_t>::iterator iter;
-            static int exec_count = 0;
             string line;
 
             in_pipe.read_end().close_on_exec();
             in_pipe.write_end().close_on_exec();
 
             lnav_data.ld_children.push_back(child_pid);
-            if (out_pipe.read_end() != -1) {
-                piper_proc *pp = new piper_proc(out_pipe.read_end(), false);
-                char desc[128];
 
-                lnav_data.ld_pipers.push_back(pp);
+            future<string> reader;
-                snprintf(desc,
+
-                        sizeof(desc), "[%d] Output of %s",
+            if (out_pipe.read_end() != -1) {
-                        exec_count++,
+                reader = ec.ec_pipe_callback(ec, cmdline, out_pipe.read_end());
-                        cmdline.c_str());
-                lnav_data.ld_file_names[desc]
-                    .with_fd(pp->get_fd())
-                    .with_detect_format(false);
-                lnav_data.ld_files_to_front.push_back(make_pair(desc, 0));
-                if (lnav_data.ld_rl_view != NULL) {
-                    lnav_data.ld_rl_view->set_alt_value(HELP_MSG_1(
-                            X, "to close the file"));
-                }
             }
 
             if (pipe_line_to) {
@@ -787,14 +774,19 @@ static string com_pipe_to(string cmdline, vector<string> &args)
                 }
             }
 
-            retval = "";
+            if (reader.valid()) {
+                retval = reader.get();
+            }
+            else {
+                retval = "";
+            }
             break;
     }
 
     return retval;
 }
 
-static string com_highlight(string cmdline, vector<string> &args)
+static string com_highlight(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting regular expression to highlight";
 
@@ -839,7 +831,7 @@ static string com_highlight(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_clear_highlight(string cmdline, vector<string> &args)
+static string com_clear_highlight(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting highlight expression to clear";
 
@@ -866,7 +858,7 @@ static string com_clear_highlight(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_help(string cmdline, vector<string> &args)
+static string com_help(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "";
 
@@ -905,9 +897,9 @@ protected:
     pcrepp pf_pcre;
 };
 
-static string com_enable_filter(string cmdline, vector<string> &args);
+static string com_enable_filter(exec_context &ec, string cmdline, vector<string> &args);
 
-static string com_filter(string cmdline, vector<string> &args)
+static string com_filter(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting regular expression to filter out";
 
@@ -924,7 +916,7 @@ static string com_filter(string cmdline, vector<string> &args)
 
         args[1] = remaining_args(cmdline, args);
         if (fs.get_filter(args[1]) != NULL) {
-            retval = com_enable_filter(cmdline, args);
+            retval = com_enable_filter(ec, cmdline, args);
         }
         else if (fs.full()) {
             retval = "error: filter limit reached, try combining "
@@ -959,7 +951,7 @@ static string com_filter(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_delete_filter(string cmdline, vector<string> &args)
+static string com_delete_filter(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a filter to delete";
 
@@ -985,7 +977,7 @@ static string com_delete_filter(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_enable_filter(string cmdline, vector<string> &args)
+static string com_enable_filter(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting disabled filter to enable";
 
@@ -1017,7 +1009,7 @@ static string com_enable_filter(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_disable_filter(string cmdline, vector<string> &args)
+static string com_disable_filter(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting enabled filter to disable";
 
@@ -1049,7 +1041,7 @@ static string com_disable_filter(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_enable_word_wrap(string cmdline, vector<string> &args)
+static string com_enable_word_wrap(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "";
 
@@ -1065,7 +1057,7 @@ static string com_enable_word_wrap(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_disable_word_wrap(string cmdline, vector<string> &args)
+static string com_disable_word_wrap(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "";
 
@@ -1083,7 +1075,7 @@ static string com_disable_word_wrap(string cmdline, vector<string> &args)
 
 static std::set<string> custom_logline_tables;
 
-static string com_create_logline_table(string cmdline, vector<string> &args)
+static string com_create_logline_table(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a table name";
 
@@ -1120,7 +1112,7 @@ static string com_create_logline_table(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_delete_logline_table(string cmdline, vector<string> &args)
+static string com_delete_logline_table(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a table name";
 
@@ -1153,7 +1145,7 @@ static string com_delete_logline_table(string cmdline, vector<string> &args)
 
 static std::set<string> custom_search_tables;
 
-static string com_create_search_table(string cmdline, vector<string> &args)
+static string com_create_search_table(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a table name";
 
@@ -1199,7 +1191,7 @@ static string com_create_search_table(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_delete_search_table(string cmdline, vector<string> &args)
+static string com_delete_search_table(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a table name";
 
@@ -1230,7 +1222,7 @@ static string com_delete_search_table(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_session(string cmdline, vector<string> &args)
+static string com_session(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a command to save to the session file";
 
@@ -1296,7 +1288,7 @@ static string com_session(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_open(string cmdline, vector<string> &args)
+static string com_open(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting file name to open";
 
@@ -1322,7 +1314,7 @@ static string com_open(string cmdline, vector<string> &args)
     vector<string> split_args;
     shlex lexer(pat);
 
-    if (!lexer.split(split_args, lnav_data.ld_local_vars.top())) {
+    if (!lexer.split(split_args, ec.ec_local_vars.top())) {
         return "error: unable to parse arguments";
     }
 
@@ -1444,7 +1436,7 @@ static string com_open(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_close(string cmdline, vector<string> &args)
+static string com_close(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: close must be run in the log or text file views";
 
@@ -1497,7 +1489,7 @@ static string com_close(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_partition_name(string cmdline, vector<string> &args)
+static string com_partition_name(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting partition name";
 
@@ -1522,7 +1514,7 @@ static string com_partition_name(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_clear_partition(string cmdline, vector<string> &args)
+static string com_clear_partition(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "";
 
@@ -1558,7 +1550,7 @@ static string com_clear_partition(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_pt_time(string cmdline, vector<string> &args)
+static string com_pt_time(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a time value";
 
@@ -1623,7 +1615,7 @@ static string com_pt_time(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_summarize(string cmdline, vector<string> &args)
+static string com_summarize(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "";
 
@@ -1794,7 +1786,7 @@ static string com_summarize(string cmdline, vector<string> &args)
                     break;
 
                 case SQLITE_ROW:
-                    sql_callback(stmt.in());
+                    ec.ec_sql_callback(ec, stmt.in());
                     break;
 
                 default:
@@ -1827,7 +1819,7 @@ static string com_summarize(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_add_test(string cmdline, vector<string> &args)
+static string com_add_test(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "";
 
@@ -1868,7 +1860,7 @@ static string com_add_test(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_switch_to_view(string cmdline, vector<string> &args)
+static string com_switch_to_view(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "";
 
@@ -1893,7 +1885,7 @@ static string com_switch_to_view(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_zoom_to(string cmdline, vector<string> &args)
+static string com_zoom_to(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "";
 
@@ -1949,7 +1941,7 @@ static string com_zoom_to(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_reset_session(string cmdline, vector<string> &args)
+static string com_reset_session(exec_context &ec, string cmdline, vector<string> &args)
 {
     if (args.empty()) {
 
@@ -1962,7 +1954,7 @@ static string com_reset_session(string cmdline, vector<string> &args)
     return "";
 }
 
-static string com_load_session(string cmdline, vector<string> &args)
+static string com_load_session(exec_context &ec, string cmdline, vector<string> &args)
 {
     if (args.empty()) {
 
@@ -1976,7 +1968,7 @@ static string com_load_session(string cmdline, vector<string> &args)
     return "";
 }
 
-static string com_save_session(string cmdline, vector<string> &args)
+static string com_save_session(exec_context &ec, string cmdline, vector<string> &args)
 {
     if (args.empty()) {
 
@@ -1988,7 +1980,7 @@ static string com_save_session(string cmdline, vector<string> &args)
     return "";
 }
 
-static string com_set_min_log_level(string cmdline, vector<string> &args)
+static string com_set_min_log_level(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting log level name";
 
@@ -2011,7 +2003,7 @@ static string com_set_min_log_level(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_hide_line(string cmdline, vector<string> &args)
+static string com_hide_line(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval;
 
@@ -2113,7 +2105,7 @@ static string com_hide_line(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_show_lines(string cmdline, vector<string> &args)
+static string com_show_lines(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "info: showing lines";
 
@@ -2131,7 +2123,7 @@ static string com_show_lines(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_rebuild(string cmdline, vector<string> &args)
+static string com_rebuild(exec_context &ec, string cmdline, vector<string> &args)
 {
     if (args.empty()) {
 
@@ -2143,7 +2135,7 @@ static string com_rebuild(string cmdline, vector<string> &args)
     return "";
 }
 
-static string com_shexec(string cmdline, vector<string> &args)
+static string com_shexec(exec_context &ec, string cmdline, vector<string> &args)
 {
     if (args.empty()) {
 
@@ -2155,7 +2147,7 @@ static string com_shexec(string cmdline, vector<string> &args)
     return "";
 }
 
-static string com_poll_now(string cmdline, vector<string> &args)
+static string com_poll_now(exec_context &ec, string cmdline, vector<string> &args)
 {
     if (args.empty()) {
 
@@ -2167,7 +2159,7 @@ static string com_poll_now(string cmdline, vector<string> &args)
     return "";
 }
 
-static string com_redraw(string cmdline, vector<string> &args)
+static string com_redraw(exec_context &ec, string cmdline, vector<string> &args)
 {
     if (args.empty()) {
 
@@ -2179,7 +2171,7 @@ static string com_redraw(string cmdline, vector<string> &args)
     return "";
 }
 
-static string com_echo(string cmdline, vector<string> &args)
+static string com_echo(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a message";
 
@@ -2219,7 +2211,7 @@ static string com_echo(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_eval(string cmdline, vector<string> &args)
+static string com_eval(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a command or query to evaluate";
 
@@ -2232,7 +2224,7 @@ static string com_eval(string cmdline, vector<string> &args)
         shlex lexer(all_args.c_str(), all_args.size());
 
         log_debug("Evaluating: %s", all_args.c_str());
-        if (!lexer.eval(expanded_cmd, lnav_data.ld_local_vars.top())) {
+        if (!lexer.eval(expanded_cmd, ec.ec_local_vars.top())) {
             return "error: invalid arguments";
         }
         log_debug("Expanded command to evaluate: %s", expanded_cmd.c_str());
@@ -2244,14 +2236,14 @@ static string com_eval(string cmdline, vector<string> &args)
         string alt_msg;
         switch (expanded_cmd[0]) {
             case ':':
-                retval = execute_command(expanded_cmd.substr(1));
+                retval = execute_command(ec, expanded_cmd.substr(1));
                 break;
             case ';':
-                retval = execute_sql(expanded_cmd.substr(1), alt_msg);
+                retval = execute_sql(ec, expanded_cmd.substr(1), alt_msg);
                 break;
             case '|':
                 retval = "info: executed file -- " + expanded_cmd.substr(1) +
-                        " -- " + execute_file(expanded_cmd.substr(1));
+                        " -- " + execute_file(ec, expanded_cmd.substr(1));
                 break;
             default:
                 retval = "error: expecting argument to start with ':', ';', "
@@ -2263,7 +2255,7 @@ static string com_eval(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_config(string cmdline, vector<string> &args)
+static string com_config(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a configuration option to read or write";
 
@@ -2331,7 +2323,7 @@ static string com_config(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_save_config(string cmdline, vector<string> &args)
+static string com_save_config(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval;
 
@@ -2344,7 +2336,7 @@ static string com_save_config(string cmdline, vector<string> &args)
     return retval;
 }
 
-static string com_reset_config(string cmdline, vector<string> &args)
+static string com_reset_config(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a configuration option to reset";
 
@@ -2668,7 +2660,7 @@ public:
     string dsvs_error_msg;
 };
 
-static string com_spectrogram(string cmdline, vector<string> &args)
+static string com_spectrogram(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a message field name";
 

src/log_format.cc

@@ -41,6 +41,7 @@
 #include "log_vtab_impl.hh"
 #include "ptimec.hh"
 #include "log_search_table.hh"
+#include "command_executor.hh"
 
 using namespace std;
 
@@ -923,6 +924,56 @@ void external_log_format::annotate(shared_buffer_ref &line,
     }
 }
 
+void external_log_format::rewrite(exec_context &ec,
+                                  shared_buffer_ref &line,
+                                  string_attrs_t &sa,
+                                  string &value_out)
+{
+    vector<logline_value>::iterator iter, bind_iter, shift_iter;
+    vector<logline_value> &values = *ec.ec_line_values;
+
+    value_out.assign(line.get_data(), line.length());
+
+    for (iter = values.begin(); iter != values.end(); ++iter) {
+        map<const intern_string_t, value_def>::iterator vd_iter;
+
+        if (!iter->lv_origin.is_valid()) {
+            log_debug("not rewriting value with invalid origin -- %s", iter->lv_name.get());
+            continue;
+        }
+
+        vd_iter = this->elf_value_defs.find(iter->lv_name);
+        if (vd_iter == this->elf_value_defs.end()) {
+            log_debug("not rewriting undefined value -- %s", iter->lv_name.get());
+            continue;
+        }
+
+        value_def &vd = vd_iter->second;
+
+        if (!vd.vd_rewriter.empty()) {
+            string field_value = iter->to_string();
+
+            field_value = execute_any(ec, vd.vd_rewriter);
+
+            struct line_range adj_origin = iter->origin_in_full_msg(
+                value_out.c_str(), value_out.length());
+
+            value_out.erase(adj_origin.lr_start, adj_origin.length());
+
+            int32_t shift_amount = field_value.length() - adj_origin.length();
+            value_out.insert(adj_origin.lr_start, field_value);
+            for (shift_iter = values.begin();
+                 shift_iter != values.end(); ++shift_iter) {
+                if (shift_iter->lv_name == iter->lv_name) {
+                    continue;
+                }
+
+                shift_iter->lv_origin.shift(adj_origin.lr_start, shift_amount);
+            }
+        }
+    }
+}
+
 static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
 {
     json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;

src/log_format.hh

@@ -56,8 +56,10 @@
 #include "intern_string.hh"
 #include "shared_buffer.hh"
 
+struct sqlite3;
 class log_format;
 class log_vtab_manager;
+struct exec_context;
 
 /**
  * Metadata for a single line in a log file.
@@ -523,6 +525,38 @@ public:
         return this->lv_sbr.length();
     }
 
+    struct line_range origin_in_full_msg(const char *msg, size_t len) {
+        if (this->lv_sub_offset == 0) {
+            return this->lv_origin;
+        }
+
+        struct line_range retval = this->lv_origin;
+        const char *last = msg;
+
+        for (int lpc = 0; lpc < this->lv_sub_offset; lpc++) {
+            const char *next = strchr(last, '\n');
+            require(next != NULL);
+
+            next += 1;
+            int amount = (next - last);
+
+            retval.lr_start += amount;
+            if (retval.lr_end != -1) {
+                retval.lr_end += amount;
+            }
+
+            last = next + 1;
+        }
+
+        if (retval.lr_end == -1) {
+            const char *eol = strchr(last, '\n');
+
+            retval.lr_end = eol - msg;
+        }
+
+        return retval;
+    };
+
     intern_string_t lv_name;
     kind_t      lv_kind;
     union value_u {
@@ -732,6 +766,13 @@ public:
                           bool annotate_module = true) const
     { };
 
+    virtual void rewrite(exec_context &ec,
+                         shared_buffer_ref &line,
+                         string_attrs_t &sa,
+                         std::string &value_out) {
+        value_out.assign(line.get_data(), line.length());
+    };
+
     virtual const logline_value_stats *stats_for_value(const intern_string_t &name) const {
         return NULL;
     };
@@ -847,6 +888,7 @@ public:
         bool vd_hidden;
         bool vd_internal;
         std::vector<std::string> vd_action_list;
+        std::string vd_rewriter;
 
         bool operator<(const value_def &rhs) const {
             return this->vd_index < rhs.vd_index;
@@ -926,6 +968,11 @@ public:
                   std::vector<logline_value> &values,
                   bool annotate_module = true) const;
 
+    void rewrite(exec_context &ec,
+                 shared_buffer_ref &line,
+                 string_attrs_t &sa,
+                 std::string &value_out);
+
     void build(std::vector<std::string> &errors);
 
     void register_vtabs(log_vtab_manager *vtab_manager,

src/log_format_loader.cc

@@ -107,6 +107,41 @@ static external_log_format::pattern *pattern_provider(yajlpp_parse_context &ypc,
     return &pat;
 }
 
+static external_log_format::value_def *value_def_provider(yajlpp_parse_context &ypc, void *root)
+{
+    external_log_format *elf = ensure_format(&ypc);
+    const intern_string_t value_name = ypc.get_path_fragment_i(2);
+
+    external_log_format::value_def &retval = elf->elf_value_defs[value_name];
+
+    retval.vd_name = value_name;
+
+    return &retval;
+}
+
+static scaling_factor *scaling_factor_provider(yajlpp_parse_context &ypc, void *root)
+{
+    external_log_format *elf = ensure_format(&ypc);
+    const intern_string_t value_name = ypc.get_path_fragment_i(2);
+    string scale_spec = ypc.get_path_fragment(5);
+
+    const intern_string_t scale_name = intern_string::lookup(scale_spec.substr(1));
+    external_log_format::value_def &value_def = elf->elf_value_defs[value_name];
+
+    value_def.vd_name = value_name;
+
+    scaling_factor &retval = value_def.vd_unit_scaling[scale_name];
+
+    if (scale_spec[0] == '/') {
+        retval.sf_op = SO_DIVIDE;
+    }
+    else if (scale_spec[0] == '*') {
+        retval.sf_op = SO_MULTIPLY;
+    }
+
+    return &retval;
+}
+
 static external_log_format::json_format_element &
 ensure_json_format_element(external_log_format *elf, int index)
 {
@@ -229,62 +264,6 @@ static int read_level_int(yajlpp_parse_context *ypc, long long val)
     return 1;
 }
 
-static int read_value_def(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
-{
-    external_log_format *elf = ensure_format(ypc);
-    const intern_string_t value_name = ypc->get_path_fragment_i(2);
-    string field_name = ypc->get_path_fragment(3);
-    string val = string((const char *)str, len);
-
-    elf->elf_value_defs[value_name].vd_name = value_name;
-    if (field_name == "kind") {
-        logline_value::kind_t kind;
-
-        if ((kind = logline_value::string2kind(val.c_str())) ==
-            logline_value::VALUE_UNKNOWN) {
-            fprintf(stderr, "error: unknown value kind %s\n", val.c_str());
-            return 0;
-        }
-        elf->elf_value_defs[value_name].vd_kind = kind;
-    }
-    else if (field_name == "unit" && ypc->get_path_fragment(4) == "field") {
-        elf->elf_value_defs[value_name].vd_unit_field = intern_string::lookup(val);
-    }
-    else if (field_name == "collate") {
-        elf->elf_value_defs[value_name].vd_collate = val;
-    }
-
-    return 1;
-}
-
-static int read_value_action(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
-{
-    external_log_format *elf = ensure_format(ypc);
-    const intern_string_t value_name = ypc->get_path_fragment_i(2);
-    string field_name = ypc->get_path_fragment(3);
-    string val = string((const char *)str, len);
-
-    elf->elf_value_defs[value_name].vd_action_list.push_back(val);
-
-    return 1;
-}
-
-static int read_value_bool(yajlpp_parse_context *ypc, int val)
-{
-    external_log_format *elf = ensure_format(ypc);
-    const intern_string_t value_name = ypc->get_path_fragment_i(2);
-    string key_name = ypc->get_path_fragment(3);
-
-    if (key_name == "identifier")
-        elf->elf_value_defs[value_name].vd_identifier = val;
-    else if (key_name == "foreign-key")
-        elf->elf_value_defs[value_name].vd_foreign_key = val;
-    else if (key_name == "hidden")
-        elf->elf_value_defs[value_name].vd_hidden = val;
-
-    return 1;
-}
-
 static int read_action_def(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
 {
     external_log_format *elf = ensure_format(ypc);
@@ -331,42 +310,6 @@ static external_log_format::sample &ensure_sample(external_log_format *elf,
     return elf->elf_samples[index];
 }
 
-static int read_scaling(yajlpp_parse_context *ypc, double val)
-{
-    external_log_format *elf = ensure_format(ypc);
-    const intern_string_t value_name = ypc->get_path_fragment_i(2);
-    string scale_spec = ypc->get_path_fragment(5);
-
-    if (scale_spec.empty()) {
-        fprintf(stderr,
-                "error:%s:%s: scaling factor field cannot be empty\n",
-                ypc->get_path_fragment(0).c_str(),
-                value_name.get());
-        return 0;
-    }
-
-    const intern_string_t scale_name = intern_string::lookup(scale_spec.substr(1));
-    struct scaling_factor &sf = elf->elf_value_defs[value_name].vd_unit_scaling[scale_name];
-
-    if (scale_spec[0] == '/') {
-        sf.sf_op = SO_DIVIDE;
-    }
-    else if (scale_spec[0] == '*') {
-        sf.sf_op = SO_MULTIPLY;
-    }
-    else {
-        fprintf(stderr,
-                "error:%s:%s: scaling factor field must start with '/' or '*'\n",
-                ypc->get_path_fragment(0).c_str(),
-                value_name.get());
-        return 0;
-    }
-
-    sf.sf_value = val;
-
-    return 1;
-}
-
 static int read_sample_line(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
 {
     external_log_format *elf = ensure_format(ypc);
@@ -476,7 +419,7 @@ static struct json_path_handler line_format_handlers[] = {
         .for_enum(&nullobj<external_log_format::json_format_element>()->jfe_align),
 
     json_path_handler("overflow")
-        .with_synopsis("abbrev")
+        .with_synopsis("abbrev|truncate|dot-dot")
         .with_description("Overflow style")
         .with_enum_values(OVERFLOW_ENUM)
         .for_enum(&nullobj<external_log_format::json_format_element>()->jfe_overflow),
@@ -484,6 +427,75 @@ static struct json_path_handler line_format_handlers[] = {
     json_path_handler()
 };
 
+static const json_path_handler_base::enum_value_t KIND_ENUM[] = {
+    {"string", logline_value::VALUE_TEXT},
+    {"integer", logline_value::VALUE_INTEGER},
+    {"float", logline_value::VALUE_FLOAT},
+    {"boolean", logline_value::VALUE_BOOLEAN},
+    {"json", logline_value::VALUE_JSON},
+    {"quoted", logline_value::VALUE_QUOTED},
+
+    json_path_handler_base::ENUM_TERMINATOR
+};
+
+static struct json_path_handler unit_handlers[] = {
+    json_path_handler("field")
+        .with_synopsis("<field-name>")
+        .with_description("The name of the field that contains the units for this field")
+        .for_field(&nullobj<external_log_format::value_def>()->vd_unit_field),
+
+    json_path_handler("scaling-factor/.*$")
+        .with_synopsis("[*,/]<unit>")
+        .with_obj_provider(scaling_factor_provider)
+        .for_field(&nullobj<scaling_factor>()->sf_value),
+
+    json_path_handler()
+};
+
+static struct json_path_handler value_def_handlers[] = {
+    json_path_handler("kind")
+        .with_synopsis("string|integer|float|boolean|json|quoted")
+        .with_description("The type of data in the field")
+        .with_enum_values(KIND_ENUM)
+        .for_enum(&nullobj<external_log_format::value_def>()->vd_kind),
+
+    json_path_handler("collate")
+        .with_synopsis("<function>")
+        .with_description("The collating function to use for this column")
+        .for_field(&nullobj<external_log_format::value_def>()->vd_collate),
+
+    json_path_handler("unit/")
+        .with_obj_provider(value_def_provider)
+        .with_children(unit_handlers),
+
+    json_path_handler("identifier")
+        .with_synopsis("<bool>")
+        .with_description("Indicates whether or not this field contains an identifier that should be highlighted")
+        .for_field(&nullobj<external_log_format::value_def>()->vd_identifier),
+
+    json_path_handler("foreign-key")
+        .with_synopsis("<bool>")
+        .with_description("Indicates whether or not this field should be treated as a foreign key for row in another table")
+        .for_field(&nullobj<external_log_format::value_def>()->vd_foreign_key),
+
+    json_path_handler("hidden")
+        .with_synopsis("<bool>")
+        .with_description("Indicates whether or not this JSON field should be hidden")
+        .for_field(&nullobj<external_log_format::value_def>()->vd_hidden),
+
+    json_path_handler("action-list#")
+        .with_synopsis("<string>")
+        .with_description("Actions to execute when this field is clicked on")
+        .for_field(&nullobj<external_log_format::value_def>()->vd_action_list),
+
+    json_path_handler("rewriter")
+        .with_synopsis("<command>")
+        .with_description("A command that will rewrite this field when pretty-printing")
+        .for_field(&nullobj<external_log_format::value_def>()->vd_rewriter),
+
+    json_path_handler()
+};
+
 struct json_path_handler format_handlers[] = {
     json_path_handler("/\\w+/regex/[^/]+/")
         .with_obj_provider(pattern_provider)
@@ -502,11 +514,11 @@ struct json_path_handler format_handlers[] = {
                       "(trace|debug\\d*|info|stats|warning|error|critical|fatal)")
         .add_cb(read_levels)
         .add_cb(read_level_int),
-    json_path_handler("/\\w+/value/.+/(kind|collate|unit/field)$", read_value_def),
+
-    json_path_handler("/\\w+/value/.+/(identifier|foreign-key|hidden)$", read_value_bool),
+    json_path_handler("/\\w+/value/[^/]+/")
-    json_path_handler("/\\w+/value/.+/unit/scaling-factor/.*$",
+        .with_obj_provider(value_def_provider)
-        read_scaling),
+        .with_children(value_def_handlers),
-    json_path_handler("/\\w+/value/.+/action-list#", read_value_action),
+
     json_path_handler("/\\w+/action/[^/]+/label", read_action_def),
     json_path_handler("/\\w+/action/[^/]+/capture-output", read_action_bool),
     json_path_handler("/\\w+/action/[^/]+/cmd#", read_action_cmd),
@@ -909,6 +921,8 @@ static void find_format_in_path(const string &path,
             meta.sm_name = script_name;
             extract_metadata_from_file(meta);
             scripts[script_name].push_back(meta);
+
+            log_debug("  found script: %s", meta.sm_path.c_str());
         }
     }
 }

src/log_vtab_impl.cc

@@ -94,6 +94,8 @@ std::string log_vtab_impl::get_table_statement(void)
         << "  log_body text hidden\n"
         << ");\n";
 
+    log_debug("log_vtab_impl.get_table_statement() -> %s", oss.str().c_str());
+
     return oss.str();
 }
 

src/readline_callbacks.cc

@@ -233,6 +233,7 @@ void rl_abort(void *dummy, readline_curses *rc)
 
 void rl_callback(void *dummy, readline_curses *rc)
 {
+    exec_context &ec = lnav_data.ld_exec_context;
     string alt_msg;
 
     lnav_data.ld_bottom_source.set_prompt("");
@@ -243,7 +244,7 @@ void rl_callback(void *dummy, readline_curses *rc)
 
     case LNM_COMMAND:
         rc->set_alt_value("");
-        rc->set_value(execute_command(rc->get_value()));
+        rc->set_value(execute_command(ec, rc->get_value()));
         break;
 
     case LNM_SEARCH:
@@ -264,10 +265,17 @@ void rl_callback(void *dummy, readline_curses *rc)
         }
         break;
 
-    case LNM_SQL:
+    case LNM_SQL: {
-        rc->set_value(execute_sql(rc->get_value(), alt_msg));
+        string result = execute_sql(ec, rc->get_value(), alt_msg);
+
+        if (!result.empty()) {
+            result = "SQL Result: " + result;
+        }
+
+        rc->set_value(result);
         rc->set_alt_value(alt_msg);
         break;
+    }
 
     case LNM_EXEC: {
         char fn_template[PATH_MAX];
@@ -294,7 +302,7 @@ void rl_callback(void *dummy, readline_curses *rc)
 
                 if ((tmpout = fdopen(fd, "w+")) != NULL) {
                     lnav_data.ld_output_stack.push(tmpout);
-                    string result = execute_file(path_and_args);
+                    string result = execute_file(ec, path_and_args);
                     string::size_type lf_index = result.find('\n');
                     if (lf_index != string::npos) {
                         result = result.substr(0, lf_index);

src/readline_curses.hh

@@ -43,6 +43,7 @@
 
 #include <map>
 #include <set>
+#include <stack>
 #include <string>
 #include <vector>
 #include <exception>
@@ -52,9 +53,14 @@
 
 #include "auto_fd.hh"
 #include "vt52_curses.hh"
+#include "log_format.hh"
+
+struct exec_context;
 
 typedef void (*readline_highlighter_t)(attr_line_t &line, int x);
 
+extern exec_context INIT_EXEC_CONTEXT;
+
 /**
  * Container for information related to different readline contexts.  Since
  * lnav uses readline for different inputs, we need a way to keep things like
@@ -62,7 +68,7 @@ typedef void (*readline_highlighter_t)(attr_line_t &line, int x);
  */
 class readline_context {
 public:
-    typedef std::string (*command_func_t)(
+    typedef std::string (*command_func_t)(exec_context &ec,
             std::string cmdline, std::vector<std::string> &args);
     typedef struct {
         const char *c_name;
@@ -96,7 +102,7 @@ public:
                 std::string cmd = iter->first;
 
                 this->rc_possibilities["__command"].insert(cmd);
-                iter->second.c_func(cmd, this->rc_prototypes[cmd]);
+                iter->second.c_func(INIT_EXEC_CONTEXT, cmd, this->rc_prototypes[cmd]);
             }
         }
 

src/session_data.cc

@@ -771,7 +771,7 @@ static int read_commands(yajlpp_parse_context *ypc, const unsigned char *str, si
             ypc->get_path_fragment(-3));
     view_index = view_name - lnav_view_strings;
     bool active = ensure_view(&lnav_data.ld_views[view_index]);
-    execute_command(cmdline);
+    execute_command(lnav_data.ld_exec_context, cmdline);
     if (!active) {
         lnav_data.ld_view_stack.pop();
     }

src/view_curses.hh

@@ -186,6 +186,15 @@ struct line_range {
         return this->contains(other.lr_start) || this->contains(other.lr_end);
     };
 
+    void shift(int32_t start, int32_t amount) {
+        if (this->lr_start >= start) {
+            this->lr_start += amount;
+        }
+        if (this->lr_end != -1 && start < this->lr_end) {
+            this->lr_end += amount;
+        }
+    };
+
     void ltrim(const char *str) {
         while (this->lr_start < this->lr_end && isspace(str[this->lr_start])) {
             this->lr_start += 1;
@@ -315,14 +324,7 @@ inline void remove_string_attr(string_attrs_t &sa, const struct line_range &lr)
 inline void shift_string_attrs(string_attrs_t &sa, int32_t start, int32_t amount)
 {
     for (string_attrs_t::iterator iter = sa.begin(); iter != sa.end(); ++iter) {
-        struct line_range *existing_lr = &iter->sa_range;
+        iter->sa_range.shift(start, amount);
-
-        if (existing_lr->lr_start >= start) {
-            existing_lr->lr_start += amount;
-        }
-        if (existing_lr->lr_end != -1 && start < existing_lr->lr_end) {
-            existing_lr->lr_end += amount;
-        }
     }
 }
 

src/yajlpp.cc

@@ -65,6 +65,17 @@ int yajlpp_static_string(yajlpp_parse_context *ypc, const unsigned char *str, si
     return 1;
 }
 
+int yajlpp_static_string_vector(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
+{
+    char *root_ptr = resolve_root(ypc);
+    vector<string> *field_ptr = (vector<string> *) root_ptr;
+
+    field_ptr->push_back(string((const char *) str, len));
+    yajlpp_validator_for_string(*ypc, *ypc->ypc_current_handler);
+
+    return 1;
+}
+
 int yajlpp_static_intern_string(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
 {
     char *root_ptr = resolve_root(ypc);
@@ -182,6 +193,20 @@ void yajlpp_validator_for_int(yajlpp_parse_context &ypc,
     }
 }
 
+void yajlpp_validator_for_double(yajlpp_parse_context &ypc,
+                                 const json_path_handler_base &jph)
+{
+    double *field_ptr = (double *) resolve_root(&ypc);
+    char buffer[1024];
+
+    if (*field_ptr < jph.jph_min_value) {
+        snprintf(buffer, sizeof(buffer),
+                 "value must be greater than %lld",
+                 jph.jph_min_value);
+        ypc.report_error(buffer);
+    }
+}
+
 int yajlpp_static_number(yajlpp_parse_context *ypc, long long num)
 {
     char *root_ptr = resolve_root(ypc);
@@ -192,6 +217,16 @@ int yajlpp_static_number(yajlpp_parse_context *ypc, long long num)
     return 1;
 }
 
+int yajlpp_static_decimal(yajlpp_parse_context *ypc, double num)
+{
+    char *root_ptr = resolve_root(ypc);
+    double *field_ptr = (double *) root_ptr;
+
+    *field_ptr = num;
+
+    return 1;
+}
+
 int yajlpp_static_bool(yajlpp_parse_context *ypc, int val)
 {
     char *root_ptr = resolve_root(ypc);
@@ -283,13 +318,32 @@ int yajlpp_parse_context::map_key(void *ctx,
                                   size_t len)
 {
     yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
-    int start, retval = 1;
+    int retval = 1;
 
     ypc->ypc_path.resize(ypc->ypc_path_index_stack.back());
     ypc->ypc_path.push_back('/');
-    start = ypc->ypc_path.size();
+    if (ypc->ypc_handlers != NULL) {
-    ypc->ypc_path.resize(ypc->ypc_path.size() + len);
+        for (size_t lpc = 0; lpc < len; lpc++) {
-    memcpy(&ypc->ypc_path[start], key, len);
+            switch (key[lpc]) {
+                case '~':
+                    ypc->ypc_path.push_back('~');
+                    ypc->ypc_path.push_back('0');
+                    break;
+                case '/':
+                    ypc->ypc_path.push_back('~');
+                    ypc->ypc_path.push_back('1');
+                    break;
+                default:
+                    ypc->ypc_path.push_back(key[lpc]);
+                    break;
+            }
+        }
+    }
+    else {
+        size_t start = ypc->ypc_path.size();
+        ypc->ypc_path.resize(ypc->ypc_path.size() + len);
+        memcpy(&ypc->ypc_path[start], key, len);
+    }
     ypc->ypc_path.push_back('\0');
 
     if (ypc->ypc_alt_callbacks.yajl_map_key != NULL) {
@@ -339,11 +393,11 @@ void yajlpp_parse_context::update_callbacks(const json_path_handler_base *orig_h
             pcre_context::capture_t *cap = this->ypc_pcre_context.all();
 
             if (jph.jph_children) {
-                if (this->ypc_path[cap->c_end - 1] != '/') {
+                if (this->ypc_path[child_start + cap->c_end - 1] != '/') {
                     continue;
                 }
 
-                this->update_callbacks(jph.jph_children, cap->c_end);
+                this->update_callbacks(jph.jph_children, child_start + cap->c_end);
             }
             else {
                 if (child_start + cap->c_end != this->ypc_path.size() - 1) {

src/yajlpp.hh

@@ -43,6 +43,7 @@
 #include <algorithm>
 
 #include "pcrepp.hh"
+#include "json_ptr.hh"
 #include "intern_string.hh"
 
 #include "yajl/api/yajl_parse.h"
@@ -129,6 +130,7 @@ struct json_path_handler_base {
 };
 
 int yajlpp_static_string(yajlpp_parse_context *, const unsigned char *, size_t);
+int yajlpp_static_string_vector(yajlpp_parse_context *, const unsigned char *, size_t);
 int yajlpp_static_intern_string(yajlpp_parse_context *, const unsigned char *, size_t);
 int yajlpp_static_enum(yajlpp_parse_context *, const unsigned char *, size_t);
 yajl_gen_status yajlpp_static_gen_string(yajlpp_gen_context &ygc,
@@ -140,8 +142,11 @@ void yajlpp_validator_for_intern_string(yajlpp_parse_context &ypc,
                                         const json_path_handler_base &jph);
 void yajlpp_validator_for_int(yajlpp_parse_context &ypc,
                               const json_path_handler_base &jph);
+void yajlpp_validator_for_double(yajlpp_parse_context &ypc,
+                                 const json_path_handler_base &jph);
 
 int yajlpp_static_number(yajlpp_parse_context *, long long);
+int yajlpp_static_decimal(yajlpp_parse_context *, double);
 
 int yajlpp_static_bool(yajlpp_parse_context *, int);
 yajl_gen_status yajlpp_static_gen_bool(yajlpp_gen_context &ygc,
@@ -257,6 +262,12 @@ struct json_path_handler : public json_path_handler_base {
         return *this;
     };
 
+    json_path_handler &for_field(std::vector<std::string> *field) {
+        this->add_cb(yajlpp_static_string_vector);
+        this->jph_simple_offset = field;
+        return *this;
+    };
+
     json_path_handler &for_field(intern_string_t *field) {
         this->add_cb(yajlpp_static_intern_string);
         this->jph_simple_offset = field;
@@ -280,6 +291,13 @@ struct json_path_handler : public json_path_handler_base {
         return *this;
     };
 
+    json_path_handler &for_field(double *field) {
+        this->add_cb(yajlpp_static_decimal);
+        this->jph_simple_offset = field;
+        this->jph_validator = yajlpp_validator_for_double;
+        return *this;
+    };
+
     json_path_handler &for_field(bool *field) {
         this->add_cb(yajlpp_static_bool);
         this->jph_simple_offset = field;
@@ -323,7 +341,8 @@ public:
         memset(&this->ypc_alt_callbacks, 0, sizeof(this->ypc_alt_callbacks));
     };
 
-    void get_path_fragment(int offset, const char **frag, size_t &len_out) const {
+    const char *get_path_fragment(int offset, char *frag_in, size_t &len_out) const {
+        const char *retval;
         size_t start, end;
 
         if (offset < 0) {
@@ -333,27 +352,36 @@ public:
         if ((offset + 1) < (int)this->ypc_path_index_stack.size()) {
             end = this->ypc_path_index_stack[offset + 1];
         }
-        else{
+        else {
             end = this->ypc_path.size() - 1;
         }
-        *frag = &this->ypc_path[start];
+        if (this->ypc_handlers != NULL) {
-        len_out = end - start;
+            len_out = json_ptr::decode(frag_in, &this->ypc_path[start], end - start);
+            retval = frag_in;
+        }
+        else {
+            retval = &this->ypc_path[start];
+            len_out = end - start;
+        }
+
+        return retval;
     }
 
     const intern_string_t get_path_fragment_i(int offset) const {
+        char fragbuf[this->ypc_path.size()];
         const char *frag;
         size_t len;
 
-        this->get_path_fragment(offset, &frag, len);
+        frag = this->get_path_fragment(offset, fragbuf, len);
         return intern_string::lookup(frag, len);
     };
 
-    std::string get_path_fragment(int offset) const
+    std::string get_path_fragment(int offset) const {
-    {
+        char fragbuf[this->ypc_path.size()];
         const char *frag;
         size_t len;
 
-        this->get_path_fragment(offset, &frag, len);
+        frag = this->get_path_fragment(offset, fragbuf, len);
         return std::string(frag, len);
     };
 

test/Makefile.am

@@ -161,6 +161,7 @@ drive_data_scanner_LDADD = \
 	../src/dhclient-summary.o \
 	../src/dump-pid-sh.o \
 	../src/partition-by-boot.o \
+	$(LIBCURL) \
 	$(PCRE_LIBS) \
 	$(SQLITE3_LIBS) \
 	-lpcrecpp \

test/drive_data_scanner.cc

@@ -51,6 +51,11 @@ using namespace std;
 
 const char *TMP_NAME = "scanned.tmp";
 
+string execute_any(exec_context &ec, const string &cmdline_with_mode)
+{
+    return "";
+}
+
 int main(int argc, char *argv[])
 {
     int  c, retval = EXIT_SUCCESS;

test/drive_logfile.cc

@@ -57,6 +57,11 @@ time_t time(time_t *_unused) {
   return 1194107018;
 }
 
+string execute_any(exec_context &ec, const string &cmdline_with_mode)
+{
+  return "";
+}
+
 int main(int argc, char *argv[]) {
   int c, retval = EXIT_SUCCESS;
   dl_mode_t mode = MODE_NONE;

test/drive_readline_curses.cc

@@ -56,6 +56,17 @@ static struct {
     volatile sig_atomic_t dd_looping;
 } drive_data;
 
+string execute_any(exec_context &ec, const string &cmdline_with_mode)
+{
+    return "";
+}
+
+struct exec_context {
+
+};
+
+exec_context INIT_EXEC_CONTEXT;
+
 static void rl_callback(void *dummy, readline_curses *rc)
 {
     string line = rc->get_value();

test/drive_sql.cc

@@ -32,6 +32,11 @@ static int sql_callback(void *ptr,
     return 0;
 }
 
+std::string execute_any(exec_context &ec, const std::string &cmdline_with_mode)
+{
+    return "";
+}
+
 int main(int argc, char *argv[])
 {
     int retval = EXIT_SUCCESS;

test/formats/jsontest/format.json

@@ -18,6 +18,12 @@
             "user" : {
                 "kind" : "string",
                 "identifier" : true
+            },
+            "msg" : {
+                "rewriter" : ";SELECT :msg || 'bork bork bork'"
+            },
+            "user" : {
+                "rewriter" : "|rewrite-user"
             }
         }
     }

test/logfile_uwsgi.0

@@ -1,6 +1,6 @@
 [pid: 88185|app: 0|req: 1/1] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:49:12 2016] POST /update_metrics => generated 47 bytes in 129 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 3)
 [pid: 88185|app: 0|req: 3/2] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:49:15 2016] POST /update_metrics => generated 47 bytes in 35 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 30)
-[pid: 88185|app: 0|req: 3/3] 127.0.0.1 () {34 vars in 617 bytes} [Sun Mar 13 22:49:15 2016] POST /endpoint2 => generated 215 bytes in 68 msecs (HTTP/1.1 200) 9 headers in 373 bytes (1 switches on core 8)
+[pid: 88185|app: 0|req: 3/3] 127.0.0.1 () {34 vars in 617 bytes} [Sun Mar 13 22:49:15 2016] POST /endpoint2 => generated 215 bytes in 68 micros (HTTP/1.1 200) 9 headers in 373 bytes (1 switches on core 8)
 [pid: 88185|app: 0|req: 4/4] 127.0.0.1 () {34 vars in 617 bytes} [Sun Mar 13 22:49:15 2016] POST /endpoint2 => generated 215 bytes in 16 msecs (HTTP/1.1 200) 9 headers in 373 bytes (1 switches on core 22)
 [pid: 88185|app: 0|req: 5/5] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:50:12 2016] POST /update_metrics => generated 47 bytes in 10 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 0)
 [pid: 88186|app: 0|req: 1/6] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:50:15 2016] POST /update_metrics => generated 47 bytes in 65 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 16)

test/test_json_format.sh

@@ -27,6 +27,36 @@ check_output "json log format is not working" <<EOF
 EOF
 
 
+run_test ${lnav_test} -n \
+    -I ${test_dir} -c ':switch-to-view pretty' \
+    ${test_dir}/logfile_json.json
+
+check_output "json log format is not working" <<EOF
+2013-09-06T20:00:48.124 TRACE trace testbork bork bork
+2013-09-06T20:00:49.124 INFO Starting up servicebork bork bork
+2013-09-06T22:00:49.124 INFO Shutting down servicebork bork bork
+user: mailto:steve@example.com
+2013-09-06T22:00:59.124 DEBUG5 Details...bork bork bork
+2013-09-06T22:00:59.124 DEBUG4 Details...bork bork bork
+2013-09-06T22:00:59.124 DEBUG3 Details...bork bork bork
+2013-09-06T22:00:59.124 DEBUG2 Details...bork bork bork
+2013-09-06T22:00:59.124 DEBUG Details...bork bork bork
+2013-09-06T22:01:49.124 STATS 1 beat per secondbork bork bork
+2013-09-06T22:01:49.124 WARNING not looking goodbork bork bork
+2013-09-06T22:01:49.124 ERROR looking badbork bork bork
+2013-09-06T22:01:49.124 CRITICAL sooo badbork bork bork
+2013-09-06T22:01:49.124 FATAL shootbork bork bork
+obj: {
+    "field1" : "hi",
+    "field2": 2
+}
+arr: [
+    "hi",
+        {"sub1": true}
+]
+EOF
+
+
 run_test ${lnav_test} -n \
     -I ${test_dir} \
     ${test_dir}/log.clog

test/test_top_status.cc

@@ -47,6 +47,11 @@ time_t time(time_t *arg)
     return current_time;
 }
 
+string execute_any(exec_context &ec, const string &cmdline_with_mode)
+{
+    return "";
+}
+
 int main(int argc, char *argv[])
 {
     int retval = EXIT_SUCCESS;