Robot
/
lnav
mirror of https://github.com/tstack/lnav.git
2
0
Fork 0
Code Issues Releases Wiki Activity

[sql] add lnav_top_file() SQL function

This commit is contained in:
Timothy Stack 2021-03-05 16:17:28 -08:00
parent 68759ada2d
commit f6128240ab
40 changed files with 807 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
NEWS
View File

@ -40,6 +40,16 @@ lnav v0.9.1:
an aggregator.
* Added a "log_time_msecs" hidden column to the log tables that returns
the timestamp as the number of milliseconds from the epoch.
* Added an "lnav_top_file()" SQL function that can be used to get the
name of the top line in the top view or NULL if the line did not come
from a file.
* Added a "mimetype" column to the lnav_file table that returns a guess as
to the MIME type of the file contents.
* Added a "content" hidden column to the lnav_file table that can be used
to read the contents of the file. The contents can then be passed to
functions that operate on XML/JSON data, like xpath() or json_tree().
* Added an "lnav_top_view" SQL VIEW that returns the row for the top view
in the lnav_views table.
Interface Changes:
* When copying log lines, the file name and time offset will be included

1
src/CMakeLists.txt
View File

@ -276,6 +276,7 @@ add_library(diag STATIC
listview_curses.cc
lnav_commands.cc
lnav_config.cc
base/lnav.gzip.cc
base/lnav_log.cc
lnav_util.cc
log_accel.cc

1
src/Makefile.am
View File

@ -177,6 +177,7 @@ noinst_HEADERS = \
field_overlay_source.hh \
file_collection.hh \
file_format.hh \
file_vtab.cfg.hh \
files_sub_source.hh \
filter_observer.hh \
filter_status_source.hh \

12
src/attr_line.hh
View File

@ -207,6 +207,18 @@ find_string_attr(const string_attrs_t &sa, string_attr_type_t type, int start =
return iter;
}
inline nonstd::optional<const string_attr*>
get_string_attr(const string_attrs_t &sa, string_attr_type_t type, int start = 0)
{
auto iter = find_string_attr(sa, type, start);
if (iter == sa.end()) {
return nonstd::nullopt;
}
return nonstd::make_optional(&(*iter));
}
template<typename T>
inline string_attrs_t::const_iterator
find_string_attr_containing(const string_attrs_t &sa, string_attr_type_t type, T x)

60
src/auto_mem.hh
View File

@ -38,6 +38,8 @@
#include <exception>
#include "base/result.h"
typedef void (*free_func_t)(void *);
/**
@ -146,4 +148,62 @@ private:
T srm_value;
};
class auto_buffer {
public:
static auto_buffer alloc(size_t size) {
return auto_buffer{ (char *) malloc(size), size };
}
auto_buffer(auto_buffer&& other) noexcept
: ab_buffer(other.ab_buffer), ab_size(other.ab_size) {
other.ab_buffer = nullptr;
other.ab_size = 0;
}
~auto_buffer() {
free(this->ab_buffer);
this->ab_buffer = nullptr;
this->ab_size = 0;
}
char *in() {
return this->ab_buffer;
}
std::pair<char *, size_t> release() {
auto retval = std::make_pair(this->ab_buffer, this->ab_size);
this->ab_buffer = nullptr;
this->ab_size = 0;
return retval;
}
size_t size() const {
return this->ab_size;
}
void expand_by(size_t amount) {
auto new_size = this->ab_size + amount;
auto new_buffer = (char *) realloc(this->ab_buffer, new_size);
if (new_buffer == nullptr) {
throw std::bad_alloc();
}
this->ab_buffer = new_buffer;
this->ab_size = new_size;
}
auto_buffer& shrink_to(size_t new_size) {
this->ab_size = new_size;
return *this;
}
private:
auto_buffer(char *buffer, size_t size) : ab_buffer(buffer), ab_size(size) {
}
char *ab_buffer;
size_t ab_size;
};
#endif

2
src/base/Makefile.am
View File

@ -26,6 +26,7 @@ noinst_HEADERS = \
is_utf8.hh \
isc.hh \
lnav_log.hh \
lnav.gzip.hh \
lrucache.hpp \
math_util.hh \
opt_util.hh \
@ -39,6 +40,7 @@ libbase_a_SOURCES = \
intern_string.cc \
is_utf8.cc \
isc.cc \
lnav.gzip.cc \
lnav_log.cc \
string_util.cc \
time_util.cc

95
src/base/lnav.gzip.cc Normal file
View File

@ -0,0 +1,95 @@
/**
* Copyright (c) 2021, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file lnav.gzip.cc
*/
#include "config.h"
#include <zlib.h>
#include "fmt/format.h"
#include "lnav.gzip.hh"
namespace lnav {
namespace gzip {
bool is_gzipped(const char *buffer, size_t len)
{
return len > 2 && buffer[0] == '\037' && buffer[1] == '\213';
}
Result<auto_buffer, std::string> uncompress(const std::string& src,
const void *buffer,
size_t size)
{
auto uncomp = auto_buffer::alloc(size * 2);
z_stream strm;
int err;
strm.next_in = (Bytef *) buffer;
strm.avail_in = size;
strm.total_out = 0;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
if ((err = inflateInit2(&strm, (16 + MAX_WBITS))) != Z_OK) {
return Err(fmt::format("invalid gzip data: {} -- {}",
src, strm.msg ? strm.msg : zError(err)));
}
bool done = false;
while (!done) {
if (strm.total_out >= uncomp.size()) {
uncomp.expand_by(size / 2);
}
strm.next_out = (Bytef *) (uncomp.in() + strm.total_out);
strm.avail_out = uncomp.size() - strm.total_out;
// Inflate another chunk.
err = inflate(&strm, Z_SYNC_FLUSH);
if (err == Z_STREAM_END) {
done = true;
} else if (err != Z_OK) {
return Err(fmt::format("unable to uncompress: {} -- {}",
src, strm.msg ? strm.msg : zError(err)));
}
}
if (inflateEnd(&strm) != Z_OK) {
return Err(fmt::format("unable to uncompress: {} -- {}",
src, strm.msg ? strm.msg : zError(err)));
}
return Ok(std::move(uncomp.shrink_to(strm.total_out)));
}
}
}

52
src/base/lnav.gzip.hh Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright (c) 2021, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file lnav.gzip.hh
*/
#ifndef lnav_gzip_hh
#define lnav_gzip_hh
#include <string>
#include "auto_mem.hh"
#include "result.h"
namespace lnav {
namespace gzip {
bool is_gzipped(const char *buffer, size_t len);
Result<auto_buffer, std::string> uncompress(const std::string& src,
const void *buffer,
size_t size);
}
}
#endif

36
src/base/result.h
View File

@ -33,7 +33,10 @@ namespace types {
E val;
};
}
template<>
struct Err<void> { };
};
template<typename T, typename CleanT = typename std::decay<T>::type>
types::Ok<CleanT> Ok(T&& val) {
@ -49,6 +52,10 @@ types::Err<CleanE> Err(E&& val) {
return types::Err<CleanE>(std::forward<E>(val));
}
inline types::Err<void> Err() {
return {};
}
template<typename T, typename E> struct Result;
namespace details {
@ -554,7 +561,7 @@ struct Storage {
void construct(types::Ok<T> ok)
{
new (&storage_) T(ok.val);
new (&storage_) T(std::move(ok.val));
initialized_ = true;
}
void construct(types::Err<E> err)
@ -796,17 +803,32 @@ struct Result {
}
template<typename Func>
Result<T, E> then(Func func) const {
Result<void, E> then(Func func) {
if (this->isOk()) {
func(this->storage().template get<T>());
func(std::move(this->storage().template get<T>()));
return Ok();
}
return *this;
return Err(std::move(this->storage().template get<E>()));
}
template<typename Func>
Result<T, E> otherwise(Func func) const {
return details::otherwise(*this, func);
Result<typename std::result_of<Func>::type, E> then(Func func) {
if (this->isOk()) {
return Ok(func(std::move(this->storage().template get<T>())));
}
return Err(std::move(this->storage().template get<E>()));
}
template<typename Func>
void otherwise(Func func) {
if (this->isOk()) {
return;
}
func(std::move(this->storage().template get<E>()));
}
template<typename Func,

4
src/command_executor.cc
View File

@ -792,8 +792,10 @@ future<string> pipe_callback(exec_context &ec, const string &cmdline, auto_fd &f
auto pp = make_shared<piper_proc>(
fd, false, open_temp_file(ghc::filesystem::temp_directory_path() /
"lnav.out.XXXXXX")
.then([](auto pair) {
.map([](auto pair) {
ghc::filesystem::remove(pair.first);
return pair;
})
.expect("Cannot create temporary file for callback")
.second);

2
src/db_sub_source.cc
View File

@ -105,7 +105,7 @@ void db_label_source::text_attrs_for_line(textview_curses &tc, int row,
this->dls_chart.chart_attrs_for_value(tc, left, this->dls_headers[lpc].hm_name, num_value, sa);
}
}
if (row_len > 2 &&
if (row_len > 2 && row_len < MAX_COLUMN_WIDTH &&
((row_value[0] == '{' && row_value[row_len - 1] == '}') ||
(row_value[0] == '[' && row_value[row_len - 1] == ']'))) {
json_ptr_walk jpw;

63
src/file_vtab.cc
View File

@ -33,12 +33,14 @@
#include <string.h>
#include "base/injector.bind.hh"
#include "base/lnav.gzip.hh"
#include "base/lnav_log.hh"
#include "file_collection.hh"
#include "logfile.hh"
#include "session_data.hh"
#include "vtab_module.hh"
#include "log_format.hh"
#include "file_vtab.cfg.hh"
using namespace std;
@ -52,9 +54,12 @@ CREATE TABLE lnav_file (
device integer, -- The device the file is stored on.
inode integer, -- The inode for the file on the device.
filepath text, -- The path to the file.
mimetype text, -- The MIME type for the file.
format text, -- The log file format for the file.
lines integer, -- The number of lines in the file.
time_offset integer -- The millisecond offset for timestamps.
time_offset integer, -- The millisecond offset for timestamps.
content BLOB HIDDEN -- The contents of the file.
);
)";
@ -88,18 +93,66 @@ CREATE TABLE lnav_file (
to_sqlite(ctx, name);
break;
case 3:
to_sqlite(ctx, format_name);
to_sqlite(ctx, fmt::format("{}", lf->get_text_format()));
break;
case 4:
to_sqlite(ctx, format_name);
break;
case 5:
to_sqlite(ctx, (int64_t) lf->size());
break;
case 5: {
case 6: {
auto tv = lf->get_time_offset();
int64_t ms = (tv.tv_sec * 1000LL) + tv.tv_usec / 1000LL;
to_sqlite(ctx, ms);
break;
}
case 7: {
auto& cfg = injector::get<const file_vtab::config&>();
auto lf_stat = lf->get_stat();
if (lf_stat.st_size > cfg.fvc_max_content_size) {
sqlite3_result_error(ctx, "file is too large", -1);
} else {
auto fd = lf->get_fd();
auto_mem<char> buf;
buf = (char *) malloc(lf_stat.st_size);
auto rc = pread(fd, buf, lf_stat.st_size, 0);
if (rc == -1) {
auto errmsg = fmt::format("unable to read file: {}",
strerror(errno));
sqlite3_result_error(ctx, errmsg.c_str(),
errmsg.length());
} else if (rc != lf_stat.st_size) {
auto errmsg = fmt::format("short read of file: {} < {}",
rc, lf_stat.st_size);
sqlite3_result_error(ctx, errmsg.c_str(),
errmsg.length());
} else if (lnav::gzip::is_gzipped(buf, rc)) {
lnav::gzip::uncompress(lf->get_unique_path(), buf, rc)
.then([ctx](auto uncomp) {
auto pair = uncomp.release();
sqlite3_result_blob64(ctx,
pair.first,
pair.second,
free);
})
.otherwise([ctx](auto msg) {
sqlite3_result_error(ctx,
msg.c_str(),
msg.size());
});
} else {
sqlite3_result_blob64(ctx, buf.release(), rc, free);
}
}
break;
}
default:
ensure(0);
break;
@ -125,9 +178,11 @@ CREATE TABLE lnav_file (
int64_t device,
int64_t inode,
std::string path,
const char *text_format,
const char *format,
int64_t lines,
int64_t time_offset) {
int64_t time_offset,
const char *content) {
auto lf = this->lf_collection.fc_files[rowid];
struct timeval tv = {
(int) (time_offset / 1000LL),

43
src/file_vtab.cfg.hh Normal file
View File

@ -0,0 +1,43 @@
/**
* Copyright (c) 2021, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file file_vtab.cfg.hh
*/
#ifndef lnav_file_vtab_cfg_hh
#define lnav_file_vtab_cfg_hh
namespace file_vtab {
struct config {
int64_t fvc_max_content_size{32 * 1024 * 1024};
};
}
#endif

4
src/highlighter.cc
View File

@ -41,7 +41,7 @@ highlighter::highlighter(const highlighter &other)
pcre_refcount(this->h_code, 1);
this->study();
this->h_attrs = other.h_attrs;
this->h_text_format = other.h_text_format;
this->h_text_formats = other.h_text_formats;
this->h_format_name = other.h_format_name;
this->h_nestable = other.h_nestable;
}
@ -67,7 +67,7 @@ highlighter &highlighter::operator=(const highlighter &other)
this->study();
this->h_format_name = other.h_format_name;
this->h_attrs = other.h_attrs;
this->h_text_format = other.h_text_format;
this->h_text_formats = other.h_text_formats;
this->h_nestable = other.h_nestable;
return *this;

6
src/highlighter.hh
View File

@ -32,6 +32,8 @@
#ifndef highlighter_hh
#define highlighter_hh
#include <set>
#include "optional.hpp"
#include "pcrepp/pcrepp.hh"
#include "text_format.hh"
@ -82,7 +84,7 @@ struct highlighter {
};
highlighter &with_text_format(text_format_t tf) {
this->h_text_format = tf;
this->h_text_formats.insert(tf);
return *this;
}
@ -121,7 +123,7 @@ struct highlighter {
pcre *h_code;
pcre_extra *h_code_extra;
int h_attrs{-1};
text_format_t h_text_format{text_format_t::TF_UNKNOWN};
std::set<text_format_t> h_text_formats;
intern_string_t h_format_name;
bool h_nestable{true};
};

4
src/init.sql
View File

@ -85,6 +85,10 @@ CREATE TABLE lnav_example_log (
log_body text hidden
);
CREATE VIEW lnav_top_view AS
SELECT * FROM lnav_views WHERE name = (
SELECT name FROM lnav_view_stack ORDER BY rowid DESC LIMIT 1);
INSERT INTO lnav_example_log VALUES
(0, null, '2017-02-03T04:05:06.100', '2017-02-03T04:05:06.100', 0, 'info', 0, null, null, null, 'hw', 2, '/tmp/log', '2017-02-03T04:05:06.100 hw(2): Hello, World!', 'Hello, World!'),
(1, null, '2017-02-03T04:05:06.200', '2017-02-03T04:05:06.200', 100, 'error', 0, null, null, null, 'gw', 4, '/tmp/log', '2017-02-03T04:05:06.200 gw(4): Goodbye, World!', 'Goodbye, World!'),

14
src/internals/config-v1.schema.json
View File

@ -34,6 +34,20 @@
}
},
"additionalProperties": false
},
"file-vtab": {
"description": "Settings related to the lnav_file virtual-table",
"title": "/tuning/file-vtab",
"type": "object",
"properties": {
"max-content-size": {
"title": "/tuning/file-vtab/max-content-size",
"description": "The maximum allowed file size for the content column",
"type": "integer",
"minimum": 0
}
},
"additionalProperties": false
}
},
"additionalProperties": false

11
src/internals/sql-ref.rst
View File

@ -1833,6 +1833,17 @@ likely(*value*)
----
.. _lnav_top_file:
lnav_top_file()
^^^^^^^^^^^^^^^
Return the name of the file that the top line in the current view came from.
----
.. _load_extension:
load_extension(*path*, *\[entry-point\]*)

12
src/listview_curses.hh
View File

@ -277,6 +277,18 @@ public:
return retval;
};
template<typename F>
auto map_top_row(F func) -> typename std::result_of<F(const attr_line_t&)>::type {
if (this->get_inner_height() == 0) {
return nonstd::nullopt;
}
std::vector<attr_line_t> top_line{1};
this->lv_source->listview_value_for_rows(*this, this->lv_top, top_line);
return func(top_line[0]);
}
/** @param win The curses window this view is attached to. */
void set_window(WINDOW *win) { this->lv_window = win; };

4
src/lnav.cc
View File

@ -2418,8 +2418,10 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
false,
open_temp_file(ghc::filesystem::temp_directory_path() /
"lnav.fifo.XXXXXX")
.then([](auto pair) {
.map([](auto pair) {
ghc::filesystem::remove(pair.first);
return pair;
})
.expect("Cannot create temporary file for FIFO")
.second);

4
src/lnav_commands.cc
View File

@ -2027,8 +2027,10 @@ static Result<string, string> com_open(exec_context &ec, string cmdline, vector<
false,
open_temp_file(ghc::filesystem::temp_directory_path() /
"lnav.fifo.XXXXXX")
.then([](auto pair) {
.map([](auto pair) {
ghc::filesystem::remove(pair.first);
return pair;
})
.expect("Cannot create temporary file for FIFO")
.second);

17
src/lnav_config.cc
View File

@ -78,6 +78,10 @@ static auto a = injector::bind<archive_manager::config>::to_instance(+[]() {
return &lnav_config.lc_archive_manager;
});
static auto fvc = injector::bind<file_vtab::config>::to_instance(+[]() {
return &lnav_config.lc_file_vtab;
});
ghc::filesystem::path dotlnav_path()
{
auto home_env = getenv("HOME");
@ -889,10 +893,23 @@ static struct json_path_container archive_handlers = {
&archive_manager::config::amc_cache_ttl),
};
static struct json_path_container file_vtab_handlers = {
yajlpp::property_handler("max-content-size")
.with_synopsis("<bytes>")
.with_description(
"The maximum allowed file size for the content column")
.with_min_value(0)
.for_field(&_lnav_config::lc_file_vtab,
&file_vtab::config::fvc_max_content_size),
};
static struct json_path_container tuning_handlers = {
yajlpp::property_handler("archive-manager")
.with_description("Settings related to opening archive files")
.with_children(archive_handlers),
yajlpp::property_handler("file-vtab")
.with_description("Settings related to the lnav_file virtual-table")
.with_children(file_vtab_handlers),
};
static set<string> SUPPORTED_CONFIG_SCHEMAS = {

2
src/lnav_config.hh
View File

@ -47,6 +47,7 @@
#include "lnav_config_fwd.hh"
#include "archive_manager.cfg.hh"
#include "file_vtab.cfg.hh"
/**
* Compute the path to a file in the user's '.lnav' directory.
@ -101,6 +102,7 @@ struct _lnav_config {
key_map lc_active_keymap;
archive_manager::config lc_archive_manager;
file_vtab::config lc_file_vtab;
};
extern struct _lnav_config lnav_config;

4
src/log_actions.cc
View File

@ -125,8 +125,10 @@ static string execute_action(log_data_helper &ldh,
false,
open_temp_file(ghc::filesystem::temp_directory_path() /
"lnav.action.XXXXXX")
.then([](auto pair) {
.map([](auto pair) {
ghc::filesystem::remove(pair.first);
return pair;
})
.expect("Cannot create temporary file for action")
.second);

4
src/logfile.cc
View File

@ -194,6 +194,7 @@ bool logfile::process_prefix(shared_buffer_ref &sbr, const line_info &li)
this->lf_index.size(),
(*iter)->get_name().get());
this->lf_text_format = text_format_t::TF_LOG;
this->lf_format = (*iter)->specialized();
this->set_format_base_time(this->lf_format.get());
this->lf_content_id = hasher()
@ -410,7 +411,7 @@ logfile::rebuild_result_t logfile::rebuild_index()
size_t old_size = this->lf_index.size();
if (old_size == 0) {
if (old_size == 0 && this->lf_text_format == text_format_t::TF_UNKNOWN) {
file_range fr = this->lf_line_buffer.get_available();
auto avail_data = this->lf_line_buffer.read_range(fr);
@ -420,6 +421,7 @@ logfile::rebuild_result_t logfile::rebuild_index()
avail_sbr.get_data(), avail_sbr.length());
})
.unwrapOr(text_format_t::TF_UNKNOWN);
log_debug("setting text format to %d", this->lf_text_format);
}
auto read_result = this->lf_line_buffer.read_range(li.li_file_range);

3
src/readline_curses.hh
View File

@ -237,6 +237,9 @@ public:
void set_value(const std::string &value)
{
this->rc_value = value;
if (this->rc_value.length() > 1024) {
this->rc_value = this->rc_value.substr(0, 1024);
}
this->rc_value_expiration = time(nullptr) + VALUE_EXPIRATION;
this->set_needs_update();
};

28
src/regexp_vtab.cc
View File

@ -72,6 +72,7 @@ CREATE TABLE regexp_capture (
pcre_context_static<30> c_context;
unique_ptr<pcre_input> c_input;
string c_content;
bool c_content_as_blob{false};
int c_index;
int c_start_index;
bool c_matched{false};
@ -157,10 +158,17 @@ CREATE TABLE regexp_capture (
}
break;
case RC_COL_VALUE:
sqlite3_result_text(ctx,
vc.c_content.c_str(),
vc.c_content.length(),
SQLITE_TRANSIENT);
if (vc.c_content_as_blob) {
sqlite3_result_blob64(ctx,
vc.c_content.c_str(),
vc.c_content.length(),
SQLITE_STATIC);
} else {
sqlite3_result_text(ctx,
vc.c_content.c_str(),
vc.c_content.length(),
SQLITE_STATIC);
}
break;
case RC_COL_PATTERN: {
auto str = vc.c_pattern.get_pattern();
@ -209,11 +217,13 @@ static int rcFilter(sqlite3_vtab_cursor *pVtabCursor,
return SQLITE_OK;
}
const char *value = (const char *) sqlite3_value_text(argv[0]);
auto byte_count = sqlite3_value_bytes(argv[0]);
auto blob = (const char *) sqlite3_value_blob(argv[0]);
pCur->c_content_as_blob = (sqlite3_value_type(argv[0]) == SQLITE_BLOB);
pCur->c_content.assign(blob, byte_count);
const char *pattern = (const char *) sqlite3_value_text(argv[1]);
pCur->c_content = value;
auto re_res = pcrepp::from_str(pattern);
if (re_res.isErr()) {
pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf(
@ -229,6 +239,8 @@ static int rcFilter(sqlite3_vtab_cursor *pVtabCursor,
pCur->c_input = make_unique<pcre_input>(pCur->c_content);
pCur->c_matched = pCur->c_pattern.match(pCur->c_context, *(pCur->c_input));
log_debug("matched %d", pCur->c_matched);
return SQLITE_OK;
}

6
src/sql_util.cc
View File

@ -352,10 +352,10 @@ int walk_sqlite_metadata(sqlite3 *db, struct sqlite_metadata_callbacks &smc)
std::string &table_name = *table_iter;
table_query = sqlite3_mprintf(
"pragma %Q.table_info(%Q)",
"pragma %Q.table_xinfo(%Q)",
iter->first.c_str(),
table_name.c_str());
if (table_query == NULL) {
if (table_query == nullptr) {
return SQLITE_NOMEM;
}
@ -375,7 +375,7 @@ int walk_sqlite_metadata(sqlite3 *db, struct sqlite_metadata_callbacks &smc)
"pragma %Q.foreign_key_list(%Q)",
iter->first.c_str(),
table_name.c_str());
if (table_query == NULL) {
if (table_query == nullptr) {
return SQLITE_NOMEM;
}

24
src/state-extension-functions.cc
View File

@ -65,6 +65,24 @@ static nonstd::optional<std::string> sql_log_top_datetime()
return buffer;
}
static nonstd::optional<std::string> sql_lnav_top_file()
{
auto top_view_opt = lnav_data.ld_view_stack.top();
if (!top_view_opt) {
return nonstd::nullopt;
}
auto top_view = top_view_opt.value();
return top_view->map_top_row([](const auto& al) {
return get_string_attr(al.get_attrs(), &logline::L_FILE) | [](const auto* sa) {
auto lf = (logfile *) sa->sa_value.sav_ptr;
return nonstd::make_optional(lf->get_filename());
};
});
}
static int64_t sql_error(const char *str)
{
throw sqlite_func_error("{}", str);
@ -86,6 +104,12 @@ int state_extension_functions(struct FuncDef **basic_funcs,
.sql_function()
),
sqlite_func_adapter<decltype(&sql_lnav_top_file), sql_lnav_top_file>::builder(
help_text("lnav_top_file",
"Return the name of the file that the top line in the current view came from.")
.sql_function()
),
sqlite_func_adapter<decltype(&sql_error), sql_error>::builder(
help_text("raise_error",
"Raises an error with the given message when executed")

33
src/text_format.cc
View File

@ -32,6 +32,7 @@
#include "config.h"
#include "pcrepp/pcrepp.hh"
#include "yajl/api/yajl_parse.h"
#include "text_format.hh"
@ -54,6 +55,14 @@ text_format_t detect_text_format(const char *str, size_t len)
)",
PCRE_MULTILINE);
static pcrepp JAVA_MATCHERS = pcrepp(
"(?:"
"^package\\s+|"
"^import\\s+|"
"^\\s*(?:public)?\\s*class\\s*(\\w+\\s+)*\\s*{"
")",
PCRE_MULTILINE);
static pcrepp C_LIKE_MATCHERS = pcrepp(
"(?:"
"^#\\s*include\\s+|"
@ -70,10 +79,26 @@ text_format_t detect_text_format(const char *str, size_t len)
")",
PCRE_MULTILINE|PCRE_CASELESS);
static pcrepp XML_MATCHERS = pcrepp(
"(?:"
R"(<\?xml(\s+\w+\s*=\s*"[^"]*")*\?>|)"
R"(</?\w+(\s+\w+\s*=\s*"[^"]*")*\s*>)"
")",
PCRE_MULTILINE|PCRE_CASELESS);
text_format_t retval = text_format_t::TF_UNKNOWN;
pcre_input pi(str, 0, len);
pcre_context_static<30> pc;
{
auto_mem<yajl_handle_t> jhandle(yajl_free);
jhandle = yajl_alloc(nullptr, nullptr, nullptr);
if (yajl_parse(jhandle, (unsigned char *) str, len) == yajl_status_ok) {
return text_format_t::TF_JSON;
}
}
if (PYTHON_MATCHERS.match(pc, pi)) {
return text_format_t::TF_PYTHON;
}
@ -82,6 +107,10 @@ text_format_t detect_text_format(const char *str, size_t len)
return text_format_t::TF_RUST;
}
if (JAVA_MATCHERS.match(pc, pi)) {
return text_format_t::TF_JAVA;
}
if (C_LIKE_MATCHERS.match(pc, pi)) {
return text_format_t::TF_C_LIKE;
}
@ -90,5 +119,9 @@ text_format_t detect_text_format(const char *str, size_t len)
return text_format_t::TF_SQL;
}
if (XML_MATCHERS.match(pc, pi)) {
return text_format_t::TF_XML;
}
return retval;
}

48
src/text_format.hh
View File

@ -36,14 +36,62 @@
#include <string>
#include "fmt/format.h"
enum class text_format_t {
TF_UNKNOWN,
TF_LOG,
TF_PYTHON,
TF_RUST,
TF_JAVA,
TF_C_LIKE,
TF_SQL,
TF_XML,
TF_JSON,
};
namespace fmt {
template<>
struct formatter<text_format_t> : formatter<string_view> {
template<typename FormatContext>
auto format(text_format_t tf, FormatContext &ctx)
{
string_view name = "unknown";
switch (tf) {
case text_format_t::TF_UNKNOWN:
name = "application/octet-stream";
break;
case text_format_t::TF_LOG:
name = "text/log";
break;
case text_format_t::TF_PYTHON:
name = "text/python";
break;
case text_format_t::TF_RUST:
name = "text/rust";
break;
case text_format_t::TF_JAVA:
name = "text/java";
break;
case text_format_t::TF_C_LIKE:
name = "text/c";
break;
case text_format_t::TF_SQL:
name = "application/sql";
break;
case text_format_t::TF_XML:
name = "text/xml";
break;
case text_format_t::TF_JSON:
name = "application/json";
break;
}
return formatter<string_view>::format(name, ctx);
}
};
}
/**
* Try to detect the format of the given text file fragment.
*

3
src/textfile_highlighters.cc
View File

@ -246,6 +246,7 @@ void setup_highlights(highlight_map_t &hm)
")"))
.with_nestable(false)
.with_text_format(text_format_t::TF_C_LIKE)
.with_text_format(text_format_t::TF_JAVA)
.with_role(view_colors::VCR_KEYWORD);
hm[{highlight_source_t::INTERNAL, "sql.0.comment"}] = highlighter(xpcre_compile(
@ -440,10 +441,12 @@ void setup_highlights(highlight_map_t &hm)
"\\b[A-Z_][A-Z0-9_]+\\b"))
.with_nestable(false)
.with_text_format(text_format_t::TF_C_LIKE)
.with_text_format(text_format_t::TF_JAVA)
.with_role(view_colors::VCR_SYMBOL);
hm[{highlight_source_t::INTERNAL, "num"}] = highlighter(xpcre_compile(
R"(\b-?(?:\d+|0x[a-zA-Z0-9]+)\b)"))
.with_nestable(false)
.with_text_format(text_format_t::TF_C_LIKE)
.with_text_format(text_format_t::TF_JAVA)
.with_role(view_colors::VCR_NUMBER);
}

4
src/textview_curses.cc
View File

@ -430,8 +430,8 @@ void textview_curses::textview_value_for_row(vis_line_t row,
tc_highlight.first.first == highlight_source_t::INTERNAL ||
tc_highlight.first.first == highlight_source_t::THEME;
if (tc_highlight.second.h_text_format != text_format_t::TF_UNKNOWN &&
source_format != tc_highlight.second.h_text_format) {
if (!tc_highlight.second.h_text_formats.empty() &&
tc_highlight.second.h_text_formats.count(source_format) == 0) {
continue;
}

7
src/time-extension-functions.cc
View File

@ -47,7 +47,7 @@
using namespace std;
static lnav::sqlite::text_buffer timeslice(sqlite3_value *time_in, nonstd::optional<const char *> slice_in_opt)
static auto_buffer timeslice(sqlite3_value *time_in, nonstd::optional<const char *> slice_in_opt)
{
thread_local date_time_scanner dts;
thread_local struct {
@ -114,9 +114,10 @@ static lnav::sqlite::text_buffer timeslice(sqlite3_value *time_in, nonstd::optio
tv.tv_sec = us / (1000LL * 1000LL);
tv.tv_usec = us % (1000LL * 1000LL);
auto ts = lnav::sqlite::text_buffer::alloc(64);
ts.tb_length = sql_strftime(ts.tb_value, ts.tb_length, tv);
auto ts = auto_buffer::alloc(64);
auto actual_length = sql_strftime(ts.in(), ts.size(), tv);
ts.shrink_to(actual_length);
return ts;
}

23
src/views_vtab.cc
View File

@ -119,6 +119,7 @@ CREATE TABLE lnav_views (
height INTEGER, -- The height of the viewport.
inner_height INTEGER, -- The number of lines in the view.
top_time DATETIME, -- The time of the top line in the view, if the content is time-based.
top_file TEXT, -- The file the top line is from.
paused INTEGER, -- Indicates if the view is paused and will not load new data.
search TEXT, -- The text to search for in the view.
filtering INTEGER -- Indicates if the view is applying filters.
@ -174,16 +175,23 @@ CREATE TABLE lnav_views (
}
break;
}
case 6:
sqlite3_result_int(ctx, tc.is_paused());
break;
case 7: {
const string &str = tc.get_current_search();
case 6: {
to_sqlite(ctx, tc.map_top_row([](const auto& al) {
return get_string_attr(al.get_attrs(), &logline::L_FILE) | [](const auto* sa) {
auto lf = (logfile *) sa->sa_value.sav_ptr;
sqlite3_result_text(ctx, str.c_str(), str.length(), SQLITE_TRANSIENT);
return nonstd::make_optional(lf->get_filename());
};
}));
break;
}
case 8: {
case 7:
sqlite3_result_int(ctx, tc.is_paused());
break;
case 8:
to_sqlite(ctx, tc.get_current_search());
break;
case 9: {
auto tss = tc.get_sub_source();
if (tss != nullptr && tss->tss_supports_filtering) {
@ -218,6 +226,7 @@ CREATE TABLE lnav_views (
int64_t height,
int64_t inner_height,
const char *top_time,
const char *top_file,
bool is_paused,
const char *search,
bool do_filtering) {

21
src/vtab_module.hh
View File

@ -172,25 +172,10 @@ inline void to_sqlite(sqlite3_context *ctx, const char *str)
}
}
namespace lnav {
namespace sqlite {
struct text_buffer {
static text_buffer alloc(size_t size) {
return text_buffer {
(char *) malloc(size),
size,
};
}
char *tb_value;
size_t tb_length;
};
}
}
inline void to_sqlite(sqlite3_context *ctx, const lnav::sqlite::text_buffer &str)
inline void to_sqlite(sqlite3_context *ctx, auto_buffer& buf)
{
sqlite3_result_text(ctx, str.tb_value, str.tb_length, free);
auto pair = buf.release();
sqlite3_result_text(ctx, pair.first, pair.second, free);
}
inline void to_sqlite(sqlite3_context *ctx, const std::string &str)

21
src/xpath_vtab.cc
View File

@ -106,6 +106,7 @@ CREATE TABLE xpath (
sqlite3_int64 c_rowid{0};
string c_xpath;
string c_value;
bool c_value_as_blob{false};
pugi::xpath_query c_query;
pugi::xml_document c_doc;
pugi::xpath_node_set c_results;
@ -253,10 +254,17 @@ CREATE TABLE xpath (
SQLITE_STATIC);
break;
case XP_COL_VALUE:
sqlite3_result_text(ctx,
vc.c_value.c_str(),
vc.c_value.length(),
SQLITE_STATIC);
if (vc.c_value_as_blob) {
sqlite3_result_blob64(ctx,
vc.c_value.c_str(),
vc.c_value.length(),
SQLITE_STATIC);
} else {
sqlite3_result_text(ctx,
vc.c_value.c_str(),
vc.c_value.length(),
SQLITE_STATIC);
}
break;
}
@ -298,7 +306,10 @@ static int rcFilter(sqlite3_vtab_cursor *pVtabCursor,
return SQLITE_OK;
}
pCur->c_value = (const char *) sqlite3_value_text(argv[1]);
pCur->c_value_as_blob = (sqlite3_value_type(argv[1]) == SQLITE_BLOB);
auto byte_count = sqlite3_value_bytes(argv[1]);
auto blob = (const char *) sqlite3_value_blob(argv[1]);
pCur->c_value.assign(blob, byte_count);
auto parse_res = pCur->c_doc.load_string(pCur->c_value.c_str());
if (!parse_res) {
pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf(

120
test/books.xml Normal file
View File

@ -0,0 +1,120 @@
<?xml version="1.0"?>
<catalog>
<book id="bk101">
<author>Gambardella, Matthew</author>
<title>XML Developer's Guide</title>
<genre>Computer</genre>
<price>44.95</price>
<publish_date>2000-10-01</publish_date>
<description>An in-depth look at creating applications
with XML.</description>
</book>
<book id="bk102">
<author>Ralls, Kim</author>
<title>Midnight Rain</title>
<genre>Fantasy</genre>
<price>5.95</price>
<publish_date>2000-12-16</publish_date>
<description>A former architect battles corporate zombies,
an evil sorceress, and her own childhood to become queen
of the world.</description>
</book>
<book id="bk103">
<author>Corets, Eva</author>
<title>Maeve Ascendant</title>
<genre>Fantasy</genre>
<price>5.95</price>
<publish_date>2000-11-17</publish_date>
<description>After the collapse of a nanotechnology
society in England, the young survivors lay the
foundation for a new society.</description>
</book>
<book id="bk104">
<author>Corets, Eva</author>
<title>Oberon's Legacy</title>
<genre>Fantasy</genre>
<price>5.95</price>
<publish_date>2001-03-10</publish_date>
<description>In post-apocalypse England, the mysterious
agent known only as Oberon helps to create a new life
for the inhabitants of London. Sequel to Maeve
Ascendant.</description>
</book>
<book id="bk105">
<author>Corets, Eva</author>
<title>The Sundered Grail</title>
<genre>Fantasy</genre>
<price>5.95</price>
<publish_date>2001-09-10</publish_date>
<description>The two daughters of Maeve, half-sisters,
battle one another for control of England. Sequel to
Oberon's Legacy.</description>
</book>
<book id="bk106">
<author>Randall, Cynthia</author>
<title>Lover Birds</title>
<genre>Romance</genre>
<price>4.95</price>
<publish_date>2000-09-02</publish_date>
<description>When Carla meets Paul at an ornithology
conference, tempers fly as feathers get ruffled.</description>
</book>
<book id="bk107">
<author>Thurman, Paula</author>
<title>Splish Splash</title>
<genre>Romance</genre>
<price>4.95</price>
<publish_date>2000-11-02</publish_date>
<description>A deep sea diver finds true love twenty
thousand leagues beneath the sea.</description>
</book>
<book id="bk108">
<author>Knorr, Stefan</author>
<title>Creepy Crawlies</title>
<genre>Horror</genre>
<price>4.95</price>
<publish_date>2000-12-06</publish_date>
<description>An anthology of horror stories about roaches,
centipedes, scorpions and other insects.</description>
</book>
<book id="bk109">
<author>Kress, Peter</author>
<title>Paradox Lost</title>
<genre>Science Fiction</genre>
<price>6.95</price>
<publish_date>2000-11-02</publish_date>
<description>After an inadvertant trip through a Heisenberg
Uncertainty Device, James Salway discovers the problems
of being quantum.</description>
</book>
<book id="bk110">
<author>O'Brien, Tim</author>
<title>Microsoft .NET: The Programming Bible</title>
<genre>Computer</genre>
<price>36.95</price>
<publish_date>2000-12-09</publish_date>
<description>Microsoft's .NET initiative is explored in
detail in this deep programmer's reference.</description>
</book>
<book id="bk111">
<author>O'Brien, Tim</author>
<title>MSXML3: A Comprehensive Guide</title>
<genre>Computer</genre>
<price>36.95</price>
<publish_date>2000-12-01</publish_date>
<description>The Microsoft MSXML3 parser is covered in
detail, with attention to XML DOM interfaces, XSLT processing,
SAX and more.</description>
</book>
<book id="bk112">
<author>Galos, Mike</author>
<title>Visual Studio 7: A Comprehensive Guide</title>
<genre>Computer</genre>
<price>49.95</price>
<publish_date>2001-04-16</publish_date>
<description>Microsoft Visual Studio 7 is explored in depth,
looking at how Visual Basic, Visual C++, C#, and ASP+ are
integrated into a comprehensive development
environment.</description>
</book>
</catalog>

5
test/expected_help.txt
View File

@ -1886,6 +1886,11 @@ Parameter
value The boolean value to return
Synopsis
lnav_top_file() -- Return the name of the file that the top line in the
current view came from.
Synopsis
load_extension(path, [entry-point]) -- Loads SQLite extensions out of the
given shared library file using the given entry point.

61
test/test_sql.sh
View File

@ -84,6 +84,67 @@ check_error_output "raise_error() does not work?" <<EOF
command-option:1: error: oops!
EOF
run_test ${lnav_test} -n \
-c ";SELECT content, length(content) FROM lnav_file" \
-c ":write-csv-to -" \
${test_dir}/logfile_empty.0
check_output "empty content not handled correctly?" <<EOF
content,length(content)
,0
EOF
run_test ${lnav_test} -n \
-c ";SELECT distinct xp.node_text FROM lnav_file, xpath('//author', content) as xp" \
-c ":write-csv-to -" \
${test_dir}/books.xml
check_output "xpath on file content not working?" <<EOF
node_text
"Gambardella, Matthew"
"Ralls, Kim"
"Corets, Eva"
"Randall, Cynthia"
"Thurman, Paula"
"Knorr, Stefan"
"Kress, Peter"
"O'Brien, Tim"
"Galos, Mike"
EOF
gzip -c ${srcdir}/logfile_json.json > logfile_json.json.gz
dd if=logfile_json.json.gz of=logfile_json-trunc.json.gz bs=64 count=2
run_test ${lnav_test} -n \
-c ";SELECT content FROM lnav_file" \
logfile_json-trunc.json.gz
check_error_output "invalid gzip file working?" <<EOF
command-option:1: error: unable to uncompress: logfile_json-trunc.json.gz -- buffer error
EOF
run_test ${lnav_test} -n \
-c ";SELECT jget(rc.content, '/ts') AS ts FROM lnav_file, regexp_capture(lnav_file.content, '.*\n') as rc" \
-c ":write-csv-to -" \
logfile_json.json.gz
check_output "jget on file content not working?" <<EOF
ts
2013-09-06T20:00:48.124817Z
2013-09-06T20:00:49.124817Z
2013-09-06T22:00:49.124817Z
2013-09-06T22:00:59.124817Z
2013-09-06T22:00:59.124817Z
2013-09-06T22:00:59.124817Z
2013-09-06T22:00:59.124817Z
2013-09-06T22:00:59.124817Z
2013-09-06T22:01:49.124817Z
2013-09-06T22:01:49.124817Z
2013-09-06T22:01:49.124817Z
2013-09-06T22:01:49.124817Z
2013-09-06T22:01:49.124817Z
EOF
run_test ${lnav_test} -n \
-c ";UPDATE lnav_file SET visible=0" \
${test_dir}/logfile_access_log.0