diff --git a/NEWS b/NEWS index db6e1c61..8a0f903f 100644 --- a/NEWS +++ b/NEWS @@ -40,6 +40,16 @@ lnav v0.9.1: an aggregator. * Added a "log_time_msecs" hidden column to the log tables that returns the timestamp as the number of milliseconds from the epoch. + * Added an "lnav_top_file()" SQL function that can be used to get the + name of the top line in the top view or NULL if the line did not come + from a file. + * Added a "mimetype" column to the lnav_file table that returns a guess as + to the MIME type of the file contents. + * Added a "content" hidden column to the lnav_file table that can be used + to read the contents of the file. The contents can then be passed to + functions that operate on XML/JSON data, like xpath() or json_tree(). + * Added an "lnav_top_view" SQL VIEW that returns the row for the top view + in the lnav_views table. Interface Changes: * When copying log lines, the file name and time offset will be included diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0c168a33..9b6f005b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -276,6 +276,7 @@ add_library(diag STATIC listview_curses.cc lnav_commands.cc lnav_config.cc + base/lnav.gzip.cc base/lnav_log.cc lnav_util.cc log_accel.cc diff --git a/src/Makefile.am b/src/Makefile.am index 3369b2de..8a4ca63f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -177,6 +177,7 @@ noinst_HEADERS = \ field_overlay_source.hh \ file_collection.hh \ file_format.hh \ + file_vtab.cfg.hh \ files_sub_source.hh \ filter_observer.hh \ filter_status_source.hh \ diff --git a/src/attr_line.hh b/src/attr_line.hh index 47be5871..0b8383bf 100644 --- a/src/attr_line.hh +++ b/src/attr_line.hh @@ -207,6 +207,18 @@ find_string_attr(const string_attrs_t &sa, string_attr_type_t type, int start = return iter; } +inline nonstd::optional +get_string_attr(const string_attrs_t &sa, string_attr_type_t type, int start = 0) +{ + auto iter = find_string_attr(sa, type, start); + + if (iter == sa.end()) { + return nonstd::nullopt; + } + + return nonstd::make_optional(&(*iter)); +} + template inline string_attrs_t::const_iterator find_string_attr_containing(const string_attrs_t &sa, string_attr_type_t type, T x) diff --git a/src/auto_mem.hh b/src/auto_mem.hh index 944efde1..6d09b9c6 100644 --- a/src/auto_mem.hh +++ b/src/auto_mem.hh @@ -38,6 +38,8 @@ #include +#include "base/result.h" + typedef void (*free_func_t)(void *); /** @@ -146,4 +148,62 @@ private: T srm_value; }; +class auto_buffer { +public: + static auto_buffer alloc(size_t size) { + return auto_buffer{ (char *) malloc(size), size }; + } + + auto_buffer(auto_buffer&& other) noexcept + : ab_buffer(other.ab_buffer), ab_size(other.ab_size) { + other.ab_buffer = nullptr; + other.ab_size = 0; + } + + ~auto_buffer() { + free(this->ab_buffer); + this->ab_buffer = nullptr; + this->ab_size = 0; + } + + char *in() { + return this->ab_buffer; + } + + std::pair release() { + auto retval = std::make_pair(this->ab_buffer, this->ab_size); + + this->ab_buffer = nullptr; + this->ab_size = 0; + return retval; + } + + size_t size() const { + return this->ab_size; + } + + void expand_by(size_t amount) { + auto new_size = this->ab_size + amount; + auto new_buffer = (char *) realloc(this->ab_buffer, new_size); + + if (new_buffer == nullptr) { + throw std::bad_alloc(); + } + + this->ab_buffer = new_buffer; + this->ab_size = new_size; + } + + auto_buffer& shrink_to(size_t new_size) { + this->ab_size = new_size; + return *this; + } +private: + auto_buffer(char *buffer, size_t size) : ab_buffer(buffer), ab_size(size) { + } + + char *ab_buffer; + size_t ab_size; +}; + #endif diff --git a/src/base/Makefile.am b/src/base/Makefile.am index f6cc6f97..a2d90f4e 100644 --- a/src/base/Makefile.am +++ b/src/base/Makefile.am @@ -26,6 +26,7 @@ noinst_HEADERS = \ is_utf8.hh \ isc.hh \ lnav_log.hh \ + lnav.gzip.hh \ lrucache.hpp \ math_util.hh \ opt_util.hh \ @@ -39,6 +40,7 @@ libbase_a_SOURCES = \ intern_string.cc \ is_utf8.cc \ isc.cc \ + lnav.gzip.cc \ lnav_log.cc \ string_util.cc \ time_util.cc diff --git a/src/base/lnav.gzip.cc b/src/base/lnav.gzip.cc new file mode 100644 index 00000000..a662998f --- /dev/null +++ b/src/base/lnav.gzip.cc @@ -0,0 +1,95 @@ +/** + * Copyright (c) 2021, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file lnav.gzip.cc + */ + +#include "config.h" + +#include + +#include "fmt/format.h" +#include "lnav.gzip.hh" + +namespace lnav { +namespace gzip { + +bool is_gzipped(const char *buffer, size_t len) +{ + return len > 2 && buffer[0] == '\037' && buffer[1] == '\213'; +} + +Result uncompress(const std::string& src, + const void *buffer, + size_t size) +{ + auto uncomp = auto_buffer::alloc(size * 2); + z_stream strm; + int err; + + strm.next_in = (Bytef *) buffer; + strm.avail_in = size; + strm.total_out = 0; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + + if ((err = inflateInit2(&strm, (16 + MAX_WBITS))) != Z_OK) { + return Err(fmt::format("invalid gzip data: {} -- {}", + src, strm.msg ? strm.msg : zError(err))); + } + + bool done = false; + + while (!done) { + if (strm.total_out >= uncomp.size()) { + uncomp.expand_by(size / 2); + } + + strm.next_out = (Bytef *) (uncomp.in() + strm.total_out); + strm.avail_out = uncomp.size() - strm.total_out; + + // Inflate another chunk. + err = inflate(&strm, Z_SYNC_FLUSH); + if (err == Z_STREAM_END) { + done = true; + } else if (err != Z_OK) { + return Err(fmt::format("unable to uncompress: {} -- {}", + src, strm.msg ? strm.msg : zError(err))); + } + } + + if (inflateEnd(&strm) != Z_OK) { + return Err(fmt::format("unable to uncompress: {} -- {}", + src, strm.msg ? strm.msg : zError(err))); + } + + return Ok(std::move(uncomp.shrink_to(strm.total_out))); +} + +} +} diff --git a/src/base/lnav.gzip.hh b/src/base/lnav.gzip.hh new file mode 100644 index 00000000..a800634e --- /dev/null +++ b/src/base/lnav.gzip.hh @@ -0,0 +1,52 @@ +/** + * Copyright (c) 2021, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file lnav.gzip.hh + */ + +#ifndef lnav_gzip_hh +#define lnav_gzip_hh + +#include + +#include "auto_mem.hh" +#include "result.h" + +namespace lnav { +namespace gzip { + +bool is_gzipped(const char *buffer, size_t len); + +Result uncompress(const std::string& src, + const void *buffer, + size_t size); + +} +} + +#endif diff --git a/src/base/result.h b/src/base/result.h index 5edc7ad6..b6dc1ca6 100644 --- a/src/base/result.h +++ b/src/base/result.h @@ -33,7 +33,10 @@ namespace types { E val; }; -} + + template<> + struct Err { }; +}; template::type> types::Ok Ok(T&& val) { @@ -49,6 +52,10 @@ types::Err Err(E&& val) { return types::Err(std::forward(val)); } +inline types::Err Err() { + return {}; +} + template struct Result; namespace details { @@ -554,7 +561,7 @@ struct Storage { void construct(types::Ok ok) { - new (&storage_) T(ok.val); + new (&storage_) T(std::move(ok.val)); initialized_ = true; } void construct(types::Err err) @@ -796,17 +803,32 @@ struct Result { } template - Result then(Func func) const { + Result then(Func func) { if (this->isOk()) { - func(this->storage().template get()); + func(std::move(this->storage().template get())); + + return Ok(); } - return *this; + return Err(std::move(this->storage().template get())); } template - Result otherwise(Func func) const { - return details::otherwise(*this, func); + Result::type, E> then(Func func) { + if (this->isOk()) { + return Ok(func(std::move(this->storage().template get()))); + } + + return Err(std::move(this->storage().template get())); + } + + template + void otherwise(Func func) { + if (this->isOk()) { + return; + } + + func(std::move(this->storage().template get())); } template pipe_callback(exec_context &ec, const string &cmdline, auto_fd &f auto pp = make_shared( fd, false, open_temp_file(ghc::filesystem::temp_directory_path() / "lnav.out.XXXXXX") - .then([](auto pair) { + .map([](auto pair) { ghc::filesystem::remove(pair.first); + + return pair; }) .expect("Cannot create temporary file for callback") .second); diff --git a/src/db_sub_source.cc b/src/db_sub_source.cc index b0ab0f61..f84145c5 100644 --- a/src/db_sub_source.cc +++ b/src/db_sub_source.cc @@ -105,7 +105,7 @@ void db_label_source::text_attrs_for_line(textview_curses &tc, int row, this->dls_chart.chart_attrs_for_value(tc, left, this->dls_headers[lpc].hm_name, num_value, sa); } } - if (row_len > 2 && + if (row_len > 2 && row_len < MAX_COLUMN_WIDTH && ((row_value[0] == '{' && row_value[row_len - 1] == '}') || (row_value[0] == '[' && row_value[row_len - 1] == ']'))) { json_ptr_walk jpw; diff --git a/src/file_vtab.cc b/src/file_vtab.cc index 2a8ee768..60e81605 100644 --- a/src/file_vtab.cc +++ b/src/file_vtab.cc @@ -33,12 +33,14 @@ #include #include "base/injector.bind.hh" +#include "base/lnav.gzip.hh" #include "base/lnav_log.hh" #include "file_collection.hh" #include "logfile.hh" #include "session_data.hh" #include "vtab_module.hh" #include "log_format.hh" +#include "file_vtab.cfg.hh" using namespace std; @@ -52,9 +54,12 @@ CREATE TABLE lnav_file ( device integer, -- The device the file is stored on. inode integer, -- The inode for the file on the device. filepath text, -- The path to the file. + mimetype text, -- The MIME type for the file. format text, -- The log file format for the file. lines integer, -- The number of lines in the file. - time_offset integer -- The millisecond offset for timestamps. + time_offset integer, -- The millisecond offset for timestamps. + + content BLOB HIDDEN -- The contents of the file. ); )"; @@ -88,18 +93,66 @@ CREATE TABLE lnav_file ( to_sqlite(ctx, name); break; case 3: - to_sqlite(ctx, format_name); + to_sqlite(ctx, fmt::format("{}", lf->get_text_format())); break; case 4: + to_sqlite(ctx, format_name); + break; + case 5: to_sqlite(ctx, (int64_t) lf->size()); break; - case 5: { + case 6: { auto tv = lf->get_time_offset(); int64_t ms = (tv.tv_sec * 1000LL) + tv.tv_usec / 1000LL; to_sqlite(ctx, ms); break; } + case 7: { + auto& cfg = injector::get(); + auto lf_stat = lf->get_stat(); + + if (lf_stat.st_size > cfg.fvc_max_content_size) { + sqlite3_result_error(ctx, "file is too large", -1); + } else { + auto fd = lf->get_fd(); + auto_mem buf; + buf = (char *) malloc(lf_stat.st_size); + auto rc = pread(fd, buf, lf_stat.st_size, 0); + + if (rc == -1) { + auto errmsg = fmt::format("unable to read file: {}", + strerror(errno)); + + sqlite3_result_error(ctx, errmsg.c_str(), + errmsg.length()); + } else if (rc != lf_stat.st_size) { + auto errmsg = fmt::format("short read of file: {} < {}", + rc, lf_stat.st_size); + + sqlite3_result_error(ctx, errmsg.c_str(), + errmsg.length()); + } else if (lnav::gzip::is_gzipped(buf, rc)) { + lnav::gzip::uncompress(lf->get_unique_path(), buf, rc) + .then([ctx](auto uncomp) { + auto pair = uncomp.release(); + + sqlite3_result_blob64(ctx, + pair.first, + pair.second, + free); + }) + .otherwise([ctx](auto msg) { + sqlite3_result_error(ctx, + msg.c_str(), + msg.size()); + }); + } else { + sqlite3_result_blob64(ctx, buf.release(), rc, free); + } + } + break; + } default: ensure(0); break; @@ -125,9 +178,11 @@ CREATE TABLE lnav_file ( int64_t device, int64_t inode, std::string path, + const char *text_format, const char *format, int64_t lines, - int64_t time_offset) { + int64_t time_offset, + const char *content) { auto lf = this->lf_collection.fc_files[rowid]; struct timeval tv = { (int) (time_offset / 1000LL), diff --git a/src/file_vtab.cfg.hh b/src/file_vtab.cfg.hh new file mode 100644 index 00000000..9803fb31 --- /dev/null +++ b/src/file_vtab.cfg.hh @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2021, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file file_vtab.cfg.hh + */ + +#ifndef lnav_file_vtab_cfg_hh +#define lnav_file_vtab_cfg_hh + +namespace file_vtab { + +struct config { + int64_t fvc_max_content_size{32 * 1024 * 1024}; +}; + +} + +#endif diff --git a/src/highlighter.cc b/src/highlighter.cc index 9d3e6cee..60e8b4d1 100644 --- a/src/highlighter.cc +++ b/src/highlighter.cc @@ -41,7 +41,7 @@ highlighter::highlighter(const highlighter &other) pcre_refcount(this->h_code, 1); this->study(); this->h_attrs = other.h_attrs; - this->h_text_format = other.h_text_format; + this->h_text_formats = other.h_text_formats; this->h_format_name = other.h_format_name; this->h_nestable = other.h_nestable; } @@ -67,7 +67,7 @@ highlighter &highlighter::operator=(const highlighter &other) this->study(); this->h_format_name = other.h_format_name; this->h_attrs = other.h_attrs; - this->h_text_format = other.h_text_format; + this->h_text_formats = other.h_text_formats; this->h_nestable = other.h_nestable; return *this; diff --git a/src/highlighter.hh b/src/highlighter.hh index 287ec412..1601f969 100644 --- a/src/highlighter.hh +++ b/src/highlighter.hh @@ -32,6 +32,8 @@ #ifndef highlighter_hh #define highlighter_hh +#include + #include "optional.hpp" #include "pcrepp/pcrepp.hh" #include "text_format.hh" @@ -82,7 +84,7 @@ struct highlighter { }; highlighter &with_text_format(text_format_t tf) { - this->h_text_format = tf; + this->h_text_formats.insert(tf); return *this; } @@ -121,7 +123,7 @@ struct highlighter { pcre *h_code; pcre_extra *h_code_extra; int h_attrs{-1}; - text_format_t h_text_format{text_format_t::TF_UNKNOWN}; + std::set h_text_formats; intern_string_t h_format_name; bool h_nestable{true}; }; diff --git a/src/init.sql b/src/init.sql index fb38c5aa..6136ac99 100644 --- a/src/init.sql +++ b/src/init.sql @@ -85,6 +85,10 @@ CREATE TABLE lnav_example_log ( log_body text hidden ); +CREATE VIEW lnav_top_view AS + SELECT * FROM lnav_views WHERE name = ( + SELECT name FROM lnav_view_stack ORDER BY rowid DESC LIMIT 1); + INSERT INTO lnav_example_log VALUES (0, null, '2017-02-03T04:05:06.100', '2017-02-03T04:05:06.100', 0, 'info', 0, null, null, null, 'hw', 2, '/tmp/log', '2017-02-03T04:05:06.100 hw(2): Hello, World!', 'Hello, World!'), (1, null, '2017-02-03T04:05:06.200', '2017-02-03T04:05:06.200', 100, 'error', 0, null, null, null, 'gw', 4, '/tmp/log', '2017-02-03T04:05:06.200 gw(4): Goodbye, World!', 'Goodbye, World!'), diff --git a/src/internals/config-v1.schema.json b/src/internals/config-v1.schema.json index 17ec4ee6..3add5369 100644 --- a/src/internals/config-v1.schema.json +++ b/src/internals/config-v1.schema.json @@ -34,6 +34,20 @@ } }, "additionalProperties": false + }, + "file-vtab": { + "description": "Settings related to the lnav_file virtual-table", + "title": "/tuning/file-vtab", + "type": "object", + "properties": { + "max-content-size": { + "title": "/tuning/file-vtab/max-content-size", + "description": "The maximum allowed file size for the content column", + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/src/internals/sql-ref.rst b/src/internals/sql-ref.rst index d3b946dc..ff1f45c7 100644 --- a/src/internals/sql-ref.rst +++ b/src/internals/sql-ref.rst @@ -1833,6 +1833,17 @@ likely(*value*) ---- +.. _lnav_top_file: + +lnav_top_file() +^^^^^^^^^^^^^^^ + + Return the name of the file that the top line in the current view came from. + + +---- + + .. _load_extension: load_extension(*path*, *\[entry-point\]*) diff --git a/src/listview_curses.hh b/src/listview_curses.hh index 49a7d5bb..d3570b84 100644 --- a/src/listview_curses.hh +++ b/src/listview_curses.hh @@ -277,6 +277,18 @@ public: return retval; }; + template + auto map_top_row(F func) -> typename std::result_of::type { + if (this->get_inner_height() == 0) { + return nonstd::nullopt; + } + + std::vector top_line{1}; + + this->lv_source->listview_value_for_rows(*this, this->lv_top, top_line); + return func(top_line[0]); + } + /** @param win The curses window this view is attached to. */ void set_window(WINDOW *win) { this->lv_window = win; }; diff --git a/src/lnav.cc b/src/lnav.cc index f17213c6..89ddc263 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -2418,8 +2418,10 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' false, open_temp_file(ghc::filesystem::temp_directory_path() / "lnav.fifo.XXXXXX") - .then([](auto pair) { + .map([](auto pair) { ghc::filesystem::remove(pair.first); + + return pair; }) .expect("Cannot create temporary file for FIFO") .second); diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index 961da13f..1774438b 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -2027,8 +2027,10 @@ static Result com_open(exec_context &ec, string cmdline, vector< false, open_temp_file(ghc::filesystem::temp_directory_path() / "lnav.fifo.XXXXXX") - .then([](auto pair) { + .map([](auto pair) { ghc::filesystem::remove(pair.first); + + return pair; }) .expect("Cannot create temporary file for FIFO") .second); diff --git a/src/lnav_config.cc b/src/lnav_config.cc index 96a63ec3..d2b99adf 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -78,6 +78,10 @@ static auto a = injector::bind::to_instance(+[]() { return &lnav_config.lc_archive_manager; }); +static auto fvc = injector::bind::to_instance(+[]() { + return &lnav_config.lc_file_vtab; +}); + ghc::filesystem::path dotlnav_path() { auto home_env = getenv("HOME"); @@ -889,10 +893,23 @@ static struct json_path_container archive_handlers = { &archive_manager::config::amc_cache_ttl), }; +static struct json_path_container file_vtab_handlers = { + yajlpp::property_handler("max-content-size") + .with_synopsis("") + .with_description( + "The maximum allowed file size for the content column") + .with_min_value(0) + .for_field(&_lnav_config::lc_file_vtab, + &file_vtab::config::fvc_max_content_size), +}; + static struct json_path_container tuning_handlers = { yajlpp::property_handler("archive-manager") .with_description("Settings related to opening archive files") .with_children(archive_handlers), + yajlpp::property_handler("file-vtab") + .with_description("Settings related to the lnav_file virtual-table") + .with_children(file_vtab_handlers), }; static set SUPPORTED_CONFIG_SCHEMAS = { diff --git a/src/lnav_config.hh b/src/lnav_config.hh index 22e979f3..7a38ba19 100644 --- a/src/lnav_config.hh +++ b/src/lnav_config.hh @@ -47,6 +47,7 @@ #include "lnav_config_fwd.hh" #include "archive_manager.cfg.hh" +#include "file_vtab.cfg.hh" /** * Compute the path to a file in the user's '.lnav' directory. @@ -101,6 +102,7 @@ struct _lnav_config { key_map lc_active_keymap; archive_manager::config lc_archive_manager; + file_vtab::config lc_file_vtab; }; extern struct _lnav_config lnav_config; diff --git a/src/log_actions.cc b/src/log_actions.cc index 53c55646..c3825824 100644 --- a/src/log_actions.cc +++ b/src/log_actions.cc @@ -125,8 +125,10 @@ static string execute_action(log_data_helper &ldh, false, open_temp_file(ghc::filesystem::temp_directory_path() / "lnav.action.XXXXXX") - .then([](auto pair) { + .map([](auto pair) { ghc::filesystem::remove(pair.first); + + return pair; }) .expect("Cannot create temporary file for action") .second); diff --git a/src/logfile.cc b/src/logfile.cc index 81bfbb06..eda97dfd 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -194,6 +194,7 @@ bool logfile::process_prefix(shared_buffer_ref &sbr, const line_info &li) this->lf_index.size(), (*iter)->get_name().get()); + this->lf_text_format = text_format_t::TF_LOG; this->lf_format = (*iter)->specialized(); this->set_format_base_time(this->lf_format.get()); this->lf_content_id = hasher() @@ -410,7 +411,7 @@ logfile::rebuild_result_t logfile::rebuild_index() size_t old_size = this->lf_index.size(); - if (old_size == 0) { + if (old_size == 0 && this->lf_text_format == text_format_t::TF_UNKNOWN) { file_range fr = this->lf_line_buffer.get_available(); auto avail_data = this->lf_line_buffer.read_range(fr); @@ -420,6 +421,7 @@ logfile::rebuild_result_t logfile::rebuild_index() avail_sbr.get_data(), avail_sbr.length()); }) .unwrapOr(text_format_t::TF_UNKNOWN); + log_debug("setting text format to %d", this->lf_text_format); } auto read_result = this->lf_line_buffer.read_range(li.li_file_range); diff --git a/src/readline_curses.hh b/src/readline_curses.hh index 20888648..b925aaa8 100644 --- a/src/readline_curses.hh +++ b/src/readline_curses.hh @@ -237,6 +237,9 @@ public: void set_value(const std::string &value) { this->rc_value = value; + if (this->rc_value.length() > 1024) { + this->rc_value = this->rc_value.substr(0, 1024); + } this->rc_value_expiration = time(nullptr) + VALUE_EXPIRATION; this->set_needs_update(); }; diff --git a/src/regexp_vtab.cc b/src/regexp_vtab.cc index 72bc1c6a..3362cb64 100644 --- a/src/regexp_vtab.cc +++ b/src/regexp_vtab.cc @@ -72,6 +72,7 @@ CREATE TABLE regexp_capture ( pcre_context_static<30> c_context; unique_ptr c_input; string c_content; + bool c_content_as_blob{false}; int c_index; int c_start_index; bool c_matched{false}; @@ -157,10 +158,17 @@ CREATE TABLE regexp_capture ( } break; case RC_COL_VALUE: - sqlite3_result_text(ctx, - vc.c_content.c_str(), - vc.c_content.length(), - SQLITE_TRANSIENT); + if (vc.c_content_as_blob) { + sqlite3_result_blob64(ctx, + vc.c_content.c_str(), + vc.c_content.length(), + SQLITE_STATIC); + } else { + sqlite3_result_text(ctx, + vc.c_content.c_str(), + vc.c_content.length(), + SQLITE_STATIC); + } break; case RC_COL_PATTERN: { auto str = vc.c_pattern.get_pattern(); @@ -209,11 +217,13 @@ static int rcFilter(sqlite3_vtab_cursor *pVtabCursor, return SQLITE_OK; } - const char *value = (const char *) sqlite3_value_text(argv[0]); + auto byte_count = sqlite3_value_bytes(argv[0]); + auto blob = (const char *) sqlite3_value_blob(argv[0]); + + pCur->c_content_as_blob = (sqlite3_value_type(argv[0]) == SQLITE_BLOB); + pCur->c_content.assign(blob, byte_count); + const char *pattern = (const char *) sqlite3_value_text(argv[1]); - - pCur->c_content = value; - auto re_res = pcrepp::from_str(pattern); if (re_res.isErr()) { pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( @@ -229,6 +239,8 @@ static int rcFilter(sqlite3_vtab_cursor *pVtabCursor, pCur->c_input = make_unique(pCur->c_content); pCur->c_matched = pCur->c_pattern.match(pCur->c_context, *(pCur->c_input)); + log_debug("matched %d", pCur->c_matched); + return SQLITE_OK; } diff --git a/src/sql_util.cc b/src/sql_util.cc index 7f6f93f8..e4d6b36c 100644 --- a/src/sql_util.cc +++ b/src/sql_util.cc @@ -352,10 +352,10 @@ int walk_sqlite_metadata(sqlite3 *db, struct sqlite_metadata_callbacks &smc) std::string &table_name = *table_iter; table_query = sqlite3_mprintf( - "pragma %Q.table_info(%Q)", + "pragma %Q.table_xinfo(%Q)", iter->first.c_str(), table_name.c_str()); - if (table_query == NULL) { + if (table_query == nullptr) { return SQLITE_NOMEM; } @@ -375,7 +375,7 @@ int walk_sqlite_metadata(sqlite3 *db, struct sqlite_metadata_callbacks &smc) "pragma %Q.foreign_key_list(%Q)", iter->first.c_str(), table_name.c_str()); - if (table_query == NULL) { + if (table_query == nullptr) { return SQLITE_NOMEM; } diff --git a/src/state-extension-functions.cc b/src/state-extension-functions.cc index 1f7ad291..e7cb6272 100644 --- a/src/state-extension-functions.cc +++ b/src/state-extension-functions.cc @@ -65,6 +65,24 @@ static nonstd::optional sql_log_top_datetime() return buffer; } +static nonstd::optional sql_lnav_top_file() +{ + auto top_view_opt = lnav_data.ld_view_stack.top(); + + if (!top_view_opt) { + return nonstd::nullopt; + } + + auto top_view = top_view_opt.value(); + return top_view->map_top_row([](const auto& al) { + return get_string_attr(al.get_attrs(), &logline::L_FILE) | [](const auto* sa) { + auto lf = (logfile *) sa->sa_value.sav_ptr; + + return nonstd::make_optional(lf->get_filename()); + }; + }); +} + static int64_t sql_error(const char *str) { throw sqlite_func_error("{}", str); @@ -86,6 +104,12 @@ int state_extension_functions(struct FuncDef **basic_funcs, .sql_function() ), + sqlite_func_adapter::builder( + help_text("lnav_top_file", + "Return the name of the file that the top line in the current view came from.") + .sql_function() + ), + sqlite_func_adapter::builder( help_text("raise_error", "Raises an error with the given message when executed") diff --git a/src/text_format.cc b/src/text_format.cc index 5ce04f8b..387a0d67 100644 --- a/src/text_format.cc +++ b/src/text_format.cc @@ -32,6 +32,7 @@ #include "config.h" #include "pcrepp/pcrepp.hh" +#include "yajl/api/yajl_parse.h" #include "text_format.hh" @@ -54,6 +55,14 @@ text_format_t detect_text_format(const char *str, size_t len) )", PCRE_MULTILINE); + static pcrepp JAVA_MATCHERS = pcrepp( + "(?:" + "^package\\s+|" + "^import\\s+|" + "^\\s*(?:public)?\\s*class\\s*(\\w+\\s+)*\\s*{" + ")", + PCRE_MULTILINE); + static pcrepp C_LIKE_MATCHERS = pcrepp( "(?:" "^#\\s*include\\s+|" @@ -70,10 +79,26 @@ text_format_t detect_text_format(const char *str, size_t len) ")", PCRE_MULTILINE|PCRE_CASELESS); + static pcrepp XML_MATCHERS = pcrepp( + "(?:" + R"(<\?xml(\s+\w+\s*=\s*"[^"]*")*\?>|)" + R"()" + ")", + PCRE_MULTILINE|PCRE_CASELESS); + text_format_t retval = text_format_t::TF_UNKNOWN; pcre_input pi(str, 0, len); pcre_context_static<30> pc; + { + auto_mem jhandle(yajl_free); + + jhandle = yajl_alloc(nullptr, nullptr, nullptr); + if (yajl_parse(jhandle, (unsigned char *) str, len) == yajl_status_ok) { + return text_format_t::TF_JSON; + } + } + if (PYTHON_MATCHERS.match(pc, pi)) { return text_format_t::TF_PYTHON; } @@ -82,6 +107,10 @@ text_format_t detect_text_format(const char *str, size_t len) return text_format_t::TF_RUST; } + if (JAVA_MATCHERS.match(pc, pi)) { + return text_format_t::TF_JAVA; + } + if (C_LIKE_MATCHERS.match(pc, pi)) { return text_format_t::TF_C_LIKE; } @@ -90,5 +119,9 @@ text_format_t detect_text_format(const char *str, size_t len) return text_format_t::TF_SQL; } + if (XML_MATCHERS.match(pc, pi)) { + return text_format_t::TF_XML; + } + return retval; } diff --git a/src/text_format.hh b/src/text_format.hh index a2246359..8f095379 100644 --- a/src/text_format.hh +++ b/src/text_format.hh @@ -36,14 +36,62 @@ #include +#include "fmt/format.h" + enum class text_format_t { TF_UNKNOWN, + TF_LOG, TF_PYTHON, TF_RUST, + TF_JAVA, TF_C_LIKE, TF_SQL, + TF_XML, + TF_JSON, }; + +namespace fmt { +template<> +struct formatter : formatter { + template + auto format(text_format_t tf, FormatContext &ctx) + { + string_view name = "unknown"; + switch (tf) { + case text_format_t::TF_UNKNOWN: + name = "application/octet-stream"; + break; + case text_format_t::TF_LOG: + name = "text/log"; + break; + case text_format_t::TF_PYTHON: + name = "text/python"; + break; + case text_format_t::TF_RUST: + name = "text/rust"; + break; + case text_format_t::TF_JAVA: + name = "text/java"; + break; + case text_format_t::TF_C_LIKE: + name = "text/c"; + break; + case text_format_t::TF_SQL: + name = "application/sql"; + break; + case text_format_t::TF_XML: + name = "text/xml"; + break; + case text_format_t::TF_JSON: + name = "application/json"; + break; + } + return formatter::format(name, ctx); + } +}; +} + /** * Try to detect the format of the given text file fragment. * diff --git a/src/textfile_highlighters.cc b/src/textfile_highlighters.cc index 12d38a9f..d58565f2 100644 --- a/src/textfile_highlighters.cc +++ b/src/textfile_highlighters.cc @@ -246,6 +246,7 @@ void setup_highlights(highlight_map_t &hm) ")")) .with_nestable(false) .with_text_format(text_format_t::TF_C_LIKE) + .with_text_format(text_format_t::TF_JAVA) .with_role(view_colors::VCR_KEYWORD); hm[{highlight_source_t::INTERNAL, "sql.0.comment"}] = highlighter(xpcre_compile( @@ -440,10 +441,12 @@ void setup_highlights(highlight_map_t &hm) "\\b[A-Z_][A-Z0-9_]+\\b")) .with_nestable(false) .with_text_format(text_format_t::TF_C_LIKE) + .with_text_format(text_format_t::TF_JAVA) .with_role(view_colors::VCR_SYMBOL); hm[{highlight_source_t::INTERNAL, "num"}] = highlighter(xpcre_compile( R"(\b-?(?:\d+|0x[a-zA-Z0-9]+)\b)")) .with_nestable(false) .with_text_format(text_format_t::TF_C_LIKE) + .with_text_format(text_format_t::TF_JAVA) .with_role(view_colors::VCR_NUMBER); } diff --git a/src/textview_curses.cc b/src/textview_curses.cc index bd3147fe..cd3c4d83 100644 --- a/src/textview_curses.cc +++ b/src/textview_curses.cc @@ -430,8 +430,8 @@ void textview_curses::textview_value_for_row(vis_line_t row, tc_highlight.first.first == highlight_source_t::INTERNAL || tc_highlight.first.first == highlight_source_t::THEME; - if (tc_highlight.second.h_text_format != text_format_t::TF_UNKNOWN && - source_format != tc_highlight.second.h_text_format) { + if (!tc_highlight.second.h_text_formats.empty() && + tc_highlight.second.h_text_formats.count(source_format) == 0) { continue; } diff --git a/src/time-extension-functions.cc b/src/time-extension-functions.cc index 394de70a..11c1a8c3 100644 --- a/src/time-extension-functions.cc +++ b/src/time-extension-functions.cc @@ -47,7 +47,7 @@ using namespace std; -static lnav::sqlite::text_buffer timeslice(sqlite3_value *time_in, nonstd::optional slice_in_opt) +static auto_buffer timeslice(sqlite3_value *time_in, nonstd::optional slice_in_opt) { thread_local date_time_scanner dts; thread_local struct { @@ -114,9 +114,10 @@ static lnav::sqlite::text_buffer timeslice(sqlite3_value *time_in, nonstd::optio tv.tv_sec = us / (1000LL * 1000LL); tv.tv_usec = us % (1000LL * 1000LL); - auto ts = lnav::sqlite::text_buffer::alloc(64); - ts.tb_length = sql_strftime(ts.tb_value, ts.tb_length, tv); + auto ts = auto_buffer::alloc(64); + auto actual_length = sql_strftime(ts.in(), ts.size(), tv); + ts.shrink_to(actual_length); return ts; } diff --git a/src/views_vtab.cc b/src/views_vtab.cc index 026f66ea..c6959017 100644 --- a/src/views_vtab.cc +++ b/src/views_vtab.cc @@ -119,6 +119,7 @@ CREATE TABLE lnav_views ( height INTEGER, -- The height of the viewport. inner_height INTEGER, -- The number of lines in the view. top_time DATETIME, -- The time of the top line in the view, if the content is time-based. + top_file TEXT, -- The file the top line is from. paused INTEGER, -- Indicates if the view is paused and will not load new data. search TEXT, -- The text to search for in the view. filtering INTEGER -- Indicates if the view is applying filters. @@ -174,16 +175,23 @@ CREATE TABLE lnav_views ( } break; } - case 6: - sqlite3_result_int(ctx, tc.is_paused()); - break; - case 7: { - const string &str = tc.get_current_search(); + case 6: { + to_sqlite(ctx, tc.map_top_row([](const auto& al) { + return get_string_attr(al.get_attrs(), &logline::L_FILE) | [](const auto* sa) { + auto lf = (logfile *) sa->sa_value.sav_ptr; - sqlite3_result_text(ctx, str.c_str(), str.length(), SQLITE_TRANSIENT); + return nonstd::make_optional(lf->get_filename()); + }; + })); break; } - case 8: { + case 7: + sqlite3_result_int(ctx, tc.is_paused()); + break; + case 8: + to_sqlite(ctx, tc.get_current_search()); + break; + case 9: { auto tss = tc.get_sub_source(); if (tss != nullptr && tss->tss_supports_filtering) { @@ -218,6 +226,7 @@ CREATE TABLE lnav_views ( int64_t height, int64_t inner_height, const char *top_time, + const char *top_file, bool is_paused, const char *search, bool do_filtering) { diff --git a/src/vtab_module.hh b/src/vtab_module.hh index 0ce8b95b..076849f5 100644 --- a/src/vtab_module.hh +++ b/src/vtab_module.hh @@ -172,25 +172,10 @@ inline void to_sqlite(sqlite3_context *ctx, const char *str) } } -namespace lnav { -namespace sqlite { -struct text_buffer { - static text_buffer alloc(size_t size) { - return text_buffer { - (char *) malloc(size), - size, - }; - } - - char *tb_value; - size_t tb_length; -}; -} -} - -inline void to_sqlite(sqlite3_context *ctx, const lnav::sqlite::text_buffer &str) +inline void to_sqlite(sqlite3_context *ctx, auto_buffer& buf) { - sqlite3_result_text(ctx, str.tb_value, str.tb_length, free); + auto pair = buf.release(); + sqlite3_result_text(ctx, pair.first, pair.second, free); } inline void to_sqlite(sqlite3_context *ctx, const std::string &str) diff --git a/src/xpath_vtab.cc b/src/xpath_vtab.cc index bf552314..26c6968d 100644 --- a/src/xpath_vtab.cc +++ b/src/xpath_vtab.cc @@ -106,6 +106,7 @@ CREATE TABLE xpath ( sqlite3_int64 c_rowid{0}; string c_xpath; string c_value; + bool c_value_as_blob{false}; pugi::xpath_query c_query; pugi::xml_document c_doc; pugi::xpath_node_set c_results; @@ -253,10 +254,17 @@ CREATE TABLE xpath ( SQLITE_STATIC); break; case XP_COL_VALUE: - sqlite3_result_text(ctx, - vc.c_value.c_str(), - vc.c_value.length(), - SQLITE_STATIC); + if (vc.c_value_as_blob) { + sqlite3_result_blob64(ctx, + vc.c_value.c_str(), + vc.c_value.length(), + SQLITE_STATIC); + } else { + sqlite3_result_text(ctx, + vc.c_value.c_str(), + vc.c_value.length(), + SQLITE_STATIC); + } break; } @@ -298,7 +306,10 @@ static int rcFilter(sqlite3_vtab_cursor *pVtabCursor, return SQLITE_OK; } - pCur->c_value = (const char *) sqlite3_value_text(argv[1]); + pCur->c_value_as_blob = (sqlite3_value_type(argv[1]) == SQLITE_BLOB); + auto byte_count = sqlite3_value_bytes(argv[1]); + auto blob = (const char *) sqlite3_value_blob(argv[1]); + pCur->c_value.assign(blob, byte_count); auto parse_res = pCur->c_doc.load_string(pCur->c_value.c_str()); if (!parse_res) { pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( diff --git a/test/books.xml b/test/books.xml new file mode 100644 index 00000000..19c4f737 --- /dev/null +++ b/test/books.xml @@ -0,0 +1,120 @@ + + + + Gambardella, Matthew + XML Developer's Guide + Computer + 44.95 + 2000-10-01 + An in-depth look at creating applications + with XML. + + + Ralls, Kim + Midnight Rain + Fantasy + 5.95 + 2000-12-16 + A former architect battles corporate zombies, + an evil sorceress, and her own childhood to become queen + of the world. + + + Corets, Eva + Maeve Ascendant + Fantasy + 5.95 + 2000-11-17 + After the collapse of a nanotechnology + society in England, the young survivors lay the + foundation for a new society. + + + Corets, Eva + Oberon's Legacy + Fantasy + 5.95 + 2001-03-10 + In post-apocalypse England, the mysterious + agent known only as Oberon helps to create a new life + for the inhabitants of London. Sequel to Maeve + Ascendant. + + + Corets, Eva + The Sundered Grail + Fantasy + 5.95 + 2001-09-10 + The two daughters of Maeve, half-sisters, + battle one another for control of England. Sequel to + Oberon's Legacy. + + + Randall, Cynthia + Lover Birds + Romance + 4.95 + 2000-09-02 + When Carla meets Paul at an ornithology + conference, tempers fly as feathers get ruffled. + + + Thurman, Paula + Splish Splash + Romance + 4.95 + 2000-11-02 + A deep sea diver finds true love twenty + thousand leagues beneath the sea. + + + Knorr, Stefan + Creepy Crawlies + Horror + 4.95 + 2000-12-06 + An anthology of horror stories about roaches, + centipedes, scorpions and other insects. + + + Kress, Peter + Paradox Lost + Science Fiction + 6.95 + 2000-11-02 + After an inadvertant trip through a Heisenberg + Uncertainty Device, James Salway discovers the problems + of being quantum. + + + O'Brien, Tim + Microsoft .NET: The Programming Bible + Computer + 36.95 + 2000-12-09 + Microsoft's .NET initiative is explored in + detail in this deep programmer's reference. + + + O'Brien, Tim + MSXML3: A Comprehensive Guide + Computer + 36.95 + 2000-12-01 + The Microsoft MSXML3 parser is covered in + detail, with attention to XML DOM interfaces, XSLT processing, + SAX and more. + + + Galos, Mike + Visual Studio 7: A Comprehensive Guide + Computer + 49.95 + 2001-04-16 + Microsoft Visual Studio 7 is explored in depth, + looking at how Visual Basic, Visual C++, C#, and ASP+ are + integrated into a comprehensive development + environment. + + diff --git a/test/expected_help.txt b/test/expected_help.txt index 3bac24a4..6e5a96fc 100644 --- a/test/expected_help.txt +++ b/test/expected_help.txt @@ -1886,6 +1886,11 @@ Parameter value The boolean value to return +Synopsis + lnav_top_file() -- Return the name of the file that the top line in the + current view came from. + + Synopsis load_extension(path, [entry-point]) -- Loads SQLite extensions out of the given shared library file using the given entry point. diff --git a/test/test_sql.sh b/test/test_sql.sh index 868ea290..996c7bc8 100644 --- a/test/test_sql.sh +++ b/test/test_sql.sh @@ -84,6 +84,67 @@ check_error_output "raise_error() does not work?" < logfile_json.json.gz +dd if=logfile_json.json.gz of=logfile_json-trunc.json.gz bs=64 count=2 + +run_test ${lnav_test} -n \ + -c ";SELECT content FROM lnav_file" \ + logfile_json-trunc.json.gz + +check_error_output "invalid gzip file working?" <