Robot
/
lnav
mirror of https://github.com/tstack/lnav.git
2
0
Fork 0
Code Issues Releases Wiki Activity

[filters] sql filter 

Related to #568
This commit is contained in:
Timothy Stack 2020-11-29 13:20:07 -08:00
parent 6d0054d3b6
commit 22c2e95df0
24 changed files with 919 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
NEWS
View File

@ -1,5 +1,7 @@
lnav v0.9.1:
Features:
* Added the ':filter-expr' command to filter log messages based on an SQL
expression. This command allows much greater control over filtering.
* Added the ':prompt' command to allow for more customization of prompts.
Combined with a custom keymapping, you can now open a prompt and prefill
it with a given value. For example, a key could be bound to the

8
src/base/intern_string.hh
View File

@ -119,6 +119,10 @@ struct string_fragment {
strncmp(this->data(), str, this->length()) == 0;
};
bool operator!=(const char *str) const {
return !(*this == str);
}
bool startswith(const char *prefix) const {
auto iter = this->begin();
@ -279,6 +283,10 @@ public:
return strcmp(this->get(), rhs) == 0;
}
bool operator!=(const char *rhs) const {
return strcmp(this->get(), rhs) != 0;
}
intern_string_t &operator=(const intern_string_t &rhs) {
this->ist_interned_string = rhs.ist_interned_string;
return *this;

8
src/filter_status_source.cc
View File

@ -231,15 +231,19 @@ size_t filter_help_status_source::statusview_fields()
if (editor.fss_editing) {
auto tf = *(fs.begin() + lv.get_selection());
auto lang = tf->get_lang() == filter_lang_t::SQL ?
"an SQL": "a regular";
if (tf->get_type() == text_filter::type_t::INCLUDE) {
this->fss_help.set_value(
" "
"Enter a regular expression to match lines to filter in:");
"Enter %s expression to match lines to filter in:",
lang);
} else {
this->fss_help.set_value(
" "
"Enter a regular expression to match lines to filter out:");
"Enter %s expression to match lines to filter out:",
lang);
}
} else if (fs.empty()) {
this->fss_help.set_value(" %s", CREATE_HELP);

169
src/filter_sub_source.cc
View File

@ -29,6 +29,7 @@
#include "config.h"
#include "base/enum_util.hh"
#include "base/func_util.hh"
#include "lnav.hh"
@ -41,9 +42,14 @@ using namespace std;
filter_sub_source::filter_sub_source()
{
this->filter_context.set_highlighter(readline_regex_highlighter)
this->fss_regex_context.set_highlighter(readline_regex_highlighter)
.set_append_character(0);
this->fss_editor.add_context(LNM_FILTER, this->filter_context);
this->fss_editor.add_context(to_underlying(filter_lang_t::REGEX),
this->fss_regex_context);
this->fss_sql_context.set_highlighter(readline_sqlite_highlighter)
.set_append_character(0);
this->fss_editor.add_context(to_underlying(filter_lang_t::SQL),
this->fss_sql_context);
this->fss_editor.set_change_action(bind_mem(
&filter_sub_source::rl_change, this));
this->fss_editor.set_perform_action(bind_mem(
@ -130,8 +136,8 @@ bool filter_sub_source::list_input_handle_key(listview_curses &lv, int ch)
shared_ptr<text_filter> tf = *(fs.begin() + lv.get_selection());
fs.delete_filter(tf->get_id());
tss->text_filters_changed();
lv.reload_data();
tss->text_filters_changed();
return true;
}
case 'i': {
@ -154,7 +160,9 @@ bool filter_sub_source::list_input_handle_key(listview_curses &lv, int ch)
this->fss_editing = true;
add_view_text_possibilities(&this->fss_editor, LNM_FILTER, "*",
add_view_text_possibilities(&this->fss_editor,
to_underlying(filter_lang_t::REGEX),
"*",
top_view);
this->fss_editor.set_window(lv.get_window());
this->fss_editor.set_visible(true);
@ -162,7 +170,7 @@ bool filter_sub_source::list_input_handle_key(listview_curses &lv, int ch)
lv.get_y() + (int) (lv.get_selection() - lv.get_top()));
this->fss_editor.set_left(22);
this->fss_editor.set_width(this->tss_view->get_width() - 8 - 1);
this->fss_editor.focus(LNM_FILTER, "", "");
this->fss_editor.focus(to_underlying(filter_lang_t::REGEX), "", "");
this->fss_filter_state = true;
ef->disable();
return true;
@ -187,7 +195,9 @@ bool filter_sub_source::list_input_handle_key(listview_curses &lv, int ch)
this->fss_editing = true;
add_view_text_possibilities(&this->fss_editor, LNM_FILTER, "*",
add_view_text_possibilities(&this->fss_editor,
to_underlying(filter_lang_t::REGEX),
"*",
top_view);
this->fss_editor.set_window(lv.get_window());
this->fss_editor.set_visible(true);
@ -195,7 +205,7 @@ bool filter_sub_source::list_input_handle_key(listview_curses &lv, int ch)
lv.get_y() + (int) (lv.get_selection() - lv.get_top()));
this->fss_editor.set_left(22);
this->fss_editor.set_width(this->tss_view->get_width() - 8 - 1);
this->fss_editor.focus(LNM_FILTER, "", "");
this->fss_editor.focus(to_underlying(filter_lang_t::REGEX), "", "");
this->fss_filter_state = true;
ef->disable();
return true;
@ -214,15 +224,20 @@ bool filter_sub_source::list_input_handle_key(listview_curses &lv, int ch)
this->fss_editing = true;
add_view_text_possibilities(&this->fss_editor, LNM_FILTER, "*",
add_view_text_possibilities(&this->fss_editor,
to_underlying(filter_lang_t::REGEX),
"*",
top_view);
add_filter_expr_possibilities(&this->fss_editor,
to_underlying(filter_lang_t::SQL),
"*");
this->fss_editor.set_window(lv.get_window());
this->fss_editor.set_visible(true);
this->fss_editor.set_y(
lv.get_y() + (int) (lv.get_selection() - lv.get_top()));
this->fss_editor.set_left(22);
this->fss_editor.set_width(this->tss_view->get_width() - 8 - 1);
this->fss_editor.focus(LNM_FILTER, "");
this->fss_editor.focus(to_underlying(tf->get_lang()), "");
this->fss_editor.rewrite_line(0, tf->get_id().c_str());
this->fss_filter_state = tf->is_enabled();
tf->disable();
@ -373,33 +388,60 @@ void filter_sub_source::rl_change(readline_curses *rc)
auto iter = fs.begin() + this->tss_view->get_selection();
shared_ptr<text_filter> tf = *iter;
string new_value = rc->get_line_buffer();
auto_mem<pcre> code;
const char *errptr;
int eoff;
if ((code = pcre_compile(new_value.c_str(),
PCRE_CASELESS,
&errptr,
&eoff,
nullptr)) == nullptr) {
lnav_data.ld_filter_help_status_source.fss_error
.set_value("error: %s", errptr);
} else {
auto &hm = top_view->get_highlights();
highlighter hl(code.release());
int color;
switch (tf->get_lang()) {
case filter_lang_t::NONE:
break;
case filter_lang_t::REGEX: {
auto_mem<pcre> code;
const char *errptr;
int eoff;
if (tf->get_type() == text_filter::EXCLUDE) {
color = COLOR_RED;
} else {
color = COLOR_GREEN;
if ((code = pcre_compile(new_value.c_str(),
PCRE_CASELESS,
&errptr,
&eoff,
nullptr)) == nullptr) {
lnav_data.ld_filter_help_status_source.fss_error
.set_value("error: %s", errptr);
} else {
auto &hm = top_view->get_highlights();
highlighter hl(code.release());
int color;
if (tf->get_type() == text_filter::EXCLUDE) {
color = COLOR_RED;
} else {
color = COLOR_GREEN;
}
hl.with_attrs(
view_colors::ansi_color_pair(COLOR_BLACK, color) | A_BLINK);
hm[{highlight_source_t::PREVIEW, "preview"}] = hl;
top_view->set_needs_update();
lnav_data.ld_filter_help_status_source.fss_error.clear();
}
break;
}
case filter_lang_t::SQL: {
auto full_sql = fmt::format("SELECT 1 WHERE {}", new_value);
auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
auto retcode = sqlite3_prepare_v3(lnav_data.ld_db.in(),
full_sql.c_str(),
full_sql.size(),
SQLITE_PREPARE_PERSISTENT,
stmt.out(),
nullptr);
if (retcode != SQLITE_OK) {
lnav_data.ld_filter_help_status_source.fss_error
.set_value("error2: %s", sqlite3_errmsg(lnav_data.ld_db));
} else {
lnav_data.ld_log_source.set_preview_sql_filter(stmt.release());
top_view->set_needs_update();
lnav_data.ld_filter_help_status_source.fss_error.clear();
}
break;
}
hl.with_attrs(
view_colors::ansi_color_pair(COLOR_BLACK, color) | A_BLINK);
hm[{highlight_source_t::PREVIEW, "preview"}] = hl;
top_view->set_needs_update();
lnav_data.ld_filter_help_status_source.fss_error.clear();
}
}
@ -411,30 +453,58 @@ void filter_sub_source::rl_perform(readline_curses *rc)
auto iter = fs.begin() + this->tss_view->get_selection();
shared_ptr<text_filter> tf = *iter;
string new_value = rc->get_value();
auto_mem<pcre> code;
const char *errptr;
int eoff;
if (new_value.empty()) {
this->rl_abort(rc);
} else if ((code = pcre_compile(new_value.c_str(),
PCRE_CASELESS,
&errptr,
&eoff,
nullptr)) == nullptr) {
this->rl_abort(rc);
} else {
tf->lf_deleted = true;
tss->text_filters_changed();
switch (tf->get_lang()) {
case filter_lang_t::NONE:
case filter_lang_t::REGEX: {
auto_mem<pcre> code;
const char *errptr;
int eoff;
auto pf = make_shared<pcre_filter>(tf->get_type(),
new_value, tf->get_index(),
code.release());
if ((code = pcre_compile(new_value.c_str(),
PCRE_CASELESS,
&errptr,
&eoff,
nullptr)) == nullptr) {
this->rl_abort(rc);
} else {
tf->lf_deleted = true;
tss->text_filters_changed();
*iter = pf;
tss->text_filters_changed();
auto pf = make_shared<pcre_filter>(tf->get_type(),
new_value, tf->get_index(),
code.release());
*iter = pf;
tss->text_filters_changed();
}
break;
}
case filter_lang_t::SQL: {
auto full_sql = fmt::format("SELECT 1 WHERE {}", new_value);
auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
auto retcode = sqlite3_prepare_v3(lnav_data.ld_db.in(),
full_sql.c_str(),
full_sql.size(),
SQLITE_PREPARE_PERSISTENT,
stmt.out(),
nullptr);
if (retcode != SQLITE_OK) {
this->rl_abort(rc);
} else {
lnav_data.ld_log_source.set_sql_filter(
new_value, stmt.release());
tss->text_filters_changed();
}
break;
}
}
}
lnav_data.ld_log_source.set_preview_sql_filter(nullptr);
lnav_data.ld_filter_help_status_source.fss_prompt.clear();
this->fss_editing = false;
this->fss_editor.set_visible(false);
@ -449,6 +519,7 @@ void filter_sub_source::rl_abort(readline_curses *rc)
auto iter = fs.begin() + this->tss_view->get_selection();
shared_ptr<text_filter> tf = *iter;
lnav_data.ld_log_source.set_preview_sql_filter(nullptr);
lnav_data.ld_filter_help_status_source.fss_prompt.clear();
lnav_data.ld_filter_help_status_source.fss_error.clear();
top_view->get_highlights().erase({highlight_source_t::PREVIEW, "preview"});

3
src/filter_sub_source.hh
View File

@ -67,7 +67,8 @@ public:
void rl_display_next(readline_curses *rc);
readline_context filter_context{"filter", nullptr, false};
readline_context fss_regex_context{"regex", nullptr, false};
readline_context fss_sql_context{"sql", nullptr, false};
readline_curses fss_editor;
plain_text_source fss_match_source;
textview_curses fss_match_view;

40
src/internals/cmd-ref.rst
View File

@ -94,6 +94,20 @@
----
.. _clear_filter_expr:
:clear-filter-expr
^^^^^^^^^^^^^^^^^^
Clear the filter expression
**See Also:**
:ref:`filter_expr`, :ref:`filter_in`, :ref:`filter_out`, :ref:`hide_lines_after`, :ref:`hide_lines_before`, :ref:`hide_unmarked_lines`, :ref:`toggle_filtering`
----
.. _clear_highlight:
:clear-highlight *pattern*
@ -507,6 +521,32 @@
----
.. _filter_expr:
:filter-expr *expr*
^^^^^^^^^^^^^^^^^^^
Set the filter expression
**Parameters:**
* **expr\*** --- The SQL expression to evaluate for each log message. The message values can be accessed using column names prefixed with a colon
**Examples:**
To set a filter expression that matched syslog messages from 'syslogd':
.. code-block:: lnav
:filter-expr :log_procname = 'syslogd'
**See Also:**
:ref:`clear_filter_expr`, :ref:`filter_in`, :ref:`filter_out`, :ref:`hide_lines_after`, :ref:`hide_lines_before`, :ref:`hide_unmarked_lines`, :ref:`toggle_filtering`
----
.. _filter_in:
:filter-in *pattern*

111
src/lnav_commands.cc
View File

@ -66,6 +66,7 @@
#include "papertrail_proc.hh"
#include "yajlpp/json_op.hh"
#include "yajlpp/yajlpp.hh"
#include "sqlite-extension-func.hh"
using namespace std;
@ -1439,6 +1440,81 @@ static Result<string, string> com_disable_filter(exec_context &ec, string cmdlin
return Ok(retval);
}
static Result<string, string> com_filter_expr(exec_context &ec, string cmdline, vector<string> &args)
{
string retval;
if (args.empty()) {
args.emplace_back("filter-expr-syms");
}
else if (args.size() > 1) {
textview_curses *tc = *lnav_data.ld_view_stack.top();
if (tc != &lnav_data.ld_views[LNV_LOG]) {
return ec.make_error("The :filter-expr command only works in the log view");
}
auto expr = remaining_args(cmdline, args);
args[1] = fmt::format("SELECT 1 WHERE {}", expr);
auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
auto retcode = sqlite3_prepare_v3(lnav_data.ld_db.in(),
args[1].c_str(), args[1].size(),
SQLITE_PREPARE_PERSISTENT,
stmt.out(),
nullptr);
if (retcode != SQLITE_OK) {
const char *errmsg = sqlite3_errmsg(lnav_data.ld_db);
return ec.make_error("{}", errmsg);
}
if (ec.ec_dry_run) {
lnav_data.ld_log_source.set_preview_sql_filter(stmt.release());
lnav_data.ld_preview_status_source.get_description()
.set_value("Matches are highlighted in the text view");
} else {
lnav_data.ld_log_source.set_preview_sql_filter(nullptr);
lnav_data.ld_log_source.set_sql_filter(expr, stmt.release());
}
lnav_data.ld_log_source.text_filters_changed();
tc->reload_data();
} else {
return ec.make_error("expecting an SQL expression");
}
return Ok(retval);
}
static string com_filter_expr_prompt(exec_context &ec, const string &cmdline)
{
textview_curses *tc = *lnav_data.ld_view_stack.top();
if (tc != &lnav_data.ld_views[LNV_LOG]) {
return "";
}
return fmt::format("{} {}",
trim(cmdline),
trim(lnav_data.ld_log_source.get_sql_filter_text()));
}
static Result<string, string> com_clear_filter_expr(exec_context &ec, string cmdline, vector<string> &args)
{
string retval;
if (args.empty()) {
} else {
if (!ec.ec_dry_run) {
lnav_data.ld_log_source.set_sql_filter("", nullptr);
lnav_data.ld_log_source.text_filters_changed();
}
}
return Ok(retval);
}
static Result<string, string> com_enable_word_wrap(exec_context &ec, string cmdline, vector<string> &args)
{
string retval;
@ -4075,6 +4151,12 @@ static void command_prompt(vector<string> &args)
add_env_possibilities(LNM_COMMAND);
add_tag_possibilities();
add_file_possibilities();
if (tc == &lnav_data.ld_views[LNV_LOG]) {
add_filter_expr_possibilities(lnav_data.ld_rl_view,
LNM_COMMAND,
"filter-expr-syms");
}
lnav_data.ld_mode = LNM_COMMAND;
lnav_data.ld_rl_view->focus(LNM_COMMAND,
cget(args, 2).value_or(":"),
@ -4577,6 +4659,35 @@ readline_context::command_t STD_COMMANDS[] = {
"last message repeated"
})
},
{
"filter-expr",
com_filter_expr,
help_text(":filter-expr")
.with_summary("Set the filter expression")
.with_parameter(help_text(
"expr",
"The SQL expression to evaluate for each log message. "
"The message values can be accessed using column names "
"prefixed with a colon"))
.with_opposites({"clear-filter-expr"})
.with_tags({"filtering"})
.with_example({
"To set a filter expression that matched syslog messages from 'syslogd'",
":log_procname = 'syslogd'"
}),
com_filter_expr_prompt,
},
{
"clear-filter-expr",
com_clear_filter_expr,
help_text(":clear-filter-expr")
.with_summary("Clear the filter expression")
.with_opposites({"filter-expr"})
.with_tags({"filtering"})
},
{
"append-to",
com_save_to,

10
src/log_vtab_impl.cc
View File

@ -306,7 +306,7 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
content_line_t cl(vt->lss->at(vc->log_cursor.lc_curr_line));
uint64_t line_number;
auto ld = vt->lss->find_data(cl, line_number);
shared_ptr<logfile> lf = ld->get_file();
shared_ptr<logfile> lf = (*ld)->get_file();
auto ll = lf->begin() + line_number;
require(col >= 0);
@ -435,7 +435,7 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
break;
case VT_COL_LOG_COMMENT: {
const map<content_line_t, bookmark_metadata> &bm = vt->lss->get_user_bookmark_metadata();
const auto &bm = vt->lss->get_user_bookmark_metadata();
auto bm_iter = bm.find(vt->lss->at(vc->log_cursor.lc_curr_line));
if (bm_iter == bm.end() || bm_iter->second.bm_comment.empty()) {
@ -483,7 +483,7 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
}
case VT_COL_FILTERS: {
auto &filter_mask = ld->ld_filter_state.lfo_filter_state.tfs_mask;
auto &filter_mask = (*ld)->ld_filter_state.lfo_filter_state.tfs_mask;
if (!filter_mask[line_number]) {
sqlite3_result_null(ctx);
@ -980,8 +980,8 @@ string log_vtab_manager::register_vtab(log_vtab_impl *vi)
vi->get_name().get());
rc = sqlite3_exec(this->vm_db,
sql,
NULL,
NULL,
nullptr,
nullptr,
errmsg.out());
if (rc != SQLITE_OK) {
retval = errmsg;

4
src/logfile.cc
View File

@ -531,7 +531,7 @@ Result<shared_buffer_ref, std::string> logfile::read_line(logfile::iterator ll)
}
}
void logfile::read_full_message(logfile::iterator ll,
void logfile::read_full_message(logfile::const_iterator ll,
shared_buffer_ref &msg_out,
int max_lines)
{
@ -602,7 +602,7 @@ ghc::filesystem::path logfile::get_path() const
return this->lf_filename;
}
size_t logfile::line_length(logfile::iterator ll, bool include_continues)
size_t logfile::line_length(logfile::const_iterator ll, bool include_continues)
{
auto next_line = ll;
size_t retval;

8
src/logfile.hh
View File

@ -227,10 +227,14 @@ public:
const_iterator begin() const { return this->lf_index.begin(); }
const_iterator cbegin() const { return this->lf_index.begin(); }
iterator end() { return this->lf_index.end(); }
const_iterator end() const { return this->lf_index.end(); }
const_iterator cend() const { return this->lf_index.end(); }
/** @return The number of lines in the index. */
size_t size() const { return this->lf_index.size(); }
@ -290,14 +294,14 @@ public:
return retval;
}
size_t line_length(iterator ll, bool include_continues = true);
size_t line_length(const_iterator ll, bool include_continues = true);
file_range get_file_range(iterator ll, bool include_continues = true) {
return {ll->get_offset(),
(ssize_t) this->line_length(ll, include_continues)};
}
void read_full_message(iterator ll, shared_buffer_ref &msg_out, int max_lines=50);
void read_full_message(const_iterator ll, shared_buffer_ref &msg_out, int max_lines=50);
enum rebuild_result_t {
RR_INVALID,

297
src/logfile_sub_source.cc
View File

@ -41,6 +41,8 @@
#include "logfile_sub_source.hh"
#include "command_executor.hh"
#include "ansi_scrubber.hh"
#include "sql_util.hh"
#include "yajlpp/yajlpp.hh"
using namespace std;
@ -98,13 +100,7 @@ static future<string> pretty_pipe_callback(exec_context &ec,
}
logfile_sub_source::logfile_sub_source()
: lss_flags(0),
lss_force_rebuild(false),
lss_token_file(NULL),
lss_min_log_level(LEVEL_UNKNOWN),
lss_marked_only(false),
lss_index_delegate(NULL),
lss_longest_line(0),
: text_sub_source(1),
lss_meta_grepper(*this),
lss_location_history(*this)
{
@ -117,14 +113,14 @@ shared_ptr<logfile> logfile_sub_source::find(const char *fn,
content_line_t &line_base)
{
iterator iter;
shared_ptr<logfile> retval = NULL;
shared_ptr<logfile> retval = nullptr;
line_base = content_line_t(0);
for (iter = this->lss_files.begin();
iter != this->lss_files.end() && retval == NULL;
iter != this->lss_files.end() && retval == nullptr;
iter++) {
logfile_data &ld = *(*iter);
if (ld.get_file() == NULL) {
if (ld.get_file() == nullptr) {
continue;
}
if (strcmp(ld.get_file()->get_filename().c_str(), fn) == 0) {
@ -175,8 +171,9 @@ void logfile_sub_source::text_value_for_line(textview_curses &tc,
}
this->lss_token_flags = flags;
this->lss_token_file = this->find(line);
this->lss_token_line = this->lss_token_file->begin() + line;
this->lss_token_file_data = this->find_data(line);
this->lss_token_file = (*this->lss_token_file_data)->get_file();
this->lss_token_line = this->lss_token_file->begin() + line;
this->lss_token_attrs.clear();
this->lss_token_values.clear();
@ -550,6 +547,20 @@ void logfile_sub_source::text_attrs_for_line(textview_curses &lv,
value_out.emplace_back(time_range, &view_curses::VC_STYLE, attrs);
}
}
if (!this->lss_token_line->is_continued() &&
this->lss_preview_filter_stmt != nullptr) {
int color;
if (this->eval_sql_filter(this->lss_preview_filter_stmt.in(),
this->lss_token_file_data,
this->lss_token_line)) {
color = COLOR_GREEN;
} else {
color = COLOR_RED;
value_out.emplace_back(line_range{0, 1}, &view_curses::VC_STYLE, A_BLINK);
}
value_out.emplace_back(line_range{0, 1}, &view_curses::VC_BACKGROUND, color);
}
}
logfile_sub_source::rebuild_result logfile_sub_source::rebuild_index()
@ -736,21 +747,21 @@ logfile_sub_source::rebuild_result logfile_sub_source::rebuild_index()
index_index++) {
content_line_t cl = (content_line_t) this->lss_index[index_index];
uint64_t line_number;
logfile_data *ld = this->find_data(cl, line_number);
auto ld = this->find_data(cl, line_number);
if (!ld->is_visible()) {
if (!(*ld)->is_visible()) {
continue;
}
auto line_iter = ld->get_file()->begin() + line_number;
auto line_iter = (*ld)->get_file()->begin() + line_number;
if (!this->tss_apply_filters ||
(!ld->ld_filter_state.excluded(filter_in_mask, filter_out_mask,
line_number) &&
this->check_extra_filters(*line_iter))) {
(!(*ld)->ld_filter_state.excluded(filter_in_mask, filter_out_mask,
line_number) &&
this->check_extra_filters(ld, line_iter))) {
this->lss_filtered_index.push_back(index_index);
if (this->lss_index_delegate != NULL) {
shared_ptr<logfile> lf = ld->get_file();
if (this->lss_index_delegate != nullptr) {
shared_ptr<logfile> lf = (*ld)->get_file();
this->lss_index_delegate->index_line(
*this, lf.get(), lf->begin() + line_number);
}
@ -882,21 +893,21 @@ void logfile_sub_source::text_filters_changed()
for (size_t index_index = 0; index_index < this->lss_index.size(); index_index++) {
content_line_t cl = (content_line_t) this->lss_index[index_index];
uint64_t line_number;
logfile_data *ld = this->find_data(cl, line_number);
auto ld = this->find_data(cl, line_number);
if (!ld->is_visible()) {
if (!(*ld)->is_visible()) {
continue;
}
auto line_iter = ld->get_file()->begin() + line_number;
auto line_iter = (*ld)->get_file()->begin() + line_number;
if (!this->tss_apply_filters ||
(!ld->ld_filter_state.excluded(filtered_in_mask, filtered_out_mask,
(!(*ld)->ld_filter_state.excluded(filtered_in_mask, filtered_out_mask,
line_number) &&
this->check_extra_filters(*line_iter))) {
this->check_extra_filters(ld, line_iter))) {
this->lss_filtered_index.push_back(index_index);
if (this->lss_index_delegate != nullptr) {
shared_ptr<logfile> lf = ld->get_file();
shared_ptr<logfile> lf = (*ld)->get_file();
this->lss_index_delegate->index_line(
*this, lf.get(), lf->begin() + line_number);
}
@ -972,6 +983,212 @@ bool logfile_sub_source::insert_file(const shared_ptr<logfile> &lf)
return true;
}
void logfile_sub_source::set_sql_filter(std::string stmt_str, sqlite3_stmt *stmt)
{
for (auto ld : *this) {
ld->ld_filter_state.lfo_filter_state.clear_filter_state(0);
}
auto old_filter = this->get_sql_filter();
if (stmt != nullptr) {
auto new_filter = std::make_shared<sql_filter>(*this, stmt_str, stmt);
if (old_filter) {
auto existing_iter = std::find(this->tss_filters.begin(),
this->tss_filters.end(),
old_filter.value());
*existing_iter = new_filter;
} else {
this->tss_filters.add_filter(new_filter);
}
} else if (old_filter) {
this->tss_filters.delete_filter(old_filter.value()->get_id());
}
}
bool logfile_sub_source::eval_sql_filter(sqlite3_stmt *stmt, iterator ld, logfile::const_iterator ll)
{
auto lf = (*ld)->get_file();
char timestamp_buffer[64];
shared_buffer_ref sbr;
lf->read_full_message(ll, sbr);
auto format = lf->get_format();
string_attrs_t sa;
vector<logline_value> values;
format->annotate(std::distance(lf->cbegin(), ll), sbr, sa, values);
sqlite3_reset(stmt);
sqlite3_clear_bindings(stmt);
auto count = sqlite3_bind_parameter_count(stmt);
for (int lpc = 0; lpc < count; lpc++) {
auto *name = sqlite3_bind_parameter_name(stmt, lpc + 1);
if (name[0] == '$') {
const char *env_value;
if ((env_value = getenv(&name[1])) != nullptr) {
sqlite3_bind_text(stmt, lpc + 1, env_value, -1, SQLITE_STATIC);
}
continue;
}
if (strcmp(name, ":log_level") == 0) {
sqlite3_bind_text(stmt,
lpc + 1,
ll->get_level_name(), -1,
SQLITE_STATIC);
continue;
}
if (strcmp(name, ":log_time") == 0) {
auto len = sql_strftime(timestamp_buffer, sizeof(timestamp_buffer),
ll->get_timeval(),
'T');
sqlite3_bind_text(stmt,
lpc + 1,
timestamp_buffer, len,
SQLITE_STATIC);
continue;
}
if (strcmp(name, ":log_mark") == 0) {
sqlite3_bind_int(stmt, lpc + 1, ll->is_marked());
continue;
}
if (strcmp(name, ":log_mark") == 0) {
sqlite3_bind_int(stmt, lpc + 1, ll->is_marked());
continue;
}
if (strcmp(name, ":log_comment") == 0) {
const auto &bm = this->get_user_bookmark_metadata();
auto cl = this->get_file_base_content_line(ld);
cl += content_line_t(std::distance(lf->cbegin(), ll));
auto bm_iter = bm.find(cl);
if (bm_iter != bm.end() && !bm_iter->second.bm_comment.empty()) {
const auto &meta = bm_iter->second;
sqlite3_bind_text(stmt,
lpc + 1,
meta.bm_comment.c_str(),
meta.bm_comment.length(),
SQLITE_STATIC);
}
continue;
}
if (strcmp(name, ":log_tags") == 0) {
const auto &bm = this->get_user_bookmark_metadata();
auto cl = this->get_file_base_content_line(ld);
cl += content_line_t(std::distance(lf->cbegin(), ll));
auto bm_iter = bm.find(cl);
if (bm_iter != bm.end() && !bm_iter->second.bm_tags.empty()) {
const auto &meta = bm_iter->second;
yajlpp_gen gen;
yajl_gen_config(gen, yajl_gen_beautify, false);
{
yajlpp_array arr(gen);
for (const auto &str : meta.bm_tags) {
arr.gen(str);
}
}
string_fragment sf = gen.to_string_fragment();
sqlite3_bind_text(stmt,
lpc + 1,
sf.data(),
sf.length(),
SQLITE_TRANSIENT);
}
continue;
}
if (strcmp(name, ":log_path") == 0) {
const auto& filename = lf->get_filename();
sqlite3_bind_text(stmt,
lpc + 1,
filename.c_str(), filename.length(),
SQLITE_STATIC);
continue;
}
if (strcmp(name, ":log_text") == 0) {
sqlite3_bind_text(stmt,
lpc + 1,
sbr.get_data(), sbr.length(),
SQLITE_STATIC);
continue;
}
if (strcmp(name, ":log_body") == 0) {
auto iter = find_string_attr(sa, &SA_BODY);
sqlite3_bind_text(stmt,
lpc + 1,
&(sbr.get_data()[iter->sa_range.lr_start]),
iter->sa_range.length(),
SQLITE_STATIC);
continue;
}
for (auto& lv : values) {
if (lv.lv_name != &name[1]) {
continue;
}
switch (lv.lv_kind) {
case logline_value::VALUE_BOOLEAN:
sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
break;
case logline_value::VALUE_FLOAT:
sqlite3_bind_double(stmt, lpc + 1, lv.lv_value.d);
break;
case logline_value::VALUE_INTEGER:
sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
break;
case logline_value::VALUE_NULL:
sqlite3_bind_null(stmt, lpc + 1);
break;
default:
sqlite3_bind_text(stmt,
lpc + 1,
lv.text_value(),
lv.text_length(),
SQLITE_TRANSIENT);
break;
}
break;
}
}
auto step_res = sqlite3_step(stmt);
sqlite3_reset(stmt);
sqlite3_clear_bindings(stmt);
switch (step_res) {
case SQLITE_OK:
case SQLITE_DONE:
return false;
}
return true;
}
bool logfile_sub_source::check_extra_filters(iterator ld, logfile::iterator ll)
{
if (this->lss_marked_only && !ll->is_marked()) {
return false;
}
if (ll->get_msg_level() < this->lss_min_log_level) {
return false;
}
if (*ll < this->lss_min_log_time) {
return false;
}
if (!(*ll <= this->lss_max_log_time)) {
return false;
}
return true;
}
void log_location_history::loc_history_append(vis_line_t top)
{
if (top >= vis_line_t(this->llh_log_source.text_line_count())) {
@ -1035,3 +1252,31 @@ log_location_history::loc_history_forward(vis_line_t current_top)
return nonstd::nullopt;
}
bool sql_filter::matches(const logfile &lf, logfile::const_iterator ll,
shared_buffer_ref &line)
{
if (ll->is_continued()) {
return false;
}
if (this->sf_filter_stmt == nullptr) {
return false;
}
auto lfp = lf.shared_from_this();
auto ld = this->sf_log_source.find_data_i(lfp);
if (ld == this->sf_log_source.end()) {
return false;
}
if (this->sf_log_source.eval_sql_filter(this->sf_filter_stmt, ld, ll)) {
return false;
}
return true;
}
std::string sql_filter::to_command()
{
return fmt::format("filter-expr {}", this->lf_id);
}

120
src/logfile_sub_source.hh
View File

@ -54,6 +54,11 @@
STRONG_INT_TYPE(uint64_t, content_line);
struct sqlite3_stmt;
extern "C" {
int sqlite3_finalize(sqlite3_stmt *pStmt);
}
class logfile_sub_source;
class index_delegate {
@ -77,12 +82,12 @@ class pcre_filter
: public text_filter {
public:
pcre_filter(type_t type, const std::string& id, size_t index, pcre *code)
: text_filter(type, id, index),
: text_filter(type, filter_lang_t::REGEX, id, index),
pf_pcre(code) { };
~pcre_filter() override = default;
bool matches(const logfile &lf, const logline &ll, shared_buffer_ref &line) override {
bool matches(const logfile &lf, logfile::const_iterator ll, shared_buffer_ref &line) override {
pcre_context_static<30> pc;
pcre_input pi(line.get_data(), 0, line.length());
@ -99,9 +104,26 @@ protected:
pcrepp pf_pcre;
};
class sql_filter : public text_filter {
public:
sql_filter(logfile_sub_source& lss, std::string stmt_str, sqlite3_stmt *stmt)
: text_filter(EXCLUDE, filter_lang_t::SQL, std::move(stmt_str), 0),
sf_filter_stmt(stmt),
sf_log_source(lss) {
}
bool matches(const logfile &lf, logfile::const_iterator ll, shared_buffer_ref &line) override;
std::string to_command() override;
auto_mem<sqlite3_stmt> sf_filter_stmt{sqlite3_finalize};
logfile_sub_source& sf_log_source;
};
class log_location_history : public location_history {
public:
log_location_history(logfile_sub_source &lss)
explicit log_location_history(logfile_sub_source &lss)
: llh_history(std::begin(this->llh_backing),
std::end(this->llh_backing)),
llh_log_source(lss) {
@ -140,7 +162,6 @@ public:
virtual void text_filters_changed();
logfile_sub_source();
virtual ~logfile_sub_source() = default;
void toggle_time_offset() {
this->lss_flags ^= F_TIME_OFFSET;
@ -454,6 +475,21 @@ public:
return retval;
}
void set_sql_filter(std::string stmt_str, sqlite3_stmt *stmt);
void set_preview_sql_filter(sqlite3_stmt *stmt) {
this->lss_preview_filter_stmt = stmt;
}
std::string get_sql_filter_text() {
auto filt = this->get_sql_filter();
if (filt) {
return filt.value()->get_id();
}
return "";
}
std::shared_ptr<logfile> find(const char *fn, content_line_t &line_base);
std::shared_ptr<logfile> find(content_line_t &line)
@ -611,11 +647,19 @@ public:
return this->lss_files.end();
};
logfile_data *find_data(content_line_t line, uint64_t &offset_out)
iterator find_data(content_line_t &line)
{
logfile_data *retval;
auto retval = this->lss_files.begin();
std::advance(retval, line / MAX_LINES_PER_FILE);
line = content_line_t(line % MAX_LINES_PER_FILE);
retval = this->lss_files[line / MAX_LINES_PER_FILE];
return retval;
};
iterator find_data(content_line_t line, uint64_t &offset_out)
{
auto retval = this->lss_files.begin();
std::advance(retval, line / MAX_LINES_PER_FILE);
offset_out = line % MAX_LINES_PER_FILE;
return retval;
@ -630,6 +674,16 @@ public:
return nonstd::nullopt;
}
iterator find_data_i(const std::shared_ptr<const logfile>& lf) {
for (auto iter = this->begin(); iter != this->end(); ++iter) {
if ((*iter)->ld_filter_state.lfo_filter_state.tfs_logfile == lf) {
return iter;
}
}
return this->end();
}
content_line_t get_file_base_content_line(iterator iter) {
ssize_t index = std::distance(this->begin(), iter);
@ -656,8 +710,8 @@ public:
for (unsigned int index : this->lss_filtered_index) {
content_line_t cl = (content_line_t) this->lss_index[index];
uint64_t line_number;
logfile_data *ld = this->find_data(cl, line_number);
std::shared_ptr<logfile> lf = ld->get_file();
auto ld = this->find_data(cl, line_number);
std::shared_ptr<logfile> lf = (*ld)->get_file();
this->lss_index_delegate->index_line(*this, lf.get(), lf->begin() + line_number);
}
@ -738,6 +792,8 @@ public:
return &this->lss_location_history;
};
bool eval_sql_filter(sqlite3_stmt *stmt, iterator ld, logfile::const_iterator ll);
static const uint64_t MAX_CONTENT_LINES = (1ULL << 40) - 1;
static const uint64_t MAX_LINES_PER_FILE = 256 * 1024 * 1024;
static const uint64_t MAX_FILES = (
@ -871,45 +927,51 @@ private:
this->lss_line_size_cache[0].first = -1;
};
bool check_extra_filters(const logline &ll) {
if (this->lss_marked_only && !ll.is_marked()) {
return false;
}
nonstd::optional<std::shared_ptr<text_filter>> get_sql_filter() {
auto iter = std::find_if(this->tss_filters.begin(),
this->tss_filters.end(),
[](const auto& filt) {
return filt->get_index() == 0;
});
return (
ll.get_msg_level() >= this->lss_min_log_level &&
!(ll < this->lss_min_log_time) &&
ll <= this->lss_max_log_time);
};
if (iter != this->tss_filters.end()) {
return *iter;
}
return nonstd::nullopt;
}
bool check_extra_filters(iterator ld, logfile::iterator ll);
size_t lss_basename_width = 0;
size_t lss_filename_width = 0;
unsigned long lss_flags;
bool lss_force_rebuild;
unsigned long lss_flags{0};
bool lss_force_rebuild{false};
std::vector<logfile_data *> lss_files;
big_array<indexed_content> lss_index;
std::vector<uint32_t> lss_filtered_index;
auto_mem<sqlite3_stmt> lss_preview_filter_stmt{sqlite3_finalize};
bookmarks<content_line_t>::type lss_user_marks;
std::map<content_line_t, bookmark_metadata> lss_user_mark_metadata;
line_flags_t lss_token_flags;
line_flags_t lss_token_flags{0};
iterator lss_token_file_data;
std::shared_ptr<logfile> lss_token_file;
std::string lss_token_value;
string_attrs_t lss_token_attrs;
std::vector<logline_value> lss_token_values;
int lss_token_shift_start;
int lss_token_shift_size;
int lss_token_shift_start{0};
int lss_token_shift_size{0};
shared_buffer lss_share_manager;
logfile::iterator lss_token_line;
std::array<std::pair<int, size_t>, LINE_SIZE_CACHE_SIZE> lss_line_size_cache;
log_level_t lss_min_log_level;
struct timeval lss_min_log_time;
struct timeval lss_max_log_time;
bool lss_marked_only;
index_delegate *lss_index_delegate;
size_t lss_longest_line;
log_level_t lss_min_log_level{LEVEL_UNKNOWN};
struct timeval lss_min_log_time{0, 0};
struct timeval lss_max_log_time{std::numeric_limits<time_t>::max(), 0};
bool lss_marked_only{false};
index_delegate *lss_index_delegate{nullptr};
size_t lss_longest_line{0};
meta_grepper lss_meta_grepper;
log_location_history lss_location_history;
};

4
src/readline_callbacks.cc
View File

@ -148,6 +148,7 @@ void rl_change(readline_curses *rc)
tc->get_highlights().erase({highlight_source_t::PREVIEW, "preview"});
tc->get_highlights().erase({highlight_source_t::PREVIEW, "bodypreview"});
lnav_data.ld_log_source.set_preview_sql_filter(nullptr);
lnav_data.ld_preview_source.clear();
lnav_data.ld_preview_status_source.get_description().clear();
@ -279,6 +280,7 @@ static void rl_search_internal(readline_curses *rc, ln_mode_t mode, bool complet
tc->get_highlights().erase({highlight_source_t::PREVIEW, "preview"});
tc->get_highlights().erase({highlight_source_t::PREVIEW, "bodypreview"});
lnav_data.ld_log_source.set_preview_sql_filter(nullptr);
tc->reload_data();
switch (mode) {
@ -461,6 +463,7 @@ void lnav_rl_abort(readline_curses *rc)
lnav_data.ld_preview_source.clear();
tc->get_highlights().erase({highlight_source_t::PREVIEW, "preview"});
tc->get_highlights().erase({highlight_source_t::PREVIEW, "bodypreview"});
lnav_data.ld_log_source.set_preview_sql_filter(nullptr);
vector<string> errors;
lnav_config = rollback_lnav_config;
@ -495,6 +498,7 @@ static void rl_callback_int(readline_curses *rc, bool is_alt)
lnav_data.ld_preview_source.clear();
tc->get_highlights().erase({highlight_source_t::PREVIEW, "preview"});
tc->get_highlights().erase({highlight_source_t::PREVIEW, "bodypreview"});
lnav_data.ld_log_source.set_preview_sql_filter(nullptr);
auto new_mode = LNM_PAGING;

15
src/readline_curses.cc
View File

@ -695,6 +695,13 @@ void readline_curses::start()
strcpy(rl_line_buffer, initial);
rl_end = strlen(initial);
rl_redisplay();
if (sendcmd(this->rc_command_pipe[RCF_SLAVE],
'l',
rl_line_buffer,
rl_end) != 0) {
perror("line: write failed");
_exit(1);
}
}
else if (sscanf(msg, "f:%d:%n", &context, &prompt_start) == 1 &&
prompt_start != 0 &&
@ -980,9 +987,13 @@ void readline_curses::check_poll_set(const vector<struct pollfd> &pollfds)
case 'l':
this->rc_line_buffer = &msg[2];
this->rc_change(this);
if (this->rc_active_context != -1) {
this->rc_change(this);
}
this->rc_matches.clear();
this->rc_display_match(this);
if (this->rc_active_context != -1) {
this->rc_display_match(this);
}
break;
case 'c':

24
src/readline_highlighters.cc
View File

@ -40,6 +40,8 @@
using namespace std;
static void readline_sqlite_highlighter_int(attr_line_t &al, int x, int skip);
static bool check_re_prev(const string &line, int x)
{
bool retval = false;
@ -358,6 +360,7 @@ void readline_command_highlighter(attr_line_t &al, int x)
static const pcrepp RE_PREFIXES(
R"(^:(filter-in|filter-out|delete-filter|enable-filter|disable-filter|highlight|clear-highlight|create-search-table\s+[^\s]+\s+))");
static const pcrepp SH_PREFIXES("^:(eval|open|append-to|write-to|write-csv-to|write-json-to)");
static const pcrepp SQL_PREFIXES("^:(filter-expr)");
static const pcrepp IDENT_PREFIXES("^:(tag|untag|delete-tags)");
static const pcrepp COLOR_PREFIXES("^:(config)");
static const pcrepp COLOR_RE("(#(?:[a-fA-F0-9]{3}|[a-fA-F0-9]{6}))");
@ -388,6 +391,10 @@ void readline_command_highlighter(attr_line_t &al, int x)
readline_shlex_highlighter(al, x);
}
pi.reset(line);
if (SQL_PREFIXES.match(pc, pi)) {
readline_sqlite_highlighter_int(al, x, 1 + pc[0]->length());
}
pi.reset(line);
if (COLOR_PREFIXES.match(pc, pi)) {
pi.reset(line);
if (COLOR_RE.match(pc, pi)) {
@ -437,12 +444,12 @@ void readline_command_highlighter(attr_line_t &al, int x)
}
}
void readline_sqlite_highlighter(attr_line_t &al, int x)
static void readline_sqlite_highlighter_int(attr_line_t &al, int x, int skip)
{
static string keyword_re_str = sql_keyword_re() + "|\\.schema|\\.msgformats";
static pcrepp keyword_pcre(keyword_re_str.c_str(), PCRE_CASELESS);
static pcrepp string_literal_pcre("'[^']*('(?:'[^']*')*|$)");
static pcrepp ident_pcre("(\\$?\\b[a-z_]\\w*)|\"([^\"]+)\"|\\[([^\\]]+)]", PCRE_CASELESS);
static pcrepp ident_pcre("(?:\\$|:)?(\\b[a-z_]\\w*)|\"([^\"]+)\"|\\[([^\\]]+)]", PCRE_CASELESS);
static const char *brackets[] = {
"[]",
@ -459,7 +466,7 @@ void readline_sqlite_highlighter(attr_line_t &al, int x)
int error_attrs = vc.attrs_for_role(view_colors::VCR_ERROR) | A_REVERSE;
pcre_context_static<30> pc;
pcre_input pi(al.get_string());
pcre_input pi(al.get_string(), skip);
string &line = al.get_string();
while (ident_pcre.match(pc, pi)) {
@ -476,7 +483,7 @@ void readline_sqlite_highlighter(attr_line_t &al, int x)
}
}
pi.reset(line);
pi.reset(line, skip);
while (keyword_pcre.match(pc, pi)) {
pcre_context::capture_t *cap = pc.all();
@ -487,7 +494,7 @@ void readline_sqlite_highlighter(attr_line_t &al, int x)
keyword_attrs);
}
for (size_t lpc = 0; lpc < line.length(); lpc++) {
for (size_t lpc = skip; lpc < line.length(); lpc++) {
switch (line[lpc]) {
case '*':
case '<':
@ -504,7 +511,7 @@ void readline_sqlite_highlighter(attr_line_t &al, int x)
}
}
pi.reset(line);
pi.reset(line, skip);
while (string_literal_pcre.match(pc, pi)) {
pcre_context::capture_t *cap = pc.all();
@ -528,6 +535,11 @@ void readline_sqlite_highlighter(attr_line_t &al, int x)
}
}
void readline_sqlite_highlighter(attr_line_t &al, int x)
{
readline_sqlite_highlighter_int(al, x, 1);
}
void readline_shlex_highlighter(attr_line_t &al, int x)
{
view_colors &vc = view_colors::singleton();

90
src/readline_possibilities.cc
View File

@ -219,6 +219,96 @@ void add_view_text_possibilities(readline_curses *rlc, int context, const string
rlc->add_possibility(context, type, bookmark_metadata::KNOWN_TAGS);
}
void add_filter_expr_possibilities(readline_curses *rlc, int context, const std::string &type)
{
static const char *BUILTIN_VARS[] = {
":log_level",
":log_time",
":log_mark",
":log_comment",
":log_tags",
":log_path",
":log_text",
":log_body",
};
textview_curses *tc = *lnav_data.ld_view_stack.top();
auto& lss = lnav_data.ld_log_source;
auto bottom = tc->get_bottom();
rlc->clear_possibilities(context, type);
rlc->add_possibility(context, type,
std::begin(BUILTIN_VARS),
std::end(BUILTIN_VARS));
for (auto curr_line = tc->get_top(); curr_line < bottom; ++curr_line) {
auto cl = lss.at(curr_line);
auto lf = lss.find(cl);
auto ll = lf->begin() + cl;
auto format = lf->get_format();
shared_buffer_ref sbr;
string_attrs_t sa;
vector<logline_value> values;
lf->read_full_message(ll, sbr);
format->annotate(cl, sbr, sa, values);
for (auto& lv : values) {
if (!lv.lv_identifier) {
continue;
}
auto_mem<char> ident(sqlite3_free);
ident = sql_quote_ident(lv.lv_name.get());
auto bound_name = fmt::format(":{}", ident);
rlc->add_possibility(context, type, bound_name);
switch (lv.lv_kind) {
case logline_value::VALUE_BOOLEAN:
case logline_value::VALUE_FLOAT:
case logline_value::VALUE_NULL:
break;
case logline_value::VALUE_INTEGER:
rlc->add_possibility(
context, type,
std::to_string(lv.lv_value.i));
break;
default: {
auto_mem<char> str;
str = sqlite3_mprintf("%.*Q", lv.text_length(), lv.text_value());
rlc->add_possibility(context, type, string(str.in()));
break;
}
}
}
}
rlc->add_possibility(context, type,
std::begin(sql_keywords),
std::end(sql_keywords));
rlc->add_possibility(context, type, sql_function_names);
for (int lpc = 0; sqlite_registration_funcs[lpc]; lpc++) {
struct FuncDef *basic_funcs;
struct FuncDefAgg *agg_funcs;
sqlite_registration_funcs[lpc](&basic_funcs, &agg_funcs);
for (int lpc2 = 0; basic_funcs && basic_funcs[lpc2].zName; lpc2++) {
const FuncDef &func_def = basic_funcs[lpc2];
rlc->add_possibility(
context,
type,
string(func_def.zName) + (func_def.nArg ? "(" : "()"));
}
for (int lpc2 = 0; agg_funcs && agg_funcs[lpc2].zName; lpc2++) {
const FuncDefAgg &func_def = agg_funcs[lpc2];
rlc->add_possibility(
context,
type,
string(func_def.zName) + (func_def.nArg ? "(" : "()"));
}
}
}
void add_env_possibilities(int context)
{
extern char **environ;

1
src/readline_possibilities.hh
View File

@ -36,6 +36,7 @@
#include "readline_curses.hh"
void add_view_text_possibilities(readline_curses *rlc, int context, const std::string &type, textview_curses *tc);
void add_filter_expr_possibilities(readline_curses *rlc, int context, const std::string &type);
void add_env_possibilities(int context);
void add_filter_possibilities(textview_curses *tc);
void add_mark_possibilities();

1
src/session_data.cc
View File

@ -1503,6 +1503,7 @@ void reset_session()
lnav_data.ld_log_source.set_marked_only(false);
lnav_data.ld_log_source.clear_min_max_log_times();
lnav_data.ld_log_source.set_min_log_level(LEVEL_UNKNOWN);
lnav_data.ld_log_source.set_sql_filter("", nullptr);
lnav_data.ld_log_source.get_user_bookmark_metadata().clear();

63
src/sql_util.cc
View File

@ -253,7 +253,7 @@ const char *sql_function_names[] = {
"julianday(",
"strftime(",
NULL
nullptr
};
multimap<std::string, help_text *> sqlite_function_help;
@ -268,6 +268,9 @@ static int handle_db_list(void *ptr,
smc = (struct sqlite_metadata_callbacks *)ptr;
smc->smc_db_list[colvalues[1]] = std::vector<std::string>();
if (!smc->smc_database_list) {
return 0;
}
return smc->smc_database_list(ptr, ncols, colvalues, colnames);
}
@ -285,6 +288,9 @@ static int handle_table_list(void *ptr,
struct table_list_data *tld = (struct table_list_data *)ptr;
(*tld->tld_iter)->second.push_back(colvalues[0]);
if (!tld->tld_callbacks->smc_table_list) {
return 0;
}
return tld->tld_callbacks->smc_table_list(tld->tld_callbacks,
ncols,
@ -297,14 +303,16 @@ int walk_sqlite_metadata(sqlite3 *db, struct sqlite_metadata_callbacks &smc)
auto_mem<char, sqlite3_free> errmsg;
int retval;
retval = sqlite3_exec(db,
"pragma collation_list",
smc.smc_collation_list,
&smc,
errmsg.out());
if (retval != SQLITE_OK) {
log_error("could not get collation list -- %s", errmsg.in());
return retval;
if (smc.smc_collation_list) {
retval = sqlite3_exec(db,
"pragma collation_list",
smc.smc_collation_list,
&smc,
errmsg.out());
if (retval != SQLITE_OK) {
log_error("could not get collation list -- %s", errmsg.in());
return retval;
}
}
retval = sqlite3_exec(db,
@ -351,14 +359,16 @@ int walk_sqlite_metadata(sqlite3 *db, struct sqlite_metadata_callbacks &smc)
return SQLITE_NOMEM;
}
retval = sqlite3_exec(db,
table_query,
smc.smc_table_info,
&smc,
errmsg.out());
if (retval != SQLITE_OK) {
log_error("could not get table info -- %s", errmsg.in());
return retval;
if (smc.smc_table_info) {
retval = sqlite3_exec(db,
table_query,
smc.smc_table_info,
&smc,
errmsg.out());
if (retval != SQLITE_OK) {
log_error("could not get table info -- %s", errmsg.in());
return retval;
}
}
table_query = sqlite3_mprintf(
@ -369,14 +379,17 @@ int walk_sqlite_metadata(sqlite3 *db, struct sqlite_metadata_callbacks &smc)
return SQLITE_NOMEM;
}
retval = sqlite3_exec(db,
table_query,
smc.smc_foreign_key_list,
&smc,
errmsg.out());
if (retval != SQLITE_OK) {
log_error("could not get foreign key list -- %s", errmsg.in());
return retval;
if (smc.smc_foreign_key_list) {
retval = sqlite3_exec(db,
table_query,
smc.smc_foreign_key_list,
&smc,
errmsg.out());
if (retval != SQLITE_OK) {
log_error("could not get foreign key list -- %s",
errmsg.in());
return retval;
}
}
}
}

12
src/textview_curses.cc
View File

@ -77,7 +77,7 @@ text_filter::revert_to_last(logfile_filter_state &lfs, size_t rollback_size)
void text_filter::add_line(
logfile_filter_state &lfs, logfile::const_iterator ll, shared_buffer_ref &line) {
bool match_state = this->matches(*lfs.tfs_logfile, *ll, line);
bool match_state = this->matches(*lfs.tfs_logfile, ll, line);
if (!ll->is_continued()) {
this->end_of_message(lfs);
@ -91,7 +91,7 @@ void text_filter::end_of_message(logfile_filter_state &lfs)
{
uint32_t mask = 0;
mask = ((uint32_t) lfs.tfs_message_matched[this->lf_index] ? 1U : 0) << this->lf_index;
mask = ((uint32_t) 1U << this->lf_index);
for (size_t lpc = 0; lpc < lfs.tfs_lines_for_message[this->lf_index]; lpc++) {
require(lfs.tfs_filter_count[this->lf_index] <=
@ -99,7 +99,11 @@ void text_filter::end_of_message(logfile_filter_state &lfs)
size_t line_number = lfs.tfs_filter_count[this->lf_index];
lfs.tfs_mask[line_number] |= mask;
if (lfs.tfs_message_matched[this->lf_index]) {
lfs.tfs_mask[line_number] |= mask;
} else {
lfs.tfs_mask[line_number] &= ~mask;
}
lfs.tfs_filter_count[this->lf_index] += 1;
if (lfs.tfs_message_matched[this->lf_index]) {
lfs.tfs_filter_hits[this->lf_index] += 1;
@ -679,7 +683,7 @@ void text_time_translator::data_reloaded(textview_curses *tc)
template class bookmark_vector<vis_line_t>;
bool empty_filter::matches(const logfile &lf, const logline &ll,
bool empty_filter::matches(const logfile &lf, logfile::const_iterator ll,
shared_buffer_ref &line)
{
return false;

57
src/textview_curses.hh
View File

@ -77,15 +77,19 @@ public:
this->tfs_index.clear();
};
void clear_filter_state(size_t index) {
this->tfs_filter_count[index] = 0;
this->tfs_filter_hits[index] = 0;
this->tfs_message_matched[index] = false;
this->tfs_lines_for_message[index] = 0;
this->tfs_last_message_matched[index] = false;
this->tfs_last_lines_for_message[index] = 0;
};
void clear_deleted_filter_state(uint32_t used_mask) {
for (int lpc = 0; lpc < MAX_FILTERS; lpc++) {
if (!(used_mask & (1L << lpc))) {
this->tfs_filter_count[lpc] = 0;
this->tfs_filter_hits[lpc] = 0;
this->tfs_message_matched[lpc] = false;
this->tfs_lines_for_message[lpc] = 0;
this->tfs_last_message_matched[lpc] = false;
this->tfs_last_lines_for_message[lpc] = 0;
this->clear_filter_state(lpc);
}
}
for (size_t lpc = 0; lpc < this->tfs_mask.size(); lpc++) {
@ -117,6 +121,12 @@ public:
std::vector<uint32_t> tfs_index;
};
enum class filter_lang_t : int {
NONE,
REGEX,
SQL,
};
class text_filter {
public:
typedef enum {
@ -129,15 +139,20 @@ public:
LFT__MASK = (MAYBE|INCLUDE|EXCLUDE)
} type_t;
text_filter(type_t type, const std::string& id, size_t index)
text_filter(type_t type, filter_lang_t lang, std::string id, size_t index)
: lf_type(type),
lf_id(id),
lf_lang(lang),
lf_id(std::move(id)),
lf_index(index) { };
virtual ~text_filter() = default;
type_t get_type() const { return this->lf_type; };
type_t get_type() const { return this->lf_type; }
filter_lang_t get_lang() const { return this->lf_lang; }
void set_type(type_t t) { this->lf_type = t; };
std::string get_id() const { return this->lf_id; };
void set_id(std::string id) {
this->lf_id = std::move(id);
}
size_t get_index() const { return this->lf_index; };
bool is_enabled() const { return this->lf_enabled; };
@ -153,7 +168,7 @@ public:
void end_of_message(logfile_filter_state &lfs);
virtual bool matches(const logfile &lf, const logline &ll, shared_buffer_ref &line) = 0;
virtual bool matches(const logfile &lf, logfile::const_iterator ll, shared_buffer_ref &line) = 0;
virtual std::string to_command() = 0;
@ -166,6 +181,7 @@ public:
protected:
bool lf_enabled{true};
type_t lf_type;
filter_lang_t lf_lang;
std::string lf_id;
size_t lf_index;
};
@ -173,10 +189,10 @@ protected:
class empty_filter : public text_filter {
public:
empty_filter(type_t type, size_t index)
: text_filter(type, "", index) {
: text_filter(type, filter_lang_t::NONE, "", index) {
}
bool matches(const logfile &lf, const logline &ll,
bool matches(const logfile &lf, logfile::const_iterator ll,
shared_buffer_ref &line) override;
std::string to_command() override;
@ -186,6 +202,9 @@ class filter_stack {
public:
typedef std::vector<std::shared_ptr<text_filter>>::iterator iterator;
explicit filter_stack(size_t reserved = 0) : fs_reserved(reserved) {
}
iterator begin() {
return this->fs_filters.begin();
}
@ -203,7 +222,8 @@ public:
};
bool full() const {
return this->fs_filters.size() == logfile_filter_state::MAX_FILTERS;
return (this->fs_reserved + this->fs_filters.size()) ==
logfile_filter_state::MAX_FILTERS;
}
nonstd::optional<size_t> next_index() {
@ -221,7 +241,9 @@ public:
used[index] = true;
}
for (size_t lpc = 0; lpc < logfile_filter_state::MAX_FILTERS; lpc++) {
for (size_t lpc = this->fs_reserved;
lpc < logfile_filter_state::MAX_FILTERS;
lpc++) {
if (!used[lpc]) {
return lpc;
}
@ -330,12 +352,13 @@ public:
};
private:
const size_t fs_reserved;
std::vector<std::shared_ptr<text_filter>> fs_filters;
};
class text_time_translator {
public:
virtual ~text_time_translator() { };
virtual ~text_time_translator() = default;
virtual int row_for_time(struct timeval time_bucket) = 0;
@ -386,6 +409,10 @@ public:
typedef long line_flags_t;
text_sub_source(size_t reserved_filters = 0)
: tss_filters(reserved_filters) {
}
void register_view(textview_curses *tc) {
this->tss_view = tc;
};

8
src/views_vtab.cc
View File

@ -512,7 +512,13 @@ CREATE TABLE lnav_view_filters (
textview_curses &tc = lnav_data.ld_views[view_index];
text_sub_source *tss = tc.get_sub_source();
filter_stack &fs = tss->get_filters();
auto iter = fs.begin() + filter_index;
auto iter = fs.begin();
for (; iter != fs.end(); ++iter) {
if ((*iter)->get_index() == filter_index) {
break;
}
}
shared_ptr<text_filter> tf = *iter;
if (new_view_index != view_index) {

33
test/test_cmds.sh
View File

@ -1,6 +1,35 @@
#! /bin/bash
run_test ${lnav_test} -n -d /tmp/lnav.err \
-c ":filter-expr :log_text LIKE '%How are%'" \
"${test_dir}/logfile_multiline.0"
check_output "filter-expr for multiline not working" <<EOF
2009-07-20 22:59:27,672:DEBUG:Hello, World!
How are you today?
EOF
run_test ${lnav_test} -n -d /tmp/lnav.err \
-c ":filter-expr :sc_bytes > 2000" \
"${test_dir}/logfile_access_log.*"
check_output "filter-expr not working" <<EOF
192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkboot.gz HTTP/1.0" 404 46210 "-" "gPXE/0.9.7"
192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
EOF
run_test ${lnav_test} -n -d /tmp/lnav.err \
-c ":filter-expr :sc_bytes # ff" \
"${test_dir}/logfile_access_log.*"
check_error_output "filter-expr error not working" <<EOF
command-option:1: error: unrecognized token: "#"
EOF
run_test ${lnav_test} -n -d /tmp/lnav.err \
-c ":goto 0" \
-c ":close" \
@ -453,14 +482,14 @@ EOF
TOO_MANY_FILTERS=""
for i in `seq 1 33`; do
for i in `seq 1 32`; do
TOO_MANY_FILTERS="$TOO_MANY_FILTERS -c ':filter-out $i'"
done
run_test eval ${lnav_test} -d /tmp/lnav.err -n \
$TOO_MANY_FILTERS \
${test_dir}/logfile_filter.0
check_error_output "able to create too many filters?" <<EOF
command-option:33: error: filter limit reached, try combining filters with a pipe symbol (e.g. foo|bar)
command-option:32: error: filter limit reached, try combining filters with a pipe symbol (e.g. foo|bar)
EOF

4
test/test_sql.sh
View File

@ -127,7 +127,7 @@ run_test ${lnav_test} -n \
check_output "view filter stats is not working?" <<EOF
view_name,filter_id,hits
log,0,2
log,1,2
EOF
run_test ${lnav_test} -n \
@ -456,7 +456,7 @@ run_test ${lnav_test} -n \
check_output "logline table is not working" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,log_hostname,log_msgid,log_pid,log_pri,log_procname,log_struct,syslog_version,log_msg_instance,col_0,TTY,PWD,USER,COMMAND
0,<NULL>,2013-11-03 09:47:02.000,0,info,0,<NULL>,<NULL>,[0],veridian,<NULL>,<NULL>,<NULL>,sudo,<NULL>,<NULL>,0,timstack,pts/6,/auto/wstimstack/rpms/lbuild/test,root,/usr/bin/tail /var/log/messages
0,<NULL>,2013-11-03 09:47:02.000,0,info,0,<NULL>,<NULL>,[1],veridian,<NULL>,<NULL>,<NULL>,sudo,<NULL>,<NULL>,0,timstack,pts/6,/auto/wstimstack/rpms/lbuild/test,root,/usr/bin/tail /var/log/messages
EOF