Robot
/
lnav
mirror of https://github.com/tstack/lnav.git
2
0
Fork 0
Code Issues Releases Wiki Activity

[logfile] move bookmark_metadata to logfiles

This commit is contained in:
Tim Stack 2022-08-19 20:01:25 -07:00
parent ad1897ba15
commit 53c9c5cb51
41 changed files with 36522 additions and 35604 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
NEWS
View File

@ -37,6 +37,14 @@ lnav v0.11.0:
within lnav (e.g. opening a file, format is detected). You
can then add SQLite TRIGGERs to this table that can perform a
task by updating other tables.
* Tags can automatically be added to messages by defining a pattern
in a log format. Under a format definition, add the tag name
into the "tags" object in a format definition. The "pattern"
property specifies the regular expression to match against a line
in a file that matches the format. If a match is found, the tag
will be applied to the log message. To restrict matches to
certain files, you can add a "paths" array whose object elements
contain a "glob" property that will be matched against file names.
* Log messages can now be detected automatically via "watch
expressions". These are SQL expressions that are executed for
each log message. If the expressions evaluates to true, an

5
docs/schemas/event-log-msg-detected-v1.schema.json
View File

@ -21,6 +21,11 @@
"description": "The path of the file containing the log message",
"type": "string"
},
"line-number": {
"title": "/line-number",
"description": "The line number in the file, starting from zero",
"type": "integer"
},
"format": {
"title": "/format",
"description": "The name of the log format that matched this log message",

43
docs/schemas/format-v1.schema.json
View File

@ -284,6 +284,49 @@
},
"additionalProperties": false
},
"tags": {
"description": "The tags to automatically apply to log messages",
"title": "/<format_name>/tags",
"type": "object",
"patternProperties": {
"([^/]+)": {
"description": "The name of the tag to apply",
"title": "/<format_name>/tags/<tag_name>",
"type": "object",
"properties": {
"paths": {
"description": "Restrict tagging to the given paths",
"title": "/<format_name>/tags/<tag_name>/paths",
"type": "array",
"items": {
"type": "object",
"properties": {
"glob": {
"title": "/<format_name>/tags/<tag_name>/paths/glob",
"description": "The glob to match against file paths",
"type": "string",
"examples": [
"*/system.log*"
]
}
},
"additionalProperties": false
}
},
"pattern": {
"title": "/<format_name>/tags/<tag_name>/pattern",
"description": "The regular expression to match against the body of the log message",
"type": "string",
"examples": [
"\\w+ is down"
]
}
},
"additionalProperties": false
}
},
"additionalProperties": false
},
"action": {
"title": "/<format_name>/action",
"type": "object",

13
docs/source/formats.rst
View File

@ -1,4 +1,3 @@
.. _log_formats:
Log Formats
@ -276,6 +275,18 @@ should be another object with the following fields:
SELECT message FROM http_status_codes
WHERE status = :sc_status) || ') '
:tags: This object contains the tags that should automatically be added to
log messages.
:pattern: The regular expression evaluated over a line in the log file as
it is read in. If there is a match, the log message the line is a part
of will have this tag added to it.
:paths: This array contains objects that define restrictions on the file
paths that the tags will be applied to. The objects in this array can
contain:
:glob: A glob pattern to check against the log files read by lnav.
.. _format_sample:
:sample: A list of objects that contain sample log messages. All formats

2
src/Makefile.am
View File

@ -359,7 +359,7 @@ libdatascanner_a_SOURCES = \
data_scanner_re.cc
# XXX The data_scanner_re optimized build is taking 30+ minutes to run for
# some reason, so we need to override the flags
libdatascanner_a_CXXFLAGS = -O1
libdatascanner_a_CXXFLAGS = -O1 -g
libdiag_a_SOURCES = \
$(THIRD_PARTY_SRCS) \

3
src/base/ansi_scrubber.cc
View File

@ -153,6 +153,7 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
last_origin_offset_end = caps->c_begin + output_size;
origin_offset += erased_size;
pi.reset(str);
pi.pi_next_offset = last_origin_offset_end;
continue;
}
@ -274,10 +275,12 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
}
sa->emplace_back(line_range{last_origin_offset_end, caps->c_begin},
SA_ORIGIN_OFFSET.value(origin_offset));
last_origin_offset_end = caps->c_begin;
origin_offset += caps->length();
}
pi.reset(str);
pi.pi_next_offset = caps->c_begin;
}
if (sa != nullptr && last_origin_offset_end > 0) {

4
src/data_scanner.cc
View File

@ -243,6 +243,10 @@ static struct {
"dot",
pcrepp("\\A(\\.)"),
},
{
"escc",
pcrepp("\\A(\\\\\\.)"),
},
{
"gbg",

1
src/data_scanner.hh
View File

@ -92,6 +92,7 @@ enum data_token_t {
DT_LINE,
DT_WHITE,
DT_DOT,
DT_ESCAPED_CHAR,
DT_GARBAGE,

71095
src/data_scanner_re.cc
View File

File diff suppressed because it is too large Load Diff

1
src/data_scanner_re.re
View File

@ -243,6 +243,7 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
("\r"?"\n"|"\\n") { RET(DT_LINE); }
SPACE+ { RET(DT_WHITE); }
"." { RET(DT_DOT); }
"\\". { RET(DT_ESCAPED_CHAR); }
. { RET(DT_GARBAGE); }
*/

6
src/document.sections.cc
View File

@ -254,8 +254,9 @@ public:
pcre_context_static<30> pc;
data_token_t dt = DT_INVALID;
auto& pi = this->sw_scanner.get_input();
size_t garbage_count = 0;
while (this->sw_scanner.tokenize2(pc, dt)) {
while (garbage_count < 1000 && this->sw_scanner.tokenize2(pc, dt)) {
element el(dt, pc);
switch (dt) {
@ -361,6 +362,9 @@ public:
case DT_WHITE:
break;
default:
if (dt == DT_GARBAGE) {
garbage_count += 1;
}
this->sw_values.emplace_back(el);
break;
}

112
src/field_overlay_source.cc
View File

@ -45,8 +45,8 @@ json_string extract(const char* str);
void
field_overlay_source::build_field_lines(const listview_curses& lv)
{
logfile_sub_source& lss = this->fos_lss;
view_colors& vc = view_colors::singleton();
auto& lss = this->fos_lss;
auto& vc = view_colors::singleton();
this->fos_lines.clear();
@ -63,7 +63,8 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
bool display = false;
if (ll->is_time_skewed()
|| ll->get_msg_level() == log_level_t::LEVEL_INVALID) {
|| ll->get_msg_level() == log_level_t::LEVEL_INVALID)
{
display = true;
}
if (!this->fos_contexts.empty()) {
@ -301,7 +302,8 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
if (curr_elf && curr_elf->elf_body_field == lv.lv_meta.lvm_name) {
field_name = LOG_BODY;
} else if (curr_elf
&& curr_elf->lf_timestamp_field == lv.lv_meta.lvm_name) {
&& curr_elf->lf_timestamp_field == lv.lv_meta.lvm_name)
{
field_name = LOG_TIME;
} else {
field_name = lv.lv_meta.lvm_name.to_string();
@ -396,7 +398,8 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
}
if (!this->fos_contexts.empty()
&& !this->fos_contexts.top().c_show_discovered) {
&& !this->fos_contexts.top().c_show_discovered)
{
return;
}
@ -441,64 +444,60 @@ field_overlay_source::build_meta_line(const listview_curses& lv,
std::vector<attr_line_t>& dst,
vis_line_t row)
{
content_line_t cl = this->fos_lss.at(row);
auto const& bm = this->fos_lss.get_user_bookmark_metadata();
view_colors& vc = view_colors::singleton();
auto iter = bm.find(cl);
auto line_meta_opt = this->fos_lss.find_bookmark_metadata(row);
if (iter != bm.end()) {
const bookmark_metadata& line_meta = iter->second;
size_t filename_width = this->fos_lss.get_filename_offset();
auto* tc = dynamic_cast<const textview_curses*>(&lv);
if (!line_meta_opt) {
return;
}
auto& vc = view_colors::singleton();
const auto& line_meta = *(line_meta_opt.value());
size_t filename_width = this->fos_lss.get_filename_offset();
const auto* tc = dynamic_cast<const textview_curses*>(&lv);
if (!line_meta.bm_comment.empty()) {
const auto* lead = line_meta.bm_tags.empty() ? " \u2514 "
: " \u251c ";
attr_line_t al;
if (!line_meta.bm_comment.empty()) {
const auto* lead = line_meta.bm_tags.empty() ? " \u2514 " : " \u251c ";
attr_line_t al;
al.with_string(lead).append(
lnav::roles::comment(line_meta.bm_comment));
al.insert(0, filename_width, ' ');
if (tc != nullptr) {
auto hl = tc->get_highlights();
auto hl_iter = hl.find({highlight_source_t::PREVIEW, "search"});
al.with_string(lead).append(lnav::roles::comment(line_meta.bm_comment));
al.insert(0, filename_width, ' ');
if (tc != nullptr) {
auto hl = tc->get_highlights();
auto hl_iter = hl.find({highlight_source_t::PREVIEW, "search"});
if (hl_iter != hl.end()) {
hl_iter->second.annotate(al, filename_width);
}
if (hl_iter != hl.end()) {
hl_iter->second.annotate(al, filename_width);
}
dst.emplace_back(al);
}
if (!line_meta.bm_tags.empty()) {
attr_line_t al;
al.with_string(" \u2514");
for (const auto& str : line_meta.bm_tags) {
al.append(1, ' ').append(
str, VC_STYLE.value(vc.attrs_for_ident(str)));
}
dst.emplace_back(al);
}
if (!line_meta.bm_tags.empty()) {
attr_line_t al;
const auto* tc = dynamic_cast<const textview_curses*>(&lv);
if (tc) {
const auto& hm = tc->get_highlights();
auto hl_iter = hm.find({highlight_source_t::PREVIEW, "search"});
if (hl_iter != hm.end()) {
hl_iter->second.annotate(al, 2);
}
}
al.insert(0, filename_width, ' ');
if (tc != nullptr) {
auto hl = tc->get_highlights();
auto hl_iter = hl.find({highlight_source_t::PREVIEW, "search"});
if (hl_iter != hl.end()) {
hl_iter->second.annotate(al, filename_width);
}
}
dst.emplace_back(al);
al.with_string(" \u2514");
for (const auto& str : line_meta.bm_tags) {
al.append(1, ' ').append(str,
VC_STYLE.value(vc.attrs_for_ident(str)));
}
if (tc != nullptr) {
const auto& hm = tc->get_highlights();
auto hl_iter = hm.find({highlight_source_t::PREVIEW, "search"});
if (hl_iter != hm.end()) {
hl_iter->second.annotate(al, 2);
}
}
al.insert(0, filename_width, ' ');
if (tc != nullptr) {
auto hl = tc->get_highlights();
auto hl_iter = hl.find({highlight_source_t::PREVIEW, "search"});
if (hl_iter != hl.end()) {
hl_iter->second.annotate(al, filename_width);
}
}
dst.emplace_back(al);
}
}
@ -532,7 +531,8 @@ field_overlay_source::list_value_for_overlay(const listview_curses& lv,
return true;
}
if (!this->fos_meta_lines.empty()) {
if (!this->fos_meta_lines.empty() && this->fos_meta_lines_row == row - 1_vl)
{
value_out = this->fos_meta_lines.front();
this->fos_meta_lines.erase(this->fos_meta_lines.begin());
@ -540,7 +540,9 @@ field_overlay_source::list_value_for_overlay(const listview_curses& lv,
}
if (row < lv.get_inner_height()) {
this->fos_meta_lines.clear();
this->build_meta_line(lv, this->fos_meta_lines, row);
this->fos_meta_lines_row = row;
}
return false;

1
src/field_overlay_source.hh
View File

@ -79,6 +79,7 @@ public:
int fos_known_key_size{0};
int fos_unknown_key_size{0};
std::vector<attr_line_t> fos_lines;
vis_line_t fos_meta_lines_row{0_vl};
std::vector<attr_line_t> fos_meta_lines;
};

192
src/formats/syslog_log.json
View File

@ -1,99 +1,99 @@
{
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
"syslog_log": {
"title": "Syslog",
"description": "The system logger format found on most posix systems.",
"url": "http://en.wikipedia.org/wiki/Syslog",
"regex": {
"std": {
"pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|(?:\\+|-)\\d{2}:\\d{2})))(?: (?<log_hostname>[a-zA-Z0-9:][^ ]+[a-zA-Z0-9]))?(?: \\[CLOUDINIT\\])?(?:(?: syslogd [\\d\\.]+|(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[: ]+|[^ :]+))(?:\\[(?<log_pid>\\d+)\\](?: \\([^\\)]+\\))?)?))):\\s*(?<body>.*)$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
},
"rfc5424": {
"pattern": "^<(?<log_pri>\\d+)>(?<syslog_version>\\d+) (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?<log_hostname>[^ ]+|-) (?<log_syslog_tag>(?<log_procname>[^ ]+|-) (?<log_pid>[^ ]+|-) (?<log_msgid>[^ ]+|-)) (?<log_struct>\\[(?:[^\\]\"]|\"(?:\\.|[^\"])+\")*\\]|-|)\\s+(?<body>.*)"
}
},
"level-field": "body",
"level": {
"error": "(?:(?:(?<![a-zA-Z]))(?:(?i)error(?:s)?)(?:(?![a-zA-Z]))|failed|failure)",
"warning": "(?:(?:(?i)warn)|not responding|init: cannot execute)"
},
"opid-field": "log_syslog_tag",
"multiline": true,
"module-field": "log_procname",
"value": {
"log_pri": {
"kind": "integer",
"foreign-key": true,
"description": "The priority level of the message"
},
"syslog_version": {
"kind": "integer",
"foreign-key": true,
"description": "The version of the syslog format used for this message"
},
"log_hostname": {
"kind": "string",
"collate": "ipaddress",
"identifier": true,
"description": "The name of the host that generated the message"
},
"log_procname": {
"kind": "string",
"identifier": true,
"description": "The name of the process that generated the message"
},
"log_pid": {
"kind": "string",
"identifier": true,
"action-list": [
"dump_pid"
],
"description": "The ID of the process that generated the message"
},
"log_syslog_tag": {
"kind": "string",
"identifier": true,
"description": "The combination of the procname and pid"
},
"log_msgid": {
"kind": "string",
"identifier": true
},
"log_struct": {
"kind": "struct"
}
},
"action": {
"dump_pid": {
"label": "Show Process Info",
"capture-output": true,
"cmd": [
"dump-pid.sh"
]
}
},
"sample": [
{
"line": "Apr 28 04:02:03 tstack-centos5 syslogd 1.4.1: restart."
},
{
"line": "Jun 27 01:47:20 Tims-MacBook-Air.local configd[17]: network changed: v4(en0-:192.168.1.8) DNS- Proxy- SMB"
},
{
"line": "Jun 20 17:26:13 ip-10-188-149-5 [CLOUDINIT] util.py[DEBUG]: Restoring selinux mode for /var/lib/cloud (recursive=False)"
},
{
"line": "<46>1 2017-04-27T07:50:47.381967+02:00 logserver rsyslogd - - [origin software=\"rsyslogd\" swVersion=\"8.4.2\" x-pid=\"900\" x-info=\"http://www.rsyslog.com\"] start"
},
{
"line": "<30>1 2017-04-27T07:59:12+02:00 nextcloud dhclient - - - DHCPREQUEST on eth0 to 192.168.1.1 port 67"
},
{
"line": "<78>1 2017-04-27T08:09:01+02:00 nextcloud CRON 1472 - - (root) CMD ( [ -x /usr/lib/php5/sessionclean ] && /usr/lib/php5/sessionclean)"
},
{
"line": "Aug 1 00:00:03 Tim-Stacks-iMac com.apple.xpc.launchd[1] (com.apple.mdworker.shared.0C000000-0700-0000-0000-000000000000[50989]): Service exited due to SIGKILL | sent by mds[198]"
}
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
"syslog_log": {
"title": "Syslog",
"description": "The system logger format found on most posix systems.",
"url": "http://en.wikipedia.org/wiki/Syslog",
"regex": {
"std": {
"pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|(?:\\+|-)\\d{2}:\\d{2})))(?: (?<log_hostname>[a-zA-Z0-9:][^ ]+[a-zA-Z0-9]))?(?: \\[CLOUDINIT\\])?(?:(?: syslogd [\\d\\.]+|(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[: ]+|[^ :]+))(?:\\[(?<log_pid>\\d+)\\](?: \\([^\\)]+\\))?)?))):\\s*(?<body>.*)$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
},
"rfc5424": {
"pattern": "^<(?<log_pri>\\d+)>(?<syslog_version>\\d+) (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?<log_hostname>[^ ]+|-) (?<log_syslog_tag>(?<log_procname>[^ ]+|-) (?<log_pid>[^ ]+|-) (?<log_msgid>[^ ]+|-)) (?<log_struct>\\[(?:[^\\]\"]|\"(?:\\.|[^\"])+\")*\\]|-|)\\s+(?<body>.*)"
}
},
"level-field": "body",
"level": {
"error": "(?:(?:(?<![a-zA-Z]))(?:(?i)error(?:s)?)(?:(?![a-zA-Z]))|failed|failure)",
"warning": "(?:(?:(?i)warn)|not responding|init: cannot execute)"
},
"opid-field": "log_syslog_tag",
"multiline": true,
"module-field": "log_procname",
"value": {
"log_pri": {
"kind": "integer",
"foreign-key": true,
"description": "The priority level of the message"
},
"syslog_version": {
"kind": "integer",
"foreign-key": true,
"description": "The version of the syslog format used for this message"
},
"log_hostname": {
"kind": "string",
"collate": "ipaddress",
"identifier": true,
"description": "The name of the host that generated the message"
},
"log_procname": {
"kind": "string",
"identifier": true,
"description": "The name of the process that generated the message"
},
"log_pid": {
"kind": "string",
"identifier": true,
"action-list": [
"dump_pid"
],
"description": "The ID of the process that generated the message"
},
"log_syslog_tag": {
"kind": "string",
"identifier": true,
"description": "The combination of the procname and pid"
},
"log_msgid": {
"kind": "string",
"identifier": true
},
"log_struct": {
"kind": "struct"
}
},
"action": {
"dump_pid": {
"label": "Show Process Info",
"capture-output": true,
"cmd": [
"dump-pid.sh"
]
}
}
},
"sample": [
{
"line": "Apr 28 04:02:03 tstack-centos5 syslogd 1.4.1: restart."
},
{
"line": "Jun 27 01:47:20 Tims-MacBook-Air.local configd[17]: network changed: v4(en0-:192.168.1.8) DNS- Proxy- SMB"
},
{
"line": "Jun 20 17:26:13 ip-10-188-149-5 [CLOUDINIT] util.py[DEBUG]: Restoring selinux mode for /var/lib/cloud (recursive=False)"
},
{
"line": "<46>1 2017-04-27T07:50:47.381967+02:00 logserver rsyslogd - - [origin software=\"rsyslogd\" swVersion=\"8.4.2\" x-pid=\"900\" x-info=\"http://www.rsyslog.com\"] start"
},
{
"line": "<30>1 2017-04-27T07:59:12+02:00 nextcloud dhclient - - - DHCPREQUEST on eth0 to 192.168.1.1 port 67"
},
{
"line": "<78>1 2017-04-27T08:09:01+02:00 nextcloud CRON 1472 - - (root) CMD ( [ -x /usr/lib/php5/sessionclean ] && /usr/lib/php5/sessionclean)"
},
{
"line": "Aug 1 00:00:03 Tim-Stacks-iMac com.apple.xpc.launchd[1] (com.apple.mdworker.shared.0C000000-0700-0000-0000-000000000000[50989]): Service exited due to SIGKILL | sent by mds[198]"
}
]
}
}

10
src/formats/vmw_log.json
View File

@ -130,6 +130,16 @@
"level": "error"
}
},
"tags": {
"test-failure": {
"paths": [
{
"glob": "*/test.log"
}
],
"pattern": "^Expected equality of these values:"
}
},
"sample": [
{
"line": "2021-05-24T20:31:05.671Z - last log rotation time, 2021-05-24T09:30:02.683Z - time the service was last started, Section for VMware ESX, pid=1000080910, version=7.0.3, build=0, option=DEBUG"

3
src/lnav.events.cc
View File

@ -87,6 +87,9 @@ const typed_json_path_container<msg_detected> msg_detected::handlers = typed_jso
yajlpp::property_handler("filename")
.with_description("The path of the file containing the log message")
.for_field(&msg_detected::md_filename),
yajlpp::property_handler("line-number")
.with_description("The line number in the file, starting from zero")
.for_field(&msg_detected::md_line_number),
yajlpp::property_handler("format")
.with_description("The name of the log format that matched this log message")
.for_field(&msg_detected::md_format),

1
src/lnav.events.hh
View File

@ -64,6 +64,7 @@ struct msg_detected {
std::string md_watch_name;
std::string md_filename;
std::string md_format;
uint32_t md_line_number;
std::string md_timestamp;
std::map<std::string, json_any_t> md_values;
std::string md_schema{SCHEMA_ID};

68
src/lnav_commands.cc
View File

@ -2869,13 +2869,12 @@ com_comment(exec_context& ec,
"The :comment command only works in the log view");
}
auto& lss = lnav_data.ld_log_source;
auto& bm = lss.get_user_bookmark_metadata();
args[1] = trim(remaining_args(cmdline, args));
tc->set_user_mark(&textview_curses::BM_META, tc->get_top(), true);
auto& line_meta = bm[lss.at(tc->get_top())];
auto& line_meta = lss.get_bookmark_metadata(tc->get_top());
line_meta.bm_comment = args[1];
lss.set_line_meta_changed();
@ -2898,14 +2897,12 @@ com_comment_prompt(exec_context& ec, const std::string& cmdline)
if (tc != &lnav_data.ld_views[LNV_LOG]) {
return "";
}
logfile_sub_source& lss = lnav_data.ld_log_source;
std::map<content_line_t, bookmark_metadata>& bm
= lss.get_user_bookmark_metadata();
auto& lss = lnav_data.ld_log_source;
auto line_meta = bm.find(lss.at(tc->get_top()));
auto line_meta_opt = lss.find_bookmark_metadata(tc->get_top());
if (line_meta != bm.end() && !line_meta->second.bm_comment.empty()) {
return trim(cmdline) + " " + trim(line_meta->second.bm_comment);
if (line_meta_opt && !line_meta_opt.value()->bm_comment.empty()) {
return trim(cmdline) + " " + trim(line_meta_opt.value()->bm_comment);
}
return "";
@ -2929,17 +2926,15 @@ com_clear_comment(exec_context& ec,
return ec.make_error(
"The :clear-comment command only works in the log view");
}
logfile_sub_source& lss = lnav_data.ld_log_source;
std::map<content_line_t, bookmark_metadata>& bm
= lss.get_user_bookmark_metadata();
auto& lss = lnav_data.ld_log_source;
auto iter = bm.find(lss.at(tc->get_top()));
if (iter != bm.end()) {
bookmark_metadata& line_meta = iter->second;
auto line_meta_opt = lss.find_bookmark_metadata(tc->get_top());
if (line_meta_opt) {
bookmark_metadata& line_meta = *(line_meta_opt.value());
line_meta.bm_comment.clear();
if (line_meta.empty()) {
bm.erase(iter);
lss.erase_bookmark_metadata(tc->get_top());
tc->set_user_mark(
&textview_curses::BM_META, tc->get_top(), false);
}
@ -2973,12 +2968,10 @@ com_tag(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
if (tc != &lnav_data.ld_views[LNV_LOG]) {
return ec.make_error("The :tag command only works in the log view");
}
logfile_sub_source& lss = lnav_data.ld_log_source;
std::map<content_line_t, bookmark_metadata>& bm
= lss.get_user_bookmark_metadata();
auto& lss = lnav_data.ld_log_source;
tc->set_user_mark(&textview_curses::BM_META, tc->get_top(), true);
bookmark_metadata& line_meta = bm[lss.at(tc->get_top())];
auto& line_meta = lss.get_bookmark_metadata(tc->get_top());
for (size_t lpc = 1; lpc < args.size(); lpc++) {
std::string tag = args[lpc];
@ -3019,13 +3012,11 @@ com_untag(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
return ec.make_error(
"The :untag command only works in the log view");
}
logfile_sub_source& lss = lnav_data.ld_log_source;
std::map<content_line_t, bookmark_metadata>& bm
= lss.get_user_bookmark_metadata();
auto& lss = lnav_data.ld_log_source;
auto iter = bm.find(lss.at(tc->get_top()));
if (iter != bm.end()) {
bookmark_metadata& line_meta = iter->second;
auto line_meta_opt = lss.find_bookmark_metadata(tc->get_top());
if (line_meta_opt) {
bookmark_metadata& line_meta = *(line_meta_opt.value());
for (size_t lpc = 1; lpc < args.size(); lpc++) {
std::string tag = args[lpc];
@ -3091,26 +3082,24 @@ com_delete_tags(exec_context& ec,
known_tags.erase(tag);
}
logfile_sub_source& lss = lnav_data.ld_log_source;
bookmark_vector<vis_line_t>& vbm
= tc->get_bookmarks()[&textview_curses::BM_META];
std::map<content_line_t, bookmark_metadata>& bm
= lss.get_user_bookmark_metadata();
auto& lss = lnav_data.ld_log_source;
auto& vbm = tc->get_bookmarks()[&textview_curses::BM_META];
for (auto iter = vbm.begin(); iter != vbm.end();) {
content_line_t cl = lss.at(*iter);
auto line_meta = bm.find(cl);
auto line_meta_opt = lss.find_bookmark_metadata(*iter);
if (line_meta == bm.end()) {
if (!line_meta_opt) {
++iter;
continue;
}
auto& line_meta = line_meta_opt.value();
for (const auto& tag : tags) {
line_meta->second.remove_tag(tag);
line_meta->remove_tag(tag);
}
if (line_meta->second.empty()) {
if (line_meta->empty()) {
lss.erase_bookmark_metadata(*iter);
size_t off = distance(vbm.begin(), iter);
tc->set_user_mark(&textview_curses::BM_META, *iter, false);
@ -3143,14 +3132,12 @@ com_partition_name(exec_context& ec,
} else {
textview_curses& tc = lnav_data.ld_views[LNV_LOG];
logfile_sub_source& lss = lnav_data.ld_log_source;
std::map<content_line_t, bookmark_metadata>& bm
= lss.get_user_bookmark_metadata();
args[1] = trim(remaining_args(cmdline, args));
tc.set_user_mark(&textview_curses::BM_META, tc.get_top(), true);
bookmark_metadata& line_meta = bm[lss.at(tc.get_top())];
auto& line_meta = lss.get_bookmark_metadata(tc.get_top());
line_meta.bm_name = args[1];
retval = "info: name set for partition";
@ -3175,7 +3162,6 @@ com_clear_partition(exec_context& ec,
textview_curses& tc = lnav_data.ld_views[LNV_LOG];
logfile_sub_source& lss = lnav_data.ld_log_source;
auto& bv = tc.get_bookmarks()[&textview_curses::BM_META];
auto& bm = lss.get_user_bookmark_metadata();
nonstd::optional<vis_line_t> part_start;
if (binary_search(bv.begin(), bv.end(), tc.get_top())) {
@ -3188,11 +3174,11 @@ com_clear_partition(exec_context& ec,
}
if (!ec.ec_dry_run) {
content_line_t cl = lss.at(part_start.value());
bookmark_metadata& line_meta = bm[cl];
auto& line_meta = lss.get_bookmark_metadata(part_start.value());
line_meta.bm_name.clear();
if (line_meta.empty()) {
lss.erase_bookmark_metadata(part_start.value());
tc.set_user_mark(
&textview_curses::BM_META, part_start.value(), false);
}

25
src/log.watch.cc
View File

@ -256,6 +256,30 @@ eval_with(logfile& lf, logfile::iterator ll)
}
continue;
}
if (strcmp(name, ":log_tags") == 0) {
const auto& bm = lf.get_bookmark_metadata();
auto bm_iter = bm.find(line_number);
if (bm_iter != bm.end() && !bm_iter->second.bm_tags.empty()) {
const auto& meta = bm_iter->second;
yajlpp_gen gen;
yajl_gen_config(gen, yajl_gen_beautify, false);
{
yajlpp_array arr(gen);
for (const auto& str : meta.bm_tags) {
arr.gen(str);
}
}
string_fragment sf = gen.to_string_fragment();
sqlite3_bind_text(
stmt, lpc + 1, sf.data(), sf.length(), SQLITE_TRANSIENT);
}
continue;
}
auto found = false;
for (const auto& lv : values.lvv_values) {
if (lv.lv_meta.lvm_name != &name[1]) {
@ -323,6 +347,7 @@ eval_with(logfile& lf, logfile::iterator ll)
watch_pair.first,
lf.get_filename(),
lf.get_format_name().to_string(),
(uint32_t) line_number,
timestamp_buffer,
};
for (const auto& lv : values.lvv_values) {

24
src/log_format.cc
View File

@ -29,6 +29,7 @@
#include <memory>
#include <fnmatch.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
@ -1964,6 +1965,23 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
vd->set_rewrite_src_name();
}
for (const auto& td_pair : this->lf_tag_defs) {
const auto& td = td_pair.second;
if (td->ftd_pattern == nullptr || td->ftd_pattern->empty()) {
errors.emplace_back(
lnav::console::user_message::error(
attr_line_t("invalid tag definition ")
.append_quoted(lnav::roles::symbol(
fmt::format(FMT_STRING("/{}/tags/{}"),
this->elf_name,
td_pair.first))))
.with_reason(
"tag definitions must have a non-empty pattern")
.with_snippets(this->get_snippets()));
}
}
if (this->elf_type == elf_type_t::ELF_TYPE_TEXT
&& this->elf_samples.empty())
{
@ -2916,5 +2934,11 @@ external_log_format::get_value_metadata() const
return retval;
}
bool
format_tag_def::path_restriction::matches(const char* fn) const
{
return fnmatch(this->p_glob.c_str(), fn, 0) == 0;
}
/* XXX */
#include "log_format_impls.cc"

7
src/log_format.hh
View File

@ -118,10 +118,7 @@ struct logline_value_meta {
{
}
bool is_hidden() const
{
return this->lvm_hidden || this->lvm_user_hidden;
}
bool is_hidden() const { return this->lvm_hidden || this->lvm_user_hidden; }
logline_value_meta& with_struct_name(intern_string_t name)
{
@ -518,6 +515,8 @@ public:
bool lf_time_ordered{true};
bool lf_specialized{false};
nonstd::optional<int64_t> lf_max_unrecognized_lines;
std::map<const intern_string_t, std::shared_ptr<format_tag_def>>
lf_tag_defs;
protected:
static std::vector<std::shared_ptr<log_format>> lf_root_formats;

25
src/log_format_fwd.hh
View File

@ -39,6 +39,7 @@
#include "base/string_attr_type.hh"
#include "byte_array.hh"
#include "log_level.hh"
#include "pcrepp/pcrepp.hh"
#include "ptimec.hh"
#include "robin_hood/robin_hood.h"
@ -158,10 +159,7 @@ public:
}
}
bool is_ignored() const
{
return this->ll_level & LEVEL_IGNORE;
}
bool is_ignored() const { return this->ll_level & LEVEL_IGNORE; }
void set_mark(bool val)
{
@ -194,10 +192,7 @@ public:
bool is_valid_utf() const { return this->ll_valid_utf; }
/** @param l The logging level. */
void set_level(log_level_t l)
{
this->ll_level = l;
};
void set_level(log_level_t l) { this->ll_level = l; };
/** @return The logging level. */
log_level_t get_level_and_flags() const
@ -307,4 +302,18 @@ private:
char ll_schema[2];
};
struct format_tag_def {
format_tag_def(std::string name) : ftd_name(name) {}
struct path_restriction {
std::string p_glob;
bool matches(const char* fn) const;
};
std::string ftd_name;
std::vector<path_restriction> ftd_paths;
std::shared_ptr<pcrepp_with_options<PCRE_DOTALL>> ftd_pattern;
};
#endif

61
src/log_format_loader.cc
View File

@ -149,6 +149,26 @@ value_def_provider(const yajlpp_provider_context& ypc, external_log_format* elf)
return retval.get();
}
static format_tag_def*
format_tag_def_provider(const yajlpp_provider_context& ypc,
external_log_format* elf)
{
const intern_string_t tag_name = ypc.get_substr_i(0);
auto iter = elf->lf_tag_defs.find(tag_name);
std::shared_ptr<format_tag_def> retval;
if (iter == elf->lf_tag_defs.end()) {
auto tag_with_hash = fmt::format(FMT_STRING("#{}"), tag_name);
retval = std::make_shared<format_tag_def>(tag_with_hash);
elf->lf_tag_defs[tag_name] = retval;
} else {
retval = iter->second;
}
return retval.get();
}
static scaling_factor*
scaling_factor_provider(const yajlpp_provider_context& ypc,
external_log_format::value_def* value_def)
@ -703,6 +723,34 @@ static struct json_path_container value_handlers = {
.with_children(value_def_handlers),
};
static struct json_path_container tag_path_handlers = {
yajlpp::property_handler("glob")
.with_synopsis("<glob>")
.with_description("The glob to match against file paths")
.with_example("*/system.log*")
.for_field(&format_tag_def::path_restriction::p_glob),
};
static struct json_path_container format_tag_def_handlers = {
yajlpp::property_handler("paths#")
.with_description("Restrict tagging to the given paths")
.for_field(&format_tag_def::ftd_paths)
.with_children(tag_path_handlers),
yajlpp::property_handler("pattern")
.with_synopsis("<regex>")
.with_description("The regular expression to match against the body of "
"the log message")
.with_example("\\w+ is down")
.for_field(&format_tag_def::ftd_pattern),
};
static struct json_path_container tag_handlers = {
yajlpp::pattern_property_handler("(?<tag_name>[^/]+)")
.with_description("The name of the tag to apply")
.with_obj_provider(format_tag_def_provider)
.with_children(format_tag_def_handlers),
};
static struct json_path_container highlight_handlers = {
yajlpp::pattern_property_handler(R"((?<highlight_name>[^/]+))")
.with_description("The definition of a highlight")
@ -850,6 +898,10 @@ struct json_path_container format_handlers = {
.with_description("The set of value definitions")
.with_children(value_handlers),
yajlpp::property_handler("tags")
.with_description("The tags to automatically apply to log messages")
.with_children(tag_handlers),
yajlpp::property_handler("action").with_children(action_handlers),
yajlpp::property_handler("sample#")
.with_description("An array of sample log messages to be tested "
@ -981,7 +1033,8 @@ write_sample_file()
= fmt::format(FMT_STRING("formats/default/{}.lnav"), meta.sm_name);
auto script_path = lnav::paths::dotlnav() / path;
if (lnav::filesystem::statp(script_path, &st) == 0
&& st.st_size == sf.length()) {
&& st.st_size == sf.length())
{
// Assume it's the right contents and move on...
continue;
}
@ -1053,7 +1106,8 @@ load_format_file(const ghc::filesystem::path& filename,
break;
}
if (offset == 0 && (rc > 2) && (buffer[0] == '#')
&& (buffer[1] == '!')) {
&& (buffer[1] == '!'))
{
// Turn it into a JavaScript comment.
buffer[0] = buffer[1] = '/';
}
@ -1095,7 +1149,8 @@ load_from_path(const ghc::filesystem::path& path,
log_warning("Empty format file: %s", filename.c_str());
} else {
for (auto iter = format_list.begin(); iter != format_list.end();
++iter) {
++iter)
{
log_info(" found format: %s", iter->get());
}
}

41
src/log_vtab_impl.cc
View File

@ -631,16 +631,15 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
if (iter != bv.begin()) {
--iter;
content_line_t part_line = vt->lss->at(*iter);
auto& bm_meta = vt->lss->get_user_bookmark_metadata();
auto meta_iter = bm_meta.find(part_line);
if (meta_iter != bm_meta.end()
&& !meta_iter->second.bm_name.empty())
auto line_meta_opt = vt->lss->find_bookmark_metadata(*iter);
if (line_meta_opt
&& !line_meta_opt.value()->bm_name.empty())
{
sqlite3_result_text(ctx,
meta_iter->second.bm_name.c_str(),
meta_iter->second.bm_name.size(),
SQLITE_TRANSIENT);
sqlite3_result_text(
ctx,
line_meta_opt.value()->bm_name.c_str(),
line_meta_opt.value()->bm_name.size(),
SQLITE_TRANSIENT);
} else {
sqlite3_result_null(ctx);
}
@ -731,13 +730,12 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
}
case VT_COL_LOG_COMMENT: {
const auto& bm = vt->lss->get_user_bookmark_metadata();
auto bm_iter = bm.find(vt->lss->at(vc->log_cursor.lc_curr_line));
if (bm_iter == bm.end() || bm_iter->second.bm_comment.empty()) {
auto line_meta_opt
= vt->lss->find_bookmark_metadata(vc->log_cursor.lc_curr_line);
if (!line_meta_opt || line_meta_opt.value()->bm_comment.empty()) {
sqlite3_result_null(ctx);
} else {
const auto& meta = bm_iter->second;
const auto& meta = *(line_meta_opt.value());
sqlite3_result_text(ctx,
meta.bm_comment.c_str(),
meta.bm_comment.length(),
@ -747,13 +745,12 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
}
case VT_COL_LOG_TAGS: {
const auto& bm = vt->lss->get_user_bookmark_metadata();
auto bm_iter = bm.find(vt->lss->at(vc->log_cursor.lc_curr_line));
if (bm_iter == bm.end() || bm_iter->second.bm_tags.empty()) {
auto line_meta_opt
= vt->lss->find_bookmark_metadata(vc->log_cursor.lc_curr_line);
if (!line_meta_opt || line_meta_opt.value()->bm_tags.empty()) {
sqlite3_result_null(ctx);
} else {
const auto& meta = bm_iter->second;
const auto& meta = *(line_meta_opt.value());
yajlpp_gen gen;
@ -1912,8 +1909,6 @@ vt_update(sqlite3_vtab* tab,
int val = sqlite3_value_int(argv[2 + VT_COL_MARK]);
vis_line_t vrowid(rowid);
std::map<content_line_t, bookmark_metadata>& bm
= vt->lss->get_user_bookmark_metadata();
const auto* part_name = sqlite3_value_text(argv[2 + VT_COL_PARTITION]);
const auto* log_comment
= sqlite3_value_text(argv[2 + VT_COL_LOG_COMMENT]);
@ -1958,12 +1953,12 @@ vt_update(sqlite3_vtab* tab,
if (binary_search(bv.begin(), bv.end(), vrowid) && !has_meta) {
vt->tc->set_user_mark(&textview_curses::BM_META, vrowid, false);
bm.erase(vt->lss->at(vrowid));
vt->lss->erase_bookmark_metadata(vrowid);
vt->lss->set_line_meta_changed();
}
if (has_meta) {
bookmark_metadata& line_meta = bm[vt->lss->at(vrowid)];
auto& line_meta = vt->lss->get_bookmark_metadata(vrowid);
vt->tc->set_user_mark(&textview_curses::BM_META, vrowid, true);
if (part_name) {

47
src/logfile.cc
View File

@ -266,6 +266,25 @@ logfile::process_prefix(shared_buffer_ref& sbr,
this->lf_content_id
= hasher().update(sbr.get_data(), sbr.length()).to_string();
for (auto& td_pair : this->lf_format->lf_tag_defs) {
bool matches = td_pair.second->ftd_paths.empty();
for (const auto& pr : td_pair.second->ftd_paths) {
if (pr.matches(this->lf_filename.c_str())) {
matches = true;
break;
}
}
if (!matches) {
continue;
}
log_info("%s: found applicable tag definition /%s/tags/%s",
this->lf_filename.c_str(),
this->lf_format->get_name().get(),
td_pair.second->ftd_name.c_str());
this->lf_applicable_taggers.emplace_back(td_pair.second);
}
/*
* We'll go ahead and assume that any previous lines were
* written out at the same time as the last one, so we need to
@ -601,8 +620,32 @@ logfile::rebuild_index(nonstd::optional<ui_clock::time_point> deadline)
break;
}
#endif
if (this->lf_format && !this->back().is_continued()) {
lnav::log::watch::eval_with(*this, this->end() - 1);
if (this->lf_format) {
if (!this->lf_applicable_taggers.empty()) {
auto sf = sbr.to_string_fragment();
for (const auto& td : this->lf_applicable_taggers) {
pcre_context_static<30> pc;
pcre_input pi(sf);
if (td->ftd_pattern->match(pc, pi, PCRE_NO_UTF8_CHECK))
{
auto curr_ll = this->end() - 1;
curr_ll->set_mark(true);
while (curr_ll->is_continued()) {
--curr_ll;
}
auto line_number = static_cast<uint32_t>(
std::distance(this->begin(), curr_ll));
this->lf_bookmark_metadata[line_number].add_tag(
td->ftd_name);
}
}
}
if (!this->back().is_continued()) {
lnav::log::watch::eval_with(*this, this->end() - 1);
}
}
if (li.li_partial) {

10
src/logfile.hh
View File

@ -47,6 +47,7 @@
#include "ArenaAlloc/arenaalloc.h"
#include "base/lnav_log.hh"
#include "base/result.h"
#include "bookmarks.hh"
#include "byte_array.hh"
#include "ghc/filesystem.hpp"
#include "line_buffer.hh"
@ -353,6 +354,12 @@ public:
void dump_stats();
robin_hood::unordered_map<uint32_t, bookmark_metadata>&
get_bookmark_metadata()
{
return this->lf_bookmark_metadata;
}
protected:
/**
* Process a line from the file.
@ -406,6 +413,9 @@ private:
nonstd::optional<std::pair<file_off_t, size_t>> lf_next_line_cache;
std::set<intern_string_t> lf_mismatched_formats;
robin_hood::unordered_map<uint32_t, bookmark_metadata> lf_bookmark_metadata;
std::vector<std::shared_ptr<format_tag_def>> lf_applicable_taggers;
};
class logline_observer {

116
src/logfile_sub_source.cc
View File

@ -518,27 +518,23 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv,
bv_iter = lower_bound(bv.begin(), bv.end(), vis_line_t(row + 1));
if (bv_iter != bv.begin()) {
--bv_iter;
content_line_t part_start_line = this->at(*bv_iter);
std::map<content_line_t, bookmark_metadata>::iterator bm_iter;
auto line_meta_opt = this->find_bookmark_metadata(*bv_iter);
if ((bm_iter = this->lss_user_mark_metadata.find(part_start_line))
!= this->lss_user_mark_metadata.end()
&& !bm_iter->second.bm_name.empty())
{
if (line_meta_opt && !line_meta_opt.value()->bm_name.empty()) {
lr.lr_start = 0;
lr.lr_end = -1;
value_out.emplace_back(
lr, logline::L_PARTITION.value(&bm_iter->second));
lr, logline::L_PARTITION.value(line_meta_opt.value()));
}
}
auto bm_iter
= this->lss_user_mark_metadata.find(this->at(vis_line_t(row)));
auto line_meta_opt = this->find_bookmark_metadata(vis_line_t(row));
if (bm_iter != this->lss_user_mark_metadata.end()) {
if (line_meta_opt) {
lr.lr_start = 0;
lr.lr_end = -1;
value_out.emplace_back(lr, logline::L_META.value(&bm_iter->second));
value_out.emplace_back(
lr, logline::L_META.value(line_meta_opt.value()));
}
}
@ -922,6 +918,20 @@ logfile_sub_source::rebuild_index(
content_line_t con_line(file_index * MAX_LINES_PER_FILE
+ line_index);
if (lf_iter->is_marked()) {
auto start_iter = lf_iter;
while (start_iter->is_continued()) {
--start_iter;
}
int start_index
= start_iter - ld->get_file_ptr()->begin();
content_line_t start_con_line(
file_index * MAX_LINES_PER_FILE + start_index);
this->lss_user_marks[&textview_curses::BM_META]
.insert_once(start_con_line);
lf_iter->set_mark(false);
}
this->lss_index.push_back(con_line);
}
@ -1436,10 +1446,10 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
continue;
}
if (strcmp(name, ":log_comment") == 0) {
const auto& bm = this->get_user_bookmark_metadata();
auto cl = this->get_file_base_content_line(ld);
cl += content_line_t(std::distance(lf->cbegin(), ll));
auto bm_iter = bm.find(cl);
const auto& bm = lf->get_bookmark_metadata();
auto line_number
= static_cast<uint32_t>(std::distance(lf->cbegin(), ll));
auto bm_iter = bm.find(line_number);
if (bm_iter != bm.end() && !bm_iter->second.bm_comment.empty()) {
const auto& meta = bm_iter->second;
sqlite3_bind_text(stmt,
@ -1451,10 +1461,10 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
continue;
}
if (strcmp(name, ":log_tags") == 0) {
const auto& bm = this->get_user_bookmark_metadata();
auto cl = this->get_file_base_content_line(ld);
cl += content_line_t(std::distance(lf->cbegin(), ll));
auto bm_iter = bm.find(cl);
const auto& bm = lf->get_bookmark_metadata();
auto line_number
= static_cast<uint32_t>(std::distance(lf->cbegin(), ll));
auto bm_iter = bm.find(line_number);
if (bm_iter != bm.end() && !bm_iter->second.bm_tags.empty()) {
const auto& meta = bm_iter->second;
yajlpp_gen gen;
@ -1681,8 +1691,8 @@ logfile_sub_source::text_clear_marks(const bookmark_type_t* bm)
for (iter = this->lss_user_marks[bm].begin();
iter != this->lss_user_marks[bm].end();)
{
auto bm_iter = this->lss_user_mark_metadata.find(*iter);
if (bm_iter != this->lss_user_mark_metadata.end()) {
auto line_meta_opt = this->find_bookmark_metadata(*iter);
if (line_meta_opt) {
++iter;
continue;
}
@ -1894,19 +1904,17 @@ bool
logfile_sub_source::meta_grepper::grep_value_for_line(vis_line_t line,
std::string& value_out)
{
content_line_t cl = this->lmg_source.at(vis_line_t(line));
std::map<content_line_t, bookmark_metadata>& user_mark_meta
= lmg_source.get_user_bookmark_metadata();
auto meta_iter = user_mark_meta.find(cl);
if (meta_iter == user_mark_meta.end()) {
auto line_meta_opt = this->lmg_source.find_bookmark_metadata(line);
if (!line_meta_opt) {
value_out.clear();
} else {
bookmark_metadata& bm = meta_iter->second;
bookmark_metadata& bm = *(line_meta_opt.value());
value_out.append(bm.bm_comment);
value_out.append("\x1c");
for (const auto& tag : bm.bm_tags) {
value_out.append(tag);
value_out.append("\x1c");
}
}
@ -2357,3 +2365,55 @@ logfile_sub_source::quiesce()
lf->quiesce();
}
}
bookmark_metadata&
logfile_sub_source::get_bookmark_metadata(content_line_t cl)
{
auto line_pair = this->find_line_with_file(cl).value();
auto line_number = static_cast<uint32_t>(
std::distance(line_pair.first->begin(), line_pair.second));
return line_pair.first->get_bookmark_metadata()[line_number];
}
nonstd::optional<bookmark_metadata*>
logfile_sub_source::find_bookmark_metadata(content_line_t cl)
{
auto line_pair = this->find_line_with_file(cl).value();
auto line_number = static_cast<uint32_t>(
std::distance(line_pair.first->begin(), line_pair.second));
auto& bm = line_pair.first->get_bookmark_metadata();
auto bm_iter = bm.find(line_number);
if (bm_iter == bm.end()) {
return nonstd::nullopt;
}
return &bm_iter->second;
}
void
logfile_sub_source::erase_bookmark_metadata(content_line_t cl)
{
auto line_pair = this->find_line_with_file(cl).value();
auto line_number = static_cast<uint32_t>(
std::distance(line_pair.first->begin(), line_pair.second));
auto& bm = line_pair.first->get_bookmark_metadata();
auto bm_iter = bm.find(line_number);
if (bm_iter != bm.end()) {
bm.erase(bm_iter);
}
}
void
logfile_sub_source::clear_bookmark_metadata()
{
for (auto& ld : *this) {
if (ld->get_file_ptr() == nullptr) {
continue;
}
ld->get_file_ptr()->get_bookmark_metadata().clear();
}
}

24
src/logfile_sub_source.hh
View File

@ -472,11 +472,30 @@ public:
return this->lss_user_marks;
}
std::map<content_line_t, bookmark_metadata>& get_user_bookmark_metadata()
bookmark_metadata& get_bookmark_metadata(content_line_t cl);
bookmark_metadata& get_bookmark_metadata(vis_line_t vl)
{
return this->lss_user_mark_metadata;
return this->get_bookmark_metadata(this->at(vl));
}
nonstd::optional<bookmark_metadata*> find_bookmark_metadata(
content_line_t cl);
nonstd::optional<bookmark_metadata*> find_bookmark_metadata(vis_line_t vl)
{
return this->find_bookmark_metadata(this->at(vl));
}
void erase_bookmark_metadata(content_line_t cl);
void erase_bookmark_metadata(vis_line_t vl)
{
this->erase_bookmark_metadata(this->at(vl));
}
void clear_bookmark_metadata();
int get_filtered_count() const
{
return this->lss_index.size() - this->lss_filtered_index.size();
@ -960,7 +979,6 @@ private:
auto_mem<sqlite3_stmt> lss_preview_filter_stmt{sqlite3_finalize};
bookmarks<content_line_t>::type lss_user_marks;
std::map<content_line_t, bookmark_metadata> lss_user_mark_metadata;
auto_mem<sqlite3_stmt> lss_marker_stmt{sqlite3_finalize};
std::string lss_marker_stmt_text;

13
src/readline_possibilities.cc
View File

@ -464,13 +464,12 @@ add_tag_possibilities()
{
logfile_sub_source& lss = lnav_data.ld_log_source;
if (lss.text_line_count() > 0) {
content_line_t cl = lss.at(lnav_data.ld_views[LNV_LOG].get_top());
const auto& user_meta = lss.get_user_bookmark_metadata();
auto meta_iter = user_meta.find(cl);
if (meta_iter != user_meta.end()) {
rc->add_possibility(
ln_mode_t::COMMAND, "line-tags", meta_iter->second.bm_tags);
auto line_meta_opt = lss.find_bookmark_metadata(
lnav_data.ld_views[LNV_LOG].get_top());
if (line_meta_opt) {
rc->add_possibility(ln_mode_t::COMMAND,
"line-tags",
line_meta_opt.value()->bm_tags);
}
}
}

67
src/session_data.cc
View File

@ -216,7 +216,8 @@ cleanup_session_data()
}
base += 1;
if (sscanf(base, "file-%63[^.].ts%d.json", hash_id, &timestamp)
== 2) {
== 2)
{
session_count[hash_id] += 1;
session_info_list.emplace_back(timestamp, hash_id, path);
}
@ -241,7 +242,8 @@ cleanup_session_data()
session_loops += 1;
if (session_loops < MAX_SESSION_FILE_COUNT
&& session_count[front.sfi_id] == 1) {
&& session_count[front.sfi_id] == 1)
{
session_info_list.splice(session_info_list.end(),
session_info_list,
session_info_list.begin());
@ -318,7 +320,8 @@ scan_sessions()
= fmt::format(FMT_STRING("view-info-{}.*.json"), session_id.value());
auto view_info_pattern = lnav::paths::dotlnav() / view_info_pattern_base;
if (glob(view_info_pattern.c_str(), 0, nullptr, view_info_list.inout())
== 0) {
== 0)
{
for (size_t lpc = 0; lpc < view_info_list->gl_pathc; lpc++) {
const char* path = view_info_list->gl_pathv[lpc];
int timestamp, ppid, rc;
@ -368,8 +371,6 @@ void
load_time_bookmarks()
{
logfile_sub_source& lss = lnav_data.ld_log_source;
std::map<content_line_t, bookmark_metadata>& bm_meta
= lss.get_user_bookmark_metadata();
auto_mem<sqlite3, sqlite_close_wrapper> db;
auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME;
auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
@ -519,7 +520,8 @@ load_time_bookmarks()
struct timeval line_tv = line_iter->get_timeval();
if ((line_tv.tv_sec != log_tv.tv_sec)
|| (line_tv.tv_usec != log_tv.tv_usec)) {
|| (line_tv.tv_usec != log_tv.tv_usec))
{
break;
}
@ -540,21 +542,23 @@ load_time_bookmarks()
.to_string();
if (line_hash == log_hash) {
auto& bm_meta = lf->get_bookmark_metadata();
auto line_number = static_cast<uint32_t>(
std::distance(lf->begin(), line_iter));
content_line_t line_cl = content_line_t(
base_content_line
+ std::distance(lf->begin(), line_iter));
base_content_line + line_number);
bool meta = false;
if (part_name != nullptr && part_name[0] != '\0') {
lss.set_user_mark(&textview_curses::BM_META,
line_cl);
bm_meta[line_cl].bm_name = part_name;
bm_meta[line_number].bm_name = part_name;
meta = true;
}
if (comment != nullptr && comment[0] != '\0') {
lss.set_user_mark(&textview_curses::BM_META,
line_cl);
bm_meta[line_cl].bm_comment = comment;
bm_meta[line_number].bm_comment = comment;
meta = true;
}
if (tags != nullptr && tags[0] != '\0') {
@ -570,7 +574,8 @@ load_time_bookmarks()
line_cl);
for (size_t lpc = 0;
lpc < tag_list.in()->u.array.len;
lpc++) {
lpc++)
{
yajl_val elem
= tag_list.in()
->u.array.values[lpc];
@ -580,7 +585,7 @@ load_time_bookmarks()
}
bookmark_metadata::KNOWN_TAGS.insert(
elem->u.string);
bm_meta[line_cl].add_tag(
bm_meta[line_number].add_tag(
elem->u.string);
}
}
@ -694,7 +699,8 @@ load_time_bookmarks()
strlen(log_time),
nullptr,
&log_tm,
log_tv)) {
log_tv))
{
continue;
}
@ -704,7 +710,8 @@ load_time_bookmarks()
struct timeval line_tv = line_iter->get_timeval();
if ((line_tv.tv_sec != log_tv.tv_sec)
|| (line_tv.tv_usec != log_tv.tv_usec)) {
|| (line_tv.tv_usec != log_tv.tv_usec))
{
break;
}
@ -973,35 +980,32 @@ save_user_bookmarks(sqlite3* db,
bookmark_vector<content_line_t>& user_marks)
{
logfile_sub_source& lss = lnav_data.ld_log_source;
std::map<content_line_t, bookmark_metadata>& bm_meta
= lss.get_user_bookmark_metadata();
bookmark_vector<content_line_t>::iterator iter;
for (iter = user_marks.begin(); iter != user_marks.end(); ++iter) {
std::map<content_line_t, bookmark_metadata>::iterator meta_iter;
content_line_t cl = *iter;
meta_iter = bm_meta.find(cl);
auto line_meta_opt = lss.find_bookmark_metadata(cl);
if (!bind_line(db, stmt, cl, lnav_data.ld_session_time)) {
continue;
}
if (meta_iter == bm_meta.end()) {
if (!line_meta_opt) {
if (sqlite3_bind_text(stmt, 5, "", 0, SQLITE_TRANSIENT)
!= SQLITE_OK) {
!= SQLITE_OK)
{
log_error("could not bind log hash -- %s", sqlite3_errmsg(db));
return;
}
} else {
if (meta_iter->second.empty()) {
bookmark_metadata& line_meta = *(line_meta_opt.value());
if (line_meta.empty()) {
continue;
}
if (sqlite3_bind_text(stmt,
5,
meta_iter->second.bm_name.c_str(),
meta_iter->second.bm_name.length(),
line_meta.bm_name.c_str(),
line_meta.bm_name.length(),
SQLITE_TRANSIENT)
!= SQLITE_OK)
{
@ -1009,11 +1013,10 @@ save_user_bookmarks(sqlite3* db,
return;
}
bookmark_metadata& line_meta = meta_iter->second;
if (sqlite3_bind_text(stmt,
6,
meta_iter->second.bm_comment.c_str(),
meta_iter->second.bm_comment.length(),
line_meta.bm_comment.c_str(),
line_meta.bm_comment.length(),
SQLITE_TRANSIENT)
!= SQLITE_OK)
{
@ -1413,7 +1416,8 @@ save_session_with_id(const std::string& session_id)
yajlpp_array file_list(handle);
for (auto& ld_file_name :
lnav_data.ld_active_files.fc_file_names) {
lnav_data.ld_active_files.fc_file_names)
{
file_list.gen(ld_file_name.first);
}
}
@ -1504,7 +1508,8 @@ save_session_with_id(const std::string& session_id)
if (lpc == LNV_LOG) {
for (const auto& format :
log_format::get_root_formats()) {
log_format::get_root_formats())
{
auto* elf = dynamic_cast<external_log_format*>(
format.get());
@ -1618,7 +1623,7 @@ reset_session()
lnav_data.ld_log_source.set_sql_filter("", nullptr);
lnav_data.ld_log_source.set_sql_marker("", nullptr);
lnav_data.ld_log_source.get_user_bookmark_metadata().clear();
lnav_data.ld_log_source.clear_bookmark_metadata();
for (auto& tc : lnav_data.ld_views) {
text_sub_source* tss = tc.get_sub_source();

7
src/text_format.cc
View File

@ -43,6 +43,9 @@ detect_text_format(string_fragment sf,
static const auto BZ2_EXT = ghc::filesystem::path(".bz2");
static const auto MD_EXT = ghc::filesystem::path(".md");
static const pcrepp MAN_MATCHERS
= pcrepp(R"(^[A-Z]+\(\d\)\s+)", PCRE_MULTILINE);
// XXX This is a pretty crude way of detecting format...
static const pcrepp PYTHON_MATCHERS = pcrepp(
"(?:"
@ -120,6 +123,10 @@ detect_text_format(string_fragment sf,
}
}
if (MAN_MATCHERS.match(pc, pi)) {
return text_format_t::TF_MAN;
}
if (PYTHON_MATCHERS.match(pc, pi)) {
return text_format_t::TF_PYTHON;
}

4
src/text_format.hh
View File

@ -50,6 +50,7 @@ enum class text_format_t {
TF_SQL,
TF_XML,
TF_JSON,
TF_MAN,
TF_MARKDOWN,
};
@ -88,6 +89,9 @@ struct formatter<text_format_t> : formatter<string_view> {
case text_format_t::TF_JSON:
name = "application/json";
break;
case text_format_t::TF_MAN:
name = "text/man";
break;
case text_format_t::TF_MARKDOWN:
name = "text/markdown";
break;

11
test/bad-config/formats/invalid-properties/format.json
View File

@ -24,6 +24,17 @@
"background-color": "also not a color"
}
},
"tags": {
"badtag": {
"paths": []
},
"badtag2": {
"pattern": ""
},
"badtag3": {
"pattern": "invalid(abc"
}
},
"search-table": {
"bad_table_regex": {
"pattern": "abc(def"

2
test/expected/expected.am
View File

@ -308,6 +308,8 @@ EXPECTED_FILES = \
$(srcdir)/%reldir%/test_meta.sh_7b75763926d832bf9784ca234a060859770aabe7.out \
$(srcdir)/%reldir%/test_meta.sh_811b1a8a176b25001a89e35b295a1117ab76969b.err \
$(srcdir)/%reldir%/test_meta.sh_811b1a8a176b25001a89e35b295a1117ab76969b.out \
$(srcdir)/%reldir%/test_meta.sh_83ac877aa9d38b25945cf96d6326a2468187c40f.err \
$(srcdir)/%reldir%/test_meta.sh_83ac877aa9d38b25945cf96d6326a2468187c40f.out \
$(srcdir)/%reldir%/test_meta.sh_a7489c1f0e001adc732b7e2ab31bb30960fda078.err \
$(srcdir)/%reldir%/test_meta.sh_a7489c1f0e001adc732b7e2ab31bb30960fda078.out \
$(srcdir)/%reldir%/test_meta.sh_c063f96398650f130941bbbf4cf63c1244fdbee5.err \

2
test/expected/test_events.sh_ed8dc44add223341c03ccb7b3e18371bdb42b710.out
View File

@ -1,3 +1,3 @@
{"content":{"$schema":"https://lnav.org/event-file-open-v1.schema.json","filename":"{test_dir}/logfile_access_log.0"}}
{"content":{"$schema":"https://lnav.org/event-file-format-detected-v1.schema.json","filename":"{test_dir}/logfile_access_log.0","format":"access_log"}}
{"content":{"$schema":"https://lnav.org/event-log-msg-detected-v1.schema.json","watch-name":"http-errors","filename":"{test_dir}/logfile_access_log.0","format":"access_log","timestamp":"2009-07-20T22:59:29.000","values":{"body":"","c_ip":"192.168.202.254","cs_method":"GET","cs_referer":"-","cs_uri_query":null,"cs_uri_stem":"/vmw/vSphere/default/vmkboot.gz","cs_user_agent":"gPXE/0.9.7","cs_username":"-","cs_version":"HTTP/1.0","sc_bytes":46210,"sc_status":404,"timestamp":"20/Jul/2009:22:59:29 +0000"}}}
{"content":{"$schema":"https://lnav.org/event-log-msg-detected-v1.schema.json","watch-name":"http-errors","filename":"{test_dir}/logfile_access_log.0","line-number":1,"format":"access_log","timestamp":"2009-07-20T22:59:29.000","values":{"body":"","c_ip":"192.168.202.254","cs_method":"GET","cs_referer":"-","cs_uri_query":null,"cs_uri_stem":"/vmw/vSphere/default/vmkboot.gz","cs_user_agent":"gPXE/0.9.7","cs_username":"-","cs_version":"HTTP/1.0","sc_bytes":46210,"sc_status":404,"timestamp":"20/Jul/2009:22:59:29 +0000"}}}

24
test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err
View File

@ -1,6 +1,19 @@
✘ error: “invalid(abc” is not a valid regular expression for property “/invalid_props_log/tags/badtag3/pattern”
reason: missing )
 --> {test_dir}/bad-config/formats/invalid-properties/format.json:35
 |  "pattern": "invalid(abc"
 --> /invalid_props_log/tags/badtag3/pattern
 | invalid(abc 
 |  ^ missing ) 
 = help: Property Synopsis
/invalid_props_log/tags/badtag3/pattern <regex>
Description
The regular expression to match against the body of the log message
Example
\w+ is down
✘ error: “abc(def” is not a valid regular expression for property “/invalid_props_log/search-table/bad_table_regex/pattern”
reason: missing )
 --> {test_dir}/bad-config/formats/invalid-properties/format.json:29
 --> {test_dir}/bad-config/formats/invalid-properties/format.json:40
 |  "pattern": "abc(def" 
 --> /invalid_props_log/search-table/bad_table_regex/pattern
 | abc(def 
@ -104,6 +117,15 @@
 = note: the following captures are available:
body, pid, timestamp
 = help: values are populated from captures in patterns, so at least one pattern must have a capture with this value name
✘ error: invalid tag definition “/invalid_props_log/tags/badtag”
reason: tag definitions must have a non-empty pattern
 --> {test_dir}/bad-config/formats/invalid-properties/format.json:4
✘ error: invalid tag definition “/invalid_props_log/tags/badtag2”
reason: tag definitions must have a non-empty pattern
 --> {test_dir}/bad-config/formats/invalid-properties/format.json:4
✘ error: invalid tag definition “/invalid_props_log/tags/badtag3”
reason: tag definitions must have a non-empty pattern
 --> {test_dir}/bad-config/formats/invalid-properties/format.json:4
✘ error: invalid value for property “/invalid_props_log/timestamp-field”
reason: “ts” was not found in the pattern at /invalid_props_log/regex/std
 --> {test_dir}/bad-config/formats/invalid-properties/format.json:4

0
test/expected/test_meta.sh_83ac877aa9d38b25945cf96d6326a2468187c40f.err Normal file
View File

37
test/expected/test_meta.sh_83ac877aa9d38b25945cf96d6326a2468187c40f.out Normal file
View File

@ -0,0 +1,37 @@
[2020-12-10 06:56:41,061] INFO [m:108] Calling 'x' with params:
[2020-12-10 06:56:41,092] DEBUG [connect.client:69] Full request text:
└ #xml-req
<?xml version='1.0' encoding='iso-8859-2'?>
<a-request>
<head>
x
</head>
<source>
x
</source>
<request id="1">
<name>
x
</name>
</request>
</a-request>
[2020-12-10 06:56:41,099] DEBUG [m:85] Full reply text:
<?xml version='1.0' encoding='iso-8859-2'?>
<a-reply>
<head>
x
</head>
<reply id="2">
<status>
<result>OK</result>
</status>
<name>
x
</name>
</reply>
<technical-track>
x
</technical-track>
</a-reply>

2
test/expected/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.out
View File

@ -1,4 +1,4 @@
Min: 0   1-23   24-48   49+ Max: 291690
Min: 0   1-23   24-48   49+ Max: 291690
 Thu Nov 03 00:15:00               
▲ 70 values in the range 0.00-3788.18
 Thu Nov 03 00:20:00

5
test/formats/xmlmsg/format.json
View File

@ -50,6 +50,11 @@
"color": "Gold1"
}
},
"tags": {
"xml-req": {
"pattern": "Full request text:"
}
},
"sample": [
{
"line": "[2020-12-10 06:56:41,477] INFO [m:108] Calling 'x' with params:",

4
test/test_meta.sh
View File

@ -97,3 +97,7 @@ run_cap_test ${lnav_test} -n \
-c ":comment foo" \
-c ";UPDATE access_log SET log_comment = null" \
${test_dir}/logfile_access_log.0
run_cap_test ${lnav_test} -d /tmp/lnav.err -n \
-I ${test_dir} \
${test_dir}/logfile_xml_msg.0