[all_logs] add log_msg_schema hidden column to the all_logs table and refine the msg format

This commit is contained in:
Timothy Stack 2015-08-26 09:41:45 -07:00
parent e95bd90dfc
commit 443053da62
45 changed files with 173 additions and 21 deletions

View File

@ -74,6 +74,7 @@ set(diag_STAT_SRCS
yajl/yajl_version.c
spookyhash/SpookyV2.cpp
all_logs_vtab.hh
auto_fd.hh
auto_mem.hh
auto_pid.hh

View File

@ -30,6 +30,7 @@
#ifndef LNAV_ALL_LOGS_VTAB_HH
#define LNAV_ALL_LOGS_VTAB_HH
#include "byte_array.hh"
#include "log_vtab_impl.hh"
#include "data_parser.hh"
@ -39,11 +40,13 @@ public:
all_logs_vtab() : log_vtab_impl(intern_string::lookup("all_logs")) {
this->alv_value_name = intern_string::lookup("log_format");
this->alv_msg_name = intern_string::lookup("log_msg_format");
this->alv_schema_name = intern_string::lookup("log_msg_schema");
}
void get_columns(std::vector<vtab_column> &cols) {
cols.push_back(vtab_column(this->alv_value_name.get(), SQLITE3_TEXT, NULL));
cols.push_back(vtab_column(this->alv_msg_name.get(), SQLITE3_TEXT, NULL));
cols.push_back(vtab_column(this->alv_value_name.get()));
cols.push_back(vtab_column(this->alv_msg_name.get()));
cols.push_back(vtab_column(this->alv_schema_name.get(), SQLITE3_TEXT, NULL, true));
};
void extract(logfile *lf, shared_buffer_ref &line,
@ -74,6 +77,14 @@ public:
tmp_shared_buffer tsb(str.c_str());
values.push_back(logline_value(this->alv_msg_name, tsb.tsb_ref, 1));
this->alv_schema_manager.invalidate_refs();
dp.dp_schema_id.to_string(this->alv_schema_buffer);
shared_buffer_ref schema_ref;
schema_ref.share(this->alv_schema_manager,
this->alv_schema_buffer,
data_parser::schema_id_t::STRING_SIZE - 1);
values.push_back(logline_value(this->alv_schema_name, schema_ref, 2));
}
bool next(log_cursor &lc, logfile_sub_source &lss) {
@ -98,7 +109,9 @@ public:
private:
intern_string_t alv_value_name;
intern_string_t alv_msg_name;
intern_string_t alv_schema_name;
shared_buffer alv_schema_manager;
char alv_schema_buffer[data_parser::schema_id_t::STRING_SIZE];
};
#endif //LNAV_ALL_LOGS_VTAB_HH

View File

@ -40,6 +40,7 @@
template<size_t COUNT, typename T = unsigned char>
struct byte_array {
static const size_t BYTE_COUNT = COUNT * sizeof(T);
static const size_t STRING_SIZE = BYTE_COUNT * 2 + 1;
byte_array() { };
@ -74,7 +75,7 @@ struct byte_array {
std::string to_string() const
{
char buffer[BYTE_COUNT * 2 + 1];
char buffer[STRING_SIZE];
this->to_string(buffer);
return std::string(buffer);

View File

@ -40,6 +40,7 @@
#include <algorithm>
#include "lnav_log.hh"
#include "lnav_util.hh"
#include "yajlpp.hh"
#include "pcrepp.hh"
#include "byte_array.hh"
@ -635,19 +636,26 @@ private:
if (!has_value) {
element_list_t ELEMENT_LIST_T(blank_value);
pcre_input &pi = this->dp_scanner->get_input();
const char *str = pi.get_string();
struct element blank;
blank.e_token = DT_QUOTED_STRING;
blank.e_capture.c_begin = blank.e_capture.c_end = pair_subs.front().e_capture.c_end;
if (blank.e_capture.c_begin < pi.pi_length) {
switch (str[blank.e_capture.c_begin]) {
case '=':
case ':':
blank.e_capture.c_begin += 1;
blank.e_capture.c_end += 1;
break;
}
}
blank_value.PUSH_BACK(blank);
pair_subs.PUSH_BACK(element(blank_value, DNT_VALUE));
}
pairs_out.PUSH_BACK(element(pair_subs, DNT_PAIR));
if (this->dp_msg_format != NULL) {
*(this->dp_msg_format) += get_string_up_to_value(pairs_out.back()) + "#";
}
}
if (pairs_out.size() == 1) {
@ -717,9 +725,6 @@ private:
// use the token ID since some columns values might vary
// between rows.
context.Update(" ", 1);
if (schema != NULL && this->dp_msg_format != NULL) {
*(this->dp_msg_format) += this->get_string_up_to_value(free_row.front()) + "#";
}
}
break;
@ -755,10 +760,23 @@ private:
if (schema != NULL && this->dp_msg_format != NULL) {
pcre_input &pi = this->dp_scanner->get_input();
for (element_list_t::iterator fiter = pairs_out.begin();
fiter != pairs_out.end();
++fiter) {
*(this->dp_msg_format) += this->get_string_up_to_value(*fiter);
this->dp_msg_format->append("#");
}
if (this->dp_msg_format_begin < pi.pi_length) {
const char *str = pi.get_string();
pcre_context::capture_t last(this->dp_msg_format_begin,
pi.pi_length);
switch (str[last.c_begin]) {
case '\'':
case '"':
last.c_begin += 1;
break;
}
*(this->dp_msg_format) += pi.get_substr(&last);
}
}
@ -867,7 +885,7 @@ private:
this->dp_format = &FORMAT_COMMA;
if (this->dp_separator == DT_COLON && hist[DT_COMMA] > 0) {
if (!((hist[DT_COLON] == hist[DT_COMMA]) ||
(hist[DT_COLON] == (hist[DT_COMMA] - 1)))) {
((hist[DT_COLON] - 1) == hist[DT_COMMA]))) {
this->dp_separator = DT_INVALID;
if (hist[DT_COLON] == 1) {
this->dp_prefix_terminator = DT_COLON;
@ -883,7 +901,6 @@ private:
void end_of_value(element_list_t &el_stack,
element_list_t &key_comps,
element_list_t &value) {
key_comps.remove_if(element_if(this->dp_format->df_terminator));
key_comps.remove_if(element_if(DT_COMMA));
value.remove_if(element_if(this->dp_format->df_terminator));
@ -936,6 +953,8 @@ private:
key_comps.end());
}
strip(value, element_if(DT_WHITE));
strip(value, element_if(DT_COLON));
strip(value, element_if(DT_WHITE));
if (!value.empty()) {
el_stack.PUSH_BACK(element(value, DNT_VALUE));
}
@ -966,6 +985,27 @@ private:
if (this->dp_msg_format_begin <= val_elem.e_capture.c_begin) {
pcre_context::capture_t leading_and_key = pcre_context::capture_t(
this->dp_msg_format_begin, val_elem.e_capture.c_begin);
const char *str = pi.get_string();
if (leading_and_key.length() >= 2) {
switch (str[leading_and_key.c_end - 1]) {
case '\'':
case '"':
leading_and_key.c_end -= 1;
switch (str[leading_and_key.c_end - 1]) {
case 'r':
case 'u':
leading_and_key.c_end -= 1;
break;
}
break;
}
switch (str[leading_and_key.c_begin]) {
case '\'':
case '"':
leading_and_key.c_begin += 1;
break;
}
}
this->dp_msg_format_begin = val_elem.e_capture.c_end;
return pi.get_substr(&leading_and_key);
}

View File

@ -15,3 +15,5 @@ pair 8:11 ^-^ c=3
num 12:13 ^ 4
val 12:13 ^ 4
pair 12:13 ^ 4
msg :a=1 b=2 c=3,4
format :a=# b=# c=#,#

View File

@ -5,3 +5,5 @@ pair 0:13 ^-----------^ current speed
key 15:15 ^
num 15:17 ^^ 38
pair 15:17 ^^ 38
msg :current speed: 38 mph
format :#: # mph

View File

@ -2,3 +2,5 @@
key 16:16 ^
quot 16:26 ^--------^ pm/runtime
pair 16:26 ^--------^ pm/runtime
msg :quoted string u'pm/runtime'
format :quoted string #

View File

@ -5,3 +5,5 @@ pair 17:23 ^----^ 0.6.16
key 25:25 ^
vers 25:31 ^----^ 1.2-a1
pair 25:31 ^----^ 1.2-a1
msg :version numbers 0.6.16 1.2-a1
format :version numbers # #

View File

@ -3,3 +3,5 @@
num 17:34 ^---------------^ 4.41074371338e-05
val 17:34 ^---------------^ 4.41074371338e-05
pair 0:34 ^--------------------------------^ kickoff_duration=4.41074371338e-05
msg :kickoff_duration=4.41074371338e-05
format :kickoff_duration=#

View File

@ -3,3 +3,5 @@
num 17:34 ^---------------^ 4.41074371338E-05
val 17:34 ^---------------^ 4.41074371338E-05
pair 0:34 ^--------------------------------^ kickoff_duration=4.41074371338E-05
msg :kickoff_duration=4.41074371338E-05
format :kickoff_duration=#

View File

@ -27,3 +27,5 @@ pair 13:16 ^-^ six
num 17:18 ^ 7
val 17:18 ^ 7
pair 17:18 ^ 7
msg :1,2,3,4,five,six,7
format :#,#,#,#,#,#,#

View File

@ -14,3 +14,5 @@ pair 6:7 ^ 4
key 17:17 ^
num 17:18 ^ 7
pair 17:18 ^ 7
msg :1 2 3 4 five six 7
format :# # # # five six #

View File

@ -3,3 +3,5 @@
quot 12:25 ^-----------^ Hello, World!
val 12:25 ^-----------^ Hello, World!
pair 0:25 ^-----------------------^ the-value: "Hello, World!
msg :the-value: "Hello, World!"
format :the-value: #

View File

@ -3,3 +3,5 @@
url 15:45 ^----------------------------^ http://www.example.com/foo-bar
val 15:45 ^----------------------------^ http://www.example.com/foo-bar
pair 0:45 ^-------------------------------------------^ this is a url: http://www.example.com/foo-bar
msg :this is a url: http://www.example.com/foo-bar
format :this is a url: #

View File

@ -15,3 +15,5 @@ pair 16:21 ^---^ foo=1
num 26:27 ^ 2
val 26:27 ^ 2
pair 22:27 ^---^ bar=2
msg :qualified:name: foo=1 bar=2
format :#:#: foo=# bar=#

View File

@ -10,3 +10,5 @@ quot 21:22 ^ b
pair 15:22 ^-----^ arg2="b
grp 5:22 ^---------------^ arg1="a", arg2="b
pair 5:22 ^---------------^ arg1="a", arg2="b
msg :func(arg1="a", arg2="b")
format :func(#)

View File

@ -29,3 +29,5 @@ pair 168:174
pair 175:176 ^ 1
grp 168:176 ^------^ 100003,1
pair 168:176 ^------^ 100003,1
msg :Succeeded authorizing right 'system.privilege.taskport.debug' by client '/usr/libexec/taskgated' [76339] for authorization created by '/usr/libexec/taskgated' [77395] (100003,1)
format :Succeeded authorizing right # by client # [#] for authorization created by # [#] (#)

View File

@ -8,3 +8,5 @@ pair 7:15 ^------^ 12:01:22
key 17:17 ^
time 17:29 ^----------^ 12:01:22.123
pair 17:29 ^----------^ 12:01:22.123
msg :12:01 12:01:22 12:01:22.123
format :# # #

View File

@ -11,6 +11,8 @@ pair 42:49 ^-----^ </el
key 51:51 ^
xmlt 51:61 ^--------^ <closed />
pair 51:61 ^--------^ <closed />
msg :<ns1:foo> <elem attr1=xyz attr2="123"> </elem> <closed />
format :# # # #
--
<ns1:foo>

View File

@ -43,6 +43,8 @@
#include "log_format_loader.hh"
#include "pretty_printer.hh"
#include "shared_buffer.hh"
#include "../src/data_parser.hh"
#include "../src/view_curses.hh"
using namespace std;
@ -161,9 +163,13 @@ int main(int argc, char *argv[])
data_scanner ds(sub_line, body.lr_start, sub_line.length());
data_parser dp(&ds);
string msg_format;
dp.dp_msg_format = &msg_format;
dp.parse();
dp.print(out, dp.dp_pairs);
fprintf(out, "msg :%s\n", sub_line.c_str() + body.lr_start);
fprintf(out, "format :%s\n", msg_format.c_str());
if (pretty_print) {
data_scanner ds2(sub_line, body.lr_start, sub_line.length());

View File

@ -5,3 +5,5 @@ pair 29:40 ^---------^ com
key 58:58 ^
num 58:62 ^--^ 8099
pair 58:62 ^--^ 8099
msg ::com.twisted:Site starting on 8099
format ::#:Site starting on #

View File

@ -14,9 +14,11 @@ wspc 68:69
sym 69:77 ^------^ HARDDISK
val 64:77 ^-----------^ VBOX HARDDISK
pair 64:77 ^-----------^ VBOX HARDDISK
key 79:82 ^-^ S/N
key 79:79 ^
sym 79:82 ^-^ S/N
coln 82:83 ^ :
sym 83:102 ^-----------------^ VBc8882b62-a0263a39
val 83:102 ^-----------------^ VBc8882b62-a0263a39
val 79:102 ^---------------------^ S/N:VBc8882b62-a0263a39
pair 79:102 ^---------------------^ S/N:VBc8882b62-a0263a39
key 104:106 ^^ FW
num 107:110 ^-^ 1.0
@ -28,3 +30,5 @@ wspc 116:117
sym 117:119 ^^ GB
val 112:119 ^-----^ 17.1 GB
pair 112:119 ^-----^ 17.1 GB
msg : Device: /dev/sda [SAT], VBOX HARDDISK, S/N:VBc8882b62-a0263a39, FW:1.0, 17.1 GB
format : Device: #], #, #, FW:#, #

View File

@ -8,3 +8,5 @@ pair 61:63 ^^
key 75:75 ^
num 75:80 ^---^ 55327
pair 75:80 ^---^ 55327
msg : bound to 10.1.10.62 -- renewal in 55327 seconds.
format : bound to # # renewal in # seconds

View File

@ -19,3 +19,5 @@ pair 83:92
path 103:110 ^-----^ /bin/ls
val 103:110 ^-----^ /bin/ls
pair 95:110 ^-------------^ COMMAND=/bin/ls
msg : stack : TTY=ttys002 ; PWD=/ ; USER=root ; COMMAND=/bin/ls
format : # : TTY=# ; PWD=# ; USER=# ; COMMAND=#

View File

@ -8,3 +8,5 @@ pair 93:102
key 116:116 ^
ipv4 116:126 ^--------^ 10.1.10.62
pair 116:126 ^--------^ 10.1.10.62
msg : Leaving mDNS multicast group on interface eth0.IPv4 with address 10.1.10.62.
format : Leaving # multicast group on interface # with address #

View File

@ -27,3 +27,5 @@ pair 162:177
num 196:199 ^-^ 144
val 196:199 ^-^ 144
pair 179:199 ^------------------^ compressor_pages 144
msg : hibernate_teardown: wired_pages 518290, free_pages 5699523, active_pages 40010, inactive_pages 0, speculative_pages 0, cleaned_pages 0, compressor_pages 144
format : hibernate_teardown: wired_pages #, free_pages #, active_pages #, inactive_pages #, speculative_pages #, cleaned_pages #, compressor_pages #

View File

@ -5,3 +5,5 @@ pair 74:83
key 88:88 ^
sym 88:92 ^--^ mDNS
pair 88:92 ^--^ mDNS
msg : New relevant interface eth0.IPv4 for mDNS.
format : New relevant interface # for #

View File

@ -0,0 +1,11 @@
Aug 25 00:30:32 Tim-Stacks-iMac.local iTunes[558]: Entered:__thr_AMMuxedDeviceDisconnected, mux-device:509
key 51:58 ^-----^ Entered
sym 59:90 ^-----------------------------^ __thr_AMMuxedDeviceDisconnected
val 59:90 ^-----------------------------^ __thr_AMMuxedDeviceDisconnected
pair 51:90 ^-------------------------------------^ Entered:__thr_AMMuxedDeviceDisconnected
key 92:102 ^--------^ mux-device
num 103:106 ^-^ 509
val 103:106 ^-^ 509
pair 92:106 ^------------^ mux-device:509
msg : Entered:__thr_AMMuxedDeviceDisconnected, mux-device:509
format : Entered:#, mux-device:#

View File

@ -5,3 +5,5 @@ pair 82:92
key 96:96 ^
sym 96:100 ^--^ eth0
pair 96:100 ^--^ eth0
msg : Withdrawing address record for 10.1.10.62 on eth0.
format : Withdrawing address record for # on #

View File

@ -8,3 +8,5 @@ pair 93:102
key 116:116 ^
ipv4 116:126 ^--------^ 10.1.10.62
pair 116:126 ^--------^ 10.1.10.62
msg : Joining mDNS multicast group on interface eth0.IPv4 with address 10.1.10.62.
format : Joining # multicast group on interface # with address #

View File

@ -24,3 +24,5 @@ wspc 124:125
word 125:127 ^^ ls
val 103:127 ^----------------------^ /usr/bin/env VAR1=foo ls
pair 95:127 ^------------------------------^ COMMAND=/usr/bin/env VAR1=foo ls
msg : stack : TTY=ttys002 ; PWD=/ ; USER=root ; COMMAND=/usr/bin/env VAR1=foo ls
format : # : TTY=# ; PWD=# ; USER=# ; COMMAND=#

View File

@ -5,3 +5,5 @@ pair 144:155
key 169:169 ^
quot 169:199 ^----------------------------^ com.json.components.JSONReader
pair 169:199 ^----------------------------^ com.json.components.JSONReader
msg : Doing prepare for resource name "Json_Reader", component "com.json.components.JSONReader"
format : Doing prepare for resource name #, component #

View File

@ -5,3 +5,5 @@ pair 61:70 ^-----
key 94:94 ^
sym 94:98 ^--^ mDNS
pair 94:98 ^--^ mDNS
msg : Interface eth0.IPv4 no longer relevant for mDNS.
format : Interface # no longer relevant for #

View File

@ -7,3 +7,5 @@ pair 43:64 ^-------------------^
path 75:91 ^--------------^ /etc/resolv.conf
val 75:91 ^--------------^ /etc/resolv.conf
pair 67:91 ^----------------------^ updated /etc/resolv.conf
msg : /sbin/dhclient-script : updated /etc/resolv.conf
format : # : updated #

View File

@ -1,8 +1,8 @@
Jun 2 00:34:32 Tim-Stacks-iMac kernel[0]: vmnet: VNetUserIf_Create: created userIf at 0xffffff802644f400.
key 43:48 ^---^ vmnet
quot 48:48 ^
val 48:48 ^
pair 43:48 ^---^ vmnet
quot 49:49 ^
val 49:49 ^
pair 43:49 ^----^ vmnet:
key 50:67 ^---------------^ VNetUserIf_Create
word 69:76 ^-----^ created
wspc 76:77 ^
@ -13,3 +13,5 @@ wspc 86:87
hex 87:105 ^----------------^ 0xffffff802644f400
val 69:105 ^----------------------------------^ created userIf at 0xffffff802644f400
pair 50:105 ^-----------------------------------------------------^ VNetUserIf_Create: created userIf at 0xffffff802644f400
msg : vmnet: VNetUserIf_Create: created userIf at 0xffffff802644f400.
format : vmnet:# VNetUserIf_Create: #

View File

@ -12,3 +12,5 @@ pair 54:63 ^-------^
pair 65:79 ^------------^ xid=0x4e17f141
grp 65:79 ^------------^ xid=0x4e17f141
pair 65:79 ^------------^ xid=0x4e17f141
msg : DHCPNAK from 10.1.10.1 (xid=0x4e17f141)
format : # from # (#)

View File

@ -4,9 +4,9 @@
val 69:73 ^--^ eth0
pair 37:73 ^----------------------------------^ 31809412.513897] [UFW BLOCK] IN=eth0
key 74:77 ^-^ OUT
quot 77:77 ^
val 77:77 ^
pair 74:77 ^-^ OUT
quot 78:78 ^
val 78:78 ^
pair 74:78 ^--^ OUT=
key 79:82 ^-^ MAC
hexd 83:124 ^---------------------------------------^ 40:40:2e:9a:ad:92:c4:71:fe:f1:b9:7f:08:00
val 83:124 ^---------------------------------------^ 40:40:2e:9a:ad:92:c4:71:fe:f1:b9:7f:08:00
@ -65,3 +65,5 @@ pair 246:258
num 264:265 ^ 0
val 264:265 ^ 0
pair 259:265 ^----^ URGP=0
msg : [31809412.513897] [UFW BLOCK] IN=eth0 OUT= MAC=40:40:2e:9a:ad:92:c4:71:fe:f1:b9:7f:08:00 SRC=69.60.116.202 DST=173.203.237.224 LEN=44 TOS=0x00 PREC=0x00 TTL=29 ID=15852 PROTO=TCP SPT=43998 DPT=3389 WINDOW=3072 RES=0x00 SYN URGP=0
format : [31809412.513897] [UFW BLOCK] IN=# OUT=# MAC=# SRC=# DST=# LEN=# TOS=# PREC=# TTL=# ID=# PROTO=# SPT=# DPT=# WINDOW=# RES=# URGP=#

View File

@ -21,3 +21,5 @@ pair 98:99
pair 101:114 ^-----------^ xid=0xd16b79d
grp 101:114 ^-----------^ xid=0xd16b79d
pair 101:114 ^-----------^ xid=0xd16b79d
msg : DHCPDISCOVER on eth0 to 255.255.255.255 port 67 interval 5 (xid=0xd16b79d)
format : # on # to # port # interval # (#)

View File

@ -18,3 +18,5 @@ pair 79:81
pair 83:96 ^-----------^ xid=0xd16b79d
grp 83:96 ^-----------^ xid=0xd16b79d
pair 83:96 ^-----------^ xid=0xd16b79d
msg : DHCPREQUEST on eth0 to 10.1.10.1 port 67 (xid=0xd16b79d)
format : # on # to # port # (#)

View File

@ -2,3 +2,5 @@
key 85:85 ^
ipv6 85:110 ^-----------------------^ fe80::22c9:d0ff:fe15:1b7c
pair 85:110 ^-----------------------^ fe80::22c9:d0ff:fe15:1b7c
msg : Invalid response packet from host fe80::22c9:d0ff:fe15:1b7c.
format : Invalid response packet from host #

View File

@ -2,3 +2,5 @@
key 85:85 ^
ipv4 85:95 ^--------^ 10.1.10.10
pair 85:95 ^--------^ 10.1.10.10
msg : Invalid response packet from host 10.1.10.10.
format : Invalid response packet from host #

View File

@ -11,3 +11,5 @@ pair 59:82 ^-------
pair 95:104 ^-------^ 0x1de9290
grp 59:104 ^-------------------------------------------^ twisted.web.server.Site instance at 0x1de9290
pair 59:104 ^-------------------------------------------^ twisted.web.server.Site instance at 0x1de9290
msg ::com.twisted:Starting factory <twisted.web.server.Site instance at 0x1de9290>
format ::#:Starting factory <#>

View File

@ -12,3 +12,5 @@ pair 54:63 ^-------^
pair 65:78 ^-----------^ xid=0xd16b79d
grp 65:78 ^-----------^ xid=0xd16b79d
pair 65:78 ^-----------^ xid=0xd16b79d
msg : DHCPACK from 10.1.10.1 (xid=0xd16b79d)
format : # from # (#)

View File

@ -5,3 +5,5 @@ pair 41:50 ^-------^
key 56:56 ^
ipv4 56:65 ^-------^ 10.1.10.1
pair 56:65 ^-------^ 10.1.10.1
msg : DHCPOFFER from 10.1.10.1
format : # from #

View File

@ -5,3 +5,5 @@ pair 86:96
key 100:100 ^
sym 100:104 ^--^ eth0
pair 100:104 ^--^ eth0
msg : Registering new address record for 10.1.10.62 on eth0.
format : Registering new address record for # on #