Robot
/
lnav
mirror of https://github.com/tstack/lnav.git
2
0
Fork 0
Code Issues Releases Wiki Activity

[perf] a few performance tweaks

This commit is contained in:
Timothy Stack 2020-04-23 06:42:48 -07:00
parent 0744a9b6de
commit 138a506b1a
21 changed files with 281 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/base/intern_string.hh
View File

@ -233,6 +233,12 @@ public:
return this->ist_interned_string->size();
}
size_t hash() const {
uintptr_t ptr = (uintptr_t) this->ist_interned_string;
return ptr;
}
std::string to_string() const {
if (this->ist_interned_string == nullptr) {
return "";
@ -267,6 +273,15 @@ private:
unsigned long hash_str(const char *str, size_t len);
namespace std {
template <>
struct hash<const intern_string_t> {
std::size_t operator()(const intern_string_t &ist) const {
return ist.hash();
}
};
}
inline bool operator<(const char *left, const intern_string_t &right) {
int rc = strncmp(left, right.get(), right.size());
return rc < 0;

28
src/filter_observer.cc
View File

@ -31,11 +31,12 @@
#include "filter_observer.hh"
void line_filter_observer::logline_new_line(const logfile &lf,
logfile::const_iterator ll,
shared_buffer_ref &sbr)
void line_filter_observer::logline_new_lines(const logfile &lf,
logfile::const_iterator ll_begin,
logfile::const_iterator ll_end,
shared_buffer_ref &sbr)
{
size_t offset = std::distance(lf.begin(), ll);
size_t offset = std::distance(lf.begin(), ll_begin);
require(&lf == this->lfo_filter_state.tfs_logfile.get());
@ -44,15 +45,18 @@ void line_filter_observer::logline_new_line(const logfile &lf,
return;
}
if (lf.get_format() != nullptr) {
lf.get_format()->get_subline(*ll, sbr);
}
for (auto &filter : this->lfo_filter_stack) {
if (filter->lf_deleted) {
continue;
for (; ll_begin != ll_end; ++ll_begin) {
if (lf.get_format() != nullptr) {
lf.get_format()->get_subline(*ll_begin, sbr);
}
if (offset >= this->lfo_filter_state.tfs_filter_count[filter->get_index()]) {
filter->add_line(this->lfo_filter_state, ll, sbr);
for (auto &filter : this->lfo_filter_stack) {
if (filter->lf_deleted) {
continue;
}
if (offset >=
this->lfo_filter_state.tfs_filter_count[filter->get_index()]) {
filter->add_line(this->lfo_filter_state, ll_begin, sbr);
}
}
}
}

5
src/filter_observer.hh
View File

@ -48,7 +48,10 @@ public:
}
};
void logline_new_line(const logfile &lf, logfile::const_iterator ll, shared_buffer_ref &sbr);
void logline_new_lines(const logfile &lf,
logfile::const_iterator ll_begin,
logfile::const_iterator ll_end,
shared_buffer_ref &sbr);
void logline_eof(const logfile &lf);;

1
src/line_buffer.cc
View File

@ -708,7 +708,6 @@ Result<line_info, string> line_buffer::load_next_line(file_range prev_line)
retval.li_partial = true;
}
this->ensure_available(offset, retval.li_file_range.fr_size);
line_start = this->get_range(offset, retval.li_file_range.fr_size);
if (retval.li_file_range.fr_size >= MAX_LINE_BUFFER_SIZE) {
retval.li_file_range.fr_size = MAX_LINE_BUFFER_SIZE - 1;

20
src/line_buffer.hh
View File

@ -210,20 +210,11 @@ public:
};
/**
* Read up to the end of file or a given delimiter.
* Attempt to load the next line into the buffer.
*
* @param offset_inout The offset in the file to start reading from. On
* return, it contains the offset where the next line should start or one
* past the size of the file.
* @param len_out On return, contains the length of the line, not including
* the delimiter.
* @param delim The character that splits lines in the input, defaults to a
* line feed.
* @return The address in the internal buffer where the line starts. The
* line is not NULL-terminated, but this method ensures there is room to NULL
* terminate the line. If any modifications are made to the line, such as
* NULL termination, the invalidate() must be called before re-reading the
* line to refresh the buffer.
* @param prev_line The range of the previous line.
* @return If the read was successful, information about the line.
* Otherwise, an error message.
*/
Result<line_info, std::string> load_next_line(file_range prev_line = {});
@ -338,7 +329,8 @@ private:
*/
time_t lb_file_time;
ssize_t lb_buffer_size; /*< The amount of cached data in the buffer. */
ssize_t lb_buffer_max; /*< The size of the buffer memory. */
ssize_t lb_buffer_max; /*< The amount of allocated memory for the
* buffer. */
bool lb_seekable; /*< Flag set for seekable file descriptors. */
off_t lb_last_line_offset; /*< */
};

25
src/lnav_util.cc
View File

@ -362,7 +362,6 @@ time_t tm2sec(const struct tm *t)
} /* must be a valid time */
}
static const int MONSPERYEAR = 12;
static const int SECSPERMIN = 60;
static const int SECSPERHOUR = 60 * SECSPERMIN;
static const int SECSPERDAY = 24 * SECSPERHOUR;
@ -373,15 +372,17 @@ static const int EPOCH_YEAR = 1970;
#define isleap(y) ((((y) % 4) == 0 && ((y) % 100) != 0) || ((y) % 400) == 0)
static const int mon_lengths[2][MONSPERYEAR] = {
{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31},
{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
} ;
static const int year_lengths[2] = {
365,
366
} ;
};
const unsigned short int mon_yday[2][13] = {
/* Normal years. */
{ 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 },
/* Leap years. */
{ 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 }
};
static void secs2wday(const struct timeval &tv, struct tm *res)
{
@ -409,7 +410,7 @@ struct tm *secs2tm(time_t *tim_p, struct tm *res)
time_t lcltime;
int y;
int yleap;
const int *ip;
const unsigned short int *ip;
/* base decision about std/dst time on current time */
lcltime = *tim_p;
@ -457,9 +458,11 @@ struct tm *secs2tm(time_t *tim_p, struct tm *res)
res->tm_year = y - YEAR_BASE;
res->tm_yday = days;
ip = mon_lengths[yleap];
for (res->tm_mon = 0; days >= ip[res->tm_mon]; ++res->tm_mon)
days -= ip[res->tm_mon];
ip = mon_yday[isleap(y)];
for (y = 11; days < (long int) ip[y]; --y)
continue;
days -= ip[y];
res->tm_mon = y;
res->tm_mday = days + 1;
res->tm_isdst = 0;

7
src/log_data_table.hh
View File

@ -223,12 +223,9 @@ public:
break;
default: {
shared_buffer_ref value_sbr;
value_sbr.subset(line,
pvalue.e_capture.c_begin, pvalue.e_capture.length());
values.emplace_back(intern_string::lookup("", 0),
logline_value::VALUE_TEXT, value_sbr);
logline_value::VALUE_TEXT, line, false, nullptr,
-1, pvalue.e_capture.c_begin, pvalue.e_capture.c_end);
break;
}

86
src/log_format.cc
View File

@ -583,7 +583,7 @@ log_format::scan_result_t external_log_format::scan(logfile &lf,
continue;
}
if (!pat->match(pc, pi)) {
if (!pat->match(pc, pi, PCRE_NO_UTF8_CHECK)) {
if (!this->lf_pattern_locks.empty() && pat_index != -1) {
log_debug("no match on pattern %d", pat_index);
curr_fmt = -1;
@ -763,7 +763,7 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line
int pat_index = this->pattern_index_for_line(line_number);
pattern &pat = *this->elf_pattern_order[pat_index];
if (!pat.p_pcre->match(pc, pi)) {
if (!pat.p_pcre->match(pc, pi, PCRE_NO_UTF8_CHECK)) {
// A continued line still needs a body.
lr.lr_start = 0;
lr.lr_end = line.length();
@ -775,14 +775,14 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line
cap = pc[pat.p_timestamp_field_index];
lr.lr_start = cap->c_begin;
lr.lr_end = cap->c_end;
sa.push_back(string_attr(lr, &logline::L_TIMESTAMP));
sa.emplace_back(lr, &logline::L_TIMESTAMP);
if (pat.p_module_field_index != -1) {
module_cap = pc[pat.p_module_field_index];
if (module_cap != NULL && module_cap->is_valid()) {
lr.lr_start = module_cap->c_begin;
lr.lr_end = module_cap->c_end;
sa.push_back(string_attr(lr, &logline::L_MODULE));
sa.emplace_back(lr, &logline::L_MODULE);
}
}
@ -790,7 +790,7 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line
if (cap != NULL && cap->is_valid()) {
lr.lr_start = cap->c_begin;
lr.lr_end = cap->c_end;
sa.push_back(string_attr(lr, &logline::L_OPID));
sa.emplace_back(lr, &logline::L_OPID);
}
}
@ -810,7 +810,6 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line
const struct scaling_factor *scaling = NULL;
pcre_context::capture_t *cap = pc[ivd.ivd_index];
const value_def &vd = *ivd.ivd_value_def;
shared_buffer_ref field;
if (ivd.ivd_unit_field_index >= 0) {
pcre_context::iterator unit_cap = pc[ivd.ivd_unit_field_index];
@ -829,18 +828,20 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line
}
}
field.subset(line, cap->c_begin, cap->length());
values.emplace_back(vd.vd_name,
vd.vd_kind,
field,
vd.vd_identifier,
scaling,
vd.vd_column,
cap->c_begin,
cap->c_end,
pat.p_module_format,
this);
if (cap->is_valid()) {
values.emplace_back(vd.vd_name,
vd.vd_kind,
line,
vd.vd_identifier,
scaling,
vd.vd_column,
cap->c_begin,
cap->c_end,
pat.p_module_format,
this);
} else {
values.emplace_back(vd.vd_name);
}
values.back().lv_hidden = vd.vd_hidden || vd.vd_user_hidden;
}
@ -1137,16 +1138,16 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
}
if (lv_iter->lv_name == this->lf_timestamp_field) {
this->jlf_line_attrs.push_back(
string_attr(lr, &logline::L_TIMESTAMP));
this->jlf_line_attrs.emplace_back(
lr, &logline::L_TIMESTAMP);
}
else if (lv_iter->lv_name == this->elf_body_field) {
this->jlf_line_attrs.push_back(
string_attr(lr, &textview_curses::SA_BODY));
this->jlf_line_attrs.emplace_back(
lr, &textview_curses::SA_BODY);
}
else if (lv_iter->lv_name == this->elf_opid_field) {
this->jlf_line_attrs.push_back(
string_attr(lr, &logline::L_OPID));
this->jlf_line_attrs.emplace_back(
lr, &logline::L_OPID);
}
lv_iter->lv_origin = lr;
used_values[distance(this->jlf_line_values.begin(),
@ -1485,26 +1486,26 @@ void external_log_format::build(std::vector<std::string> &errors) {
stable_sort(this->elf_level_pairs.begin(), this->elf_level_pairs.end());
for (auto &elf_value_def : this->elf_value_defs) {
for (auto vd : this->elf_value_def_order) {
std::vector<std::string>::iterator act_iter;
if (!elf_value_def.second->vd_internal &&
elf_value_def.second->vd_column == -1) {
elf_value_def.second->vd_column = this->elf_column_count++;
if (!vd->vd_internal &&
vd->vd_column == -1) {
vd->vd_column = this->elf_column_count++;
}
if (elf_value_def.second->vd_kind == logline_value::VALUE_UNKNOWN) {
elf_value_def.second->vd_kind = logline_value::VALUE_TEXT;
if (vd->vd_kind == logline_value::VALUE_UNKNOWN) {
vd->vd_kind = logline_value::VALUE_TEXT;
}
for (act_iter = elf_value_def.second->vd_action_list.begin();
act_iter != elf_value_def.second->vd_action_list.end();
for (act_iter = vd->vd_action_list.begin();
act_iter != vd->vd_action_list.end();
++act_iter) {
if (this->lf_action_defs.find(*act_iter) ==
this->lf_action_defs.end()) {
errors.push_back("error:" +
this->elf_name.to_string() + ":" +
elf_value_def.first.get() +
vd->vd_name.get() +
": cannot find action -- " + (*act_iter));
}
}
@ -1860,19 +1861,18 @@ public:
const external_log_format &elf = this->elt_format;
cols.resize(elf.elf_column_count);
for (const auto &elf_value_def : elf.elf_value_defs) {
const auto &vd = *elf_value_def.second;
pair<int, unsigned int> type_pair = log_vtab_impl::logline_value_to_sqlite_type(vd.vd_kind);
for (const auto &vd : elf.elf_value_def_order) {
pair<int, unsigned int> type_pair = log_vtab_impl::logline_value_to_sqlite_type(vd->vd_kind);
if (vd.vd_column == -1) {
if (vd->vd_column == -1) {
continue;
}
cols[vd.vd_column].vc_name = vd.vd_name.get();
cols[vd.vd_column].vc_type = type_pair.first;
cols[vd.vd_column].vc_subtype = type_pair.second;
cols[vd.vd_column].vc_collator = vd.vd_collate.c_str();
cols[vd.vd_column].vc_comment = vd.vd_description;
cols[vd->vd_column].vc_name = vd->vd_name.get();
cols[vd->vd_column].vc_type = type_pair.first;
cols[vd->vd_column].vc_subtype = type_pair.second;
cols[vd->vd_column].vc_collator = vd->vd_collate.c_str();
cols[vd->vd_column].vc_comment = vd->vd_description;
}
};
@ -1882,7 +1882,7 @@ public:
for (const auto &elf_value_def : this->elt_format.elf_value_defs) {
if (elf_value_def.second->vd_foreign_key) {
keys_inout.push_back(elf_value_def.first.to_string());
keys_inout.emplace_back(elf_value_def.first.to_string());
}
}
};

29
src/log_format.hh
View File

@ -376,8 +376,8 @@ public:
};
logline_value(const intern_string_t name, kind_t kind, shared_buffer_ref &sbr,
bool ident=false, const scaling_factor *scaling=NULL,
int col=-1, int start=-1, int end=-1, bool from_module=false,
bool ident, const scaling_factor *scaling,
int col, int start, int end, bool from_module=false,
const log_format *format=NULL)
: lv_name(name), lv_kind(kind),
lv_identifier(ident), lv_column(col), lv_hidden(false), lv_sub_offset(0),
@ -385,7 +385,7 @@ public:
lv_from_module(from_module),
lv_format(format)
{
if (sbr.get_data() == NULL) {
if (sbr.get_data() == nullptr) {
this->lv_kind = kind = VALUE_NULL;
}
@ -395,24 +395,25 @@ public:
case VALUE_TEXT:
case VALUE_QUOTED:
case VALUE_TIMESTAMP:
this->lv_sbr = sbr;
this->lv_sbr.subset(sbr, start, end - start);
break;
case VALUE_NULL:
break;
case VALUE_INTEGER:
strtonum(this->lv_value.i, sbr.get_data(), sbr.length());
strtonum(this->lv_value.i, sbr.get_data_at(start), end - start);
if (scaling != NULL) {
scaling->scale(this->lv_value.i);
}
break;
case VALUE_FLOAT: {
char scan_value[sbr.length() + 1];
ssize_t len = end - start;
char scan_value[len + 1];
memcpy(scan_value, sbr.get_data(), sbr.length());
scan_value[sbr.length()] = '\0';
memcpy(scan_value, sbr.get_data_at(start), len);
scan_value[len] = '\0';
this->lv_value.d = strtod(scan_value, NULL);
if (scaling != NULL) {
scaling->scale(this->lv_value.d);
@ -421,8 +422,8 @@ public:
}
case VALUE_BOOLEAN:
if (strncmp(sbr.get_data(), "true", sbr.length()) == 0 ||
strncmp(sbr.get_data(), "yes", sbr.length()) == 0) {
if (strncmp(sbr.get_data_at(start), "true", end - start) == 0 ||
strncmp(sbr.get_data_at(start), "yes", end - start) == 0) {
this->lv_value.i = 1;
}
else {
@ -682,7 +683,7 @@ public:
*
* @return The log format name.
*/
virtual intern_string_t get_name(void) const = 0;
virtual const intern_string_t get_name() const = 0;
virtual bool match_name(const std::string &filename) { return true; };
@ -949,7 +950,7 @@ public:
this->jlf_line_offsets.reserve(128);
};
intern_string_t get_name(void) const {
const intern_string_t get_name(void) const {
return this->elf_name;
};
@ -1203,7 +1204,9 @@ public:
std::map<std::string, std::shared_ptr<pattern>> elf_patterns;
std::vector<std::shared_ptr<pattern>> elf_pattern_order;
std::vector<sample> elf_samples;
std::map<const intern_string_t, std::shared_ptr<value_def>> elf_value_defs;
std::unordered_map<const intern_string_t, std::shared_ptr<value_def>>
elf_value_defs;
std::vector<std::shared_ptr<value_def>> elf_value_def_order;
std::vector<std::shared_ptr<value_def>> elf_numeric_value_defs;
int elf_column_count;
double elf_timestamp_divisor;

30
src/log_format_impls.cc
View File

@ -116,7 +116,7 @@ class generic_log_format : public log_format {
return get_pcre_log_formats()[pat_index].name;
}
intern_string_t get_name() const {
const intern_string_t get_name() const {
return intern_string::lookup("generic_log");
};
@ -386,8 +386,8 @@ public:
this->lf_time_ordered = false;
};
intern_string_t get_name(void) const {
static intern_string_t name = intern_string::lookup("bro");
const intern_string_t get_name(void) const {
static const intern_string_t name(intern_string::lookup("bro"));
return this->blf_format_name.empty() ? name : this->blf_format_name;
};
@ -633,8 +633,6 @@ public:
string_fragment sf = *iter;
logline_value::kind_t kind = fd.fd_kind;
struct line_range lr(sf.sf_begin, sf.sf_end);
if (sf == this->blf_empty_field) {
sf.clear();
} else if (sf == this->blf_unset_field) {
@ -642,18 +640,28 @@ public:
kind = logline_value::VALUE_NULL;
}
auto lr = line_range(sf.sf_begin, sf.sf_end);
if (fd.fd_name == TS) {
sa.emplace_back(lr, &logline::L_TIMESTAMP);
} else if (fd.fd_name == UID) {
sa.emplace_back(lr, &logline::L_OPID);
}
shared_buffer_ref value_ref;
value_ref.subset(sbr, sf.sf_begin, sf.length());
values.emplace_back(fd.fd_name, kind, value_ref,
fd.fd_identifier, nullptr, iter.index(),
lr.lr_start, lr.lr_end, false,
this);
if (lr.is_valid()) {
values.emplace_back(fd.fd_name,
kind,
sbr,
fd.fd_identifier,
nullptr,
iter.index(),
lr.lr_start,
lr.lr_end,
false,
this);
} else {
values.emplace_back(fd.fd_name);
}
}
};

12
src/log_format_loader.cc
View File

@ -114,14 +114,18 @@ static external_log_format::value_def *value_def_provider(const yajlpp_provider_
{
const intern_string_t value_name = ypc.get_substr_i(0);
auto &retval = elf->elf_value_defs[value_name];
auto iter = elf->elf_value_defs.find(value_name);
shared_ptr<external_log_format::value_def> retval;
if (retval.get() == nullptr) {
if (iter == elf->elf_value_defs.end()) {
retval = make_shared<external_log_format::value_def>();
retval->vd_name = value_name;
elf->elf_value_defs[value_name] = retval;
elf->elf_value_def_order.emplace_back(retval);
} else {
retval = iter->second;
}
retval->vd_name = value_name;
return retval.get();
}

8
src/log_search_table.hh
View File

@ -157,11 +157,9 @@ public:
values.emplace_back(instance_name, this->lst_instance);
values.back().lv_column = next_column++;
for (int lpc = 0; lpc < this->lst_regex.get_capture_count(); lpc++) {
pcre_context::capture_t *cap = this->lst_match_context[lpc];
shared_buffer_ref value_sbr;
value_sbr.subset(line, cap->c_begin, cap->length());
values.emplace_back(empty, this->lst_column_types[lpc], value_sbr);
auto cap = this->lst_match_context[lpc];
values.emplace_back(empty, this->lst_column_types[lpc], line,
false, nullptr, -1, cap->c_begin, cap->c_end);
values.back().lv_column = next_column++;
}
};

38
src/log_vtab_impl.cc
View File

@ -483,31 +483,35 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
}
case VT_COL_FILTERS: {
auto &filters = vt->lss->get_filters();
auto &filter_state = ld->ld_filter_state;
yajlpp_gen gen;
auto &filter_mask = ld->ld_filter_state.lfo_filter_state.tfs_mask;
yajl_gen_config(gen, yajl_gen_beautify, false);
if (!filter_mask[line_number]) {
sqlite3_result_null(ctx);
} else {
auto &filters = vt->lss->get_filters();
yajlpp_gen gen;
{
yajlpp_array arr(gen);
yajl_gen_config(gen, yajl_gen_beautify, false);
for (auto &filter : filters) {
if (filter->lf_deleted) {
continue;
}
{
yajlpp_array arr(gen);
uint32_t mask = (1UL << filter->get_index());
for (auto &filter : filters) {
if (filter->lf_deleted) {
continue;
}
if (filter_state.lfo_filter_state.tfs_mask[line_number] &
mask) {
arr.gen(filter->get_index());
uint32_t mask = (1UL << filter->get_index());
if (filter_mask[line_number] & mask) {
arr.gen(filter->get_index());
}
}
}
}
to_sqlite(ctx, gen.to_string_fragment());
sqlite3_result_subtype(ctx, 'J');
to_sqlite(ctx, gen.to_string_fragment());
sqlite3_result_subtype(ctx, 'J');
}
break;
}

41
src/logfile.cc
View File

@ -393,11 +393,9 @@ logfile::rebuild_result_t logfile::rebuild_index()
old_size = 0;
}
for (auto iter = this->begin() + old_size;
iter != this->end(); ++iter) {
if (this->lf_logline_observer != nullptr) {
this->lf_logline_observer->logline_new_line(*this, iter, sbr);
}
if (this->lf_logline_observer != nullptr) {
this->lf_logline_observer->logline_new_lines(
*this, this->begin() + old_size, this->end(), sbr);
}
if (this->lf_logfile_observer != nullptr) {
@ -517,24 +515,31 @@ void logfile::set_logline_observer(logline_observer *llo)
void logfile::reobserve_from(iterator iter)
{
if (this->lf_logline_observer != NULL) {
for (; iter != this->end(); ++iter) {
off_t offset = std::distance(this->begin(), iter);
for (; iter != this->end(); ++iter) {
off_t offset = std::distance(this->begin(), iter);
if (this->lf_logfile_observer != NULL) {
this->lf_logfile_observer->logfile_indexing(
*this, offset, this->size());
}
this->read_line(iter).then([this, iter](auto sbr) {
this->lf_logline_observer->logline_new_line(*this, iter, sbr);
});
if (iter->get_sub_offset() > 0) {
continue;
}
if (this->lf_logfile_observer != NULL) {
if (this->lf_logfile_observer != nullptr) {
this->lf_logfile_observer->logfile_indexing(
*this, this->size(), this->size());
*this, offset, this->size());
}
this->read_line(iter).then([this, iter](auto sbr) {
auto iter_end = iter + 1;
while (iter_end != this->end() && iter_end->get_sub_offset() != 0) {
++iter_end;
}
this->lf_logline_observer->logline_new_lines(
*this, iter, iter_end, sbr);
});
}
if (this->lf_logfile_observer != nullptr) {
this->lf_logfile_observer->logfile_indexing(
*this, this->size(), this->size());
this->lf_logline_observer->logline_eof(*this);
}
}

6
src/logfile.hh
View File

@ -402,7 +402,11 @@ public:
virtual void logline_restart(const logfile &lf, size_t rollback_size) = 0;
virtual void logline_new_line(const logfile &lf, logfile::const_iterator ll, shared_buffer_ref &sbr) = 0;
virtual void logline_new_lines(
const logfile &lf,
logfile::const_iterator ll_begin,
logfile::const_iterator ll_end,
shared_buffer_ref &sbr) = 0;
virtual void logline_eof(const logfile &lf) = 0;
};

4
src/shared_buffer.cc
View File

@ -68,8 +68,8 @@ bool shared_buffer_ref::subset(shared_buffer_ref &other, off_t offset, size_t le
if (offset != -1) {
this->sb_owner = other.sb_owner;
this->sb_length = len;
if (this->sb_owner == NULL) {
if ((this->sb_data = (char *)malloc(this->sb_length)) == NULL) {
if (this->sb_owner == nullptr) {
if ((this->sb_data = (char *)malloc(this->sb_length)) == nullptr) {
return false;
}

14
test/formats/jsontest/format.json
View File

@ -16,19 +16,19 @@
"timestamp-field": "ts",
"body-field" : "msg",
"value" : {
"user" : {
"kind" : "string",
"identifier" : true,
"rewriter" : "|rewrite-user"
},
"msg" : {
"rewriter" : ";SELECT :msg || 'bork bork bork'"
},
"arr" : {
"kind" : "json"
},
"obj" : {
"kind" : "json"
},
"arr" : {
"kind" : "json"
"user" : {
"kind" : "string",
"identifier" : true,
"rewriter" : "|rewrite-user"
}
}
}

12
test/test_cmds.sh
View File

@ -516,9 +516,7 @@ check_output "write-json-to is not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"c_ip": "192.168.202.254",
"cs_method": "GET",
"cs_referer": "-",
@ -539,9 +537,7 @@ check_output "write-json-to is not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"c_ip": "192.168.202.254",
"cs_method": "GET",
"cs_referer": "-",
@ -562,9 +558,7 @@ check_output "write-json-to is not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"c_ip": "192.168.202.254",
"cs_method": "GET",
"cs_referer": "-",

16
test/test_format_loader.sh
View File

@ -58,12 +58,12 @@ run_test ${lnav_test} -n \
check_output "levels are not correct?" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters
0,<NULL>,2016-06-30 12:00:01.000,0,trace,0,<NULL>,<NULL>,[]
1,<NULL>,2016-06-30 12:00:02.000,1000,debug,0,<NULL>,<NULL>,[]
2,<NULL>,2016-06-30 12:00:03.000,1000,debug2,0,<NULL>,<NULL>,[]
3,<NULL>,2016-06-30 12:00:04.000,1000,debug3,0,<NULL>,<NULL>,[]
4,<NULL>,2016-06-30 12:00:05.000,1000,info,0,<NULL>,<NULL>,[]
5,<NULL>,2016-06-30 12:00:06.000,1000,warning,0,<NULL>,<NULL>,[]
6,<NULL>,2016-06-30 12:00:07.000,1000,fatal,0,<NULL>,<NULL>,[]
7,<NULL>,2016-06-30 12:00:08.000,1000,info,0,<NULL>,<NULL>,[]
0,<NULL>,2016-06-30 12:00:01.000,0,trace,0,<NULL>,<NULL>,<NULL>
1,<NULL>,2016-06-30 12:00:02.000,1000,debug,0,<NULL>,<NULL>,<NULL>
2,<NULL>,2016-06-30 12:00:03.000,1000,debug2,0,<NULL>,<NULL>,<NULL>
3,<NULL>,2016-06-30 12:00:04.000,1000,debug3,0,<NULL>,<NULL>,<NULL>
4,<NULL>,2016-06-30 12:00:05.000,1000,info,0,<NULL>,<NULL>,<NULL>
5,<NULL>,2016-06-30 12:00:06.000,1000,warning,0,<NULL>,<NULL>,<NULL>
6,<NULL>,2016-06-30 12:00:07.000,1000,fatal,0,<NULL>,<NULL>,<NULL>
7,<NULL>,2016-06-30 12:00:08.000,1000,info,0,<NULL>,<NULL>,<NULL>
EOF

122
test/test_json_format.sh
View File

@ -138,19 +138,19 @@ run_test ${lnav_test} -n \
check_output "log levels not working" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,arr,obj,user
0,<NULL>,2013-09-06 20:00:48.124,0,trace,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
2,<NULL>,2013-09-06 20:00:49.124,1000,info,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
4,<NULL>,2013-09-06 22:00:49.124,7200000,info,0,<NULL>,<NULL>,[],<NULL>,<NULL>,steve@example.com
7,<NULL>,2013-09-06 22:00:59.124,10000,debug5,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
9,<NULL>,2013-09-06 22:00:59.124,0,debug4,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
11,<NULL>,2013-09-06 22:00:59.124,0,debug3,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
13,<NULL>,2013-09-06 22:00:59.124,0,debug2,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
15,<NULL>,2013-09-06 22:00:59.124,0,debug,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
17,<NULL>,2013-09-06 22:01:49.124,50000,stats,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
19,<NULL>,2013-09-06 22:01:49.124,0,warning,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
21,<NULL>,2013-09-06 22:01:49.124,0,error,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
23,<NULL>,2013-09-06 22:01:49.124,0,critical,0,<NULL>,<NULL>,[],<NULL>,<NULL>,<NULL>
25,<NULL>,2013-09-06 22:01:49.124,0,fatal,0,<NULL>,<NULL>,[],"[""hi"", {""sub1"": true}]","{ ""field1"" : ""hi"", ""field2"": 2 }",<NULL>
0,<NULL>,2013-09-06 20:00:48.124,0,trace,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
2,<NULL>,2013-09-06 20:00:49.124,1000,info,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
4,<NULL>,2013-09-06 22:00:49.124,7200000,info,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,steve@example.com
7,<NULL>,2013-09-06 22:00:59.124,10000,debug5,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
9,<NULL>,2013-09-06 22:00:59.124,0,debug4,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
11,<NULL>,2013-09-06 22:00:59.124,0,debug3,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
13,<NULL>,2013-09-06 22:00:59.124,0,debug2,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
15,<NULL>,2013-09-06 22:00:59.124,0,debug,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
17,<NULL>,2013-09-06 22:01:49.124,50000,stats,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
19,<NULL>,2013-09-06 22:01:49.124,0,warning,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
21,<NULL>,2013-09-06 22:01:49.124,0,error,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
23,<NULL>,2013-09-06 22:01:49.124,0,critical,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
25,<NULL>,2013-09-06 22:01:49.124,0,fatal,0,<NULL>,<NULL>,<NULL>,"[""hi"", {""sub1"": true}]","{ ""field1"" : ""hi"", ""field2"": 2 }",<NULL>
EOF
@ -171,9 +171,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -187,9 +185,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -203,9 +199,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": "steve@example.com"
@ -219,9 +213,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -235,9 +227,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -251,9 +241,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -267,9 +255,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -283,9 +269,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -299,9 +283,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -315,9 +297,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -331,9 +311,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -347,9 +325,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": null,
"obj": null,
"user": null
@ -363,9 +339,7 @@ check_output "json output not working" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"arr": [
"hi",
{
@ -400,10 +374,10 @@ run_test ${lnav_test} -n -d /tmp/lnav.err \
${test_dir}/logfile_json2.json
check_output "log levels not working" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,cl,user
0,<NULL>,2013-09-06 20:00:49.124,0,info,0,<NULL>,<NULL>,[],com.exmaple.foo,<NULL>
1,<NULL>,2013-09-06 22:00:49.124,7200000,info,0,<NULL>,<NULL>,[],com.exmaple.foo,steve@example.com
3,<NULL>,2013-09-06 22:01:49.124,60000,error,0,<NULL>,<NULL>,[],com.exmaple.foo,<NULL>
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,user,cl
0,<NULL>,2013-09-06 20:00:49.124,0,info,0,<NULL>,<NULL>,<NULL>,<NULL>,com.exmaple.foo
1,<NULL>,2013-09-06 22:00:49.124,7200000,info,0,<NULL>,<NULL>,<NULL>,steve@example.com,com.exmaple.foo
3,<NULL>,2013-09-06 22:01:49.124,60000,error,0,<NULL>,<NULL>,<NULL>,<NULL>,com.exmaple.foo
EOF
@ -465,20 +439,20 @@ run_test ${lnav_test} -n \
${test_dir}/logfile_nested_json.json
check_output "log levels not working" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,@fields/trace#,@fields/user
0,<NULL>,2013-09-06 20:00:48.124,0,trace,0,<NULL>,<NULL>,[],<NULL>,<NULL>
2,<NULL>,2013-09-06 20:00:49.124,1000,info,0,<NULL>,<NULL>,[],<NULL>,<NULL>
4,<NULL>,2013-09-06 22:00:49.124,7200000,info,0,<NULL>,<NULL>,[],<NULL>,steve@example.com
7,<NULL>,2013-09-06 22:00:59.124,10000,debug5,0,<NULL>,<NULL>,[],<NULL>,<NULL>
9,<NULL>,2013-09-06 22:00:59.124,0,debug4,0,<NULL>,<NULL>,[],<NULL>,<NULL>
11,<NULL>,2013-09-06 22:00:59.124,0,debug3,0,<NULL>,<NULL>,[],<NULL>,<NULL>
13,<NULL>,2013-09-06 22:00:59.124,0,debug2,0,<NULL>,<NULL>,[],<NULL>,<NULL>
15,<NULL>,2013-09-06 22:00:59.124,0,debug,0,<NULL>,<NULL>,[],<NULL>,<NULL>
17,<NULL>,2013-09-06 22:01:49.124,50000,stats,0,<NULL>,<NULL>,[],<NULL>,<NULL>
19,<NULL>,2013-09-06 22:01:49.124,0,warning,0,<NULL>,<NULL>,[],<NULL>,<NULL>
21,<NULL>,2013-09-06 22:01:49.124,0,error,0,<NULL>,<NULL>,[],<NULL>,<NULL>
23,<NULL>,2013-09-06 22:01:49.124,0,critical,0,<NULL>,<NULL>,[],<NULL>,<NULL>
25,<NULL>,2013-09-06 22:01:49.124,0,fatal,0,<NULL>,<NULL>,[],line:1,<NULL>
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,@fields/user,@fields/trace#
0,<NULL>,2013-09-06 20:00:48.124,0,trace,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
2,<NULL>,2013-09-06 20:00:49.124,1000,info,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
4,<NULL>,2013-09-06 22:00:49.124,7200000,info,0,<NULL>,<NULL>,<NULL>,steve@example.com,<NULL>
7,<NULL>,2013-09-06 22:00:59.124,10000,debug5,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
9,<NULL>,2013-09-06 22:00:59.124,0,debug4,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
11,<NULL>,2013-09-06 22:00:59.124,0,debug3,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
13,<NULL>,2013-09-06 22:00:59.124,0,debug2,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
15,<NULL>,2013-09-06 22:00:59.124,0,debug,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
17,<NULL>,2013-09-06 22:01:49.124,50000,stats,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
19,<NULL>,2013-09-06 22:01:49.124,0,warning,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
21,<NULL>,2013-09-06 22:01:49.124,0,error,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
23,<NULL>,2013-09-06 22:01:49.124,0,critical,0,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>
25,<NULL>,2013-09-06 22:01:49.124,0,fatal,0,<NULL>,<NULL>,<NULL>,<NULL>,line:1
EOF
@ -512,10 +486,10 @@ run_test ${lnav_test} -n \
${test_dir}/logfile_json3.json
check_output "json log3 format is not working" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,client_ip,details1,details2,details3,request/method,request/size,request/uri,response/status
0,<NULL>,2017-03-24 20:06:26.240,0,info,0,<NULL>,<NULL>,[],1.1.1.1,<NULL>,<NULL>,<NULL>,GET,166,/example/uri/5,200
1,<NULL>,2017-03-24 20:12:47.764,381524,critical,0,<NULL>,<NULL>,[],1.1.1.1,<NULL>,<NULL>,<NULL>,GET,166,/example/uri/5,500
2,<NULL>,2017-03-24 20:15:31.694,163930,warning,0,<NULL>,<NULL>,[],1.1.1.1,"{""foo"": ""bar""}","{""foo"": ""bar""}","{""foo"": ""bar""}",GET,166,/example/uri/5,400
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,client_ip,request/method,request/uri,request/size,response/status,details1,details2,details3
0,<NULL>,2017-03-24 20:06:26.240,0,info,0,<NULL>,<NULL>,<NULL>,1.1.1.1,GET,/example/uri/5,166,200,<NULL>,<NULL>,<NULL>
1,<NULL>,2017-03-24 20:12:47.764,381524,critical,0,<NULL>,<NULL>,<NULL>,1.1.1.1,GET,/example/uri/5,166,500,<NULL>,<NULL>,<NULL>
2,<NULL>,2017-03-24 20:15:31.694,163930,warning,0,<NULL>,<NULL>,<NULL>,1.1.1.1,GET,/example/uri/5,166,400,"{""foo"": ""bar""}","{""foo"": ""bar""}","{""foo"": ""bar""}"
EOF
run_test ${lnav_test} -n \

40
test/test_sql.sh
View File

@ -124,11 +124,11 @@ run_test env TZ=UTC ${lnav_test} -n \
check_output "bro logs are not recognized?" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,bro_ts,bro_uid,bro_id_orig_h,bro_id_orig_p,bro_id_resp_h,bro_id_resp_p,bro_trans_depth,bro_method,bro_host,bro_uri,bro_referrer,bro_version,bro_user_agent,bro_request_body_len,bro_response_body_len,bro_status_code,bro_status_msg,bro_info_code,bro_info_msg,bro_tags,bro_username,bro_password,bro_proxied,bro_orig_fuids,bro_orig_filenames,bro_orig_mime_types,bro_resp_fuids,bro_resp_filenames,bro_resp_mime_types
0,<NULL>,2011-11-03 00:19:26.452,0,info,0,<NULL>,<NULL>,[],1320279566.452687,CwFs1P2UcUdlSxD2La,192.168.2.76,52026,132.235.215.119,80,1,GET,www.reddit.com,/,<NULL>,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,109978,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,Ftw3fJ2JJF3ntMTL2,<NULL>,text/html
1,<NULL>,2011-11-03 00:19:26.831,379,info,0,<NULL>,<NULL>,[],1320279566.831619,CJxSUgkInyKSHiju1,192.168.2.76,52030,72.21.211.173,80,1,GET,e.thumbs.redditmedia.com,/E-pbDbmiBclPkDaX.jpg,http://www.reddit.com/,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,2300,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,FFTf9Zdgk3YkfCKo3,<NULL>,image/jpeg
2,<NULL>,2011-11-03 00:19:26.831,0,info,0,<NULL>,<NULL>,[],1320279566.831563,CJwUi9bdB9c1lLW44,192.168.2.76,52029,72.21.211.173,80,1,GET,f.thumbs.redditmedia.com,/BP5bQfy4o-C7cF6A.jpg,http://www.reddit.com/,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,2272,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,FfXtOj3o7aub4vbs2j,<NULL>,image/jpeg
3,<NULL>,2011-11-03 00:19:26.831,0,info,0,<NULL>,<NULL>,[],1320279566.831473,CoX7zA3OJKGUOSCBY2,192.168.2.76,52027,72.21.211.173,80,1,GET,e.thumbs.redditmedia.com,/SVUtep3Rhg5FTRn4.jpg,http://www.reddit.com/,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,2562,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,F21Ybs3PTqS6O4Q2Zh,<NULL>,image/jpeg
4,<NULL>,2011-11-03 00:19:26.831,0,info,0,<NULL>,<NULL>,[],1320279566.831643,CT0JIh479jXIGt0Po1,192.168.2.76,52031,72.21.211.173,80,1,GET,f.thumbs.redditmedia.com,/uuy31444rLSyKdHS.jpg,http://www.reddit.com/,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,1595,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,Fdk0MZ1wQmKWAJ4WH4,<NULL>,image/jpeg
0,<NULL>,2011-11-03 00:19:26.452,0,info,0,<NULL>,<NULL>,<NULL>,1320279566.452687,CwFs1P2UcUdlSxD2La,192.168.2.76,52026,132.235.215.119,80,1,GET,www.reddit.com,/,<NULL>,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,109978,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,Ftw3fJ2JJF3ntMTL2,<NULL>,text/html
1,<NULL>,2011-11-03 00:19:26.831,379,info,0,<NULL>,<NULL>,<NULL>,1320279566.831619,CJxSUgkInyKSHiju1,192.168.2.76,52030,72.21.211.173,80,1,GET,e.thumbs.redditmedia.com,/E-pbDbmiBclPkDaX.jpg,http://www.reddit.com/,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,2300,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,FFTf9Zdgk3YkfCKo3,<NULL>,image/jpeg
2,<NULL>,2011-11-03 00:19:26.831,0,info,0,<NULL>,<NULL>,<NULL>,1320279566.831563,CJwUi9bdB9c1lLW44,192.168.2.76,52029,72.21.211.173,80,1,GET,f.thumbs.redditmedia.com,/BP5bQfy4o-C7cF6A.jpg,http://www.reddit.com/,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,2272,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,FfXtOj3o7aub4vbs2j,<NULL>,image/jpeg
3,<NULL>,2011-11-03 00:19:26.831,0,info,0,<NULL>,<NULL>,<NULL>,1320279566.831473,CoX7zA3OJKGUOSCBY2,192.168.2.76,52027,72.21.211.173,80,1,GET,e.thumbs.redditmedia.com,/SVUtep3Rhg5FTRn4.jpg,http://www.reddit.com/,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,2562,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,F21Ybs3PTqS6O4Q2Zh,<NULL>,image/jpeg
4,<NULL>,2011-11-03 00:19:26.831,0,info,0,<NULL>,<NULL>,<NULL>,1320279566.831643,CT0JIh479jXIGt0Po1,192.168.2.76,52031,72.21.211.173,80,1,GET,f.thumbs.redditmedia.com,/uuy31444rLSyKdHS.jpg,http://www.reddit.com/,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,1595,200,OK,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,Fdk0MZ1wQmKWAJ4WH4,<NULL>,image/jpeg
EOF
run_test env TZ=UTC ${lnav_test} -n \
@ -138,7 +138,7 @@ run_test env TZ=UTC ${lnav_test} -n \
check_output "bro logs are not recognized?" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,bro_ts,bro_uid,bro_id_orig_h,bro_id_orig_p,bro_id_resp_h,bro_id_resp_p,bro_trans_depth,bro_method,bro_host,bro_uri,bro_referrer,bro_version,bro_user_agent,bro_request_body_len,bro_response_body_len,bro_status_code,bro_status_msg,bro_info_code,bro_info_msg,bro_tags,bro_username,bro_password,bro_proxied,bro_orig_fuids,bro_orig_filenames,bro_orig_mime_types,bro_resp_fuids,bro_resp_filenames,bro_resp_mime_types
118,<NULL>,2011-11-03 00:19:49.337,18,error,0,<NULL>,<NULL>,[],1320279589.337053,CBHHuR1xFnm5C5CQBc,192.168.2.76,52074,74.125.225.76,80,1,GET,i4.ytimg.com,/vi/gDbg_GeuiSY/hqdefault.jpg,<NULL>,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,893,404,Not Found,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,F2GiAw3j1m22R2yIg2,<NULL>,image/jpeg
118,<NULL>,2011-11-03 00:19:49.337,18,error,0,<NULL>,<NULL>,<NULL>,1320279589.337053,CBHHuR1xFnm5C5CQBc,192.168.2.76,52074,74.125.225.76,80,1,GET,i4.ytimg.com,/vi/gDbg_GeuiSY/hqdefault.jpg,<NULL>,1.1,Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1,0,893,404,Not Found,<NULL>,<NULL>,,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,<NULL>,F2GiAw3j1m22R2yIg2,<NULL>,image/jpeg
EOF
run_test ${lnav_test} -n \
@ -333,9 +333,9 @@ run_test ${lnav_test} -n \
check_output "access_log table is not working" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,c_ip,cs_method,cs_referer,cs_uri_query,cs_uri_stem,cs_user_agent,cs_username,cs_version,sc_bytes,sc_status
0,<NULL>,2009-07-20 22:59:26.000,0,info,0,<NULL>,<NULL>,[],192.168.202.254,GET,-,<NULL>,/vmw/cgi/tramp,gPXE/0.9.7,-,HTTP/1.0,134,200
1,<NULL>,2009-07-20 22:59:29.000,3000,error,0,<NULL>,<NULL>,[],192.168.202.254,GET,-,<NULL>,/vmw/vSphere/default/vmkboot.gz,gPXE/0.9.7,-,HTTP/1.0,46210,404
2,<NULL>,2009-07-20 22:59:29.000,0,info,0,<NULL>,<NULL>,[],192.168.202.254,GET,-,<NULL>,/vmw/vSphere/default/vmkernel.gz,gPXE/0.9.7,-,HTTP/1.0,78929,200
0,<NULL>,2009-07-20 22:59:26.000,0,info,0,<NULL>,<NULL>,<NULL>,192.168.202.254,GET,-,<NULL>,/vmw/cgi/tramp,gPXE/0.9.7,-,HTTP/1.0,134,200
1,<NULL>,2009-07-20 22:59:29.000,3000,error,0,<NULL>,<NULL>,<NULL>,192.168.202.254,GET,-,<NULL>,/vmw/vSphere/default/vmkboot.gz,gPXE/0.9.7,-,HTTP/1.0,46210,404
2,<NULL>,2009-07-20 22:59:29.000,0,info,0,<NULL>,<NULL>,<NULL>,192.168.202.254,GET,-,<NULL>,/vmw/vSphere/default/vmkernel.gz,gPXE/0.9.7,-,HTTP/1.0,78929,200
EOF
@ -346,7 +346,7 @@ run_test ${lnav_test} -n \
check_output "loglevel collator is not working" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,c_ip,cs_method,cs_referer,cs_uri_query,cs_uri_stem,cs_user_agent,cs_username,cs_version,sc_bytes,sc_status
1,<NULL>,2009-07-20 22:59:29.000,3000,error,0,<NULL>,<NULL>,[],192.168.202.254,GET,-,<NULL>,/vmw/vSphere/default/vmkboot.gz,gPXE/0.9.7,-,HTTP/1.0,46210,404
1,<NULL>,2009-07-20 22:59:29.000,3000,error,0,<NULL>,<NULL>,<NULL>,192.168.202.254,GET,-,<NULL>,/vmw/vSphere/default/vmkboot.gz,gPXE/0.9.7,-,HTTP/1.0,46210,404
EOF
run_test ${lnav_test} -n \
@ -371,10 +371,10 @@ run_test ${lnav_test} -n \
check_output "syslog_log table is not working" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,log_hostname,log_msgid,log_pid,log_pri,log_procname,log_struct,syslog_version
0,<NULL>,2013-11-03 09:23:38.000,0,error,0,<NULL>,<NULL>,[],veridian,<NULL>,7998,<NULL>,automount,<NULL>,<NULL>
1,<NULL>,2013-11-03 09:23:38.000,0,info,0,<NULL>,<NULL>,[],veridian,<NULL>,16442,<NULL>,automount,<NULL>,<NULL>
2,<NULL>,2013-11-03 09:23:38.000,0,error,0,<NULL>,<NULL>,[],veridian,<NULL>,7999,<NULL>,automount,<NULL>,<NULL>
3,<NULL>,2013-11-03 09:47:02.000,1404000,info,0,<NULL>,<NULL>,[],veridian,<NULL>,<NULL>,<NULL>,sudo,<NULL>,<NULL>
0,<NULL>,2013-11-03 09:23:38.000,0,error,0,<NULL>,<NULL>,<NULL>,veridian,<NULL>,7998,<NULL>,automount,<NULL>,<NULL>
1,<NULL>,2013-11-03 09:23:38.000,0,info,0,<NULL>,<NULL>,<NULL>,veridian,<NULL>,16442,<NULL>,automount,<NULL>,<NULL>
2,<NULL>,2013-11-03 09:23:38.000,0,error,0,<NULL>,<NULL>,<NULL>,veridian,<NULL>,7999,<NULL>,automount,<NULL>,<NULL>
3,<NULL>,2013-11-03 09:47:02.000,1404000,info,0,<NULL>,<NULL>,<NULL>,veridian,<NULL>,<NULL>,<NULL>,sudo,<NULL>,<NULL>
EOF
@ -394,7 +394,7 @@ run_test ${lnav_test} -n \
check_output "log_time collation is wrong" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,log_hostname,log_msgid,log_pid,log_pri,log_procname,log_struct,syslog_version
3,<NULL>,2013-11-03 09:47:02.000,1404000,info,0,<NULL>,<NULL>,[],veridian,<NULL>,<NULL>,<NULL>,sudo,<NULL>,<NULL>
3,<NULL>,2013-11-03 09:47:02.000,1404000,info,0,<NULL>,<NULL>,<NULL>,veridian,<NULL>,<NULL>,<NULL>,sudo,<NULL>,<NULL>
EOF
@ -808,9 +808,7 @@ check_output "write-json-to isn't working?" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"contextid": "82e87195d704585501",
"data": "http://localhost:8086|/|<samlp:Response xmlns:samlp=\"urn:oasis:names:tc:SAML:2.0:protocol\" ID=\"s2daac0735bf476f4560aab81104b623bedfb0cbc0\" InResponseTo=\"84cbf2be33f6410bbe55877545a93f02\" Version=\"2.0\" IssueInstant=\"2014-06-15T01:04:52Z\" Destination=\"http://localhost:8086/api/1/rest/admin/org/530e42ccd6f45fd16d0d0717/saml/consume\"><saml:Issuer xmlns:saml=\"urn:oasis:names:tc:SAML:2.0:assertion\">http://openam.vagrant.dev/openam</saml:Issuer><samlp:Status xmlns:samlp=\"urn:oasis:names:tc:SAML:2.0:protocol\">\\\\n<samlp:StatusCode xmlns:samlp=\"urn:oasis:names:tc:SAML:2.0:protocol\"\\\\nValue=\"urn:oasis:names:tc:SAML:2.0:status:Success\">\\\\n</samlp:StatusCode>\\\\n</samlp:Status><saml:Assertion xmlns:saml=\"urn:oasis:names:tc:SAML:2.0:assertion\" ID=\"s2a0bee0da937e236167e99b209802056033816ac2\" IssueInstant=\"2014-06-15T01:04:52Z\" Version=\"2.0\">\\\\n<saml:Issuer>http://openam.vagrant.dev/openam</saml:Issuer><ds:Signature xmlns:ds=\"http://www.w3.org/2000/09/xmldsig#\">\\\\n<ds:SignedInfo>\\\\n<ds:CanonicalizationMethod Algorithm=\"http://www.w3.org/2001/10/xml-exc-c14n#\"/>\\\\n<ds:SignatureMethod Algorithm=\"http://www.w3.org/2000/09/xmldsig#rsa-sha1\"/>\\\\n<ds:Reference URI=\"#s2a0bee0da937e236167e99b209802056033816ac2\">\\\\n<ds:Transforms>\\\\n<ds:Transform Algorithm=\"http://www.w3.org/2000/09/xmldsig#enveloped-signature\"/>\\\\n<ds:Transform Algorithm=\"http://www.w3.org/2001/10/xml-exc-c14n#\"/>\\\\n</ds:Transforms>\\\\n<ds:DigestMethod Algorithm=\"http://www.w3.org/2000/09/xmldsig#sha1\"/>\\\\n<ds:DigestValue>4uSmVzjovUdQd3px/RcnoxQBsqE=</ds:DigestValue>\\\\n</ds:Reference>\\\\n</ds:SignedInfo>\\\\n<ds:SignatureValue>\\\\nhm/grge36uA6j1OWif2bTcvVTwESjmuJa27NxepW0AiV5YlcsHDl7RAIk6k/CjsSero3bxGbm56m\\\\nYncOEi9F1Tu7dS0bfx+vhm/kKTPgwZctf4GWn4qQwP+KeoZywbNj9ShsYJ+zPKzXwN4xBSuPjMxP\\\\nNf5szzjEWpOndQO/uDs=\\\\n</ds:SignatureValue>\\\\n<ds:KeyInfo>\\\\n<ds:X509Data>\\\\n<ds:X509Certificate>\\\\nMIICQDCCAakCBEeNB0swDQYJKoZIhvcNAQEEBQAwZzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNh\\\\nbGlmb3JuaWExFDASBgNVBAcTC1NhbnRhIENsYXJhMQwwCgYDVQQKEwNTdW4xEDAOBgNVBAsTB09w\\\\nZW5TU08xDTALBgNVBAMTBHRlc3QwHhcNMDgwMTE1MTkxOTM5WhcNMTgwMTEyMTkxOTM5WjBnMQsw\\\\nCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxMLU2FudGEgQ2xhcmExDDAK\\\\nBgNVBAoTA1N1bjEQMA4GA1UECxMHT3BlblNTTzENMAsGA1UEAxMEdGVzdDCBnzANBgkqhkiG9w0B\\\\nAQEFAAOBjQAwgYkCgYEArSQc/U75GB2AtKhbGS5piiLkmJzqEsp64rDxbMJ+xDrye0EN/q1U5Of+\\\\nRkDsaN/igkAvV1cuXEgTL6RlafFPcUX7QxDhZBhsYF9pbwtMzi4A4su9hnxIhURebGEmxKW9qJNY\\\\nJs0Vo5+IgjxuEWnjnnVgHTs1+mq5QYTA7E6ZyL8CAwEAATANBgkqhkiG9w0BAQQFAAOBgQB3Pw/U\\\\nQzPKTPTYi9upbFXlrAKMwtFf2OW4yvGWWvlcwcNSZJmTJ8ARvVYOMEVNbsT4OFcfu2/PeYoAdiDA\\\\ncGy/F2Zuj8XJJpuQRSE6PtQqBuDEHjjmOQJ0rV/r8mO1ZCtHRhpZ5zYRjhRC9eCbjx9VrFax0JDC\\\\n/FfwWigmrW0Y0Q==\\\\n</ds:X509Certificate>\\\\n</ds:X509Data>\\\\n</ds:KeyInfo>\\\\n</ds:Signature><saml:Subject>\\\\n<saml:NameID Format=\"urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress\" NameQualifier=\"http://openam.vagrant.dev/openam\">user@example.com</saml:NameID><saml:SubjectConfirmation Method=\"urn:oasis:names:tc:SAML:2.0:cm:bearer\">\\\\n<saml:SubjectConfirmationData InResponseTo=\"84cbf2be33f6410bbe55877545a93f02\" NotOnOrAfter=\"2014-06-15T01:14:52Z\" Recipient=\"http://localhost:8086/api/1/rest/admin/org/530e42ccd6f45fd16d0d0717/saml/consume\"/></saml:SubjectConfirmation>\\\\n</saml:Subject><saml:Conditions NotBefore=\"2014-06-15T00:54:52Z\" NotOnOrAfter=\"2014-06-15T01:14:52Z\">\\\\n<saml:AudienceRestriction>\\\\n<saml:Audience>http://localhost:8086</saml:Audience>\\\\n</saml:AudienceRestriction>\\\\n</saml:Conditions>\\\\n<saml:AuthnStatement AuthnInstant=\"2014-06-15T01:00:25Z\" SessionIndex=\"s2f9b4d4b453d12b40ef3905cc959cdb40579c2301\"><saml:AuthnContext><saml:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport</saml:AuthnContextClassRef></saml:AuthnContext></saml:AuthnStatement></saml:Assertion></samlp:Response>",
"domain": "dc=openam",
@ -831,9 +829,7 @@ check_output "write-json-to isn't working?" <<EOF
"log_mark": 0,
"log_comment": null,
"log_tags": null,
"log_filters": [
],
"log_filters": null,
"contextid": "ec5708a7f199678a01",
"data": "vagrant|/",
"domain": "dc=openam",
@ -987,7 +983,7 @@ run_test ${lnav_test} -n \
check_output "access_log not found within syslog file" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,c_ip,cs_method,cs_referer,cs_uri_query,cs_uri_stem,cs_user_agent,cs_username,cs_version,sc_bytes,sc_status
1,<NULL>,2015-03-24 14:02:50.000,6927348000,info,0,<NULL>,<NULL>,[],127.0.0.1,GET,<NULL>,<NULL>,/includes/js/combined-javascript.js,<NULL>,-,HTTP/1.1,65508,200
1,<NULL>,2015-03-24 14:02:50.000,6927348000,info,0,<NULL>,<NULL>,<NULL>,127.0.0.1,GET,<NULL>,<NULL>,/includes/js/combined-javascript.js,<NULL>,-,HTTP/1.1,65508,200
EOF