[spectro] add spectro view support for sql results

This commit is contained in:
Timothy Stack 2016-03-30 20:18:28 -07:00
parent cad311f557
commit 9c364bf48e
10 changed files with 324 additions and 120 deletions

View File

@ -40,6 +40,7 @@
#include "shlex.hh"
#include "command_executor.hh"
#include "db_sub_source.hh"
using namespace std;
@ -518,49 +519,17 @@ int sql_callback(sqlite3_stmt *stmt)
}
for (lpc = 0; lpc < ncols; lpc++) {
const char *value = (const char *)sqlite3_column_text(stmt, lpc);
double num_value = 0.0;
size_t value_len;
dls.push_column(value);
if (value == NULL) {
value_len = 0;
}
else {
value_len = strlen(value);
}
if (value != NULL &&
(dls.dls_headers[lpc] == "log_line" ||
dls.dls_headers[lpc] == "min(log_line)")) {
(dls.dls_headers[lpc].hm_name == "log_line" ||
dls.dls_headers[lpc].hm_name == "min(log_line)")) {
int line_number = -1;
if (sscanf(value, "%d", &line_number) == 1) {
lss.text_mark(&BM_QUERY, line_number, true);
}
}
if (value != NULL && dls.dls_headers_to_graph[lpc]) {
if (sscanf(value, "%lf", &num_value) != 1) {
num_value = 0.0;
}
chart.add_value(dls.dls_headers[lpc], num_value);
}
else if (value_len > 2 &&
((value[0] == '{' && value[value_len - 1] == '}') ||
(value[0] == '[' && value[value_len - 1] == ']'))) {
json_ptr_walk jpw;
if (jpw.parse(value, value_len) == yajl_status_ok &&
jpw.complete_parse() == yajl_status_ok) {
for (json_ptr_walk::walk_list_t::iterator iter = jpw.jpw_values.begin();
iter != jpw.jpw_values.end();
++iter) {
if (iter->wt_type == yajl_t_number &&
sscanf(iter->wt_value.c_str(), "%lf", &num_value) == 1) {
chart.add_value(iter->wt_ptr, num_value);
chart.with_attrs_for_ident(iter->wt_ptr, vc.attrs_for_ident(iter->wt_ptr));
}
}
}
}
}
return retval;

View File

@ -40,9 +40,11 @@
#include "hist_source.hh"
#include "log_vtab_impl.hh"
class db_label_source : public text_sub_source {
class db_label_source : public text_sub_source, public text_time_translator {
public:
db_label_source() { };
db_label_source() : dls_time_column_index(-1) {
};
~db_label_source() { };
@ -57,10 +59,10 @@ public:
size_t text_line_width(textview_curses &curses) {
size_t retval = 0;
for (std::vector<size_t>::iterator iter = this->dls_column_sizes.begin();
iter != this->dls_column_sizes.end();
for (std::vector<header_meta>::iterator iter = this->dls_headers.begin();
iter != this->dls_headers.end();
++iter) {
retval += *iter;
retval += iter->hm_column_size;
}
return retval;
};
@ -81,15 +83,15 @@ public:
}
for (int lpc = 0; lpc < (int)this->dls_rows[row].size();
lpc++) {
int padding = (this->dls_column_sizes[lpc] -
int padding = (this->dls_headers[lpc].hm_column_size -
strlen(this->dls_rows[row][lpc]) -
1);
if (this->dls_column_types[lpc] != SQLITE3_TEXT) {
if (this->dls_headers[lpc].hm_column_type != SQLITE3_TEXT) {
label_out.append(padding, ' ');
}
label_out.append(this->dls_rows[row][lpc]);
if (this->dls_column_types[lpc] == SQLITE3_TEXT) {
if (this->dls_headers[lpc].hm_column_type == SQLITE3_TEXT) {
label_out.append(padding, ' ');
}
label_out.append(1, ' ');
@ -104,26 +106,26 @@ public:
if (row >= (int)this->dls_rows.size()) {
return;
}
for (size_t lpc = 0; lpc < this->dls_column_sizes.size() - 1; lpc++) {
for (size_t lpc = 0; lpc < this->dls_headers.size() - 1; lpc++) {
if (row % 2 == 0) {
sa.push_back(string_attr(lr2, &view_curses::VC_STYLE, A_BOLD));
}
lr.lr_start += this->dls_column_sizes[lpc] - 1;
lr.lr_start += this->dls_headers[lpc].hm_column_size - 1;
lr.lr_end = lr.lr_start + 1;
sa.push_back(string_attr(lr, &view_curses::VC_GRAPHIC, ACS_VLINE));
lr.lr_start += 1;
}
int left = 0;
for (size_t lpc = 0; lpc < this->dls_column_sizes.size(); lpc++) {
for (size_t lpc = 0; lpc < this->dls_headers.size(); lpc++) {
const char *row_value = this->dls_rows[row][lpc];
size_t row_len = strlen(row_value);
if (this->dls_headers_to_graph[lpc]) {
if (this->dls_headers[lpc].hm_graphable) {
double num_value;
if (sscanf(row_value, "%lf", &num_value) == 1) {
this->dls_chart.chart_attrs_for_value(tc, left, this->dls_headers[lpc], num_value, sa);
this->dls_chart.chart_attrs_for_value(tc, left, this->dls_headers[lpc].hm_name, num_value, sa);
}
}
if (row_len > 2 &&
@ -148,12 +150,35 @@ public:
}
}
void push_header(const std::string &colstr, int type, bool graphable)
{
this->dls_headers.push_back(header_meta(colstr));
header_meta &hm = this->dls_headers.back();
hm.hm_column_size = colstr.length() + 1;
hm.hm_column_type = type;
hm.hm_graphable = graphable;
if (colstr == "log_time") {
this->dls_time_column_index = this->dls_headers.size() - 1;
}
}
/* TODO: add support for left and right justification... numbers should */
/* be right justified and strings should be left. */
void push_column(const char *colstr)
{
view_colors &vc = view_colors::singleton();
int index = this->dls_rows.back().size();
double num_value = 0.0;
size_t value_len;
if (colstr == NULL) {
value_len = 0;
}
else {
value_len = strlen(colstr);
}
if (colstr == NULL) {
colstr = NULL_STR;
}
@ -164,34 +189,51 @@ public:
}
}
this->dls_rows.back().push_back(colstr);
if (this->dls_rows.back().size() > this->dls_column_sizes.size()) {
this->dls_column_sizes.push_back(1);
if (index == this->dls_time_column_index) {
date_time_scanner dts;
struct timeval tv;
if (!dts.convert_to_timeval(colstr, -1, tv)) {
tv.tv_sec = -1;
tv.tv_usec = -1;
}
this->dls_time_column.push_back(tv);
}
this->dls_rows.back().push_back(colstr);
this->dls_headers[index].hm_column_size =
std::max(this->dls_headers[index].hm_column_size, strlen(colstr) + 1);
if (colstr != NULL && this->dls_headers[index].hm_graphable) {
if (sscanf(colstr, "%lf", &num_value) != 1) {
num_value = 0.0;
}
this->dls_chart.add_value(this->dls_headers[index].hm_name, num_value);
}
else if (value_len > 2 &&
((colstr[0] == '{' && colstr[value_len - 1] == '}') ||
(colstr[0] == '[' && colstr[value_len - 1] == ']'))) {
json_ptr_walk jpw;
if (jpw.parse(colstr, value_len) == yajl_status_ok &&
jpw.complete_parse() == yajl_status_ok) {
for (json_ptr_walk::walk_list_t::iterator iter = jpw.jpw_values.begin();
iter != jpw.jpw_values.end();
++iter) {
if (iter->wt_type == yajl_t_number &&
sscanf(iter->wt_value.c_str(), "%lf", &num_value) == 1) {
this->dls_chart.add_value(iter->wt_ptr, num_value);
this->dls_chart.with_attrs_for_ident(
iter->wt_ptr, vc.attrs_for_ident(iter->wt_ptr));
}
}
}
}
this->dls_column_sizes[index] =
std::max(this->dls_column_sizes[index], strlen(colstr) + 1);
};
void push_header(const std::string &colstr, int type, bool graphable)
{
int index = this->dls_headers.size();
this->dls_headers.push_back(colstr);
if (this->dls_headers.size() > this->dls_column_sizes.size()) {
this->dls_column_sizes.push_back(1);
}
this->dls_column_sizes[index] =
std::max(this->dls_column_sizes[index], colstr.length() + 1);
this->dls_column_types.push_back(type);
this->dls_headers_to_graph.push_back(graphable);
}
void clear(void)
{
void clear(void) {
this->dls_chart.clear();
this->dls_headers.clear();
this->dls_headers_to_graph.clear();
this->dls_column_types.clear();
for (size_t row = 0; row < this->dls_rows.size(); row++) {
for (size_t col = 0; col < this->dls_rows[row].size(); col++) {
if (this->dls_rows[row][col] != NULL_STR) {
@ -200,11 +242,11 @@ public:
}
}
this->dls_rows.clear();
this->dls_column_sizes.clear();
}
this->dls_time_column.clear();
};
long column_name_to_index(const std::string &name) const {
std::vector<std::string>::const_iterator iter;
std::vector<header_meta>::const_iterator iter;
iter = std::find(this->dls_headers.begin(),
this->dls_headers.end(),
@ -214,14 +256,54 @@ public:
}
return std::distance(this->dls_headers.begin(), iter);
}
};
int row_for_time(time_t time_bucket) {
std::vector<struct timeval>::iterator iter;
iter = std::lower_bound(this->dls_time_column.begin(),
this->dls_time_column.end(),
time_bucket);
if (iter != this->dls_time_column.end()) {
return std::distance(this->dls_time_column.begin(), iter);
}
return -1;
};
time_t time_for_row(int row) {
if (row < 0 || row >= this->dls_time_column.size()) {
return -1;
}
return this->dls_time_column[row].tv_sec;
};
struct header_meta {
header_meta(const std::string &name = "")
: hm_name(name),
hm_column_type(SQLITE3_TEXT),
hm_graphable(false),
hm_log_time(false),
hm_column_size(0) {
};
bool operator==(const std::string &name) const {
return this->hm_name == name;
};
const std::string hm_name;
int hm_column_type;
bool hm_graphable;
bool hm_log_time;
size_t hm_column_size;
};
stacked_bar_chart<std::string> dls_chart;
std::vector<std::string> dls_headers;
std::vector<int> dls_headers_to_graph;
std::vector<int> dls_column_types;
std::vector<header_meta> dls_headers;
std::vector<std::vector<const char *> > dls_rows;
std::vector<size_t> dls_column_sizes;
std::vector<struct timeval> dls_time_column;
int dls_time_column_index;
static const char *NULL_STR;
};
@ -267,7 +349,7 @@ public:
jpw.complete_parse() == yajl_status_ok) {
{
const std::string &header = this->dos_labels->dls_headers[col];
const std::string &header = this->dos_labels->dls_headers[col].hm_name;
this->dos_lines.push_back(" JSON Column: " + header);
retval += 1;
@ -351,19 +433,19 @@ public:
string_attrs_t &sa = value_out.get_attrs();
for (size_t lpc = 0;
lpc < this->dos_labels->dls_column_sizes.size();
lpc < this->dos_labels->dls_headers.size();
lpc++) {
int before, total_fill =
dls->dls_column_sizes[lpc] -
dls->dls_headers[lpc].length();
dls->dls_headers[lpc].hm_column_size -
dls->dls_headers[lpc].hm_name.length();
struct line_range header_range(line.length(),
line.length() +
dls->dls_column_sizes[lpc]);
dls->dls_headers[lpc].hm_column_size);
int attrs =
vc.attrs_for_ident(dls->dls_headers[lpc]) | A_UNDERLINE;
if (!this->dos_labels->dls_headers_to_graph[lpc]) {
vc.attrs_for_ident(dls->dls_headers[lpc].hm_name) | A_UNDERLINE;
if (!this->dos_labels->dls_headers[lpc].hm_graphable) {
attrs = A_UNDERLINE;
}
sa.push_back(string_attr(header_range, &view_curses::VC_STYLE,
@ -372,7 +454,7 @@ public:
before = total_fill / 2;
total_fill -= before;
line.append(before, ' ');
line.append(dls->dls_headers[lpc]);
line.append(dls->dls_headers[lpc].hm_name);
line.append(total_fill, ' ');
}

View File

@ -397,11 +397,10 @@ public:
ci.ci_stats.update(amount);
};
protected:
struct bucket_stats_t {
bucket_stats_t() :
bs_min_value(std::numeric_limits<double>::max()),
bs_max_value(0)
bs_min_value(std::numeric_limits<double>::max()),
bs_max_value(0)
{
};
@ -428,6 +427,14 @@ protected:
double bs_max_value;
};
const bucket_stats_t &get_stats_for(const T &ident) {
const chart_ident &ci = this->find_ident(ident);
return ci.ci_stats;
};
protected:
struct chart_ident {
chart_ident(const T &ident) : ci_ident(ident) { };

View File

@ -208,16 +208,17 @@ void handle_paging_key(int ch)
tc = lnav_data.ld_view_stack.top();
tc->set_needs_update();
if (ch == 'Q') {
text_time_translator *ttt = dynamic_cast<text_time_translator *>(lnav_data.ld_last_view->get_sub_source());
textview_curses &log_view = lnav_data.ld_views[LNV_LOG];
text_time_translator *src_view = dynamic_cast<text_time_translator *>(lnav_data.ld_last_view->get_sub_source());
text_time_translator *dst_view = dynamic_cast<text_time_translator *>(tc->get_sub_source());
time_t last_time = 0;
lss = &lnav_data.ld_log_source;
if (ttt != NULL) {
last_time = ttt->time_for_row(lnav_data.ld_last_view->get_top());
vis_line_t new_log_top = lss->find_from_time(last_time);
if (src_view != NULL && dst_view != NULL) {
last_time = src_view->time_for_row(lnav_data.ld_last_view->get_top());
if (last_time != -1) {
int new_top = dst_view->row_for_time(last_time);
log_view.set_top(new_log_top);
tc->set_top(vis_line_t(new_top));
}
}
}
lnav_data.ld_scroll_broadcaster.invoke(tc);
@ -244,13 +245,15 @@ void handle_paging_key(int ch)
}
else {
textview_curses *tc = lnav_data.ld_last_view;
text_time_translator *ttt = dynamic_cast<text_time_translator *>(tc->get_sub_source());
textview_curses *top_tc = lnav_data.ld_view_stack.top();
text_time_translator *dst_view = dynamic_cast<text_time_translator *>(tc->get_sub_source());
text_time_translator *src_view = dynamic_cast<text_time_translator *>(top_tc->get_sub_source());
lnav_data.ld_last_view = NULL;
if (ttt != NULL) {
time_t log_top = lnav_data.ld_top_time;
if (src_view != NULL && dst_view != NULL) {
time_t top_time = src_view->time_for_row(top_tc->get_top());
tc->set_top(vis_line_t(ttt->row_for_time(log_top)));
tc->set_top(vis_line_t(dst_view->row_for_time(top_time)));
}
ensure_view(tc);
}

View File

@ -52,6 +52,7 @@
#include "log_search_table.hh"
#include "shlex.hh"
#include "yajl/api/yajl_parse.h"
#include "db_sub_source.hh"
using namespace std;
@ -420,15 +421,15 @@ static void json_write_row(yajl_gen handle, int row)
db_label_source &dls = lnav_data.ld_db_row_source;
yajlpp_map obj_map(handle);
for (size_t col = 0; col < dls.dls_column_types.size(); col++) {
obj_map.gen(dls.dls_headers[col]);
for (size_t col = 0; col < dls.dls_headers.size(); col++) {
obj_map.gen(dls.dls_headers[col].hm_name);
if (dls.dls_rows[row][col] == db_label_source::NULL_STR) {
obj_map.gen();
continue;
}
switch (dls.dls_column_types[col]) {
switch (dls.dls_headers[col].hm_column_type) {
case SQLITE_FLOAT:
case SQLITE_INTEGER:
yajl_gen_number(handle, dls.dls_rows[row][col],
@ -530,7 +531,7 @@ static string com_save_to(string cmdline, vector<string> &args)
if (args[0] == "write-csv-to") {
std::vector<std::vector<const char *> >::iterator row_iter;
std::vector<const char *>::iterator iter;
std::vector<string>::iterator hdr_iter;
std::vector<db_label_source::header_meta>::iterator hdr_iter;
bool first = true;
for (hdr_iter = dls.dls_headers.begin();
@ -539,7 +540,7 @@ static string com_save_to(string cmdline, vector<string> &args)
if (!first) {
fprintf(outfile, ",");
}
csv_write_string(outfile, *hdr_iter);
csv_write_string(outfile, hdr_iter->hm_name);
first = false;
}
fprintf(outfile, "\n");
@ -2555,6 +2556,95 @@ public:
bool lsvs_found;
};
class db_spectro_value_source : public spectrogram_value_source {
public:
db_spectro_value_source(string colname)
: dsvs_colname(colname),
dsvs_begin_time(0),
dsvs_end_time(0),
dsvs_found(false) {
this->update_stats();
};
void update_stats() {
this->dsvs_begin_time = 0;
this->dsvs_end_time = 0;
this->dsvs_stats.clear();
db_label_source &dls = lnav_data.ld_db_row_source;
stacked_bar_chart<string> &chart = dls.dls_chart;
date_time_scanner dts;
this->dsvs_column_index = dls.column_name_to_index(this->dsvs_colname);
if (dls.dls_time_column_index == -1 ||
this->dsvs_column_index == -1 ||
!dls.dls_headers[this->dsvs_column_index].hm_graphable ||
dls.dls_rows.empty()) {
this->dsvs_found = false;
return;
}
stacked_bar_chart<string>::bucket_stats_t bs = chart.get_stats_for(this->dsvs_colname);
this->dsvs_begin_time = dls.dls_time_column.front().tv_sec;
this->dsvs_end_time = dls.dls_time_column.back().tv_sec;
this->dsvs_stats.lvs_min_value = bs.bs_min_value;
this->dsvs_stats.lvs_max_value = bs.bs_max_value;
this->dsvs_stats.lvs_count = dls.dls_rows.size();
this->dsvs_found = true;
};
void spectro_bounds(spectrogram_bounds &sb_out) {
db_label_source &dls = lnav_data.ld_db_row_source;
if (dls.text_line_count() == 0) {
return;
}
this->update_stats();
sb_out.sb_begin_time = this->dsvs_begin_time;
sb_out.sb_end_time = this->dsvs_end_time;
sb_out.sb_min_value_out = this->dsvs_stats.lvs_min_value;
sb_out.sb_max_value_out = this->dsvs_stats.lvs_max_value;
sb_out.sb_count = this->dsvs_stats.lvs_count;
};
void spectro_row(spectrogram_request &sr, spectrogram_row &row_out) {
db_label_source &dls = lnav_data.ld_db_row_source;
int begin_row = dls.row_for_time(sr.sr_begin_time);
int end_row = dls.row_for_time(sr.sr_end_time);
if (begin_row == -1) {
begin_row = 0;
}
if (end_row == -1) {
end_row = dls.dls_rows.size();
}
for (int lpc = begin_row; lpc < end_row; lpc++) {
double value = 0.0;
sscanf(dls.dls_rows[lpc][this->dsvs_column_index], "%lf", &value);
row_out.add_value(sr, value, false);
}
};
void spectro_mark(textview_curses &tc,
time_t begin_time, time_t end_time,
double range_min, double range_max) {
};
string dsvs_colname;
logline_value_stats dsvs_stats;
time_t dsvs_begin_time;
time_t dsvs_end_time;
int dsvs_column_index;
bool dsvs_found;
};
static string com_spectrogram(string cmdline, vector<string> &args)
{
string retval = "error: expecting a message field name";
@ -2562,13 +2652,10 @@ static string com_spectrogram(string cmdline, vector<string> &args)
if (args.empty()) {
args.push_back("numeric-colname");
}
else if (lnav_data.ld_view_stack.top() != &lnav_data.ld_views[LNV_LOG] &&
lnav_data.ld_view_stack.top() != &lnav_data.ld_views[LNV_SPECTRO]) {
retval = "error: this command can only be run from the log or spectrogram views";
}
else if (args.size() == 2) {
intern_string_t colname = intern_string::lookup(remaining_args(cmdline, args));
string colname = remaining_args(cmdline, args);
spectrogram_source &ss = lnav_data.ld_spectro_source;
bool found = false;
ss.ss_granularity = ZOOM_LEVELS[lnav_data.ld_zoom_level];
if (ss.ss_value_source != NULL) {
@ -2577,18 +2664,39 @@ static string com_spectrogram(string cmdline, vector<string> &args)
}
ss.invalidate();
auto_ptr<log_spectro_value_source> lsvs(new log_spectro_value_source(colname));
if (lnav_data.ld_view_stack.top() == &lnav_data.ld_views[LNV_DB]) {
auto_ptr<db_spectro_value_source> dsvs(
new db_spectro_value_source(colname));
if (!lsvs->lsvs_found) {
retval = "error: unknown numeric message field -- " + colname.to_string();
if (!dsvs->dsvs_found) {
retval = "error: unknown numeric message field -- " + colname;
}
else {
ss.ss_value_source = dsvs.release();
found = true;
}
}
else {
ss.ss_value_source = lsvs.release();
auto_ptr<log_spectro_value_source> lsvs(
new log_spectro_value_source(intern_string::lookup(colname)));
if (!lsvs->lsvs_found) {
retval = "error: unknown numeric message field -- " + colname;
}
else {
ss.ss_value_source = lsvs.release();
found = true;
}
}
if (found) {
ensure_view(&lnav_data.ld_views[LNV_SPECTRO]);
lnav_data.ld_rl_view->set_alt_value(HELP_MSG_2(z, Z, "to zoom in/out"));
lnav_data.ld_rl_view->set_alt_value(
HELP_MSG_2(z, Z, "to zoom in/out"));
retval = "info: visualizing field -- " + colname.to_string();
retval = "info: visualizing field -- " + colname;
}
}

View File

@ -251,6 +251,16 @@ struct tm *secs2tm(time_t *tim_p, struct tm *res);
extern const char *std_time_fmt[];
inline
bool operator<(const struct timeval &left, time_t right) {
return left.tv_sec < right;
};
inline
bool operator<(time_t left, const struct timeval &right) {
return left < right.tv_sec;
};
struct date_time_scanner {
date_time_scanner() : dts_keep_base_tz(false),
dts_local_time(false),
@ -322,6 +332,20 @@ struct date_time_scanner {
struct timeval &tv_out,
bool convert_local = true);
bool convert_to_timeval(const char *time_src,
size_t time_len,
struct timeval &tv_out) {
struct exttm tm;
if (time_len == -1) {
time_len = strlen(time_src);
}
if (this->scan(time_src, time_len, NULL, &tm, tv_out) != NULL) {
return true;
}
return false;
};
bool convert_to_timeval(const std::string &time_src,
struct timeval &tv_out) {
struct exttm tm;

View File

@ -191,7 +191,8 @@ public:
this->ldt_format_impl->extract(lf, line, values);
values.push_back(logline_value(instance_name, this->ldt_instance));
values.back().lv_column = next_column++;
logline_value &lv = values.back();
lv.lv_column = next_column++;
for (data_parser::element_list_t::iterator pair_iter =
this->ldt_pairs.begin();
pair_iter != this->ldt_pairs.end();

View File

@ -445,7 +445,6 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
break;
case logline_value::VALUE_JSON:
case logline_value::VALUE_TEXT: {
sqlite3_result_text(ctx,
lv_iter->text_value(),
lv_iter->text_length(),

View File

@ -75,7 +75,7 @@ public:
* source of data for a text view.
*/
class logfile_sub_source
: public text_sub_source {
: public text_sub_source, public text_time_translator {
public:
static bookmark_type_t BM_ERRORS;
@ -344,6 +344,14 @@ public:
return this->find_from_time(tv);
};
time_t time_for_row(int row) {
return this->find_line(this->at(vis_line_t(row)))->get_time();
};
int row_for_time(time_t time_bucket) {
return this->find_from_time(time_bucket);
};
content_line_t at(vis_line_t vl) {
return this->lss_index[this->lss_filtered_index[vl]];
};

View File

@ -431,7 +431,10 @@ public:
return;
}
time_t diff = std::max((time_t) 1, sb.sb_end_time - sb.sb_begin_time + 1);
time_t grain_begin_time = rounddown(sb.sb_begin_time, this->ss_granularity);
time_t grain_end_time = roundup_size(sb.sb_end_time, this->ss_granularity);
time_t diff = std::max((time_t) 1, grain_end_time - grain_begin_time);
this->ss_cached_line_count =
(diff + this->ss_granularity - 1) / this->ss_granularity;