Robot
/
lnav
mirror of https://github.com/tstack/lnav.git
2
0
Fork 0
Code Issues Releases Wiki Activity

[spectro] improve error messages and fix some issues with invalidation

This commit is contained in:
Timothy Stack 2022-07-06 11:56:29 -07:00
parent 812fa08055
commit d6c9b0036a
19 changed files with 250 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
release/loggen.py
View File

@ -159,6 +159,6 @@ while True:
fp.write(next(gen))
#if random.uniform(0.0, 1.0) < 0.010:
# fp.truncate(0)
time.sleep(random.uniform(0.05, 0.10))
time.sleep(random.uniform(0.01, 0.02))
#if random.uniform(0.0, 1.0) < 0.001:
# os.remove(fname)

9
src/base/attr_line.hh
View File

@ -501,6 +501,15 @@ public:
return *this;
}
attr_line_t& append_quoted(const attr_line_t& al)
{
this->al_string.append("\u201c");
this->append(al);
this->al_string.append("\u201d");
return *this;
}
template<typename S>
attr_line_t& append_quoted(S s)
{

54
src/base/itertools.hh
View File

@ -93,6 +93,16 @@ struct mapper {
F m_func;
};
template<typename F>
struct flat_mapper {
F fm_func;
};
template<typename F>
struct for_eacher {
F fe_func;
};
template<typename R, typename T>
struct folder {
R f_func;
@ -243,6 +253,20 @@ map(F func)
return details::mapper<F>{func};
}
template<typename F>
inline details::flat_mapper<F>
flat_map(F func)
{
return details::flat_mapper<F>{func};
}
template<typename F>
inline details::for_eacher<F>
for_each(F func)
{
return details::for_eacher<F>{func};
}
inline auto
deref()
{
@ -534,6 +558,36 @@ operator|(T in, const lnav::itertools::details::sorted& sorter)
return in;
}
template<typename T,
typename F,
std::enable_if_t<lnav::func::is_invocable<F, T>::value, int> = 0>
auto
operator|(nonstd::optional<T> in,
const lnav::itertools::details::flat_mapper<F>& mapper) ->
typename std::remove_const_t<typename std::remove_reference_t<
decltype(lnav::func::invoke(mapper.fm_func, in.value()))>>
{
if (!in) {
return nonstd::nullopt;
}
return lnav::func::invoke(mapper.fm_func, in.value());
}
template<typename T,
typename F,
std::enable_if_t<lnav::func::is_invocable<F, T>::value, int> = 0>
void
operator|(nonstd::optional<T> in,
const lnav::itertools::details::for_eacher<F>& eacher)
{
if (!in) {
return;
}
lnav::func::invoke(eacher.fe_func, in.value());
}
template<typename T,
typename F,
std::enable_if_t<lnav::func::is_invocable<F, T>::value, int> = 0>

1
src/db_sub_source.cc
View File

@ -205,6 +205,7 @@ db_label_source::push_column(const char* colstr)
}
if (!this->dls_time_column.empty() && tv < this->dls_time_column.back())
{
this->dls_time_column_invalidated_at = this->dls_time_column.size();
this->dls_time_column_index = -1;
this->dls_time_column.clear();
} else {

26
src/db_sub_source.hh
View File

@ -43,21 +43,20 @@ class db_label_source
: public text_sub_source
, public text_time_translator {
public:
~db_label_source()
{
this->clear();
}
~db_label_source() override { this->clear(); }
bool has_log_time_column() const { return !this->dls_time_column.empty(); }
size_t text_line_count() { return this->dls_rows.size(); }
size_t text_line_count() override { return this->dls_rows.size(); }
size_t text_size_for_line(textview_curses& tc, int line, line_flags_t flags)
size_t text_size_for_line(textview_curses& tc,
int line,
line_flags_t flags) override
{
return this->text_line_width(tc);
}
size_t text_line_width(textview_curses& curses)
size_t text_line_width(textview_curses& curses) override
{
size_t retval = 0;
@ -70,9 +69,11 @@ public:
void text_value_for_line(textview_curses& tc,
int row,
std::string& label_out,
line_flags_t flags);
line_flags_t flags) override;
void text_attrs_for_line(textview_curses& tc, int row, string_attrs_t& sa);
void text_attrs_for_line(textview_curses& tc,
int row,
string_attrs_t& sa) override;
void push_header(const std::string& colstr, int type, bool graphable);
@ -83,9 +84,10 @@ public:
nonstd::optional<size_t> column_name_to_index(
const std::string& name) const;
nonstd::optional<vis_line_t> row_for_time(struct timeval time_bucket);
nonstd::optional<vis_line_t> row_for_time(
struct timeval time_bucket) override;
nonstd::optional<struct timeval> time_for_row(vis_line_t row)
nonstd::optional<struct timeval> time_for_row(vis_line_t row) override
{
if ((row < 0_vl) || (((size_t) row) >= this->dls_time_column.size())) {
return nonstd::nullopt;
@ -106,7 +108,6 @@ public:
int hm_column_type{SQLITE3_TEXT};
unsigned int hm_sub_type{0};
bool hm_graphable{false};
bool hm_log_time{false};
size_t hm_column_size{0};
};
@ -116,6 +117,7 @@ public:
std::vector<struct timeval> dls_time_column;
std::vector<size_t> dls_cell_width;
int dls_time_column_index{-1};
nonstd::optional<size_t> dls_time_column_invalidated_at;
static const char* NULL_STR;
};

2
src/internals/cmd-ref.rst
View File

@ -1240,7 +1240,7 @@
:spectrogram *field-name*
^^^^^^^^^^^^^^^^^^^^^^^^^
Visualize the given message field using a spectrogram
Visualize the given message field or database column using a spectrogram
**Parameters**
* **field-name\*** --- The name of the numeric field to visualize.

27
src/lnav_commands.cc
View File

@ -4410,7 +4410,7 @@ com_spectrogram(exec_context& ec,
} else if (ec.ec_dry_run) {
retval = "";
} else if (args.size() == 2) {
std::string colname = remaining_args(cmdline, args);
auto colname = remaining_args(cmdline, args);
auto& ss = *lnav_data.ld_spectro_source;
bool found = false;
@ -4422,17 +4422,17 @@ com_spectrogram(exec_context& ec,
ss.invalidate();
if (*lnav_data.ld_view_stack.top() == &lnav_data.ld_views[LNV_DB]) {
std::unique_ptr<db_spectro_value_source> dsvs(
new db_spectro_value_source(colname));
auto dsvs = std::make_unique<db_spectro_value_source>(colname);
if (!dsvs->dsvs_error_msg.empty()) {
return ec.make_error("{}", dsvs->dsvs_error_msg);
if (dsvs->dsvs_error_msg) {
return Err(
dsvs->dsvs_error_msg.value().with_snippets(ec.ec_source));
}
ss.ss_value_source = dsvs.release();
found = true;
} else {
std::unique_ptr<log_spectro_value_source> lsvs(
new log_spectro_value_source(intern_string::lookup(colname)));
auto lsvs = std::make_unique<log_spectro_value_source>(
intern_string::lookup(colname));
if (!lsvs->lsvs_found) {
return ec.make_error("unknown numeric message field -- {}",
@ -5594,7 +5594,8 @@ readline_context::command_t STD_COMMANDS[] = {
com_spectrogram,
help_text(":spectrogram")
.with_summary("Visualize the given message field using a spectrogram")
.with_summary("Visualize the given message field or database column "
"using a spectrogram")
.with_parameter(help_text(
"field-name", "The name of the numeric field to visualize."))
.with_example(
@ -5605,12 +5606,10 @@ readline_context::command_t STD_COMMANDS[] = {
help_text(":quit").with_summary("Quit lnav")}};
static std::unordered_map<char const*, std::vector<char const*>> aliases
= {{"quit", {"q", "q!"}},
{"write-table-to",
{
"write-cols-to",
}}};
static std::unordered_map<char const*, std::vector<char const*>> aliases = {
{"quit", {"q", "q!"}},
{"write-table-to", {"write-cols-to"}},
};
void
init_lnav_commands(readline_context::command_map_t& cmd_map)

120
src/spectro_impls.cc
View File

@ -29,6 +29,7 @@
#include "spectro_impls.hh"
#include "base/itertools.hh"
#include "lnav.hh"
#include "logfile_sub_source.hh"
@ -43,40 +44,50 @@ public:
std::string& value_out,
line_flags_t flags) override
{
this->fss_delegate->text_value_for_line(
tc, this->fss_lines[line], value_out, flags);
this->fss_lines | lnav::itertools::nth(line)
| lnav::itertools::for_each([&](const auto row) {
this->fss_delegate->text_value_for_line(
tc, *row, value_out, flags);
});
}
size_t text_size_for_line(textview_curses& tc,
int line,
line_flags_t raw) override
{
return this->fss_delegate->text_size_for_line(
tc, this->fss_lines[line], raw);
return this->fss_lines | lnav::itertools::nth(line)
| lnav::itertools::map([&](const auto row) {
return this->fss_delegate->text_size_for_line(tc, *row, raw);
})
| lnav::itertools::unwrap_or(size_t{0});
}
void text_attrs_for_line(textview_curses& tc,
int line,
string_attrs_t& value_out) override
{
this->fss_delegate->text_attrs_for_line(
tc, this->fss_lines[line], value_out);
this->fss_lines | lnav::itertools::nth(line)
| lnav::itertools::for_each([&](const auto row) {
this->fss_delegate->text_attrs_for_line(tc, *row, value_out);
});
}
nonstd::optional<vis_line_t> row_for_time(
struct timeval time_bucket) override
{
return dynamic_cast<text_time_translator*>(this->fss_delegate)
->row_for_time(time_bucket);
return this->fss_time_delegate->row_for_time(time_bucket);
}
nonstd::optional<struct timeval> time_for_row(vis_line_t row) override
{
return dynamic_cast<text_time_translator*>(this->fss_delegate)
->time_for_row(this->fss_lines[row]);
return this->fss_lines | lnav::itertools::nth(row)
| lnav::itertools::flat_map([this](const auto row) {
return this->fss_time_delegate->time_for_row(*row);
});
}
text_sub_source* fss_delegate;
text_time_translator* fss_time_delegate;
std::vector<vis_line_t> fss_lines;
};
@ -202,6 +213,7 @@ log_spectro_value_source::spectro_row(spectrogram_request& sr,
.value_or(lss.text_line_count());
retval->fss_delegate = &lss;
retval->fss_time_delegate = &lss;
for (const auto& msg_info : lss.window_at(begin_line, end_line)) {
const auto& ll = msg_info.get_logline();
if (ll.get_time() >= sr.sr_end_time) {
@ -311,36 +323,103 @@ db_spectro_value_source::update_stats()
this->dsvs_end_time = 0;
this->dsvs_stats.clear();
db_label_source& dls = lnav_data.ld_db_row_source;
stacked_bar_chart<std::string>& chart = dls.dls_chart;
auto& dls = lnav_data.ld_db_row_source;
auto& chart = dls.dls_chart;
date_time_scanner dts;
this->dsvs_column_index = dls.column_name_to_index(this->dsvs_colname);
if (!dls.has_log_time_column()) {
this->dsvs_error_msg
= "no 'log_time' column found or not in ascending order, "
"unable to create spectrogram";
if (dls.dls_time_column_invalidated_at) {
static const auto order_by_help
= attr_line_t()
.append(lnav::roles::keyword("ORDER BY"))
.append(" ")
.append(lnav::roles::variable("log_time"))
.append(" ")
.append(lnav::roles::keyword("ASC"));
this->dsvs_error_msg
= lnav::console::user_message::error(
"Cannot generate spectrogram for database results")
.with_reason(
attr_line_t()
.append("The ")
.append_quoted(lnav::roles::variable("log_time"))
.append(
" column is not in ascending order between "
"rows {} and {}",
dls.dls_time_column_invalidated_at.value()
- 1,
dls.dls_time_column_invalidated_at.value()))
.with_note(
attr_line_t("An ascending ")
.append_quoted(lnav::roles::variable("log_time"))
.append(
" column is needed to render a spectrogram"))
.with_help(attr_line_t("Add an ")
.append_quoted(order_by_help)
.append(" clause to your ")
.append(lnav::roles::keyword("SELECT"))
.append(" statement"));
} else {
this->dsvs_error_msg
= lnav::console::user_message::error(
"Cannot generate spectrogram for database results")
.with_reason(
attr_line_t()
.append("No ")
.append_quoted(lnav::roles::variable("log_time"))
.append(" column found in the result set"))
.with_note(
attr_line_t("An ascending ")
.append_quoted(lnav::roles::variable("log_time"))
.append(
" column is needed to render a spectrogram"))
.with_help(
attr_line_t("Include a ")
.append_quoted(lnav::roles::variable("log_time"))
.append(" column in your ")
.append(" statement. Use an ")
.append(lnav::roles::keyword("AS"))
.append(
" directive to alias a computed timestamp"));
}
return;
}
if (!this->dsvs_column_index) {
this->dsvs_error_msg = "unknown column -- " + this->dsvs_colname;
this->dsvs_error_msg
= lnav::console::user_message::error(
"Cannot generate spectrogram for database results")
.with_reason(attr_line_t("unknown column -- ")
.append_quoted(lnav::roles::variable(
this->dsvs_colname)))
.with_help("Expecting a numeric column to visualize");
return;
}
if (!dls.dls_headers[this->dsvs_column_index.value()].hm_graphable) {
this->dsvs_error_msg = "column is not numeric -- " + this->dsvs_colname;
this->dsvs_error_msg
= lnav::console::user_message::error(
"Cannot generate spectrogram for database results")
.with_reason(attr_line_t()
.append_quoted(lnav::roles::variable(
this->dsvs_colname))
.append(" is not a numeric column"))
.with_help("Only numeric columns can be visualized");
return;
}
if (dls.dls_rows.empty()) {
this->dsvs_error_msg = "empty result set";
this->dsvs_error_msg
= lnav::console::user_message::error(
"Cannot generate spectrogram for database results")
.with_reason("Result set is empty");
return;
}
stacked_bar_chart<std::string>::bucket_stats_t bs
= chart.get_stats_for(this->dsvs_colname);
auto bs = chart.get_stats_for(this->dsvs_colname);
this->dsvs_begin_time = dls.dls_time_column.front().tv_sec;
this->dsvs_end_time = dls.dls_time_column.back().tv_sec;
@ -392,6 +471,7 @@ db_spectro_value_source::spectro_row(spectrogram_request& sr,
auto retval = std::make_unique<filtered_sub_source>();
retval->fss_delegate = &dls;
retval->fss_time_delegate = &dls;
auto begin_row = dls.row_for_time({sr.sr_begin_time, 0}).value_or(0_vl);
auto end_row = dls.row_for_time({sr.sr_end_time, 0})
.value_or(dls.dls_rows.size());

2
src/spectro_impls.hh
View File

@ -81,7 +81,7 @@ public:
time_t dsvs_begin_time{0};
time_t dsvs_end_time{0};
nonstd::optional<size_t> dsvs_column_index;
std::string dsvs_error_msg;
nonstd::optional<lnav::console::user_message> dsvs_error_msg;
};
#endif

45
src/spectro_source.cc
View File

@ -210,24 +210,30 @@ spectrogram_source::list_value_for_overlay(const listview_curses& lv,
+ this->ss_cursor_column.value() * sr.sr_column_size;
auto range_max = range_min + sr.sr_column_size;
const auto desc = fmt::format(
FMT_STRING("{} value{} in the range {:.6Lg}-{:.6Lg} "),
bucket.rb_counter,
bucket.rb_counter == 1 ? "" : "s",
range_min,
range_max);
auto desc = attr_line_t()
.append(lnav::roles::number(
fmt::to_string(bucket.rb_counter)))
.append(lnav::roles::comment(fmt::format(
FMT_STRING(" value{} in the range "),
bucket.rb_counter == 1 ? "" : "s")))
.append(lnav::roles::number(
fmt::format(FMT_STRING("{:.2Lf}"), range_min)))
.append(lnav::roles::comment("-"))
.append(lnav::roles::number(
fmt::format(FMT_STRING("{:.2Lf}"), range_max)))
.append(" ");
auto mark_offset = this->ss_cursor_column.value();
auto mark_is_before = true;
if (this->ss_cursor_column.value() + desc.size() > width) {
mark_offset -= desc.size();
if (this->ss_cursor_column.value() + desc.length() + 1 > width) {
mark_offset -= desc.length();
mark_is_before = false;
}
value_out.append(mark_offset, ' ');
if (mark_is_before) {
value_out.append("\u25b2 ");
}
value_out.append(lnav::roles::comment(desc));
value_out.append(desc);
if (!mark_is_before) {
value_out.append("\u25b2 ");
}
@ -257,7 +263,10 @@ spectrogram_source::list_value_for_overlay(const listview_curses& lv,
this->cache_bounds();
if (this->ss_cached_line_count == 0) {
value_out.append(lnav::roles::error("error: no log data"));
value_out
.append(lnav::roles::error("error: no data available, use the "))
.append_quoted(lnav::roles::keyword(":spectrogram"))
.append(lnav::roles::error(" command to visualize numeric data"));
return true;
}
@ -433,12 +442,23 @@ spectrogram_source::text_attrs_for_line(textview_curses& tc,
}
}
void
spectrogram_source::reset_details_source()
{
if (this->ss_details_view != nullptr) {
this->ss_details_view->set_sub_source(this->ss_no_details_source);
}
this->ss_details_source.reset();
}
void
spectrogram_source::cache_bounds()
{
if (this->ss_value_source == nullptr) {
this->ss_cached_bounds.sb_count = 0;
this->ss_cached_bounds.sb_begin_time = 0;
this->ss_cursor_column = nonstd::nullopt;
this->reset_details_source();
return;
}
@ -454,6 +474,8 @@ spectrogram_source::cache_bounds()
if (sb.sb_count == 0) {
this->ss_cached_line_count = 0;
this->ss_cursor_column = nonstd::nullopt;
this->reset_details_source();
return;
}
@ -533,7 +555,8 @@ spectrogram_source::text_is_row_selectable(textview_curses& tc, vis_line_t row)
void
spectrogram_source::text_selection_changed(textview_curses& tc)
{
if (this->ss_value_source == nullptr) {
if (this->ss_value_source == nullptr || this->text_line_count() == 0) {
this->ss_cursor_column = nonstd::nullopt;
return;
}

2
src/spectro_source.hh
View File

@ -171,6 +171,8 @@ public:
const spectrogram_row& load_row(const listview_curses& lv, int row);
void reset_details_source();
textview_curses* ss_details_view;
text_sub_source* ss_no_details_source;
exec_context* ss_exec_context;

2
src/view_curses.hh
View File

@ -291,6 +291,8 @@ private:
/** Private constructor that initializes the member fields. */
view_colors();
view_colors(const view_colors&) = delete;
struct dyn_pair {
int dp_color_pair;
};

1
src/view_helpers.cc
View File

@ -1100,6 +1100,7 @@ void
hist_index_delegate::index_complete(logfile_sub_source& lss)
{
this->hid_view.reload_data();
lnav_data.ld_views[LNV_SPECTRO].reload_data();
}
static std::vector<breadcrumb::possibility>

3
test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out
View File

@ -1416,7 +1416,8 @@ lnav@googlegroups.com[1] support@lnav.org[2]
:spectrogram field-name
══════════════════════════════════════════════════════════════════════
Visualize the given message field using a spectrogram
Visualize the given message field or database column using a
spectrogram
Parameter
field-name The name of the numeric field to
visualize.

8
test/expected/test_sql.sh_26c0d94d7837792144f2d0f866fb3c12a0bd410d.err
View File

@ -1,6 +1,6 @@
✘ error: no 'log_time' column found or not in ascending order, unable to create spectrogram
✘ error: Cannot generate spectrogram for database results
reason: No “log_time” column found in the result set
 --> command-option:2
 | :spectrogram sc_bytes 
 = help: :spectrogram field-name
══════════════════════════════════════════════════════════════════════
Visualize the given message field using a spectrogram
 = note: An ascending “log_time” column is needed to render a spectrogram
 = help: Include a “log_time” column in your statement. Use an AS directive to alias a computed timestamp

7
test/expected/test_sql.sh_4090f96ea11a344c1e2939211da778992dab47d8.err
View File

@ -1,6 +1,5 @@
✘ error: unknown column -- sc_byes
✘ error: Cannot generate spectrogram for database results
reason: unknown column -- “sc_byes”
 --> command-option:2
 | :spectrogram sc_byes 
 = help: :spectrogram field-name
══════════════════════════════════════════════════════════════════════
Visualize the given message field using a spectrogram
 = help: Expecting a numeric column to visualize

7
test/expected/test_sql.sh_57427f3c4b4ec785ffff7c5802c10db0d3e547cf.err
View File

@ -1,6 +1,5 @@
✘ error: column is not numeric -- c_ip
✘ error: Cannot generate spectrogram for database results
reason: “c_ip” is not a numeric column
 --> command-option:2
 | :spectrogram c_ip 
 = help: :spectrogram field-name
══════════════════════════════════════════════════════════════════════
Visualize the given message field using a spectrogram
 = help: Only numeric columns can be visualized

2
test/expected/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.out
View File

@ -1,4 +1,4 @@
Min: 0   1-23   24-48   49+ Max: 291690
 Thu Nov 03 00:15:00               
▲ 70 values in the range 0-3788.18 
▲ 70 values in the range 0.00-3788.18
 Thu Nov 03 00:20:00

8
test/expected/test_sql.sh_9ceccab07fbf7130bffe3c201c710719e4a3e9af.err
View File

@ -1,6 +1,6 @@
✘ error: no 'log_time' column found or not in ascending order, unable to create spectrogram
✘ error: Cannot generate spectrogram for database results
reason: The “log_time” column is not in ascending order between rows 1 and 2
 --> command-option:2
 | :spectrogram sc_bytes 
 = help: :spectrogram field-name
══════════════════════════════════════════════════════════════════════
Visualize the given message field using a spectrogram
 = note: An ascending “log_time” column is needed to render a spectrogram
 = help: Add an “ORDER BY log_time ASC” clause to your SELECT statement