Robot
/
lnav
mirror of https://github.com/tstack/lnav.git
2
0
Fork 0
Code Issues Releases Wiki Activity

[files] make file scanning async

This commit is contained in:
Timothy Stack 2020-10-28 21:22:56 -07:00
parent dfd18a4be5
commit 7b77a612c2
22 changed files with 435 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/CMakeLists.txt
View File

@ -349,6 +349,7 @@ add_library(diag STATIC
doc_status_source.hh
elem_to_json.hh
base/enum_util.hh
base/future_util.hh
field_overlay_source.hh
file_vtab.hh
files_sub_source.hh

11
src/auto_fd.hh
View File

@ -90,7 +90,8 @@ public:
* @param af The source of the file descriptor.
*/
auto_fd(auto_fd && af)
: af_fd(af.release()) { };
: af_fd(af.release()) {
};
/**
* Const copy constructor. The file descriptor from the source will be
@ -115,7 +116,7 @@ public:
};
/** @return The file descriptor as a plain integer. */
operator int(void) const { return this->af_fd; };
operator int() const { return this->af_fd; };
/**
* Replace the current descriptor with the given one. The current
@ -150,7 +151,7 @@ public:
*
* @return A pointer to the internal integer.
*/
int *out(void)
int *out()
{
this->reset();
return &this->af_fd;
@ -161,7 +162,7 @@ public:
*
* @return The file descriptor.
*/
int release(void)
int release()
{
int retval = this->af_fd;
@ -172,7 +173,7 @@ public:
/**
* @return The file descriptor.
*/
int get(void) const
int get() const
{
return this->af_fd;
};

1
src/base/Makefile.am
View File

@ -9,6 +9,7 @@ noinst_LIBRARIES = libbase.a
noinst_HEADERS = \
enum_util.hh \
file_range.hh \
future_util.hh \
humanize.hh \
intern_string.hh \
is_utf8.hh \

69
src/base/future_util.hh Normal file
View File

@ -0,0 +1,69 @@
/**
* Copyright (c) 2020, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef lnav_future_util_hh
#define lnav_future_util_hh
#include <future>
template<class T>
std::future<std::decay_t<T>> make_ready_future( T&& t ) {
std::promise<std::decay_t<T>> pr;
auto r = pr.get_future();
pr.set_value(std::forward<T>(t));
return r;
}
template<typename T>
class future_queue {
public:
future_queue(std::function<void(const T&)> processor)
: fq_processor(processor) {};
~future_queue() {
this->pop_to();
}
void push_back(std::future<T>&& f) {
this->fq_deque.emplace_back(std::move(f));
this->pop_to(8);
}
void pop_to(size_t size = 0) {
while (this->fq_deque.size() > size) {
this->fq_processor(this->fq_deque.front().get());
this->fq_deque.pop_front();
}
}
std::function<void(const T&)> fq_processor;
std::deque<std::future<T>> fq_deque;
};
#endif

15
src/base/lnav_log.cc
View File

@ -109,6 +109,18 @@ static pthread_mutex_t lnav_log_mutex = PTHREAD_MUTEX_INITIALIZER;
std::vector<log_state_dumper*> log_state_dumper::DUMPER_LIST;
std::vector<log_crash_recoverer*> log_crash_recoverer::CRASH_LIST;
struct thid {
static uint32_t COUNTER;
thid() : t_id(COUNTER++) {}
uint32_t t_id;
};
uint32_t thid::COUNTER = 0;
thread_local thid current_thid;
static struct {
size_t lr_length;
off_t lr_frag_start;
@ -278,7 +290,7 @@ void log_msg(lnav_log_level_t level, const char *src_file, int line_number,
auto line = log_alloc();
prefix_size = snprintf(
line, MAX_LOG_LINE_SIZE,
"%4d-%02d-%02dT%02d:%02d:%02d.%03d %s %s:%d ",
"%4d-%02d-%02dT%02d:%02d:%02d.%03d %s t%u %s:%d ",
localtm.tm_year + 1900,
localtm.tm_mon + 1,
localtm.tm_mday,
@ -287,6 +299,7 @@ void log_msg(lnav_log_level_t level, const char *src_file, int line_number,
localtm.tm_sec,
(int)(curr_time.tv_usec / 1000),
LEVEL_NAMES[to_underlying(level)],
current_thid.t_id,
src_file,
line_number);
rc = vsnprintf(&line[prefix_size], MAX_LOG_LINE_SIZE - prefix_size,

4
src/command_executor.cc
View File

@ -672,7 +672,7 @@ void execute_init_commands(exec_context &ec, vector<pair<Result<string, string>,
lnav_data.ld_pt_search.substr(3),
lnav_data.ld_pt_min_time,
lnav_data.ld_pt_max_time));
lnav_data.ld_file_names[lnav_data.ld_pt_search]
lnav_data.ld_active_files.fc_file_names[lnav_data.ld_pt_search]
.with_fd(pt->copy_fd());
lnav_data.ld_curl_looper.add_request(pt.release());
#endif
@ -791,7 +791,7 @@ future<string> pipe_callback(exec_context &ec, const string &cmdline, auto_fd &f
sizeof(desc), "[%d] Output of %s",
exec_count++,
cmdline.c_str());
lnav_data.ld_file_names[desc]
lnav_data.ld_active_files.fc_file_names[desc]
.with_fd(pp->get_fd())
.with_include_in_session(false)
.with_detect_format(false);

14
src/file_vtab.cc
View File

@ -68,11 +68,11 @@ CREATE TABLE lnav_file (
};
iterator begin() {
return lnav_data.ld_files.begin();
return lnav_data.ld_active_files.fc_files.begin();
}
iterator end() {
return lnav_data.ld_files.end();
return lnav_data.ld_active_files.fc_files.end();
}
int get_column(const cursor &vc, sqlite3_context *ctx, int col) {
@ -138,7 +138,7 @@ CREATE TABLE lnav_file (
int64_t lines,
int64_t time_offset,
bool visible) {
auto lf = lnav_data.ld_files[rowid];
auto lf = lnav_data.ld_active_files.fc_files[rowid];
struct timeval tv = {
(int) (time_offset / 1000LL),
(int) (time_offset / (1000LL * 1000LL)),
@ -152,15 +152,15 @@ CREATE TABLE lnav_file (
"real file paths cannot be updated, only symbolic ones");
}
auto iter = lnav_data.ld_file_names.find(lf->get_filename());
auto iter = lnav_data.ld_active_files.fc_file_names.find(lf->get_filename());
if (iter != lnav_data.ld_file_names.end()) {
if (iter != lnav_data.ld_active_files.fc_file_names.end()) {
auto loo = iter->second;
lnav_data.ld_file_names.erase(iter);
lnav_data.ld_active_files.fc_file_names.erase(iter);
loo.loo_include_in_session = true;
lnav_data.ld_file_names[path] = loo;
lnav_data.ld_active_files.fc_file_names[path] = loo;
lf->set_filename(path);
init_session();

14
src/files_sub_source.cc
View File

@ -52,12 +52,12 @@ bool files_sub_source::list_input_handle_key(listview_curses &lv, int ch)
case KEY_ENTER:
case '\r': {
if (lnav_data.ld_files.empty()) {
if (lnav_data.ld_active_files.fc_files.empty()) {
return true;
}
auto& lss = lnav_data.ld_log_source;
auto &lf = lnav_data.ld_files[lv.get_selection()];
auto &lf = lnav_data.ld_active_files.fc_files[lv.get_selection()];
if (!lf->is_visible()) {
lf->show();
@ -86,11 +86,11 @@ bool files_sub_source::list_input_handle_key(listview_curses &lv, int ch)
}
case ' ': {
if (lnav_data.ld_files.empty()) {
if (lnav_data.ld_active_files.fc_files.empty()) {
return true;
}
auto &lf = lnav_data.ld_files[lv.get_selection()];
auto &lf = lnav_data.ld_active_files.fc_files[lv.get_selection()];
lf->set_visibility(!lf->is_visible());
auto top_view = *lnav_data.ld_view_stack.top();
auto tss = top_view->get_sub_source();
@ -115,7 +115,7 @@ void files_sub_source::list_input_handle_scroll_out(listview_curses &lv)
size_t files_sub_source::text_line_count()
{
return lnav_data.ld_files.size();
return lnav_data.ld_active_files.fc_files.size();
}
size_t files_sub_source::text_line_width(textview_curses &curses)
@ -127,7 +127,7 @@ void files_sub_source::text_value_for_line(textview_curses &tc, int line,
std::string &value_out,
text_sub_source::line_flags_t flags)
{
const auto &lf = lnav_data.ld_files[line];
const auto &lf = lnav_data.ld_active_files.fc_files[line];
char start_time[64] = "", end_time[64] = "";
if (lf->get_format() != nullptr) {
@ -148,7 +148,7 @@ void files_sub_source::text_attrs_for_line(textview_curses &tc, int line,
auto &vcolors = view_colors::singleton();
bool selected = lnav_data.ld_mode == LNM_FILES && line == tc.get_selection();
int bg = selected ? COLOR_WHITE : COLOR_BLACK;
auto &lf = lnav_data.ld_files[line];
auto &lf = lnav_data.ld_active_files.fc_files[line];
chtype visible = lf->is_visible() ? ACS_DIAMOND : ' ';
value_out.emplace_back(line_range{2, 3}, &view_curses::VC_GRAPHIC, visible);

7
src/filter_status_source.cc
View File

@ -220,15 +220,16 @@ size_t filter_help_status_source::statusview_fields()
"Disable Filtering" :
"Enable Filtering");
}
} else if (lnav_data.ld_mode == LNM_FILES) {
if (lnav_data.ld_files.empty()) {
} else if (lnav_data.ld_mode == LNM_FILES &&
lnav_data.ld_session_loaded) {
if (lnav_data.ld_active_files.fc_files.empty()) {
this->fss_help.clear();
return;
}
auto &lv = lnav_data.ld_files_view;
auto sel = lv.get_selection();
auto &lf = lnav_data.ld_files[sel];
auto &lf = lnav_data.ld_active_files.fc_files[sel];
this->fss_help.set_value(" %s%s %s",
ENABLE_HELP,

376
src/lnav.cc
View File

@ -127,6 +127,7 @@
#include "regexp_vtab.hh"
#include "fstat_vtab.hh"
#include "textfile_highlighters.hh"
#include "base/future_util.hh"
#ifdef HAVE_LIBCURL
#include <curl/curl.h>
@ -154,6 +155,7 @@
#endif
using namespace std;
using namespace std::literals::chrono_literals;
static multimap<lnav_flags_t, string> DEFAULT_FILES;
@ -233,17 +235,6 @@ static std::vector<std::string> DEFAULT_DB_KEY_NAMES = {
const static size_t MAX_STDIN_CAPTURE_SIZE = 10 * 1024 * 1024;
static void regenerate_unique_file_names()
{
unique_path_generator upg;
for (const auto& lf : lnav_data.ld_files) {
upg.add_source(shared_ptr<logfile>(lf));
}
upg.generate();
}
bool setup_logline_table(exec_context &ec)
{
// Hidden columns don't show up in the table_info pragma.
@ -380,7 +371,7 @@ public:
};
void logfile_indexing(logfile &lf, off_t off, size_t total)
void logfile_indexing(const shared_ptr<logfile>& lf, off_t off, size_t total)
{
static sig_atomic_t index_counter = 0;
@ -401,26 +392,23 @@ public:
}
if (!lnav_data.ld_looping) {
throw logfile::error(lf.get_filename(), EINTR);
throw logfile::error(lf->get_filename(), EINTR);
}
};
private:
void do_update(logfile &lf)
void do_update(const shared_ptr<logfile>& lf)
{
lnav_data.ld_top_source.update_time();
for (auto &sc : lnav_data.ld_status) {
sc.do_update();
}
if (!lnav_data.ld_session_loaded && lnav_data.ld_mode == LNM_FILES) {
auto iter = std::find_if(lnav_data.ld_files.begin(),
lnav_data.ld_files.end(),
[&lf](auto elem) {
return elem.get() == &lf;
});
auto iter = std::find(lnav_data.ld_active_files.fc_files.begin(),
lnav_data.ld_active_files.fc_files.end(), lf);
if (iter != lnav_data.ld_files.end()) {
auto index = std::distance(lnav_data.ld_files.begin(),
if (iter != lnav_data.ld_active_files.fc_files.end()) {
auto index = std::distance(lnav_data.ld_active_files.fc_files.begin(),
iter);
lnav_data.ld_files_view.set_selection(vis_line_t(index));
lnav_data.ld_files_view.reload_data();
@ -496,15 +484,7 @@ public:
void closed_file(const shared_ptr<logfile> &lf) {
log_info("closed text file: %s", lf->get_filename().c_str());
if (!lf->is_valid_filename()) {
lnav_data.ld_file_names.erase(lf->get_filename());
}
auto file_iter = find(lnav_data.ld_files.begin(),
lnav_data.ld_files.end(),
lf);
lnav_data.ld_files.erase(file_iter);
regenerate_unique_file_names();
lnav_data.ld_active_files.close_file(lf);
};
void promote_file(const shared_ptr<logfile> &lf) {
@ -581,20 +561,21 @@ void rebuild_indexes()
}
}
for (auto file_iter = lnav_data.ld_files.begin();
file_iter != lnav_data.ld_files.end(); ) {
for (auto file_iter = lnav_data.ld_active_files.fc_files.begin();
file_iter != lnav_data.ld_active_files.fc_files.end(); ) {
auto lf = *file_iter;
if ((!lf->exists() || lf->is_closed())) {
log_info("closed log file: %s", lf->get_filename().c_str());
if (!lf->is_valid_filename()) {
lnav_data.ld_file_names.erase(lf->get_filename());
lnav_data.ld_active_files.fc_file_names.erase(lf->get_filename());
}
lnav_data.ld_text_source.remove(lf);
lnav_data.ld_log_source.remove_file(lf);
file_iter = lnav_data.ld_files.erase(file_iter);
file_iter = lnav_data.ld_active_files.fc_files.erase(file_iter);
lnav_data.ld_active_files.fc_files_generation += 1;
regenerate_unique_file_names();
lnav_data.ld_active_files.regenerate_unique_file_names();
}
else {
++file_iter;
@ -619,7 +600,7 @@ void rebuild_indexes()
unordered_map<string, list<shared_ptr<logfile>>> id_to_files;
bool reload = false;
for (const auto &lf : lnav_data.ld_files) {
for (const auto &lf : lnav_data.ld_active_files.fc_files) {
if (!lf->is_visible()) {
continue;
}
@ -691,7 +672,7 @@ static bool append_default_files(lnav_flags_t flag)
else {
logfile_open_options default_loo;
lnav_data.ld_file_names[abspath.in()] = default_loo;
lnav_data.ld_active_files.fc_file_names[abspath.in()] = default_loo;
}
}
else if (stat(path.c_str(), &st) == 0) {
@ -955,6 +936,54 @@ struct same_file {
const struct stat &sf_stat;
};
void file_collection::close_file(const std::shared_ptr<logfile> &lf)
{
if (!lf->is_valid_filename()) {
this->fc_file_names.erase(lf->get_filename());
}
auto file_iter = find(this->fc_files.begin(),
this->fc_files.end(),
lf);
if (file_iter != this->fc_files.end()) {
this->fc_files.erase(file_iter);
this->fc_files_generation += 1;
}
this->regenerate_unique_file_names();
}
void file_collection::regenerate_unique_file_names()
{
unique_path_generator upg;
for (const auto& lf : this->fc_files) {
upg.add_source(shared_ptr<logfile>(lf));
}
upg.generate();
}
void file_collection::merge(const file_collection& other)
{
this->fc_name_to_errors.insert(other.fc_name_to_errors.begin(),
other.fc_name_to_errors.end());
this->fc_file_names.insert(other.fc_file_names.begin(),
other.fc_file_names.end());
if (!other.fc_files.empty()) {
this->fc_files.insert(this->fc_files.end(),
other.fc_files.begin(),
other.fc_files.end());
this->fc_files_generation += 1;
}
for (auto& pair : other.fc_renamed_files) {
pair.first->set_filename(pair.second);
}
this->fc_closed_files.insert(other.fc_closed_files.begin(),
other.fc_closed_files.end());
this->fc_other_files.insert(other.fc_other_files.begin(),
other.fc_other_files.end());
}
/**
* Try to load the given file as a log file. If the file has not already been
* loaded, it will be loaded. If the file has already been loaded, the file
@ -964,15 +993,17 @@ struct same_file {
* @param fd An already-opened descriptor for 'filename'.
* @param required Specifies whether or not the file must exist and be valid.
*/
static bool watch_logfile(string filename, logfile_open_options &loo, bool required)
std::future<file_collection>
file_collection::watch_logfile(const string& filename, logfile_open_options &loo, bool required)
{
static loading_observer obs;
file_collection retval;
struct stat st;
int rc;
bool retval = false;
if (lnav_data.ld_closed_files.count(filename)) {
return retval;
if (this->fc_closed_files.count(filename)) {
return make_ready_future(retval);
}
if (loo.loo_fd != -1) {
@ -986,13 +1017,12 @@ static bool watch_logfile(string filename, logfile_open_options &loo, bool requi
if (S_ISDIR(st.st_mode) && lnav_data.ld_flags & LNF_RECURSIVE) {
string wilddir = filename + "/*";
if (lnav_data.ld_file_names.find(wilddir) ==
lnav_data.ld_file_names.end()) {
if (this->fc_file_names.find(wilddir) == this->fc_file_names.end()) {
logfile_open_options default_loo;
lnav_data.ld_file_names[wilddir] = default_loo;
retval.fc_file_names[wilddir] = default_loo;
}
return retval;
return make_ready_future(retval);
}
if (!S_ISREG(st.st_mode)) {
if (required) {
@ -1000,40 +1030,40 @@ static bool watch_logfile(string filename, logfile_open_options &loo, bool requi
errno = EINVAL;
}
else {
return retval;
return make_ready_future(retval);
}
}
}
if (rc == -1) {
if (required) {
throw logfile::error(filename, errno);
}
else{
return retval;
retval.fc_name_to_errors[filename] = strerror(errno);
}
return make_ready_future(retval);
}
auto file_iter = find_if(lnav_data.ld_files.begin(),
lnav_data.ld_files.end(),
auto file_iter = find_if(this->fc_files.begin(),
this->fc_files.end(),
same_file(st));
if (file_iter == lnav_data.ld_files.end()) {
if (find(lnav_data.ld_other_files.begin(),
lnav_data.ld_other_files.end(),
filename) == lnav_data.ld_other_files.end()) {
if (file_iter == this->fc_files.end()) {
if (this->fc_other_files.find(filename) != this->fc_other_files.end()) {
return make_ready_future(retval);
}
return std::async(std::launch::async, [filename, &loo]() {
file_format_t ff = detect_file_format(filename);
file_collection retval;
switch (ff) {
case file_format_t::FF_SQLITE_DB:
lnav_data.ld_other_files.push_back(filename);
attach_sqlite_db(lnav_data.ld_db.in(), filename);
retval = true;
retval.fc_other_files[filename] = "SQLite Database";
break;
case file_format_t::FF_ARCHIVE: {
archive_manager::walk_archive_files(filename,
[&filename](const auto& tmp_path,
const auto& entry) {
retval.fc_other_files[filename] = "Archive";
archive_manager::walk_archive_files(
filename, [&filename, &retval](const auto& tmp_path,
const auto& entry) {
auto ext = entry.path().extension();
if (ext == ".jar" || ext == ".war" || ext == ".zip") {
return;
@ -1054,43 +1084,48 @@ static bool watch_logfile(string filename, logfile_open_options &loo, bool requi
filename.c_str(),
entry.path().c_str());
// TODO add some heuristics for hiding files
lnav_data.ld_file_names[entry.path().string()] =
retval.fc_file_names[entry.path().string()] =
logfile_open_options()
.with_filename(custom_name.string())
.with_visibility(is_visible)
.with_non_utf_visibility(false)
.with_visible_size_limit(128 * 1024);
});
lnav_data.ld_other_files.emplace_back(filename);
retval = true;
break;
}
default:
/* It's a new file, load it in. */
shared_ptr<logfile> lf = make_shared<logfile>(filename, loo);
log_info("loading new file: filename=%s",
filename.c_str());
lf->set_logfile_observer(&obs);
lnav_data.ld_files.push_back(lf);
lnav_data.ld_text_source.push_back(lf);
regenerate_unique_file_names();
/* It's a new file, load it in. */
try {
shared_ptr<logfile> lf = make_shared<logfile>(filename,
loo);
retval = true;
lf->set_logfile_observer(&obs);
retval.fc_files.push_back(lf);
} catch (logfile::error& e) {
retval.fc_name_to_errors[filename] = e.what();
}
break;
}
return retval;
});
}
else {
auto lf = *file_iter;
if (lf->is_valid_filename() && lf->get_filename() != filename) {
/* The file is already loaded, but has been found under a different
* name. We just need to update the stored file name.
*/
retval.fc_renamed_files.emplace_back(lf, filename);
}
}
else if ((*file_iter)->is_valid_filename()) {
/* The file is already loaded, but has been found under a different
* name. We just need to update the stored file name.
*/
(*file_iter)->set_filename(filename);
}
return retval;
return make_ready_future(retval);
}
/**
@ -1099,14 +1134,18 @@ static bool watch_logfile(string filename, logfile_open_options &loo, bool requi
* @param path The glob pattern to expand.
* @param required Passed to watch_logfile.
*/
static void expand_filename(const string& path, logfile_open_options &loo, bool required)
file_collection file_collection::expand_filename(const string& path, logfile_open_options &loo, bool required)
{
static_root_mem<glob_t, globfree> gl;
file_collection retval;
if (is_url(path.c_str())) {
return;
return retval;
}
else if (glob(path.c_str(), GLOB_NOCHECK, nullptr, gl.inout()) == 0) {
future_queue<file_collection> fq([&retval](auto& fc) {
retval.merge(fc);
});
int lpc;
if (gl->gl_pathc == 1 /*&& gl.gl_matchc == 0*/) {
@ -1132,55 +1171,68 @@ static void expand_filename(const string& path, logfile_open_options &loo, bool
}
}
else if (required || access(abspath.in(), R_OK) == 0) {
watch_logfile(abspath.in(), loo, required);
fq.push_back(watch_logfile(abspath.in(), loo, required));
}
}
}
}
bool rescan_files(bool required)
{
map<string, logfile_open_options>::iterator iter;
bool retval = false;
for (iter = lnav_data.ld_file_names.begin();
iter != lnav_data.ld_file_names.end();
iter++) {
if (iter->second.loo_fd == -1) {
expand_filename(iter->first, iter->second, required);
if (lnav_data.ld_flags & LNF_ROTATED) {
string path = iter->first + ".*";
expand_filename(path, iter->second, false);
}
} else {
retval = retval ||
watch_logfile(iter->first, iter->second, required);
}
}
for (auto file_iter = lnav_data.ld_files.begin();
file_iter != lnav_data.ld_files.end(); ) {
auto lf = *file_iter;
if ((!lf->exists() || lf->is_closed())) {
log_info("Log file no longer exists or is closed: %s",
lf->get_filename().c_str());
return true;
}
else {
++file_iter;
}
}
return retval;
}
file_collection file_collection::rescan_files(bool required)
{
file_collection retval;
future_queue<file_collection> fq([&retval](auto& fc) {
retval.merge(fc);
});
for (auto& pair : this->fc_file_names) {
if (pair.second.loo_fd == -1) {
retval.merge(this->expand_filename(pair.first, pair.second, required));
if (lnav_data.ld_flags & LNF_ROTATED) {
string path = pair.first + ".*";
retval.merge(this->expand_filename(path, pair.second, false));
}
} else {
fq.push_back(watch_logfile(pair.first, pair.second, required));
}
}
fq.pop_to();
return retval;
}
bool update_active_files(const file_collection& new_files)
{
for (const auto& lf : new_files.fc_files) {
lnav_data.ld_text_source.push_back(lf);
}
lnav_data.ld_active_files.merge(new_files);
if (!new_files.fc_files.empty()) {
lnav_data.ld_active_files.regenerate_unique_file_names();
}
return true;
}
bool rescan_files(bool req)
{
bool done = false;
do {
auto fc = lnav_data.ld_active_files.rescan_files(req);
update_active_files(fc);
done = fc.fc_file_names.empty();
} while (!done);
return true;
}
class lnav_behavior : public mouse_behavior {
public:
lnav_behavior() {};
void mouse_event(int button, bool release, int x, int y)
void mouse_event(int button, bool release, int x, int y) override
{
textview_curses *tc = *(lnav_data.ld_view_stack.top());
struct mouse_event me;
@ -1557,7 +1609,6 @@ static void looper()
set_scroll_action(sb.get_functor());
lnav_data.ld_views[lpc].set_search_action(
textview_curses::action(update_hits));
using std::placeholders::_1;
lnav_data.ld_views[lpc].tc_state_event_handler = [](auto &&tc) {
lnav_data.ld_bottom_source.update_search_term(tc);
};
@ -1657,6 +1708,17 @@ static void looper()
lnav_data.ld_mode = LNM_FILES;
timer.start_fade(index_counter, 1);
log_debug("rescan started");
file_collection active_copy;
active_copy.merge(lnav_data.ld_active_files);
future<file_collection> rescan_future =
std::async(std::launch::async,
&file_collection::rescan_files,
active_copy,
false);
bool initial_rescan_completed = false;
while (lnav_data.ld_looping) {
vector<struct pollfd> pollfds;
struct timeval to = { 0, 333000 };
@ -1669,7 +1731,21 @@ static void looper()
layout_views();
rescan_files();
if (rescan_future.wait_for(0s) == std::future_status::ready) {
auto new_files = rescan_future.get();
if (!initial_rescan_completed &&
new_files.fc_file_names.empty()) {
initial_rescan_completed = true;
}
update_active_files(new_files);
active_copy.clear();
active_copy.merge(lnav_data.ld_active_files);
rescan_future = std::async(std::launch::async,
&file_collection::rescan_files,
active_copy,
false);
}
rebuild_indexes();
lnav_data.ld_view_stack.do_update();
@ -1774,8 +1850,8 @@ static void looper()
}
static bool initial_build = false;
if (!initial_build || timer.fade_diff(index_counter) == 0) {
if (initial_rescan_completed &&
(!initial_build || timer.fade_diff(index_counter) == 0)) {
if (lnav_data.ld_mode == LNM_PAGING) {
timer.start_fade(index_counter, 1);
}
@ -1802,7 +1878,7 @@ static void looper()
}
if (!initial_build &&
lnav_data.ld_log_source.text_line_count() == 0 &&
!lnav_data.ld_other_files.empty()) {
!lnav_data.ld_active_files.fc_other_files.empty()) {
ensure_view(&lnav_data.ld_views[LNV_SCHEMA]);
}
@ -1812,7 +1888,7 @@ static void looper()
}
if (lnav_data.ld_log_source.text_line_count() > 0 ||
lnav_data.ld_text_source.text_line_count() > 0 ||
!lnav_data.ld_other_files.empty()) {
!lnav_data.ld_active_files.fc_other_files.empty()) {
initial_build = true;
}
@ -1895,7 +1971,7 @@ static void looper()
if (lnav_data.ld_view_stack.vs_views.empty() ||
(lnav_data.ld_view_stack.vs_views.size() == 1 &&
starting_view_stack_size == 2 &&
lnav_data.ld_file_names.size() ==
lnav_data.ld_active_files.fc_file_names.size() ==
lnav_data.ld_text_source.size())) {
lnav_data.ld_looping = false;
}
@ -2400,7 +2476,7 @@ int main(int argc, char *argv[])
retval = EXIT_FAILURE;
}
}
} while (lnav_data.ld_file_names.empty() &&
} while (lnav_data.ld_active_files.fc_file_names.empty() &&
change_to_parent_dir());
if (chdir(start_dir) == -1) {
@ -2471,13 +2547,13 @@ int main(int argc, char *argv[])
else if (is_url(argv[lpc])) {
unique_ptr<url_loader> ul(new url_loader(argv[lpc]));
lnav_data.ld_file_names[argv[lpc]]
lnav_data.ld_active_files.fc_file_names[argv[lpc]]
.with_fd(ul->copy_fd());
lnav_data.ld_curl_looper.add_request(ul.release());
}
#endif
else if (is_glob(argv[lpc])) {
lnav_data.ld_file_names[argv[lpc]] = default_loo;
lnav_data.ld_active_files.fc_file_names[argv[lpc]] = default_loo;
}
else if (stat(argv[lpc], &st) == -1) {
fprintf(stderr,
@ -2486,6 +2562,13 @@ int main(int argc, char *argv[])
strerror(errno));
retval = EXIT_FAILURE;
}
else if (access(argv[lpc], R_OK) == -1) {
fprintf(stderr,
"Cannot read file: %s -- %s\n",
argv[lpc],
strerror(errno));
retval = EXIT_FAILURE;
}
else if (S_ISFIFO(st.st_mode)) {
auto_fd fifo_fd;
@ -2511,7 +2594,7 @@ int main(int argc, char *argv[])
snprintf(desc, sizeof(desc),
"FIFO [%d]",
lnav_data.ld_fifo_counter++);
lnav_data.ld_file_names[desc]
lnav_data.ld_active_files.fc_file_names[desc]
.with_fd(fifo_out_fd);
lnav_data.ld_pipers.push_back(fifo_piper);
}
@ -2526,17 +2609,16 @@ int main(int argc, char *argv[])
if (dir_wild[dir_wild.size() - 1] == '/') {
dir_wild.resize(dir_wild.size() - 1);
}
lnav_data.ld_file_names[dir_wild + "/*"] = default_loo;
lnav_data.ld_active_files.fc_file_names[dir_wild + "/*"] = default_loo;
}
else {
lnav_data.ld_file_names[abspath.in()] = default_loo;
lnav_data.ld_active_files.fc_file_names[abspath.in()] = default_loo;
}
}
if (lnav_data.ld_flags & LNF_CHECK_CONFIG) {
rescan_files(true);
for (auto &ld_file : lnav_data.ld_files) {
auto lf = ld_file;
for (auto &lf : lnav_data.ld_active_files.fc_files) {
logfile::rebuild_result_t rebuild_result;
do {
@ -2622,7 +2704,7 @@ int main(int argc, char *argv[])
stdin_reader = make_shared<piper_proc>(
STDIN_FILENO, lnav_data.ld_flags & LNF_TIMESTAMP, stdin_out_fd);
lnav_data.ld_file_names["stdin"]
lnav_data.ld_active_files.fc_file_names["stdin"]
.with_fd(stdin_out_fd)
.with_include_in_session(false);
lnav_data.ld_pipers.push_back(stdin_reader);
@ -2634,7 +2716,7 @@ int main(int argc, char *argv[])
}
}
if (lnav_data.ld_file_names.empty() &&
if (lnav_data.ld_active_files.fc_file_names.empty() &&
lnav_data.ld_commands.empty() &&
lnav_data.ld_pt_search.empty() &&
!(lnav_data.ld_flags & LNF_HELP)) {
@ -2647,8 +2729,6 @@ int main(int argc, char *argv[])
}
else {
try {
rescan_files(true);
log_info("startup: %s", VCS_PACKAGE_STRING);
log_host_info();
log_info("Libraries:");
@ -2676,9 +2756,8 @@ int main(int argc, char *argv[])
log_info(" %s", cmd_iter->c_str());
}
log_info(" files:");
for (auto file_iter =
lnav_data.ld_file_names.begin();
file_iter != lnav_data.ld_file_names.end();
for (auto file_iter = lnav_data.ld_active_files.fc_file_names.begin();
file_iter != lnav_data.ld_active_files.fc_file_names.end();
++file_iter) {
log_info(" %s", file_iter->first.c_str());
}
@ -2688,6 +2767,17 @@ int main(int argc, char *argv[])
textview_curses *log_tc, *text_tc, *tc;
bool found_error = false;
rescan_files(true);
if (!lnav_data.ld_active_files.fc_name_to_errors.empty()) {
for (const auto& pair : lnav_data.ld_active_files.fc_name_to_errors) {
fprintf(stderr,
"error: unable to read file: %s -- %s\n",
pair.first.c_str(),
pair.second.c_str());
}
return EXIT_FAILURE;
}
init_session();
lnav_data.ld_exec_context.ec_output_stack.back() = stdout;
alerter::singleton().enabled(false);

32
src/lnav.hh
View File

@ -215,6 +215,32 @@ struct key_repeat_history {
};
};
struct file_collection {
std::map<std::string, std::string> fc_name_to_errors;
std::map<std::string, logfile_open_options> fc_file_names;
std::vector<std::shared_ptr<logfile>> fc_files;
int fc_files_generation{0};
std::vector<std::pair<std::shared_ptr<logfile>, std::string>>
fc_renamed_files;
std::set<std::string> fc_closed_files;
std::map<std::string, std::string> fc_other_files;
void clear() {
this->fc_name_to_errors.clear();
this->fc_file_names.clear();
this->fc_files.clear();
this->fc_closed_files.clear();
this->fc_other_files.clear();
}
file_collection rescan_files(bool required = false);
file_collection expand_filename(const std::string& path, logfile_open_options &loo, bool required);
std::future<file_collection>
watch_logfile(const std::string& filename, logfile_open_options &loo, bool required);
void merge(const file_collection &other);
void close_file(const std::shared_ptr<logfile> &lf);
void regenerate_unique_file_names();
};
struct _lnav_data {
std::map<std::string, std::list<session_pair_t>> ld_session_id;
time_t ld_session_time;
@ -227,10 +253,7 @@ struct _lnav_data {
bool ld_cmd_init_done;
bool ld_session_loaded;
std::vector<ghc::filesystem::path> ld_config_paths;
std::map<std::string, logfile_open_options> ld_file_names;
std::vector<std::shared_ptr<logfile>> ld_files;
std::list<std::string> ld_other_files;
std::set<std::string> ld_closed_files;
file_collection ld_active_files;
std::list<std::pair<std::string, int> > ld_files_to_front;
std::string ld_pt_search;
time_t ld_pt_min_time;
@ -344,6 +367,7 @@ void layout_views();
bool setup_logline_table(exec_context &ec);
bool rescan_files(bool required = false);
bool update_active_files(const file_collection& new_files);
void wait_for_children();

26
src/lnav_commands.cc
View File

@ -94,7 +94,7 @@ static string refresh_pt_search()
}
#ifdef HAVE_LIBCURL
for (const auto &lf : lnav_data.ld_files) {
for (const auto &lf : lnav_data.ld_active_files.fc_files) {
if (startswith(lf->get_filename(), "pt:")) {
lf->close();
}
@ -109,7 +109,7 @@ static string refresh_pt_search()
lnav_data.ld_pt_search.substr(3),
lnav_data.ld_pt_min_time,
lnav_data.ld_pt_max_time));
lnav_data.ld_file_names[lnav_data.ld_pt_search]
lnav_data.ld_active_files.fc_file_names[lnav_data.ld_pt_search]
.with_fd(pt->copy_fd());
lnav_data.ld_curl_looper.add_request(pt.release());
@ -1804,8 +1804,8 @@ static Result<string, string> com_open(exec_context &ec, string cmdline, vector<
}
}
auto file_iter = lnav_data.ld_files.begin();
for (; file_iter != lnav_data.ld_files.end(); ++file_iter) {
auto file_iter = lnav_data.ld_active_files.fc_files.begin();
for (; file_iter != lnav_data.ld_active_files.fc_files.end(); ++file_iter) {
auto lf = *file_iter;
if (lf->get_filename() == fn) {
@ -1820,7 +1820,7 @@ static Result<string, string> com_open(exec_context &ec, string cmdline, vector<
}
}
}
if (file_iter == lnav_data.ld_files.end()) {
if (file_iter == lnav_data.ld_active_files.fc_files.end()) {
logfile_open_options default_loo;
auto_mem<char> abspath;
struct stat st;
@ -1832,7 +1832,7 @@ static Result<string, string> com_open(exec_context &ec, string cmdline, vector<
if (!ec.ec_dry_run) {
auto ul = make_unique<url_loader>(fn);
lnav_data.ld_file_names[fn]
lnav_data.ld_active_files.fc_file_names[fn]
.with_fd(ul->copy_fd());
lnav_data.ld_curl_looper.add_request(ul.release());
lnav_data.ld_files_to_front.emplace_back(fn, top);
@ -1877,7 +1877,7 @@ static Result<string, string> com_open(exec_context &ec, string cmdline, vector<
snprintf(desc, sizeof(desc),
"FIFO [%d]",
lnav_data.ld_fifo_counter++);
lnav_data.ld_file_names[desc]
lnav_data.ld_active_files.fc_file_names[desc]
.with_fd(fifo_out_fd);
lnav_data.ld_pipers.push_back(fifo_piper);
}
@ -1899,7 +1899,7 @@ static Result<string, string> com_open(exec_context &ec, string cmdline, vector<
fn);
}
else if (access(fn.c_str(), R_OK) == -1) {
return ec.make_error("error: cannot read file {} -- {}", fn,
return ec.make_error("cannot read file {} -- {}", fn,
strerror(errno));
}
else {
@ -1990,9 +1990,9 @@ static Result<string, string> com_open(exec_context &ec, string cmdline, vector<
lnav_data.ld_files_to_front.end(),
files_to_front.begin(),
files_to_front.end());
lnav_data.ld_file_names.insert(file_names.begin(), file_names.end());
lnav_data.ld_active_files.fc_file_names.insert(file_names.begin(), file_names.end());
for (const auto &fn : closed_files) {
lnav_data.ld_closed_files.erase(fn);
lnav_data.ld_active_files.fc_closed_files.erase(fn);
}
}
@ -2049,8 +2049,8 @@ static Result<string, string> com_close(exec_context &ec, string cmdline, vector
if (is_url(fn.c_str())) {
lnav_data.ld_curl_looper.close_request(fn);
}
lnav_data.ld_file_names.erase(fn);
lnav_data.ld_closed_files.insert(fn);
lnav_data.ld_active_files.fc_file_names.erase(fn);
lnav_data.ld_active_files.fc_closed_files.insert(fn);
retval = "info: closed -- " + fn;
}
}
@ -2104,7 +2104,7 @@ static Result<string, string> com_file_visibility(exec_context &ec, string cmdli
lexer.split(args, ec.create_resolver());
args.erase(args.begin());
for (const auto &lf : lnav_data.ld_files) {
for (const auto &lf : lnav_data.ld_active_files.fc_files) {
if (lf.get() == nullptr) {
continue;
}

1
src/lnav_util.hh
View File

@ -43,6 +43,7 @@
#include "spookyhash/SpookyV2.h"
#include <future>
#include <string>
#include <vector>
#include <sstream>

2
src/log_actions.cc
View File

@ -135,7 +135,7 @@ static string execute_action(log_data_helper &ldh,
sizeof(desc), "[%d] Output of %s",
exec_count++,
action.ad_cmdline[0].c_str());
lnav_data.ld_file_names[desc]
lnav_data.ld_active_files.fc_file_names[desc]
.with_fd(pp->get_fd());
lnav_data.ld_files_to_front.push_back({ desc, 0 });
}

4
src/log_format.cc
View File

@ -504,6 +504,10 @@ log_format::scan_result_t external_log_format::scan(logfile &lf,
yajl_handle handle = this->jlf_yajl_handle.in();
json_log_userdata jlu(sbr);
if (!this->lf_specialized && dst.size() >= 3) {
return log_format::SCAN_NO_MATCH;
}
if (li.li_partial) {
log_debug("skipping partial line at offset %d", li.li_file_range.fr_offset);
return log_format::SCAN_INCOMPLETE;

27
src/logfile.cc
View File

@ -59,8 +59,9 @@ logfile::logfile(const string &filename, logfile_open_options &loo)
{
require(!filename.empty());
this->lf_options = loo;
memset(&this->lf_stat, 0, sizeof(this->lf_stat));
if (loo.loo_fd == -1) {
if (this->lf_options.loo_fd == -1) {
char resolved_path[PATH_MAX];
errno = 0;
@ -76,14 +77,14 @@ logfile::logfile(const string &filename, logfile_open_options &loo)
throw error(filename, EINVAL);
}
if ((loo.loo_fd = open(resolved_path, O_RDONLY)) == -1) {
if ((this->lf_options.loo_fd = open(resolved_path, O_RDONLY)) == -1) {
throw error(filename, errno);
}
loo.loo_fd.close_on_exec();
this->lf_options.loo_fd.close_on_exec();
log_info("Creating logfile: fd=%d; size=%" PRId64 "; mtime=%" PRId64 "; filename=%s",
(int) loo.loo_fd,
(int) this->lf_options.loo_fd,
(long long) this->lf_stat.st_size,
(long long) this->lf_stat.st_mtime,
filename.c_str());
@ -91,7 +92,7 @@ logfile::logfile(const string &filename, logfile_open_options &loo)
this->lf_valid_filename = true;
}
else {
log_perror(fstat(loo.loo_fd, &this->lf_stat));
log_perror(fstat(this->lf_options.loo_fd, &this->lf_stat));
this->lf_valid_filename = false;
}
@ -101,10 +102,9 @@ logfile::logfile(const string &filename, logfile_open_options &loo)
}
this->lf_content_id = hash_string(this->lf_filename);
this->lf_line_buffer.set_fd(loo.loo_fd);
this->lf_line_buffer.set_fd(this->lf_options.loo_fd);
this->lf_index.reserve(INDEX_RESERVE_INCREMENT);
this->lf_options = loo;
this->lf_is_visible = loo.loo_is_visible;
ensure(this->invariant());
@ -291,8 +291,11 @@ logfile::rebuild_result_t logfile::rebuild_index()
if (st.st_size < this->lf_stat.st_size ||
(this->lf_stat.st_size == st.st_size &&
this->lf_stat.st_mtime != st.st_mtime)) {
log_info("overwritten file detected, closing -- %s",
this->lf_filename.c_str());
log_info("overwritten file detected, closing -- %s new: %" PRId64
"/%" PRId64 " old: %" PRId64 "/%" PRId64,
this->lf_filename.c_str(),
st.st_size, st.st_mtime,
this->lf_stat.st_size, this->lf_stat.st_mtime);
this->close();
return RR_NO_NEW_LINES;
}
@ -426,7 +429,7 @@ logfile::rebuild_result_t logfile::rebuild_index()
if (this->lf_logfile_observer != nullptr) {
this->lf_logfile_observer->logfile_indexing(
*this,
this->shared_from_this(),
this->lf_line_buffer.get_read_offset(li.li_file_range.next_offset()),
st.st_size);
}
@ -563,7 +566,7 @@ void logfile::reobserve_from(iterator iter)
if (this->lf_logfile_observer != nullptr) {
this->lf_logfile_observer->logfile_indexing(
*this, offset, this->size());
this->shared_from_this(), offset, this->size());
}
this->read_line(iter).then([this, iter](auto sbr) {
@ -578,7 +581,7 @@ void logfile::reobserve_from(iterator iter)
}
if (this->lf_logfile_observer != nullptr) {
this->lf_logfile_observer->logfile_indexing(
*this, this->size(), this->size());
this->shared_from_this(), this->size(), this->size());
this->lf_logline_observer->logline_eof(*this);
}
}

16
src/logfile.hh
View File

@ -71,7 +71,9 @@ public:
* @param off The current offset in the file being processed.
* @param total The total size of the file.
*/
virtual void logfile_indexing(logfile &lf, off_t off, size_t total) = 0;
virtual void logfile_indexing(const std::shared_ptr<logfile>& lf,
off_t off,
size_t total) = 0;
};
struct logfile_open_options {
@ -135,7 +137,9 @@ struct logfile_activity {
/**
* Container for the lines in a log file and some metadata.
*/
class logfile : public unique_path_source {
class logfile :
public unique_path_source,
public std::enable_shared_from_this<logfile> {
public:
class error : public std::exception {
@ -182,9 +186,11 @@ public:
/** @param filename The new filename for this log file. */
void set_filename(const std::string &filename)
{
this->lf_filename = filename;
ghc::filesystem::path p(filename);
this->lf_basename = p.filename();
if (this->lf_filename != filename) {
this->lf_filename = filename;
ghc::filesystem::path p(filename);
this->lf_basename = p.filename();
}
};
const std::string &get_content_id() const { return this->lf_content_id; };

2
src/readline_callbacks.cc
View File

@ -624,7 +624,7 @@ static void rl_callback_int(void *dummy, readline_curses *rc, bool is_alt)
"Output of %s (%s)",
path_and_args.c_str(),
timestamp);
lnav_data.ld_file_names[desc]
lnav_data.ld_active_files.fc_file_names[desc]
.with_fd(fd_copy)
.with_include_in_session(false)
.with_detect_format(false);

2
src/readline_possibilities.cc
View File

@ -274,7 +274,7 @@ void add_file_possibilities()
rc->clear_possibilities(LNM_COMMAND, "visible-files");
rc->clear_possibilities(LNM_COMMAND, "hidden-files");
for (const auto& lf : lnav_data.ld_files) {
for (const auto& lf : lnav_data.ld_active_files.fc_files) {
if (lf.get() == nullptr) {
continue;
}

4
src/session_data.cc
View File

@ -302,7 +302,7 @@ static nonstd::optional<std::string> compute_session_id()
context.Init(0, 0);
hash_updater updater(&context);
for (auto &ld_file_name : lnav_data.ld_file_names) {
for (auto &ld_file_name : lnav_data.ld_active_files.fc_file_names) {
if (!ld_file_name.second.loo_include_in_session) {
continue;
}
@ -1276,7 +1276,7 @@ static void save_session_with_id(const std::string session_id)
{
yajlpp_array file_list(handle);
for (auto &ld_file_name : lnav_data.ld_file_names) {
for (auto &ld_file_name : lnav_data.ld_active_files.fc_file_names) {
file_list.gen(ld_file_name.first);
}
}

22
test/test_cmds.sh
View File

@ -1,5 +1,17 @@
#! /bin/bash
run_test ${lnav_test} -n -d /tmp/lnav.err \
-c ":goto 0" \
-c ":close" \
-c ":goto 0" \
"${test_dir}/logfile_access_log.*"
check_output "close not sticking" <<EOF
10.112.81.15 - - [15/Feb/2013:06:00:31 +0000] "-" 400 0 "-" "-"
EOF
run_test ${lnav_test} -n -d /tmp/lnav.err \
-c ":goto 0" \
-c ":hide-file" \
@ -498,16 +510,6 @@ check_output "open non-existent is not working" <<EOF
EOF
run_test ${lnav_test} -n \
-c ":goto 0" \
-c ":close" \
-c ":goto 0" \
"${test_dir}/logfile_access_log.*"
check_output "close not sticking" <<EOF
10.112.81.15 - - [15/Feb/2013:06:00:31 +0000] "-" 400 0 "-" "-"
EOF
run_test ${lnav_test} -n \
-c ":goto 1" \

4
test/test_logfile.sh
View File

@ -54,12 +54,12 @@ chmod ugo-r unreadable.log
run_test ${lnav_test} -n unreadable.log
sed -e "s|/.*/unreadable.log|unreadable.log|g" `test_err_filename` \
sed -e "s|/.*/unreadable.log|unreadable.log|g" `test_err_filename` | head -1 \
> test_logfile.unreadable.out
mv test_logfile.unreadable.out `test_err_filename`
check_error_output "able to read an unreadable log file?" <<EOF
error: Permission denied -- 'unreadable.log'
Cannot read file: unreadable.log -- Permission denied
EOF
run_test ${lnav_test} -n 'unreadable.*'