Robot
/
lnav
mirror of https://github.com/tstack/lnav.git
2
0
Fork 0
Code Issues Releases Wiki Activity

[pcap] initial pcap support 

Fixes #12
This commit is contained in:
Timothy Stack 2021-11-05 15:13:16 -07:00
parent c52240a25d
commit 8bb034eeeb
40 changed files with 740 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/c-cpp.yml vendored
View File

@ -65,6 +65,7 @@ jobs:
libbz2-dev
libcurl4-openssl-dev
libreadline-dev
tshark
zlib1g-dev
- name: autogen
run: ./autogen.sh

4
NEWS
View File

@ -1,3 +1,7 @@
lnav v0.11.0:
Features:
* Add support for pcap(3) files using tshark(1).
lnav v0.10.1:
Features:
* Added ":show-only-this-file" command that hides all files except the

3
README
View File

@ -15,7 +15,7 @@ efficiently zero in on problems.
PREREQUISITES
-------------
The following software packages are required to build lnav:
The following software packages are required to build/run lnav:
gcc/clang - A C++14-compatible compiler.
libpcre - The Perl Compatible Regular Expression (PCRE) library.
@ -28,6 +28,7 @@ The following software packages are required to build lnav:
libcurl - The cURL library for downloading files from URLs. Version
7.23.0 or higher is required.
libarchive - The libarchive library for opening archive files, like zip/tgz.
wireshark - The 'tshark' program is used to interpret pcap files.
INSTALLATION

1
README.md
View File

@ -124,6 +124,7 @@ The following software packages are required to build lnav:
- bz2 - The bzip2 compression library.
- libcurl - The cURL library for downloading files from URLs. Version 7.23.0 or higher is required.
- libarchive - The libarchive library for opening archive files, like zip/tgz.
- wireshark - The 'tshark' program is used to interpret pcap files.
#### Build

3
TESTS_ENVIRONMENT.in
View File

@ -26,6 +26,9 @@ export BZIP2_CMD
XZ_CMD="@XZ_CMD@"
export XZ_CMD
TSHARK_CMD="@TSHARK_CMD@"
export TSHARK_CMD
LIBARCHIVE_LIBS="@LIBARCHIVE_LIBS@"
export LIBARCHIVE_LIBS

2
aminclude_static.am
View File

@ -1,6 +1,6 @@
# aminclude_static.am generated automatically by Autoconf
# from AX_AM_MACROS_STATIC on Mon Oct 18 09:00:48 PDT 2021
# from AX_AM_MACROS_STATIC on Wed Nov 3 22:10:14 PDT 2021
# Code coverage

3
configure.ac
View File

@ -52,7 +52,7 @@ dnl you are building it)
AS_CASE([x$srcdir],
[x/*],
AS_VAR_SET(abssrcdir, $srcdir),
AS_VAR_SET(abssrcdir, `pwd`/$srcdir)
AS_VAR_SET(abssrcdir, `cd $srcdir; pwd`)
)
AC_SUBST(abssrcdir)
@ -83,6 +83,7 @@ AC_PATH_PROG(BZIP2_CMD, [bzip2])
AC_PATH_PROG(RE2C_CMD, [re2c])
AM_CONDITIONAL(HAVE_RE2C, test x"$RE2C_CMD" != x"")
AC_PATH_PROG(XZ_CMD, [xz])
AC_PATH_PROG(TSHARK_CMD, [tshark])
AC_CHECK_SIZEOF(off_t)
AC_CHECK_SIZEOF(size_t)

16
docs/schemas/format-v1.schema.json
View File

@ -74,11 +74,27 @@
"description": "A regular expression that restricts this format to log files with a matching name",
"type": "string"
},
"mime-types": {
"title": "/<format_name>/mime-types",
"description": "A list of mime-types this format should be used for",
"type": "array",
"items": {
"type": "string",
"enum": [
"application/vnd.tcpdump.pcap"
]
}
},
"level-field": {
"title": "/<format_name>/level-field",
"description": "The name of the level field in the log message pattern",
"type": "string"
},
"level-pointer": {
"title": "/<format_name>/level-pointer",
"description": "A regular-expression that matches the JSON-pointer of the level property",
"type": "string"
},
"timestamp-field": {
"title": "/<format_name>/timestamp-field",
"description": "The name of the timestamp field in the log message pattern",

1
snapcraft.yaml
View File

@ -74,4 +74,5 @@ parts:
- libxml2
- locales-all
- ssh
- tshark
- xclip

2
src/CMakeLists.txt
View File

@ -280,6 +280,7 @@ add_library(
data_scanner_re.cc
data_parser.cc
papertrail_proc.cc
pcap_manager.cc
ptimec_rt.cc
pretty_printer.cc
pugixml/pugixml.cpp
@ -387,6 +388,7 @@ add_library(
logfile_stats.hh
optional.hpp
papertrail_proc.hh
pcap_manager.hh
plain_text_source.hh
pretty_printer.hh
preview_status_source.hh

2
src/Makefile.am
View File

@ -226,6 +226,7 @@ noinst_HEADERS = \
mapbox/variant_visitor.hpp \
optional.hpp \
papertrail_proc.hh \
pcap_manager.hh \
piper_proc.hh \
plain_text_source.hh \
pretty_printer.hh \
@ -355,6 +356,7 @@ libdiag_a_SOURCES = \
network-extension-functions.cc \
data_parser.cc \
papertrail_proc.cc \
pcap_manager.cc \
pretty_printer.cc \
ptimec_rt.cc \
readline_callbacks.cc \

29
src/base/auto_pid.hh
View File

@ -39,6 +39,7 @@
#include "base/result.h"
#include "base/lnav_log.hh"
#include "mapbox/variant.hpp"
enum class process_state {
RUNNING,
@ -54,10 +55,12 @@ public:
auto_pid(const auto_pid &other) = delete;
auto_pid(auto_pid &&other) noexcept: ap_child(std::move(other).release())
auto_pid(auto_pid &&other) noexcept
: ap_child(std::move(other).release()),
ap_status(other.ap_status)
{};
~auto_pid()
~auto_pid() noexcept
{ this->reset(); };
auto_pid &operator=(auto_pid &&other) noexcept
@ -103,6 +106,24 @@ public:
return WEXITSTATUS(this->ap_status);
}
using poll_result = mapbox::util::variant<
auto_pid<process_state::RUNNING>,
auto_pid<process_state::FINISHED>
>;
poll_result poll() && {
if (this->ap_child != -1) {
auto rc = waitpid(this->ap_child, &this->ap_status, WNOHANG);
if (rc <= 0) {
return std::move(*this);
}
}
return auto_pid<process_state::FINISHED>(
std::exchange(this->ap_child, -1), this->ap_status);
}
auto_pid<process_state::FINISHED> wait_for_child(int options = 0) &&
{
if (this->ap_child != -1) {
@ -116,11 +137,11 @@ public:
std::exchange(this->ap_child, -1), this->ap_status);
}
void reset(pid_t child = -1)
void reset(pid_t child = -1) noexcept
{
if (this->ap_child != child) {
this->ap_status = 0;
if (this->ap_child != -1) {
if (ProcState == process_state::RUNNING && this->ap_child != -1) {
log_debug("sending SIGTERM to child: %d", this->ap_child);
kill(this->ap_child, SIGTERM);
}

7
src/base/future_util.hh
View File

@ -63,7 +63,7 @@ public:
/**
* @param processor The function to execute with the result of a future.
*/
explicit future_queue(std::function<void(const T&)> processor)
explicit future_queue(std::function<void(T&)> processor)
: fq_processor(processor) {};
~future_queue() {
@ -90,12 +90,13 @@ public:
*/
void pop_to(size_t size = 0) {
while (this->fq_deque.size() > size) {
this->fq_processor(this->fq_deque.front().get());
auto v = this->fq_deque.front().get();
this->fq_processor(v);
this->fq_deque.pop_front();
}
}
std::function<void(const T&)> fq_processor;
std::function<void(T&)> fq_processor;
std::deque<std::future<T>> fq_deque;
};

90
src/file_collection.cc
View File

@ -44,10 +44,31 @@
#include "tailer/tailer.looper.hh"
#include "service_tags.hh"
#include "lnav_util.hh"
#include "pcap_manager.hh"
static std::mutex REALPATH_CACHE_MUTEX;
static std::unordered_map<std::string, std::string> REALPATH_CACHE;
child_poll_result_t child_poller::poll(file_collection& fc)
{
if (!this->cp_child) {
return child_poll_result_t::FINISHED;
}
auto poll_res = std::move(this->cp_child.value()).poll();
this->cp_child = nonstd::nullopt;
return poll_res.match(
[this](auto_pid<process_state::RUNNING>& alive) {
this->cp_child = std::move(alive);
return child_poll_result_t::ALIVE;
},
[this, &fc](auto_pid<process_state::FINISHED>& finished) {
this->cp_finalizer(fc, finished);
return child_poll_result_t::FINISHED;
}
);
}
void file_collection::close_files(const std::vector<std::shared_ptr<logfile>> &files)
{
for (const auto& lf : files) {
@ -109,6 +130,7 @@ void file_collection::regenerate_unique_file_names()
switch (pair.second.ofd_format) {
case file_format_t::FF_UNKNOWN:
case file_format_t::FF_ARCHIVE:
case file_format_t::FF_PCAP:
case file_format_t::FF_SQLITE_DB: {
auto bn = ghc::filesystem::path(pair.first).filename().string();
if (bn.length() > this->fc_largest_path_length) {
@ -126,7 +148,7 @@ void file_collection::regenerate_unique_file_names()
}
}
void file_collection::merge(const file_collection &other)
void file_collection::merge(file_collection &other)
{
this->fc_recursive = this->fc_recursive || other.fc_recursive;
this->fc_rotated = this->fc_rotated || other.fc_rotated;
@ -153,6 +175,13 @@ void file_collection::merge(const file_collection &other)
other.fc_closed_files.end());
this->fc_other_files.insert(other.fc_other_files.begin(),
other.fc_other_files.end());
if (!other.fc_child_pollers.empty()) {
this->fc_child_pollers.insert(
this->fc_child_pollers.begin(),
std::make_move_iterator(other.fc_child_pollers.begin()),
std::make_move_iterator(other.fc_child_pollers.end()));
other.fc_child_pollers.clear();
}
}
/**
@ -195,7 +224,7 @@ file_collection::watch_logfile(const std::string &filename,
int rc;
if (this->fc_closed_files.count(filename)) {
return lnav::futures::make_ready_future(retval);
return lnav::futures::make_ready_future(std::move(retval));
}
if (loo.loo_fd != -1) {
@ -212,14 +241,14 @@ file_collection::watch_logfile(const std::string &filename,
this->fc_file_names.end()) {
retval.fc_file_names.emplace(wilddir, logfile_open_options());
}
return lnav::futures::make_ready_future(retval);
return lnav::futures::make_ready_future(std::move(retval));
}
if (!S_ISREG(st.st_mode)) {
if (required) {
rc = -1;
errno = EINVAL;
} else {
return lnav::futures::make_ready_future(retval);
return lnav::futures::make_ready_future(std::move(retval));
}
}
auto err_iter = this->fc_name_to_errors.find(filename);
@ -236,7 +265,7 @@ file_collection::watch_logfile(const std::string &filename,
std::string(strerror(errno)),
});
}
return lnav::futures::make_ready_future(retval);
return lnav::futures::make_ready_future(std::move(retval));
}
auto stat_iter = find_if(this->fc_new_stats.begin(),
@ -248,7 +277,7 @@ file_collection::watch_logfile(const std::string &filename,
if (stat_iter != this->fc_new_stats.end()) {
// this file is probably a link that we have already scanned in this
// pass.
return lnav::futures::make_ready_future(retval);
return lnav::futures::make_ready_future(std::move(retval));
}
this->fc_new_stats.emplace_back(st);
@ -258,7 +287,7 @@ file_collection::watch_logfile(const std::string &filename,
if (file_iter == this->fc_files.end()) {
if (this->fc_other_files.find(filename) != this->fc_other_files.end()) {
return lnav::futures::make_ready_future(retval);
return lnav::futures::make_ready_future(std::move(retval));
}
auto func = [filename, st, loo, prog = this->fc_progress, errs = this->fc_name_to_errors]() mutable {
@ -271,11 +300,52 @@ file_collection::watch_logfile(const std::string &filename,
auto ff = detect_file_format(filename);
loo.loo_file_format = ff;
switch (ff) {
case file_format_t::FF_SQLITE_DB:
retval.fc_other_files[filename].ofd_format = ff;
break;
case file_format_t::FF_PCAP: {
auto res = pcap_manager::convert(filename);
if (res.isOk()) {
auto convert_res = res.unwrap();
loo.loo_fd = std::move(convert_res.cr_destination);
retval.fc_child_pollers.emplace_back(child_poller{
std::move(convert_res.cr_child),
[filename, st, error_queue = convert_res.cr_error_queue](auto& fc, auto& child) {
if (child.was_normal_exit() && child.exit_status() == EXIT_SUCCESS) {
log_info("pcap[%d] exited normally", child.in());
return;
}
log_error("pcap[%d] exited with %d", child.in(), child.status());
fc.fc_name_to_errors.emplace(filename, file_error_info{
st.st_mtime,
fmt::format("{}", fmt::join(*error_queue, "\n")),
});
},
});
auto open_res = logfile::open(filename, loo);
if (open_res.isOk()) {
retval.fc_files.push_back(open_res.unwrap());
} else {
retval.fc_name_to_errors.emplace(
filename, file_error_info{
st.st_mtime,
open_res.unwrapErr(),
});
}
} else {
retval.fc_name_to_errors.emplace(filename, file_error_info{
st.st_mtime,
res.unwrapErr(),
});
}
break;
}
case file_format_t::FF_ARCHIVE: {
nonstd::optional<std::list<archive_manager::extract_progress>::iterator>
prog_iter_opt;
@ -381,7 +451,7 @@ file_collection::watch_logfile(const std::string &filename,
}
}
return lnav::futures::make_ready_future(retval);
return lnav::futures::make_ready_future(std::move(retval));
}
/**
@ -440,7 +510,7 @@ void file_collection::expand_filename(lnav::futures::future_queue<file_collectio
"Initializing...";
}
fq.push_back(lnav::futures::make_ready_future(retval));
fq.push_back(lnav::futures::make_ready_future(std::move(retval)));
return;
}
@ -486,7 +556,7 @@ void file_collection::expand_filename(lnav::futures::future_queue<file_collectio
errmsg,
});
}
fq.push_back(lnav::futures::make_ready_future(retval));
fq.push_back(lnav::futures::make_ready_future(std::move(retval)));
}
continue;
} else {

37
src/file_collection.hh
View File

@ -37,6 +37,7 @@
#include <list>
#include <string>
#include <utility>
#include <forward_list>
#include "safe/safe.h"
@ -71,6 +72,39 @@ struct file_error_info {
const std::string fei_description;
};
struct file_collection;
enum class child_poll_result_t {
ALIVE,
FINISHED,
};
class child_poller {
public:
explicit child_poller(auto_pid<process_state::RUNNING> child,
std::function<void(file_collection&, auto_pid<process_state::FINISHED>&)> finalizer)
: cp_child(std::move(child)), cp_finalizer(std::move(finalizer)) {
}
child_poller(child_poller&& other) noexcept
: cp_child(std::move(other.cp_child)),
cp_finalizer(std::move(other.cp_finalizer)) {}
child_poller& operator=(child_poller&& other) noexcept {
this->cp_child = std::move(other.cp_child);
this->cp_finalizer = std::move(other.cp_finalizer);
return *this;
}
~child_poller() noexcept = default;
child_poll_result_t poll(file_collection& fc);
private:
nonstd::optional<auto_pid<process_state::RUNNING>> cp_child;
std::function<void(file_collection&, auto_pid<process_state::FINISHED>&)> cp_finalizer;
};
struct file_collection {
bool fc_invalidate_merge{false};
@ -88,6 +122,7 @@ struct file_collection {
std::set<std::string> fc_synced_files;
std::shared_ptr<safe_scan_progress> fc_progress;
std::vector<struct stat> fc_new_stats;
std::list<child_poller> fc_child_pollers;
size_t fc_largest_path_length{0};
file_collection()
@ -116,7 +151,7 @@ struct file_collection {
watch_logfile(const std::string &filename, logfile_open_options &loo,
bool required);
void merge(const file_collection &other);
void merge(file_collection &other);
void close_files(const std::vector<std::shared_ptr<logfile>> &files);

62
src/file_format.cc
View File

@ -31,12 +31,70 @@
#include "config.h"
#include <unordered_map>
#include "base/intern_string.hh"
#include "base/lnav_log.hh"
#include "auto_fd.hh"
#include "lnav_util.hh"
#include "file_format.hh"
#include "archive_manager.hh"
static bool is_pcap_header(uint8_t *buffer)
{
size_t offset = 0;
if (buffer[0] == 0x0a &&
buffer[1] == 0x0d &&
buffer[2] == 0x0d &&
buffer[3] == 0x0a) {
offset += sizeof(uint32_t) * 2;
if (buffer[offset + 0] == 0x1a &&
buffer[offset + 1] == 0x2b &&
buffer[offset + 2] == 0x3c &&
buffer[offset + 3] == 0x4d) {
return true;
}
if (buffer[offset + 0] == 0x4d &&
buffer[offset + 1] == 0x3c &&
buffer[offset + 2] == 0x2b &&
buffer[offset + 3] == 0x1a) {
return true;
}
return false;
}
if (buffer[0] == 0xa1 &&
buffer[1] == 0xb2 &&
buffer[2] == 0xc3 &&
buffer[3] == 0xd4) {
return true;
}
if (buffer[0] == 0xd4 &&
buffer[1] == 0xc3 &&
buffer[2] == 0xb2 &&
buffer[3] == 0xa1) {
return true;
}
if (buffer[0] == 0xa1 &&
buffer[1] == 0xb2 &&
buffer[2] == 0x3c &&
buffer[3] == 0x4d) {
return true;
}
if (buffer[0] == 0x4d &&
buffer[1] == 0x3c &&
buffer[2] == 0xb2 &&
buffer[3] == 0xa1) {
return true;
}
return false;
}
file_format_t detect_file_format(const ghc::filesystem::path &filename)
{
if (archive_manager::is_archive(filename)) {
@ -47,7 +105,7 @@ file_format_t detect_file_format(const ghc::filesystem::path &filename)
auto_fd fd;
if ((fd = openp(filename, O_RDONLY)) != -1) {
char buffer[32];
uint8_t buffer[32];
ssize_t rc;
if ((rc = read(fd, buffer, sizeof(buffer))) > 0) {
@ -56,6 +114,8 @@ file_format_t detect_file_format(const ghc::filesystem::path &filename)
if (header_frag.startswith(SQLITE3_HEADER)) {
retval = file_format_t::FF_SQLITE_DB;
} else if (rc > 24 && is_pcap_header(buffer)) {
retval = file_format_t::FF_PCAP;
}
}
}

7
src/file_format.hh
View File

@ -35,10 +35,11 @@
#include "fmt/format.h"
#include "ghc/filesystem.hpp"
enum class file_format_t {
enum class file_format_t : int {
FF_UNKNOWN,
FF_SQLITE_DB,
FF_ARCHIVE,
FF_PCAP,
FF_REMOTE,
};
@ -58,6 +59,9 @@ struct formatter<file_format_t> : formatter<string_view> {
case file_format_t::FF_ARCHIVE:
name = "\U0001F5C4 Archive";
break;
case file_format_t::FF_PCAP:
name = "\U0001F5A5 Pcap";
break;
case file_format_t::FF_REMOTE:
name = "\U0001F5A5 Remote";
break;
@ -69,5 +73,4 @@ struct formatter<file_format_t> : formatter<string_view> {
};
}
#endif

1
src/formats/formats.am
View File

@ -22,6 +22,7 @@ FORMAT_FILES = \
$(srcdir)/%reldir%/openstack_log.json \
$(srcdir)/%reldir%/page_log.json \
$(srcdir)/%reldir%/papertrail_log.json \
$(srcdir)/%reldir%/pcap_log.json \
$(srcdir)/%reldir%/snaplogic_log.json \
$(srcdir)/%reldir%/sssd_log.json \
$(srcdir)/%reldir%/strace_log.json \

76
src/formats/pcap_log.json Normal file
View File

@ -0,0 +1,76 @@
{
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
"pcap_log": {
"json": true,
"description": "pcap log format",
"mime-types": [
"application/vnd.tcpdump.pcap"
],
"multiline": false,
"line-format": [
{ "field": "time" },
" ",
{
"field": "source",
"min-width": 15,
"align": "right"
},
" → ",
{
"field": "destination",
"min-width": 15,
"align": "left"
},
" ",
{
"field": "protocol",
"min-width": 7,
"align": "left"
},
" ",
{
"field": "length",
"min-width": 4,
"align": "right"
},
" ",
{ "field": "info" }
],
"level": {
"warning": "^6291456$",
"error": "^8388608$"
},
"timestamp-field": "time",
"level-pointer": "/_ws_expert__ws_expert_severity$",
"body-field": "info",
"hide-extra": true,
"value": {
"source": {
"kind": "string",
"foreign-key": true,
"collate": "ipaddress",
"identifier": true
},
"destination": {
"kind": "string",
"foreign-key": true,
"collate": "ipaddress",
"identifier": true
},
"protocol": {
"kind": "string",
"identifier": true
},
"length": {
"kind": "integer"
},
"info": {
"kind": "string"
},
"layers": {
"kind": "json",
"hidden": true
}
}
}
}

51
src/lnav.cc
View File

@ -907,7 +907,7 @@ static void clear_last_user_mark(listview_curses *lv)
}
}
bool update_active_files(const file_collection& new_files)
bool update_active_files(file_collection& new_files)
{
static loading_observer obs;
@ -936,6 +936,10 @@ bool update_active_files(const file_collection& new_files)
!new_files.fc_name_to_errors.empty()) {
lnav_data.ld_active_files.regenerate_unique_file_names();
}
lnav_data.ld_child_pollers.insert(
lnav_data.ld_child_pollers.begin(),
std::make_move_iterator(lnav_data.ld_active_files.fc_child_pollers.begin()),
std::make_move_iterator(lnav_data.ld_active_files.fc_child_pollers.end()));
return true;
}
@ -1230,13 +1234,22 @@ static void gather_pipers()
++iter;
}
}
for (auto iter = lnav_data.ld_child_pollers.begin();
iter != lnav_data.ld_child_pollers.end();) {
if (iter->poll(lnav_data.ld_active_files) == child_poll_result_t::FINISHED) {
iter = lnav_data.ld_child_pollers.erase(iter);
} else {
++iter;
}
}
}
static void wait_for_pipers()
{
for (;;) {
gather_pipers();
if (lnav_data.ld_pipers.empty()) {
if (lnav_data.ld_pipers.empty() && lnav_data.ld_child_pollers.empty()) {
log_debug("all pipers finished");
break;
}
@ -1244,8 +1257,9 @@ static void wait_for_pipers()
usleep(10000);
rebuild_indexes();
}
log_debug("%d pipers still active",
lnav_data.ld_pipers.size());
log_debug("%d pipers and %d children still active",
lnav_data.ld_pipers.size(),
lnav_data.ld_child_pollers.size());
}
}
@ -1537,7 +1551,7 @@ static void looper()
future<file_collection> rescan_future =
std::async(std::launch::async,
&file_collection::rescan_files,
active_copy,
std::move(active_copy),
false);
bool initial_rescan_completed = false;
int session_stage = 0;
@ -1617,7 +1631,7 @@ static void looper()
(session_stage < 2 || ui_clock::now() >= next_rescan_time)) {
rescan_future = std::async(std::launch::async,
&file_collection::rescan_files,
active_copy,
std::move(active_copy),
false);
}
@ -1828,7 +1842,7 @@ static void looper()
else {
timer.start_fade(index_counter, 3);
}
log_debug("initial build rebuild");
// log_debug("initial build rebuild");
changes += rebuild_indexes(loop_deadline);
if (!initial_build &&
lnav_data.ld_log_source.text_line_count() == 0 &&
@ -1947,6 +1961,7 @@ static void looper()
if (lnav_data.ld_child_terminated) {
lnav_data.ld_child_terminated = false;
log_info("checking for terminated child processes");
for (auto iter = lnav_data.ld_children.begin();
iter != lnav_data.ld_children.end();
++iter) {
@ -2352,7 +2367,7 @@ int main(int argc, char *argv[])
if (lnav_data.ld_flags & LNF_SECURE_MODE) {
if ((sqlite3_set_authorizer(lnav_data.ld_db.in(),
sqlite_authorizer, NULL)) != SQLITE_OK) {
sqlite_authorizer, nullptr)) != SQLITE_OK) {
fprintf(stderr, "error: unable to attach sqlite authorizer\n");
exit(EXIT_FAILURE);
}
@ -2483,11 +2498,6 @@ int main(int argc, char *argv[])
load_format_extra(lnav_data.ld_db.in(), lnav_data.ld_config_paths, loader_errors);
load_format_vtabs(lnav_data.ld_vtab_manager.get(), loader_errors);
if (!loader_errors.empty()) {
print_errors(loader_errors);
return EXIT_FAILURE;
}
auto _vtab_cleanup = finally([] {
static const char *VIRT_TABLES = R"(
SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
@ -2533,6 +2543,11 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
}
});
if (!loader_errors.empty()) {
print_errors(loader_errors);
return EXIT_FAILURE;
}
if (!(lnav_data.ld_flags & LNF_CHECK_CONFIG)) {
DEFAULT_FILES.insert(make_pair(LNF_SYSLOG, string("var/log/messages")));
DEFAULT_FILES.insert(
@ -2919,6 +2934,16 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
clooper.process_all();
});
rebuild_indexes_repeatedly();
if (!lnav_data.ld_active_files.fc_name_to_errors.empty()) {
for (const auto& pair : lnav_data.ld_active_files.fc_name_to_errors) {
fprintf(stderr,
"error: unable to open file: %s -- %s\n",
pair.first.c_str(),
pair.second.fei_description.c_str());
}
return EXIT_FAILURE;
}
for (auto &pair : cmd_results) {
if (pair.first.isErr()) {

3
src/lnav.hh
View File

@ -207,6 +207,7 @@ struct lnav_data_t {
bool ld_session_loaded;
std::vector<ghc::filesystem::path> ld_config_paths;
file_collection ld_active_files;
std::list<child_poller> ld_child_pollers;
std::list<std::pair<std::string, int> > ld_files_to_front;
std::string ld_pt_search;
time_t ld_pt_min_time;
@ -314,7 +315,7 @@ void rebuild_indexes_repeatedly();
bool setup_logline_table(exec_context &ec);
bool rescan_files(bool required = false);
bool update_active_files(const file_collection& new_files);
bool update_active_files(file_collection& new_files);
void wait_for_children();

85
src/log_format.cc
View File

@ -76,7 +76,7 @@ struct line_range logline_value::origin_in_full_msg(const char *msg, ssize_t len
for (int lpc = 0; lpc < this->lv_sub_offset; lpc++) {
const auto *next = (const char *) memchr(last, '\n', msg_end - last);
require(next != NULL);
require(next != nullptr);
next += 1;
int amount = (next - last);
@ -172,7 +172,7 @@ std::string logline_value::to_string() const
if (this->lv_sbr.empty()) {
return this->lv_intern_string.to_string();
}
return std::string(this->lv_sbr.get_data(), this->lv_sbr.length());
return {this->lv_sbr.get_data(), this->lv_sbr.length()};
case value_kind_t::VALUE_QUOTED:
case value_kind_t::VALUE_W3C_QUOTED:
@ -190,10 +190,10 @@ std::string logline_value::to_string() const
unquoted_len = unquote_func(unquoted_str,
this->lv_sbr.get_data(),
this->lv_sbr.length());
return std::string(unquoted_str, unquoted_len);
return {unquoted_str, unquoted_len};
}
default:
return std::string(this->lv_sbr.get_data(), this->lv_sbr.length());
return {this->lv_sbr.get_data(), this->lv_sbr.length()};
}
}
break;
@ -220,7 +220,7 @@ std::string logline_value::to_string() const
break;
}
return std::string(buffer);
return {buffer};
}
vector<std::shared_ptr<log_format>> log_format::lf_root_formats;
@ -258,7 +258,7 @@ bool log_format::next_format(pcre_format *fmt, int &index, int &locked_index)
if (locked_index == -1) {
index += 1;
if (fmt[index].name == NULL) {
if (fmt[index].name == nullptr) {
retval = false;
}
}
@ -282,7 +282,7 @@ const char *log_format::log_scanf(uint32_t line_number,
...)
{
int curr_fmt = -1;
const char *retval = NULL;
const char *retval = nullptr;
bool done = false;
pcre_input pi(line, 0, len);
pcre_context_static<128> pc;
@ -294,7 +294,7 @@ const char *log_format::log_scanf(uint32_t line_number,
pi.reset(line, 0, len);
if (!fmt[curr_fmt].pcre.match(pc, pi, PCRE_NO_UTF8_CHECK)) {
retval = NULL;
retval = nullptr;
}
else {
pcre_context::capture_t *ts = pc[fmt[curr_fmt].pf_timestamp_index];
@ -307,7 +307,7 @@ const char *log_format::log_scanf(uint32_t line_number,
}
retval = this->lf_date_time.scan(
pi.get_substr_start(ts), ts->length(), NULL, tm_out, *tv_out);
pi.get_substr_start(ts), ts->length(), nullptr, tm_out, *tv_out);
if (retval) {
if (curr_fmt != pat_index) {
@ -385,20 +385,18 @@ void log_format::check_for_new_year(std::vector<logline> &dst, exttm etm,
*/
struct json_log_userdata {
json_log_userdata(shared_buffer_ref &sbr)
: jlu_format(NULL), jlu_line(NULL), jlu_base_line(NULL),
jlu_sub_line_count(1), jlu_handle(NULL), jlu_line_value(NULL),
jlu_line_size(0), jlu_sub_start(0), jlu_shared_buffer(sbr) {
: jlu_shared_buffer(sbr) {
};
external_log_format *jlu_format;
const logline *jlu_line;
logline *jlu_base_line;
int jlu_sub_line_count;
yajl_handle jlu_handle;
const char *jlu_line_value;
size_t jlu_line_size;
size_t jlu_sub_start;
external_log_format *jlu_format{nullptr};
const logline *jlu_line{nullptr};
logline *jlu_base_line{nullptr};
int jlu_sub_line_count{1};
yajl_handle jlu_handle{nullptr};
const char *jlu_line_value{nullptr};
size_t jlu_line_size{0};
size_t jlu_sub_start{0};
shared_buffer_ref &jlu_shared_buffer;
};
@ -609,7 +607,7 @@ bool external_log_format::scan_for_partial(shared_buffer_ref &sbr, size_t &len_o
auto pat = this->elf_pattern_order[this->last_pattern_index()];
pcre_input pi(sbr.get_data(), 0, sbr.length());
if (!this->elf_multiline) {
if (!this->lf_multiline) {
len_out = pat->p_pcre->match_partial(pi);
return true;
}
@ -640,6 +638,10 @@ log_format::scan_result_t external_log_format::scan(logfile &lf,
if (li.li_partial) {
log_debug("skipping partial line at offset %d", li.li_file_range.fr_offset);
if (this->lf_specialized) {
ll.set_level(LEVEL_INVALID);
dst.emplace_back(ll);
}
return log_format::SCAN_INCOMPLETE;
}
@ -661,7 +663,14 @@ log_format::scan_result_t external_log_format::scan(logfile &lf,
if (yajl_parse(handle, line_data, sbr.length()) == yajl_status_ok &&
yajl_complete_parse(handle) == yajl_status_ok) {
if (ll.get_time() == 0) {
return log_format::SCAN_NO_MATCH;
if (this->lf_specialized) {
ll.set_ignore(true);
dst.emplace_back(ll);
return log_format::SCAN_MATCH;
} else {
log_debug("no match! %.*s", sbr.length(), line_data);
return log_format::SCAN_NO_MATCH;
}
}
jlu.jlu_sub_line_count += this->jlf_line_format_init_count;
@ -857,7 +866,7 @@ log_format::scan_result_t external_log_format::scan(logfile &lf,
return log_format::SCAN_MATCH;
}
if (this->lf_specialized && !this->elf_multiline) {
if (this->lf_specialized && !this->lf_multiline) {
auto& last_line = dst.back();
dst.emplace_back(li.li_file_range.fr_offset,
@ -940,7 +949,7 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line
lr.lr_start = 0;
lr.lr_end = line.length();
sa.emplace_back(lr, &SA_BODY);
if (!this->elf_multiline) {
if (!this->lf_multiline) {
auto len = pat.p_pcre->match_partial(pi);
sa.emplace_back(line_range{(int) len, -1},
&SA_INVALID,
@ -1112,6 +1121,17 @@ static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str,
jlu->jlu_format->lf_timestamp_flags = tm_out.et_flags & ~ETF_MACHINE_ORIENTED;
jlu->jlu_base_line->set_time(tv_out);
}
else if (!jlu->jlu_format->elf_level_pointer.empty()) {
pcre_context_static<30> pc;
pcre_input pi(field_name);
if (jlu->jlu_format->elf_level_pointer.match(pc, pi)) {
pcre_input pi_level((const char *) str, 0, len);
pcre_context::capture_t level_cap = {0, (int) len};
jlu->jlu_base_line->set_level(jlu->jlu_format->convert_level(pi_level, &level_cap));
}
}
else if (jlu->jlu_format->elf_level_field == field_name) {
pcre_input pi((const char *) str, 0, len);
pcre_context::capture_t level_cap = {0, (int) len};
@ -1767,7 +1787,7 @@ void external_log_format::build(std::vector<std::string> &errors) {
}
found = true;
if (ts_len == -1 ||
dts.scan(ts, ts_len, custom_formats, &tm, tv) == NULL) {
dts.scan(ts, ts_len, custom_formats, &tm, tv) == nullptr) {
errors.push_back("error:" +
this->elf_name.to_string() +
":invalid sample -- " +
@ -1776,9 +1796,9 @@ void external_log_format::build(std::vector<std::string> &errors) {
this->elf_name.to_string() +
":unrecognized timestamp format -- " + ts);
if (custom_formats == NULL) {
if (custom_formats == nullptr) {
for (int lpc = 0;
PTIMEC_FORMATS[lpc].pf_fmt != NULL; lpc++) {
PTIMEC_FORMATS[lpc].pf_fmt != nullptr; lpc++) {
off_t off = 0;
PTIMEC_FORMATS[lpc].pf_func(&tm, ts, off, ts_len);
@ -1789,7 +1809,7 @@ void external_log_format::build(std::vector<std::string> &errors) {
}
}
else {
for (int lpc = 0; custom_formats[lpc] != NULL; lpc++) {
for (int lpc = 0; custom_formats[lpc] != nullptr; lpc++) {
off_t off = 0;
ptime_fmt(custom_formats[lpc], &tm, ts, off,
@ -2238,6 +2258,15 @@ bool external_log_format::match_name(const string &filename)
return this->elf_filename_pcre->match(pc, pi);
}
bool external_log_format::match_mime_type(const file_format_t ff) const
{
if (ff == file_format_t::FF_UNKNOWN && this->elf_mime_types.empty()) {
return true;
}
return this->elf_mime_types.count(ff) == 1;
}
int log_format::pattern_index_for_line(uint64_t line_number) const
{
auto iter = lower_bound(this->lf_pattern_locks.cbegin(),

9
src/log_format.hh
View File

@ -60,6 +60,7 @@
#include "log_level.hh"
#include "line_buffer.hh"
#include "log_format_fwd.hh"
#include "file_format.hh"
struct sqlite3;
class logfile;
@ -346,6 +347,13 @@ public:
virtual bool match_name(const std::string &filename) { return true; };
virtual bool match_mime_type(const file_format_t ff) const {
if (ff == file_format_t::FF_UNKNOWN) {
return true;
}
return false;
};
enum scan_result_t {
SCAN_MATCH,
SCAN_NO_MATCH,
@ -456,6 +464,7 @@ public:
int pattern_index_for_line(uint64_t line_number) const;
uint8_t lf_mod_index{0};
bool lf_multiline{true};
date_time_scanner lf_date_time;
std::vector<pattern_for_lines> lf_pattern_locks;
intern_string_t lf_timestamp_field{intern_string::lookup("timestamp", -1)};

6
src/log_format_ext.hh
View File

@ -116,7 +116,6 @@ public:
elf_timestamp_divisor(1.0),
elf_level_field(intern_string::lookup("level", -1)),
elf_body_field(intern_string::lookup("body", -1)),
elf_multiline(true),
elf_container(false),
elf_has_module_format(false),
elf_builtin_format(false),
@ -134,6 +133,8 @@ public:
bool match_name(const std::string &filename);
bool match_mime_type(const file_format_t ff) const;
scan_result_t scan(logfile &lf,
std::vector<logline> &dst,
const line_info &offset,
@ -349,6 +350,7 @@ public:
std::set<std::string> elf_source_path;
std::list<intern_string_t> elf_collision;
std::string elf_file_pattern;
std::set<file_format_t> elf_mime_types;
std::shared_ptr<pcrepp> elf_filename_pcre;
std::map<std::string, std::shared_ptr<pattern>> elf_patterns;
std::vector<std::shared_ptr<pattern>> elf_pattern_order;
@ -360,12 +362,12 @@ public:
int elf_column_count;
double elf_timestamp_divisor;
intern_string_t elf_level_field;
pcrepp elf_level_pointer;
intern_string_t elf_body_field;
intern_string_t elf_module_id_field;
intern_string_t elf_opid_field;
std::map<log_level_t, level_pattern> elf_level_patterns;
std::vector<std::pair<int64_t, log_level_t> > elf_level_pairs;
bool elf_multiline;
bool elf_container;
bool elf_has_module_format;
bool elf_builtin_format;

35
src/log_format_loader.cc
View File

@ -41,6 +41,7 @@
#include <string>
#include "fmt/format.h"
#include "file_format.hh"
#include "base/paths.hh"
#include "base/string_util.hh"
@ -174,7 +175,7 @@ static int read_format_bool(yajlpp_parse_context *ypc, int val)
else if (field_name == "hide-extra")
elf->jlf_hide_extra = val;
else if (field_name == "multiline")
elf->elf_multiline = val;
elf->lf_multiline = val;
return 1;
}
@ -229,6 +230,21 @@ static int read_format_field(yajlpp_parse_context *ypc, const unsigned char *str
else if (field_name == "level-field") {
elf->elf_level_field = intern_string::lookup(value);
}
else if (field_name == "level-pointer") {
auto pcre_res = pcrepp::from_str(value);
if (pcre_res.isErr()) {
ypc->ypc_error_reporter(
*ypc,
lnav_log_level_t::ERROR,
fmt::format("error:{}:{}:invalid regular expression for level-pointer -- {}",
ypc->ypc_source,
ypc->get_line_number(),
pcre_res.unwrapErr().ce_msg).c_str());
} else {
elf->elf_level_pointer = pcre_res.unwrap();
}
}
else if (field_name == "timestamp-field") {
elf->lf_timestamp_field = intern_string::lookup(value);
}
@ -245,6 +261,12 @@ static int read_format_field(yajlpp_parse_context *ypc, const unsigned char *str
else if (field_name == "opid-field") {
elf->elf_opid_field = intern_string::lookup(value);
}
else if (field_name == "mime-types") {
auto value_opt = ypc->ypc_current_handler->to_enum_value(value);
if (value_opt) {
elf->elf_mime_types.insert((file_format_t) *value_opt);
}
}
return 1;
}
@ -655,6 +677,12 @@ static struct json_path_container search_table_handlers = {
.with_children(search_table_def_handlers)
};
static const json_path_handler_base::enum_value_t MIME_TYPE_ENUM[] = {
{ "application/vnd.tcpdump.pcap", file_format_t::FF_PCAP, },
json_path_handler_base::ENUM_TERMINATOR
};
struct json_path_container format_handlers = {
yajlpp::property_handler("regex")
.with_description("The set of regular expressions used to match log messages")
@ -674,8 +702,13 @@ struct json_path_container format_handlers = {
.with_description("The value to divide a numeric timestamp by in a JSON log."),
json_path_handler("file-pattern", read_format_field)
.with_description("A regular expression that restricts this format to log files with a matching name"),
json_path_handler("mime-types#", read_format_field)
.with_description("A list of mime-types this format should be used for")
.with_enum_values(MIME_TYPE_ENUM),
json_path_handler("level-field", read_format_field)
.with_description("The name of the level field in the log message pattern"),
json_path_handler("level-pointer", read_format_field)
.with_description("A regular-expression that matches the JSON-pointer of the level property"),
json_path_handler("timestamp-field", read_format_field)
.with_description("The name of the timestamp field in the log message pattern"),
json_path_handler("body-field", read_format_field)

19
src/logfile.cc
View File

@ -200,6 +200,15 @@ bool logfile::process_prefix(shared_buffer_ref &sbr, const line_info &li)
iter != root_formats.end() && (found != log_format::SCAN_MATCH);
++iter) {
if (!(*iter)->match_name(this->lf_filename)) {
log_debug("(%s) does not match file name: %s",
(*iter)->get_name().get(),
this->lf_filename.c_str());
continue;
}
if (!(*iter)->match_mime_type(this->lf_options.loo_file_format)) {
log_debug("(%s) does not match file format: %d",
(*iter)->get_name().get(),
this->lf_options.loo_file_format);
continue;
}
@ -232,8 +241,12 @@ bool logfile::process_prefix(shared_buffer_ref &sbr, const line_info &li)
logline &last_line = this->lf_index[this->lf_index.size() - 1];
for (size_t lpc = 0; lpc < this->lf_index.size() - 1; lpc++) {
this->lf_index[lpc].set_time(last_line.get_time());
this->lf_index[lpc].set_millis(last_line.get_millis());
if (this->lf_format->lf_multiline) {
this->lf_index[lpc].set_time(last_line.get_time());
this->lf_index[lpc].set_millis(last_line.get_millis());
} else {
this->lf_index[lpc].set_ignore(true);
}
}
break;
}
@ -254,7 +267,7 @@ bool logfile::process_prefix(shared_buffer_ref &sbr, const line_info &li)
logline &second_to_last = this->lf_index[prescan_size - 1];
logline &latest = this->lf_index[prescan_size];
if (latest < second_to_last) {
if (!second_to_last.is_ignored() && latest < second_to_last) {
if (this->lf_format->lf_time_ordered) {
this->lf_out_of_time_order_count += 1;
for (size_t lpc = prescan_size;

8
src/logfile_fwd.hh
View File

@ -36,6 +36,7 @@
#include <string>
#include "auto_fd.hh"
#include "file_format.hh"
using ui_clock = std::chrono::steady_clock;
@ -103,6 +104,12 @@ struct logfile_open_options {
return *this;
}
logfile_open_options &with_file_format(file_format_t ff) {
this->loo_file_format = ff;
return *this;
}
std::string loo_filename;
auto_fd loo_fd;
logfile_name_source loo_source{logfile_name_source::USER};
@ -112,6 +119,7 @@ struct logfile_open_options {
bool loo_non_utf_is_visible{true};
ssize_t loo_visible_size_limit{-1};
bool loo_tail{true};
file_format_t loo_file_format{file_format_t::FF_UNKNOWN};
};
#endif

16
src/logfile_sub_source.cc
View File

@ -824,6 +824,10 @@ logfile_sub_source::rebuild_result logfile_sub_source::rebuild_index(nonstd::opt
}
for (size_t line_index = 0; line_index < lf->size(); line_index++) {
if ((*lf)[line_index].is_ignored()) {
continue;
}
content_line_t con_line(ld->ld_file_index * MAX_LINES_PER_FILE +
line_index);
@ -873,13 +877,15 @@ logfile_sub_source::rebuild_result logfile_sub_source::rebuild_index(nonstd::opt
break;
}
int file_index = ld->ld_file_index;
int line_index = lf_iter - ld->get_file_ptr()->begin();
if (!lf_iter->is_ignored()) {
int file_index = ld->ld_file_index;
int line_index = lf_iter - ld->get_file_ptr()->begin();
content_line_t con_line(file_index * MAX_LINES_PER_FILE +
line_index);
content_line_t con_line(file_index * MAX_LINES_PER_FILE +
line_index);
this->lss_index.push_back(con_line);
this->lss_index.push_back(con_line);
}
merge.next();
index_off += 1;

129
src/pcap_manager.cc Normal file
View File

@ -0,0 +1,129 @@
/**
* Copyright (c) 2021, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file pcap_manager.cc
*/
#include "config.h"
#include <vector>
#include <thread>
#include <unistd.h>
#include "pcap_manager.hh"
#include "lnav_util.hh"
#include "line_buffer.hh"
namespace pcap_manager {
Result<convert_result, std::string>
convert(const std::string &filename)
{
log_info("attempting to convert pcap file -- %s", filename.c_str());
auto outfile = TRY(open_temp_file(
ghc::filesystem::temp_directory_path() / "lnav.pcap.XXXXXX"));
ghc::filesystem::remove(outfile.first);
auto err_pipe = TRY(auto_pipe::for_child_fd(STDERR_FILENO));
auto child = TRY(lnav::pid::from_fork());
err_pipe.after_fork(child.in());
if (child.in_child()) {
auto dev_null = open("/dev/null", O_RDONLY);
dup2(dev_null, STDIN_FILENO);
dup2(outfile.second, STDOUT_FILENO);
const char *args[] = {
"tshark",
"-T", "ek",
"-P",
"-V",
"-t", "ad",
"-r", filename.c_str(),
nullptr,
};
execvp("tshark", (char **) args);
if (errno == ENOENT) {
fprintf(stderr,
"pcap support requires 'tshark' v3+ to be installed\n");
} else {
fprintf(stderr,
"failed to execute 'tshark' -- %s\n",
strerror(errno));
}
_exit(EXIT_FAILURE);
}
auto error_queue = std::make_shared<std::vector<std::string>>();
std::thread err_reader([err = std::move(err_pipe.read_end()), error_queue, child_pid=child.in()]() mutable {
line_buffer lb;
file_range pipe_range;
bool done = false;
lb.set_fd(err);
while (!done) {
auto load_res = lb.load_next_line(pipe_range);
if (load_res.isErr()) {
done = true;
} else {
auto li = load_res.unwrap();
pipe_range = li.li_file_range;
if (li.li_file_range.empty()) {
done = true;
} else {
lb.read_range(li.li_file_range).then([error_queue, child_pid](auto sbr) {
auto line_str = string_fragment(sbr.get_data(),
0,
sbr.length());
line_str.trim("\n");
if (error_queue->size() < 5) {
error_queue->emplace_back(line_str.to_string());
}
log_debug("pcap[%d]: %.*s",
child_pid, line_str.length(), line_str.data());
});
}
}
}
});
err_reader.detach();
log_info("started tshark %d to process file", child.in());
return Ok(convert_result{
std::move(child), auto_fd(outfile.second), error_queue,
});
}
}

55
src/pcap_manager.hh Normal file
View File

@ -0,0 +1,55 @@
/**
* Copyright (c) 2021, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file pcap_manager.hh
*/
#ifndef lnav_pcap_manager_hh
#define lnav_pcap_manager_hh
#include <string>
#include <vector>
#include "base/result.h"
#include "base/auto_pid.hh"
#include "auto_fd.hh"
namespace pcap_manager {
struct convert_result {
auto_pid<process_state::RUNNING> cr_child;
auto_fd cr_destination;
std::shared_ptr<std::vector<std::string>> cr_error_queue;
};
Result<convert_result, std::string>
convert(const std::string& filename);
}
#endif

6
src/pcrepp/pcrepp.hh
View File

@ -224,6 +224,12 @@ public:
pi_length(s.length()),
pi_string(s.data()) {};
pcre_input(const intern_string_t& s)
: pi_offset(0),
pi_next_offset(0),
pi_length(s.size()),
pi_string(s.get()) {};
pcre_input(const string_fragment &&) = delete;
pcre_input(const std::string &str, size_t off = 0)

8
src/sql_util.cc
View File

@ -583,10 +583,12 @@ static void sqlite_logger(void *dummy, int code, const char *msg)
break;
}
log_msg(level, __FILE__, __LINE__, "%s", msg);
log_msg(level, __FILE__, __LINE__, "(%d) %s", code, msg);
ensure(code != 21);
}
void sql_install_logger(void)
void sql_install_logger()
{
#ifdef SQLITE_CONFIG_LOG
sqlite3_config(SQLITE_CONFIG_LOG, sqlite_logger, NULL);
@ -869,7 +871,7 @@ int sqlite_authorizer(void *pUserData, int action_code, const char *detail1,
return SQLITE_OK;
}
string sql_keyword_re(void)
string sql_keyword_re()
{
string retval = "(?:";
bool first = true;

6
src/tailer/tailer.looper.cc
View File

@ -329,7 +329,7 @@ tailer::looper::host_tailer::for_host(const std::string& netloc)
args.push_back(nullptr);
execvp(cfg.c_ssh_cmd.c_str(), args.data());
exit(EXIT_FAILURE);
_exit(EXIT_FAILURE);
}
std::vector<std::string> error_queue;
@ -406,7 +406,7 @@ tailer::looper::host_tailer::for_host(const std::string& netloc)
args.push_back(nullptr);
execvp(cfg.c_ssh_cmd.c_str(), args.data());
exit(EXIT_FAILURE);
_exit(EXIT_FAILURE);
}
return Ok(std::make_shared<host_tailer>(
@ -568,7 +568,7 @@ void tailer::looper::host_tailer::loop_body()
if (read_res.isErr()) {
log_error("read error: %s", read_res.unwrapErr().c_str());
exit(EXIT_FAILURE);
return;
}
auto packet = read_res.unwrap();

3
test/Makefile.am
View File

@ -240,6 +240,8 @@ dist_noinst_DATA = \
datafile_simple.19 \
datafile_simple.20 \
datafile_xml.0 \
dhcp.pcapng \
dhcp-trunc.pcapng \
expected_help.txt \
listview_output.0 \
listview_output.1 \
@ -273,6 +275,7 @@ dist_noinst_DATA = \
logfile_glog.0 \
logfile_haproxy.0 \
logfile_invalid_json.json \
logfile_invalid_json2.json \
logfile_journald.json \
logfile_json.json \
logfile_json2.json \

BIN
test/dhcp-trunc.pcapng Normal file
View File

Binary file not shown.

BIN
test/dhcp.pcapng Normal file
View File

Binary file not shown.

2
test/formats/nestedjson/format.json
View File

@ -2,7 +2,7 @@
"ntest_log" : {
"title" : "Test JSON Log",
"json" : true,
"file-pattern" : "logfile_(nested|invalid)_json\\.json",
"file-pattern" : "logfile_(nested|invalid)_json\\d*\\.json",
"description" : "Test config",
"line-format" : [
{ "field" : "ts" },

3
test/logfile_invalid_json2.json Normal file
View File

@ -0,0 +1,3 @@
{"ts": "2013-09-06T20:00:48.124817Z", "@fields": { "lvl": "TRACE", "msg": "trace test"}}
{"ts": "2013-09-06T20:00:49.124817Z", "@fields": { "lvl": "INFO", "msg": "Starting up service"}}
{"ts": "2013-09-06T22:00:49.124817Z", "@fields": { "lvl": "INFO", "msg":

16
test/test_json_format.sh
View File

@ -533,3 +533,19 @@ parse error: premature EOF
2013-09-06T22:00:59.222 DEBUG4 Details...
@fields: { "lvl": "DEBUG4", "msg": "Details..."}
EOF
run_test ${lnav_test} -n \
-d /tmp/lnav.err \
-I ${test_dir} \
${test_dir}/logfile_invalid_json2.json
check_output "json log format is not working" <<EOF
2013-09-06T20:00:48.124 TRACE trace test
@fields: { "lvl": "TRACE", "msg": "trace test"}
2013-09-06T20:00:49.124 INFO Starting up service
@fields: { "lvl": "INFO", "msg": "Starting up service"}
[offset: 186] {"ts": "2013-09-06T22:00:49.124817Z", "@fields": { "lvl": "INFO", "msg":
parse error: premature EOF
{"ts": "2013-09-06T22:00:49.124
(right here) ------^
EOF

17
test/test_logfile.sh
View File

@ -3,6 +3,23 @@
echo ${top_srcdir}
echo ${top_builddir}
if test x"${TSHARK_CMD}" != x""; then
run_test ${lnav_test} -n ${test_dir}/dhcp.pcapng
check_output "pcap file is not recognized" <<EOF
2004-12-05T11:16:24.317 0.0.0.0 → 255.255.255.255 DHCP 314 DHCP Discover - Transaction ID 0x3d1d
2004-12-05T11:16:24.317 192.168.0.1 → 192.168.0.10 DHCP 342 DHCP Offer - Transaction ID 0x3d1d
2004-12-05T11:16:24.387 0.0.0.0 → 255.255.255.255 DHCP 314 DHCP Request - Transaction ID 0x3d1e
2004-12-05T11:16:24.387 192.168.0.1 → 192.168.0.10 DHCP 342 DHCP ACK - Transaction ID 0x3d1e
EOF
run_test ${lnav_test} -n ${test_dir}/dhcp-trunc.pcapng
check_error_output "truncated pcap file is not recognized" <<EOF
error: unable to open file: {test_dir}/dhcp-trunc.pcapng -- tshark: The file "{test_dir}/dhcp-trunc.pcapng" appears to have been cut short in the middle of a packet.
EOF
fi
run_test ${lnav_test} -d /tmp/lnav.err -n -w logfile_stdin.0.log \
-c ':shexec sleep 1 && touch -t 200711030923 logfile_stdin.0.log' <<EOF
2013-06-06T19:13:20.123 Hi