mirror of https://github.com/tstack/lnav.git
parent
906494ebfa
commit
f9f797fc9d
5
NEWS
5
NEWS
|
@ -24,6 +24,11 @@ lnav v0.10.2:
|
|||
within lnav (e.g. opening a file, format is detected). You
|
||||
can then add SQLite TRIGGERs to this table that can perform a
|
||||
task by updating other tables.
|
||||
* Log messages can now be detected automatically via "watch
|
||||
expressions". These are SQL expressions that are executed for
|
||||
each log message. If the expressions evaluates to true, an
|
||||
event is published to the "lnav_events" table that includes the
|
||||
message contents.
|
||||
* Added a "top_meta" column to the lnav_views table that contains
|
||||
metadata related to the top line in the view.
|
||||
* Added a "log_opid" hidden column to all log tables that contains
|
||||
|
|
|
@ -637,6 +637,40 @@
|
|||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"log": {
|
||||
"description": "Log message settings",
|
||||
"title": "/log",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"watch-expressions": {
|
||||
"description": "Log message watch expressions",
|
||||
"title": "/log/watch-expressions",
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"([\\w\\-]+)": {
|
||||
"description": "A log message watch expression",
|
||||
"title": "/log/watch-expressions/<watch_name>",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"expr": {
|
||||
"title": "/log/watch-expressions/<watch_name>/expr",
|
||||
"description": "The SQL expression to execute for each input line. If expression evaluates to true, a 'log message detected' event will be published.",
|
||||
"type": "string"
|
||||
},
|
||||
"enabled": {
|
||||
"title": "/log/watch-expressions/<watch_name>/enabled",
|
||||
"description": "Indicates whether or not this expression should be evaluated during log processing.",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"global": {
|
||||
"description": "Global variable definitions",
|
||||
"title": "/global",
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
{
|
||||
"$id": "https://lnav.org/event-log-msg-detected-v1.schema.json",
|
||||
"title": "https://lnav.org/event-log-msg-detected-v1.schema.json",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"description": "Event fired when a log message is detected by a watch expression.",
|
||||
"properties": {
|
||||
"$schema": {
|
||||
"title": "/$schema",
|
||||
"type": "string",
|
||||
"examples": [
|
||||
"https://lnav.org/event-log-msg-detected-v1.schema.json"
|
||||
]
|
||||
},
|
||||
"watch-name": {
|
||||
"title": "/watch-name",
|
||||
"description": "The name of the watch expression that matched this log message",
|
||||
"type": "string"
|
||||
},
|
||||
"filename": {
|
||||
"title": "/filename",
|
||||
"description": "The path of the file containing the log message",
|
||||
"type": "string"
|
||||
},
|
||||
"format": {
|
||||
"title": "/format",
|
||||
"description": "The name of the log format that matched this log message",
|
||||
"type": "string"
|
||||
},
|
||||
"timestamp": {
|
||||
"title": "/timestamp",
|
||||
"description": "The timestamp of the log message",
|
||||
"type": "string"
|
||||
},
|
||||
"values": {
|
||||
"description": "The log message values captured by the log format",
|
||||
"title": "/values",
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"([\\w\\-]+)": {
|
||||
"title": "/values/<name>",
|
||||
"type": [
|
||||
"boolean",
|
||||
"integer",
|
||||
"string"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
|
@ -38,7 +38,7 @@ A valid **lnav** configuration file must contain an object with the
|
|||
Options
|
||||
-------
|
||||
|
||||
The following configuration options can be used to customize **lnav** to
|
||||
The following configuration options can be used to customize the **lnav** UI to
|
||||
your liking. The options can be changed using the :code:`:config` command.
|
||||
|
||||
.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/keymap
|
||||
|
@ -201,6 +201,53 @@ Reference
|
|||
.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/keymap-defs/patternProperties/([\w\-]+)
|
||||
|
||||
|
||||
Log Handling
|
||||
------------
|
||||
|
||||
The handling of logs is largely determined by the
|
||||
:ref:`log file formats<log_formats>`, this section covers options that are not
|
||||
specific to a particular format.
|
||||
|
||||
Watch Expressions
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
Watch expressions can be used to fire an event when a log message matches a
|
||||
condition. You can then install a listener for these events and trigger an
|
||||
action to be performed. For example, to automate filtering based on
|
||||
identifiers, a watch expression can match messages that mention the ID and then
|
||||
a trigger can install a filter for that ID. Creating a watch expression is
|
||||
done by adding an entry into the :code:`/log/watch-expressions` configuration
|
||||
tree. For example, to create a watch named "dhcpdiscover" that matches
|
||||
DHCPDISCOVER messages from the :code:`dhclient` daemon, you would run the
|
||||
following:
|
||||
|
||||
.. code-block:: lnav
|
||||
|
||||
:config /log/watch-expressions/dhcpdiscover/expr :log_procname = 'dhclient' AND startswith(:log_body, 'DHCPDISCOVER')
|
||||
|
||||
The watch expression can refer to column names in the log message by prefixing
|
||||
them with a colon. The expression is evaluated by passing the log message
|
||||
fields as bound parameters and not against a table. The easiest way to test
|
||||
out an expression is with the :ref:`mark_expr` command, since it will behave
|
||||
similarly. After changing the configuration, you'll need to restart lnav
|
||||
for the effect to take place. You can then query the :code:`lnav_events`
|
||||
table to see any generated
|
||||
:code:`https://lnav.org/event-log-msg-detected-v1.schema.json` events from the
|
||||
logs that were loaded:
|
||||
|
||||
.. code-block:: custsqlite
|
||||
|
||||
;SELECT * FROM lnav_events
|
||||
|
||||
From there, you can create a SQLite trigger on the :code:`lnav_events` table
|
||||
that will examine the event contents and perform an action. See the
|
||||
:ref:`Events` section for more information on handling events.
|
||||
|
||||
Reference
|
||||
^^^^^^^^^
|
||||
|
||||
.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/watch-expressions/patternProperties/([\w\-]+)
|
||||
|
||||
.. _tuning:
|
||||
|
||||
Tuning
|
||||
|
|
|
@ -48,3 +48,6 @@ The following tables describe the schema of the event JSON objects.
|
|||
|
||||
.. jsonschema:: ../schemas/event-file-format-detected-v1.schema.json#
|
||||
:lift_description:
|
||||
|
||||
.. jsonschema:: ../schemas/event-log-msg-detected-v1.schema.json#
|
||||
:lift_description:
|
||||
|
|
|
@ -23,6 +23,7 @@ if ! test -d lnav; then
|
|||
fi
|
||||
|
||||
cd ~/github/lnav
|
||||
git restore .
|
||||
git pull --rebase
|
||||
|
||||
if test -n "$SRC_VERSION"; then
|
||||
|
@ -33,12 +34,13 @@ saved_PATH=${PATH}
|
|||
export PATH=${FAKE_ROOT}/bin:${PATH}
|
||||
saved_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
|
||||
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${FAKE_ROOT}/lib
|
||||
./autogen.sh
|
||||
if test ! -f "configure"; then
|
||||
./autogen.sh
|
||||
rm -rf ~/github/lbuild
|
||||
mkdir -p ~/github/lbuild
|
||||
|
||||
rm -rf ~/github/lbuild
|
||||
mkdir -p ~/github/lbuild
|
||||
|
||||
cd ~/github/lbuild
|
||||
cd ~/github/lbuild
|
||||
fi
|
||||
|
||||
TARGET_FILE='/vagrant/lnav-linux.zip'
|
||||
if test x"${OS}" != x"FreeBSD"; then
|
||||
|
|
|
@ -288,6 +288,7 @@ add_library(
|
|||
lnav_commands.cc
|
||||
lnav_config.cc
|
||||
lnav_util.cc
|
||||
log.watch.cc
|
||||
log_accel.cc
|
||||
log_actions.cc
|
||||
log_data_helper.cc
|
||||
|
@ -391,6 +392,7 @@ add_library(
|
|||
lnav.management_cli.hh
|
||||
lnav_config.hh
|
||||
lnav_config_fwd.hh
|
||||
log.watch.hh
|
||||
log_actions.hh
|
||||
log_data_helper.hh
|
||||
log_data_table.hh
|
||||
|
@ -401,6 +403,7 @@ add_library(
|
|||
log_gutter_source.hh
|
||||
log_level.hh
|
||||
log_search_table.hh
|
||||
logfile_sub_source.cfg.hh
|
||||
logfile.hh
|
||||
logfile_fwd.hh
|
||||
logfile_stats.hh
|
||||
|
@ -472,6 +475,9 @@ add_library(
|
|||
ww898/cp_utf8.hpp
|
||||
log_level_re.cc
|
||||
|
||||
third-party/ArenaAlloc/arenaalloc.h
|
||||
third-party/ArenaAlloc/arenaallocimpl.h
|
||||
|
||||
third-party/CLI/StringTools.hpp
|
||||
third-party/CLI/App.hpp
|
||||
third-party/CLI/Macros.hpp
|
||||
|
|
|
@ -229,6 +229,7 @@ noinst_HEADERS = \
|
|||
lnav_config.hh \
|
||||
lnav_config_fwd.hh \
|
||||
lnav_util.hh \
|
||||
log.watch.hh \
|
||||
log_accel.hh \
|
||||
log_actions.hh \
|
||||
log_data_helper.hh \
|
||||
|
@ -245,6 +246,7 @@ noinst_HEADERS = \
|
|||
logfile.cfg.hh \
|
||||
logfile_fwd.hh \
|
||||
logfile_sub_source.hh \
|
||||
logfile_sub_source.cfg.hh \
|
||||
mapbox/recursive_wrapper.hpp \
|
||||
mapbox/variant.hpp \
|
||||
mapbox/variant_io.hpp \
|
||||
|
@ -398,6 +400,7 @@ libdiag_a_SOURCES = \
|
|||
lnav_commands.cc \
|
||||
lnav_config.cc \
|
||||
lnav_util.cc \
|
||||
log.watch.cc \
|
||||
log_accel.cc \
|
||||
log_actions.cc \
|
||||
log_data_helper.cc \
|
||||
|
|
|
@ -60,7 +60,7 @@ all_logs_vtab::get_columns(std::vector<vtab_column>& cols) const
|
|||
}
|
||||
|
||||
void
|
||||
all_logs_vtab::extract(std::shared_ptr<logfile> lf,
|
||||
all_logs_vtab::extract(logfile* lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref& line,
|
||||
std::vector<logline_value>& values)
|
||||
|
|
|
@ -46,7 +46,7 @@ public:
|
|||
|
||||
void get_columns(std::vector<vtab_column>& cols) const override;
|
||||
|
||||
void extract(std::shared_ptr<logfile> lf,
|
||||
void extract(logfile* lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref& line,
|
||||
std::vector<logline_value>& values) override;
|
||||
|
|
|
@ -54,7 +54,13 @@ add_library(
|
|||
snippet_highlighters.hh
|
||||
string_attr_type.hh
|
||||
strnatcmp.h
|
||||
time_util.hh)
|
||||
time_util.hh
|
||||
|
||||
../third-party/xxHash/xxhash.h
|
||||
../third-party/xxHash/xxh_x86dispatch.h
|
||||
../third-party/xxHash/xxhash.c
|
||||
../third-party/xxHash/xxh_x86dispatch.c
|
||||
)
|
||||
|
||||
target_include_directories(base PUBLIC . .. ../fmtlib ../third-party
|
||||
${CMAKE_CURRENT_BINARY_DIR}/..)
|
||||
|
|
|
@ -78,7 +78,11 @@ libbase_a_SOURCES = \
|
|||
string_attr_type.cc \
|
||||
string_util.cc \
|
||||
strnatcmp.c \
|
||||
time_util.cc
|
||||
time_util.cc \
|
||||
../third-party/xxHash/xxhash.h \
|
||||
../third-party/xxHash/xxh_x86dispatch.h \
|
||||
../third-party/xxHash/xxhash.c \
|
||||
../third-party/xxHash/xxh_x86dispatch.c
|
||||
|
||||
check_PROGRAMS = \
|
||||
test_base
|
||||
|
|
|
@ -430,6 +430,12 @@ attr_line_t::rtrim()
|
|||
auto index = this->al_string.length();
|
||||
|
||||
for (; index > 0; index--) {
|
||||
if (find_string_attr_containing(
|
||||
this->al_attrs, &SA_PREFORMATTED, index - 1)
|
||||
!= this->al_attrs.end())
|
||||
{
|
||||
break;
|
||||
}
|
||||
if (!isspace(this->al_string[index - 1])) {
|
||||
break;
|
||||
}
|
||||
|
@ -446,6 +452,10 @@ attr_line_t::erase(size_t pos, size_t len)
|
|||
if (len == std::string::npos) {
|
||||
len = this->al_string.size() - pos;
|
||||
}
|
||||
if (len == 0) {
|
||||
return *this;
|
||||
}
|
||||
|
||||
this->al_string.erase(pos, len);
|
||||
|
||||
shift_string_attrs(this->al_attrs, pos, -((int32_t) len));
|
||||
|
@ -501,7 +511,11 @@ line_range::shift(int32_t start, int32_t amount)
|
|||
}
|
||||
} else if (this->lr_end != -1) {
|
||||
if (start < this->lr_end) {
|
||||
this->lr_end = std::max(this->lr_start, this->lr_end + amount);
|
||||
if (amount < 0 && amount < (start - this->lr_end)) {
|
||||
this->lr_end = start;
|
||||
} else {
|
||||
this->lr_end = std::max(this->lr_start, this->lr_end + amount);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -36,6 +36,7 @@
|
|||
#include <string.h>
|
||||
|
||||
#include "config.h"
|
||||
#include "xxHash/xxhash.h"
|
||||
|
||||
const static int TABLE_SIZE = 4095;
|
||||
|
||||
|
@ -68,14 +69,7 @@ intern_string::get_table_lifetime()
|
|||
unsigned long
|
||||
hash_str(const char* str, size_t len)
|
||||
{
|
||||
unsigned long retval = 5381;
|
||||
|
||||
for (size_t lpc = 0; lpc < len; lpc++) {
|
||||
/* retval * 33 + c */
|
||||
retval = ((retval << 5) + retval) + (unsigned char) str[lpc];
|
||||
}
|
||||
|
||||
return retval;
|
||||
return XXH3_64bits(str, len);
|
||||
}
|
||||
|
||||
const intern_string*
|
||||
|
|
|
@ -562,4 +562,11 @@ to_string_fragment(const std::string& s)
|
|||
return string_fragment(s.c_str(), 0, s.length());
|
||||
}
|
||||
|
||||
struct frag_hasher {
|
||||
size_t operator()(const string_fragment& sf) const
|
||||
{
|
||||
return hash_str(sf.data(), sf.length());
|
||||
}
|
||||
};
|
||||
|
||||
#endif
|
||||
|
|
|
@ -67,6 +67,8 @@ struct find {
|
|||
T f_value;
|
||||
};
|
||||
|
||||
struct first {};
|
||||
|
||||
struct second {};
|
||||
|
||||
template<typename F>
|
||||
|
@ -155,6 +157,12 @@ find(T value)
|
|||
};
|
||||
}
|
||||
|
||||
inline details::first
|
||||
first()
|
||||
{
|
||||
return details::first{};
|
||||
}
|
||||
|
||||
inline details::second
|
||||
second()
|
||||
{
|
||||
|
@ -346,6 +354,19 @@ operator|(const C& in, const lnav::itertools::details::nth indexer)
|
|||
return nonstd::nullopt;
|
||||
}
|
||||
|
||||
template<typename C>
|
||||
std::vector<typename C::key_type>
|
||||
operator|(const C& in, const lnav::itertools::details::first indexer)
|
||||
{
|
||||
std::vector<typename C::key_type> retval;
|
||||
|
||||
for (const auto& pair : in) {
|
||||
retval.emplace_back(pair.first);
|
||||
}
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
template<typename C>
|
||||
nonstd::optional<typename C::value_type>
|
||||
operator|(const C& in, const lnav::itertools::details::max_value maxer)
|
||||
|
|
|
@ -48,6 +48,7 @@ dump_internals(const char* internals_dir)
|
|||
&root_format_handler,
|
||||
&lnav::events::file::open::handlers,
|
||||
&lnav::events::file::format_detected::handlers,
|
||||
&lnav::events::log::msg_detected::handlers,
|
||||
})
|
||||
{
|
||||
dump_schema_to(*handlers, internals_dir);
|
||||
|
|
|
@ -248,7 +248,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv)
|
|||
if (!lf->get_pattern_regex(cl).empty()) {
|
||||
attr_line_t pattern_al;
|
||||
std::string& pattern_str = pattern_al.get_string();
|
||||
pattern_str = " Pattern: " + lf->get_pattern_name(cl) + " = ";
|
||||
pattern_str = " Pattern: " + lf->get_pattern_path(cl) + " = ";
|
||||
int skip = pattern_str.length();
|
||||
pattern_str += lf->get_pattern_regex(cl);
|
||||
lnav::snippets::regex_highlighter(
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
"url": "http://en.wikipedia.org/wiki/Syslog",
|
||||
"regex": {
|
||||
"std": {
|
||||
"pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|(?:\\+|-)\\d{2}:\\d{2})))(?: (?<log_hostname>[a-zA-Z0-9:][^ ]+[a-zA-Z0-9]))?(?: \\[CLOUDINIT\\])?(?:(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[: ]+|[^ :]+))(?:\\[(?<log_pid>\\d+)\\])?):\\s*(?<body>.*))$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
|
||||
"pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|(?:\\+|-)\\d{2}:\\d{2})))(?: (?<log_hostname>[a-zA-Z0-9:][^ ]+[a-zA-Z0-9]))?(?: \\[CLOUDINIT\\])?(?:(?: syslogd [\\d\\.]+|(?: (?<log_syslog_tag>(?<log_procname>(?:[^\\[: ]+|[^ :]+))(?:\\[(?<log_pid>\\d+)\\])?))):\\s*(?<body>.*)$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
|
||||
},
|
||||
"rfc5424": {
|
||||
"pattern": "^<(?<log_pri>\\d+)>(?<syslog_version>\\d+) (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?<log_hostname>[^ ]+|-) (?<log_syslog_tag>(?<log_procname>[^ ]+|-) (?<log_pid>[^ ]+|-) (?<log_msgid>[^ ]+|-)) (?<log_struct>\\[(?:[^\\]\"]|\"(?:\\.|[^\"])+\")*\\]|-|)\\s+(?<body>.*)"
|
||||
|
@ -73,6 +73,9 @@
|
|||
}
|
||||
},
|
||||
"sample": [
|
||||
{
|
||||
"line": "Apr 28 04:02:03 tstack-centos5 syslogd 1.4.1: restart."
|
||||
},
|
||||
{
|
||||
"line": "Jun 27 01:47:20 Tims-MacBook-Air.local configd[17]: network changed: v4(en0-:192.168.1.8) DNS- Proxy- SMB"
|
||||
},
|
||||
|
|
|
@ -52,6 +52,7 @@
|
|||
#include "fmtlib/fmt/format.h"
|
||||
#include "line_buffer.hh"
|
||||
|
||||
static const ssize_t INITIAL_REQUEST_SIZE = 16 * 1024;
|
||||
static const ssize_t DEFAULT_INCREMENT = 128 * 1024;
|
||||
static const ssize_t MAX_COMPRESSED_BUFFER_SIZE = 32 * 1024 * 1024;
|
||||
|
||||
|
@ -655,7 +656,7 @@ line_buffer::load_next_line(file_range prev_line)
|
|||
|
||||
auto offset = prev_line.next_offset();
|
||||
ssize_t request_size = this->lb_buffer_size == 0 ? DEFAULT_INCREMENT
|
||||
: 16 * 1024;
|
||||
: INITIAL_REQUEST_SIZE;
|
||||
retval.li_file_range.fr_offset = offset;
|
||||
while (!done) {
|
||||
char *line_start, *lf;
|
||||
|
@ -838,3 +839,20 @@ line_buffer::gz_indexed::indexDict::apply(z_streamp s)
|
|||
inflateSetDictionary(s, this->index, GZ_WINSIZE);
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool
|
||||
line_buffer::is_likely_to_flush(file_range prev_line)
|
||||
{
|
||||
auto avail = this->get_available();
|
||||
|
||||
if (prev_line.fr_offset < avail.fr_offset) {
|
||||
return true;
|
||||
}
|
||||
auto prev_line_end = prev_line.fr_offset + prev_line.fr_size;
|
||||
auto avail_end = avail.fr_offset + avail.fr_size;
|
||||
if (avail_end < prev_line_end) {
|
||||
return true;
|
||||
}
|
||||
auto remaining = avail_end - prev_line_end;
|
||||
return remaining < INITIAL_REQUEST_SIZE;
|
||||
}
|
||||
|
|
|
@ -210,6 +210,8 @@ public:
|
|||
|
||||
file_range get_available();
|
||||
|
||||
bool is_likely_to_flush(file_range prev_line);
|
||||
|
||||
void clear()
|
||||
{
|
||||
this->lb_buffer_size = 0;
|
||||
|
|
62
src/lnav.cc
62
src/lnav.cc
|
@ -1382,6 +1382,9 @@ looper()
|
|||
if (lnav_data.ld_filter_source.fss_editing) {
|
||||
lnav_data.ld_filter_source.fss_match_view.set_needs_update();
|
||||
}
|
||||
breadcrumb_view.do_update();
|
||||
// These updates need to be done last so their readline views can
|
||||
// put the cursor in the right place.
|
||||
switch (lnav_data.ld_mode) {
|
||||
case ln_mode_t::FILTER:
|
||||
case ln_mode_t::SEARCH_FILTERS:
|
||||
|
@ -1396,7 +1399,6 @@ looper()
|
|||
default:
|
||||
break;
|
||||
}
|
||||
breadcrumb_view.do_update();
|
||||
if (lnav_data.ld_mode != ln_mode_t::FILTER
|
||||
&& lnav_data.ld_mode != ln_mode_t::FILES)
|
||||
{
|
||||
|
@ -1889,6 +1891,34 @@ main(int argc, char* argv[])
|
|||
log_install_handlers();
|
||||
sql_install_logger();
|
||||
|
||||
if (sqlite3_open(":memory:", lnav_data.ld_db.out()) != SQLITE_OK) {
|
||||
fprintf(stderr, "error: unable to create sqlite memory database\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
{
|
||||
int register_collation_functions(sqlite3 * db);
|
||||
|
||||
register_sqlite_funcs(lnav_data.ld_db.in(), sqlite_registration_funcs);
|
||||
register_collation_functions(lnav_data.ld_db.in());
|
||||
}
|
||||
|
||||
register_environ_vtab(lnav_data.ld_db.in());
|
||||
{
|
||||
static auto vtab_modules
|
||||
= injector::get<std::vector<std::shared_ptr<vtab_module_base>>>();
|
||||
|
||||
for (const auto& mod : vtab_modules) {
|
||||
mod->create(lnav_data.ld_db.in());
|
||||
}
|
||||
}
|
||||
|
||||
register_views_vtab(lnav_data.ld_db.in());
|
||||
register_regexp_vtab(lnav_data.ld_db.in());
|
||||
register_xpath_vtab(lnav_data.ld_db.in());
|
||||
register_fstat_vtab(lnav_data.ld_db.in());
|
||||
lnav::events::register_events_tab(lnav_data.ld_db.in());
|
||||
|
||||
#ifdef HAVE_LIBCURL
|
||||
curl_global_init(CURL_GLOBAL_DEFAULT);
|
||||
#endif
|
||||
|
@ -2245,11 +2275,6 @@ main(int argc, char* argv[])
|
|||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
if (sqlite3_open(":memory:", lnav_data.ld_db.out()) != SQLITE_OK) {
|
||||
fprintf(stderr, "error: unable to create sqlite memory database\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (lnav_data.ld_flags & LNF_SECURE_MODE) {
|
||||
if ((sqlite3_set_authorizer(
|
||||
lnav_data.ld_db.in(), sqlite_authorizer, nullptr))
|
||||
|
@ -2268,29 +2293,6 @@ main(int argc, char* argv[])
|
|||
"/usr/share/terminfo:/lib/terminfo:/usr/share/lib/terminfo",
|
||||
0);
|
||||
|
||||
{
|
||||
int register_collation_functions(sqlite3 * db);
|
||||
|
||||
register_sqlite_funcs(lnav_data.ld_db.in(), sqlite_registration_funcs);
|
||||
register_collation_functions(lnav_data.ld_db.in());
|
||||
}
|
||||
|
||||
register_environ_vtab(lnav_data.ld_db.in());
|
||||
{
|
||||
static auto vtab_modules
|
||||
= injector::get<std::vector<std::shared_ptr<vtab_module_base>>>();
|
||||
|
||||
for (const auto& mod : vtab_modules) {
|
||||
mod->create(lnav_data.ld_db.in());
|
||||
}
|
||||
}
|
||||
|
||||
register_views_vtab(lnav_data.ld_db.in());
|
||||
register_regexp_vtab(lnav_data.ld_db.in());
|
||||
register_xpath_vtab(lnav_data.ld_db.in());
|
||||
register_fstat_vtab(lnav_data.ld_db.in());
|
||||
lnav::events::register_events_tab(lnav_data.ld_db.in());
|
||||
|
||||
lnav_data.ld_vtab_manager = std::make_unique<log_vtab_manager>(
|
||||
lnav_data.ld_db, lnav_data.ld_views[LNV_LOG], lnav_data.ld_log_source);
|
||||
|
||||
|
@ -2637,7 +2639,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
|
|||
"error:%s:%ld:line did not match format %s\n",
|
||||
lf->get_filename().c_str(),
|
||||
line_number,
|
||||
fmt->get_pattern_name(line_number).c_str());
|
||||
fmt->get_pattern_path(line_number).c_str());
|
||||
fprintf(stderr,
|
||||
"error:%s:%ld: line -- %s\n",
|
||||
lf->get_filename().c_str(),
|
||||
|
|
|
@ -65,6 +65,41 @@ const typed_json_path_container<format_detected> format_detected::handlers = typ
|
|||
|
||||
} // namespace file
|
||||
|
||||
namespace log {
|
||||
|
||||
const std::string msg_detected::SCHEMA_ID
|
||||
= "https://lnav.org/event-log-msg-detected-v1.schema.json";
|
||||
|
||||
static const json_path_container msg_values_handlers = {
|
||||
yajlpp::pattern_property_handler("(?<name>[\\w\\-]+)")
|
||||
.with_synopsis("<name>")
|
||||
.for_field(&msg_detected::md_values),
|
||||
};
|
||||
|
||||
const typed_json_path_container<msg_detected> msg_detected::handlers = typed_json_path_container<msg_detected>{
|
||||
yajlpp::property_handler("$schema").for_field(&msg_detected::md_schema)
|
||||
.with_example(msg_detected::SCHEMA_ID),
|
||||
yajlpp::property_handler("watch-name")
|
||||
.with_description("The name of the watch expression that matched this log message")
|
||||
.for_field(&msg_detected::md_watch_name),
|
||||
yajlpp::property_handler("filename")
|
||||
.with_description("The path of the file containing the log message")
|
||||
.for_field(&msg_detected::md_filename),
|
||||
yajlpp::property_handler("format")
|
||||
.with_description("The name of the log format that matched this log message")
|
||||
.for_field(&msg_detected::md_format),
|
||||
yajlpp::property_handler("timestamp")
|
||||
.with_description("The timestamp of the log message")
|
||||
.for_field(&msg_detected::md_timestamp),
|
||||
yajlpp::property_handler("values")
|
||||
.with_description("The log message values captured by the log format")
|
||||
.with_children(msg_values_handlers),
|
||||
}
|
||||
.with_schema_id2(msg_detected::SCHEMA_ID)
|
||||
.with_description2("Event fired when a log message is detected by a watch expression.");
|
||||
|
||||
} // namespace log
|
||||
|
||||
int
|
||||
register_events_tab(sqlite3* db)
|
||||
{
|
||||
|
|
|
@ -58,6 +58,22 @@ struct format_detected {
|
|||
|
||||
} // namespace file
|
||||
|
||||
namespace log {
|
||||
|
||||
struct msg_detected {
|
||||
std::string md_watch_name;
|
||||
std::string md_filename;
|
||||
std::string md_format;
|
||||
std::string md_timestamp;
|
||||
std::map<std::string, json_any_t> md_values;
|
||||
std::string md_schema{SCHEMA_ID};
|
||||
|
||||
static const std::string SCHEMA_ID;
|
||||
static const typed_json_path_container<msg_detected> handlers;
|
||||
};
|
||||
|
||||
} // namespace log
|
||||
|
||||
int register_events_tab(sqlite3* db);
|
||||
|
||||
namespace details {
|
||||
|
|
|
@ -151,6 +151,7 @@ struct subcmd_format_t {
|
|||
um.with_note(
|
||||
ext_lformat->elf_pattern_order
|
||||
| lnav::itertools::map(&external_log_format::pattern::p_name)
|
||||
| lnav::itertools::map(&intern_string_t::to_string)
|
||||
| lnav::itertools::fold(
|
||||
symbol_reducer, attr_line_t{"the available regexes are:"}));
|
||||
|
||||
|
@ -169,6 +170,7 @@ struct subcmd_format_t {
|
|||
um.with_note(
|
||||
(ext_lformat->elf_pattern_order
|
||||
| lnav::itertools::map(&external_log_format::pattern::p_name)
|
||||
| lnav::itertools::map(&intern_string_t::to_string)
|
||||
| lnav::itertools::similar_to(this->sf_regex_name)
|
||||
| lnav::itertools::fold(symbol_reducer, attr_line_t{}))
|
||||
.add_header("did you mean one of the following?"));
|
||||
|
@ -193,7 +195,8 @@ struct subcmd_format_t {
|
|||
.append(": ")
|
||||
.join(ext_format->elf_pattern_order
|
||||
| lnav::itertools::map(
|
||||
&external_log_format::pattern::p_name),
|
||||
&external_log_format::pattern::p_name)
|
||||
| lnav::itertools::map(&intern_string_t::to_string),
|
||||
VC_ROLE.value(role_t::VCR_SYMBOL),
|
||||
", ");
|
||||
}
|
||||
|
@ -512,7 +515,7 @@ struct subcmd_format_t {
|
|||
if (get_meta_res.is<lnav::session::regex101::no_entry>()) {
|
||||
lnav::session::regex101::insert_entry({
|
||||
format_regex_pair.first->get_name().to_string(),
|
||||
format_regex_pair.second->p_name,
|
||||
format_regex_pair.second->p_name.to_string(),
|
||||
upsert_info.cr_permalink_fragment,
|
||||
upsert_info.cr_delete_code,
|
||||
});
|
||||
|
|
|
@ -4366,6 +4366,9 @@ com_reset_config(exec_context& ec,
|
|||
yajlpp_parse_context ypc(INPUT_SRC, &lnav_config_handlers);
|
||||
std::string option = args[1];
|
||||
|
||||
while (!option.empty() && option.back() == '/') {
|
||||
option.pop_back();
|
||||
}
|
||||
lnav_config = rollback_lnav_config;
|
||||
ypc.set_path(option).with_obj(lnav_config);
|
||||
ypc.ypc_active_paths.insert(option);
|
||||
|
|
|
@ -93,6 +93,9 @@ static auto tc = injector::bind<tailer::config>::to_instance(
|
|||
static auto scc = injector::bind<sysclip::config>::to_instance(
|
||||
+[]() { return &lnav_config.lc_sysclip; });
|
||||
|
||||
static auto lsc = injector::bind<logfile_sub_source_ns::config>::to_instance(
|
||||
+[]() { return &lnav_config.lc_log_source; });
|
||||
|
||||
bool
|
||||
check_experimental(const char* feature_name)
|
||||
{
|
||||
|
@ -863,6 +866,11 @@ static const struct json_path_container theme_defs_handlers = {
|
|||
paths_out.emplace_back(iter.first);
|
||||
}
|
||||
})
|
||||
.with_obj_deleter(
|
||||
+[](const yajlpp_provider_context& ypc, _lnav_config* root) {
|
||||
root->lc_ui_theme_defs.erase(
|
||||
ypc.ypc_extractor.get_substr("theme_name"));
|
||||
})
|
||||
.with_children(theme_def_handlers),
|
||||
};
|
||||
|
||||
|
@ -1064,6 +1072,51 @@ static const struct json_path_container sysclip_handlers = {
|
|||
.with_children(sysclip_impls_handlers),
|
||||
};
|
||||
|
||||
static const struct json_path_container log_source_watch_expr_handlers = {
|
||||
yajlpp::property_handler("expr")
|
||||
.with_synopsis("<SQL-expression>")
|
||||
.with_description("The SQL expression to execute for each input line. "
|
||||
"If expression evaluates to true, a 'log message "
|
||||
"detected' event will be published.")
|
||||
.for_field(&logfile_sub_source_ns::watch_expression::we_expr),
|
||||
yajlpp::property_handler("enabled")
|
||||
.with_description("Indicates whether or not this expression should be "
|
||||
"evaluated during log processing.")
|
||||
.for_field(&logfile_sub_source_ns::watch_expression::we_enabled),
|
||||
};
|
||||
|
||||
static const struct json_path_container log_source_watch_handlers = {
|
||||
yajlpp::pattern_property_handler("(?<watch_name>[\\w\\-]+)")
|
||||
.with_synopsis("<name>")
|
||||
.with_description("A log message watch expression")
|
||||
.with_obj_provider<logfile_sub_source_ns::watch_expression,
|
||||
_lnav_config>(
|
||||
[](const yajlpp_provider_context& ypc, _lnav_config* root) {
|
||||
auto& retval = root->lc_log_source
|
||||
.c_watch_exprs[ypc.ypc_extractor.get_substr(
|
||||
"watch_name")];
|
||||
return &retval;
|
||||
})
|
||||
.with_path_provider<_lnav_config>(
|
||||
[](struct _lnav_config* cfg, std::vector<std::string>& paths_out) {
|
||||
for (const auto& iter : cfg->lc_log_source.c_watch_exprs) {
|
||||
paths_out.emplace_back(iter.first);
|
||||
}
|
||||
})
|
||||
.with_obj_deleter(
|
||||
+[](const yajlpp_provider_context& ypc, _lnav_config* root) {
|
||||
root->lc_log_source.c_watch_exprs.erase(
|
||||
ypc.ypc_extractor.get_substr("watch_name"));
|
||||
})
|
||||
.with_children(log_source_watch_expr_handlers),
|
||||
};
|
||||
|
||||
static const struct json_path_container log_source_handlers = {
|
||||
yajlpp::property_handler("watch-expressions")
|
||||
.with_description("Log message watch expressions")
|
||||
.with_children(log_source_watch_handlers),
|
||||
};
|
||||
|
||||
static const struct json_path_container tuning_handlers = {
|
||||
yajlpp::property_handler("archive-manager")
|
||||
.with_description("Settings related to opening archive files")
|
||||
|
@ -1124,22 +1177,26 @@ read_id(yajlpp_parse_context* ypc, const unsigned char* str, size_t len)
|
|||
}
|
||||
|
||||
const json_path_container lnav_config_handlers = json_path_container {
|
||||
json_path_handler("$schema", read_id)
|
||||
.with_synopsis("<schema-uri>")
|
||||
.with_description("The URI that specifies the schema that describes this type of file")
|
||||
.with_example(DEFAULT_CONFIG_SCHEMA),
|
||||
json_path_handler("$schema", read_id)
|
||||
.with_synopsis("<schema-uri>")
|
||||
.with_description("The URI that specifies the schema that describes this type of file")
|
||||
.with_example(DEFAULT_CONFIG_SCHEMA),
|
||||
|
||||
yajlpp::property_handler("tuning")
|
||||
.with_description("Internal settings")
|
||||
.with_children(tuning_handlers),
|
||||
yajlpp::property_handler("tuning")
|
||||
.with_description("Internal settings")
|
||||
.with_children(tuning_handlers),
|
||||
|
||||
yajlpp::property_handler("ui")
|
||||
.with_description("User-interface settings")
|
||||
.with_children(ui_handlers),
|
||||
yajlpp::property_handler("ui")
|
||||
.with_description("User-interface settings")
|
||||
.with_children(ui_handlers),
|
||||
|
||||
yajlpp::property_handler("global")
|
||||
.with_description("Global variable definitions")
|
||||
.with_children(global_var_handlers),
|
||||
yajlpp::property_handler("log")
|
||||
.with_description("Log message settings")
|
||||
.with_children(log_source_handlers),
|
||||
|
||||
yajlpp::property_handler("global")
|
||||
.with_description("Global variable definitions")
|
||||
.with_children(global_var_handlers),
|
||||
}
|
||||
.with_schema_id(*SUPPORTED_CONFIG_SCHEMAS.cbegin());
|
||||
|
||||
|
@ -1248,7 +1305,7 @@ load_config_from(_lnav_config& lconfig,
|
|||
}
|
||||
}
|
||||
|
||||
static void
|
||||
static bool
|
||||
load_default_config(struct _lnav_config& config_obj,
|
||||
const std::string& path,
|
||||
const bin_src_file& bsf,
|
||||
|
@ -1276,16 +1333,22 @@ load_default_config(struct _lnav_config& config_obj,
|
|||
if (ypc_builtin.parse(bsf.to_string_fragment()) == yajl_status_ok) {
|
||||
ypc_builtin.complete_parse();
|
||||
}
|
||||
|
||||
return path == "*" || ypc_builtin.ypc_active_paths.empty();
|
||||
}
|
||||
|
||||
static void
|
||||
static bool
|
||||
load_default_configs(struct _lnav_config& config_obj,
|
||||
const std::string& path,
|
||||
std::vector<lnav::console::user_message>& errors)
|
||||
{
|
||||
auto retval = false;
|
||||
|
||||
for (auto& bsf : lnav_config_json) {
|
||||
load_default_config(config_obj, path, bsf, errors);
|
||||
retval = load_default_config(config_obj, path, bsf, errors) || retval;
|
||||
}
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
void
|
||||
|
@ -1298,21 +1361,23 @@ load_config(const std::vector<ghc::filesystem::path>& extra_paths,
|
|||
auto sample_path = lnav::paths::dotlnav() / "configs" / "default"
|
||||
/ fmt::format(FMT_STRING("{}.sample"), bsf.get_name());
|
||||
|
||||
auto fd = auto_fd(lnav::filesystem::openp(
|
||||
sample_path, O_WRONLY | O_TRUNC | O_CREAT, 0644));
|
||||
auto sf = bsf.to_string_fragment();
|
||||
if (fd == -1 || write(fd.get(), sf.data(), sf.length()) == -1) {
|
||||
auto write_res = lnav::filesystem::write_file(sample_path,
|
||||
bsf.to_string_fragment());
|
||||
if (write_res.isErr()) {
|
||||
fprintf(stderr,
|
||||
"error:unable to write default config file: %s -- %s\n",
|
||||
sample_path.c_str(),
|
||||
strerror(errno));
|
||||
write_res.unwrapErr().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
log_info("loading builtin configuration into default");
|
||||
load_default_configs(lnav_default_config, "*", errors);
|
||||
log_info("loading builtin configuration into base");
|
||||
load_default_configs(lnav_config, "*", errors);
|
||||
|
||||
log_info("loading user configuration files");
|
||||
for (const auto& extra_path : extra_paths) {
|
||||
auto config_path = extra_path / "configs/*/*.json";
|
||||
static_root_mem<glob_t, globfree> gl;
|
||||
|
@ -1342,6 +1407,7 @@ load_config(const std::vector<ghc::filesystem::path>& extra_paths,
|
|||
}
|
||||
}
|
||||
|
||||
log_info("loading user configuration");
|
||||
load_config_from(lnav_config, user_config, errors);
|
||||
}
|
||||
|
||||
|
@ -1356,6 +1422,36 @@ reset_config(const std::string& path)
|
|||
std::vector<lnav::console::user_message> errors;
|
||||
|
||||
load_default_configs(lnav_config, path, errors);
|
||||
if (path != "*") {
|
||||
static const auto INPUT_SRC = intern_string::lookup("input");
|
||||
|
||||
yajlpp_parse_context ypc(INPUT_SRC, &lnav_config_handlers);
|
||||
ypc.set_path(path)
|
||||
.with_obj(lnav_config)
|
||||
.with_error_reporter([&errors](const auto& ypc, auto msg) {
|
||||
errors.push_back(msg);
|
||||
});
|
||||
ypc.ypc_active_paths.insert(path);
|
||||
ypc.update_callbacks();
|
||||
const json_path_handler_base* jph = ypc.ypc_current_handler;
|
||||
|
||||
if (!ypc.ypc_handler_stack.empty()) {
|
||||
jph = ypc.ypc_handler_stack.back();
|
||||
}
|
||||
|
||||
if (jph != nullptr && jph->jph_children && jph->jph_obj_deleter) {
|
||||
pcre_context_static<30> pc;
|
||||
auto key_start = ypc.ypc_path_index_stack.back();
|
||||
pcre_input pi(&ypc.ypc_path[key_start + 1],
|
||||
0,
|
||||
ypc.ypc_path.size() - key_start - 2);
|
||||
yajlpp_provider_context provider_ctx{{pc, pi},
|
||||
static_cast<size_t>(-1)};
|
||||
jph->jph_regex->match(pc, pi);
|
||||
|
||||
jph->jph_obj_deleter(provider_ctx, ypc.ypc_obj_stack.top());
|
||||
}
|
||||
}
|
||||
|
||||
reload_config(errors);
|
||||
|
||||
|
@ -1367,35 +1463,38 @@ reset_config(const std::string& path)
|
|||
std::string
|
||||
save_config()
|
||||
{
|
||||
yajlpp_gen gen;
|
||||
auto filename = fmt::format(FMT_STRING("config.json.{}.tmp"), getpid());
|
||||
auto user_config_tmp = lnav::paths::dotlnav() / filename;
|
||||
auto user_config = lnav::paths::dotlnav() / "config.json";
|
||||
|
||||
yajl_gen_config(gen, yajl_gen_beautify, true);
|
||||
yajlpp_gen gen;
|
||||
yajlpp_gen_context ygc(gen, lnav_config_handlers);
|
||||
std::vector<std::string> errors;
|
||||
|
||||
ygc.with_default_obj(lnav_default_config).with_obj(lnav_config);
|
||||
ygc.gen();
|
||||
|
||||
{
|
||||
auto_fd fd;
|
||||
auto config_str = gen.to_string_fragment().to_string();
|
||||
char errbuf[1024];
|
||||
auto* tree = yajl_tree_parse(config_str.c_str(), errbuf, sizeof(errbuf));
|
||||
|
||||
if ((fd = lnav::filesystem::openp(
|
||||
user_config_tmp, O_WRONLY | O_CREAT | O_TRUNC, 0600))
|
||||
== -1)
|
||||
{
|
||||
return "error: unable to save configuration -- "
|
||||
+ std::string(strerror(errno));
|
||||
}
|
||||
|
||||
string_fragment bits = gen.to_string_fragment();
|
||||
|
||||
log_perror(write(fd, bits.data(), bits.length()));
|
||||
if (tree == nullptr) {
|
||||
return fmt::format(
|
||||
FMT_STRING("error: unable to save configuration -- {}"), errbuf);
|
||||
}
|
||||
|
||||
rename(user_config_tmp.c_str(), user_config.c_str());
|
||||
yajl_cleanup_tree(tree);
|
||||
|
||||
yajlpp_gen clean_gen;
|
||||
|
||||
yajl_gen_config(clean_gen, yajl_gen_beautify, true);
|
||||
yajl_gen_tree(clean_gen, tree);
|
||||
|
||||
auto write_res = lnav::filesystem::write_file(
|
||||
user_config, clean_gen.to_string_fragment());
|
||||
if (write_res.isErr()) {
|
||||
return fmt::format(
|
||||
FMT_STRING("error: unable to write configuration file: {} -- {}"),
|
||||
user_config.string(),
|
||||
write_res.unwrapErr());
|
||||
}
|
||||
|
||||
return "info: configuration saved";
|
||||
}
|
||||
|
@ -1407,7 +1506,7 @@ reload_config(std::vector<lnav::console::user_message>& errors)
|
|||
|
||||
while (curr != nullptr) {
|
||||
auto reporter = [&errors](const void* cfg_value,
|
||||
const std::string& errmsg) {
|
||||
const lnav::console::user_message& errmsg) {
|
||||
auto cb = [&cfg_value, &errors, &errmsg](
|
||||
const json_path_handler_base& jph,
|
||||
const std::string& path,
|
||||
|
@ -1431,7 +1530,8 @@ reload_config(std::vector<lnav::console::user_message>& errors)
|
|||
.with_snippet(
|
||||
lnav::console::snippet::from(
|
||||
loc_iter->second.sl_source, "")
|
||||
.with_line(loc_iter->second.sl_line_number)));
|
||||
.with_line(loc_iter->second.sl_line_number))
|
||||
.with_help(jph.get_help_text(path)));
|
||||
};
|
||||
|
||||
for (const auto& jph : lnav_config_handlers.jpc_children) {
|
||||
|
|
|
@ -48,6 +48,7 @@
|
|||
#include "lnav_config_fwd.hh"
|
||||
#include "log_level.hh"
|
||||
#include "logfile.cfg.hh"
|
||||
#include "logfile_sub_source.cfg.hh"
|
||||
#include "styling.hh"
|
||||
#include "sysclip.cfg.hh"
|
||||
#include "tailer/tailer.looper.cfg.hh"
|
||||
|
@ -101,6 +102,7 @@ struct _lnav_config {
|
|||
lnav::logfile::config lc_logfile;
|
||||
tailer::config lc_tailer;
|
||||
sysclip::config lc_sysclip;
|
||||
logfile_sub_source_ns::config lc_log_source;
|
||||
};
|
||||
|
||||
extern struct _lnav_config lnav_config;
|
||||
|
|
|
@ -35,10 +35,12 @@
|
|||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
#include "base/lnav.console.hh"
|
||||
|
||||
class lnav_config_listener {
|
||||
public:
|
||||
using error_reporter
|
||||
= const std::function<void(const void*, const std::string msg)>;
|
||||
using error_reporter = const std::function<void(
|
||||
const void*, const lnav::console::user_message& msg)>;
|
||||
|
||||
lnav_config_listener()
|
||||
{
|
||||
|
@ -48,9 +50,7 @@ public:
|
|||
|
||||
virtual ~lnav_config_listener() = default;
|
||||
|
||||
virtual void reload_config(error_reporter& reporter){
|
||||
|
||||
};
|
||||
virtual void reload_config(error_reporter& reporter) {}
|
||||
|
||||
static lnav_config_listener* LISTENER_LIST;
|
||||
|
||||
|
|
|
@ -0,0 +1,355 @@
|
|||
/**
|
||||
* Copyright (c) 2022, Timothy Stack
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* * Neither the name of Timothy Stack nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "log.watch.hh"
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include "base/injector.hh"
|
||||
#include "bound_tags.hh"
|
||||
#include "lnav.events.hh"
|
||||
#include "lnav_config_fwd.hh"
|
||||
#include "log_format.hh"
|
||||
#include "logfile_sub_source.cfg.hh"
|
||||
#include "readline_highlighters.hh"
|
||||
#include "sql_util.hh"
|
||||
|
||||
namespace lnav {
|
||||
namespace log {
|
||||
namespace watch {
|
||||
|
||||
struct compiled_watch_expr {
|
||||
auto_mem<sqlite3_stmt> cwe_stmt{sqlite3_finalize};
|
||||
bool cwe_enabled{true};
|
||||
};
|
||||
|
||||
struct expressions : public lnav_config_listener {
|
||||
void reload_config(error_reporter& reporter) override
|
||||
{
|
||||
auto& lnav_db = injector::get<auto_mem<sqlite3, sqlite_close_wrapper>&,
|
||||
sqlite_db_tag>();
|
||||
|
||||
if (lnav_db.in() == nullptr) {
|
||||
log_warning("db not initialized yet!");
|
||||
return;
|
||||
}
|
||||
|
||||
const auto& cfg = injector::get<const logfile_sub_source_ns::config&>();
|
||||
|
||||
this->e_watch_exprs.clear();
|
||||
for (const auto& pair : cfg.c_watch_exprs) {
|
||||
auto stmt_str = fmt::format(FMT_STRING("SELECT 1 WHERE {}"),
|
||||
pair.second.we_expr);
|
||||
compiled_watch_expr cwe;
|
||||
|
||||
log_info("preparing watch expression: %s", stmt_str.c_str());
|
||||
auto retcode = sqlite3_prepare_v2(lnav_db,
|
||||
stmt_str.c_str(),
|
||||
stmt_str.size(),
|
||||
cwe.cwe_stmt.out(),
|
||||
nullptr);
|
||||
if (retcode != SQLITE_OK) {
|
||||
auto sql_al = attr_line_t(pair.second.we_expr)
|
||||
.with_attr_for_all(SA_PREFORMATTED.value())
|
||||
.with_attr_for_all(
|
||||
VC_ROLE.value(role_t::VCR_QUOTED_CODE));
|
||||
readline_sqlite_highlighter(sql_al, -1);
|
||||
intern_string_t watch_expr_path = intern_string::lookup(
|
||||
fmt::format(FMT_STRING("/log/watch-expressions/{}/expr"),
|
||||
pair.first));
|
||||
auto snippet = lnav::console::snippet::from(
|
||||
source_location(watch_expr_path), sql_al);
|
||||
|
||||
auto um = lnav::console::user_message::error(
|
||||
"SQL expression is invalid")
|
||||
.with_reason(sqlite3_errmsg(lnav_db))
|
||||
.with_snippet(snippet);
|
||||
|
||||
reporter(&pair.second.we_expr, um);
|
||||
continue;
|
||||
}
|
||||
|
||||
this->e_watch_exprs.emplace(pair.first, std::move(cwe));
|
||||
}
|
||||
}
|
||||
|
||||
std::map<std::string, compiled_watch_expr> e_watch_exprs;
|
||||
};
|
||||
|
||||
static expressions exprs;
|
||||
|
||||
void
|
||||
eval_with(logfile& lf, logfile::iterator ll)
|
||||
{
|
||||
if (std::none_of(exprs.e_watch_exprs.begin(),
|
||||
exprs.e_watch_exprs.end(),
|
||||
[](const auto& elem) { return elem.second.cwe_enabled; }))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
static auto& lnav_db
|
||||
= injector::get<auto_mem<sqlite3, sqlite_close_wrapper>&,
|
||||
sqlite_db_tag>();
|
||||
|
||||
char timestamp_buffer[64] = "";
|
||||
shared_buffer_ref sbr, raw_sbr;
|
||||
lf.read_full_message(ll, sbr);
|
||||
auto format = lf.get_format();
|
||||
string_attrs_t sa;
|
||||
std::vector<logline_value> values;
|
||||
auto line_number = std::distance(lf.begin(), ll);
|
||||
format->annotate(line_number, sbr, sa, values);
|
||||
|
||||
for (auto& watch_pair : exprs.e_watch_exprs) {
|
||||
if (!watch_pair.second.cwe_enabled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto* stmt = watch_pair.second.cwe_stmt.in();
|
||||
sqlite3_reset(stmt);
|
||||
|
||||
auto count = sqlite3_bind_parameter_count(stmt);
|
||||
auto missing_column = false;
|
||||
for (int lpc = 0; lpc < count; lpc++) {
|
||||
const auto* name = sqlite3_bind_parameter_name(stmt, lpc + 1);
|
||||
|
||||
if (name[0] == '$') {
|
||||
const char* env_value;
|
||||
|
||||
if ((env_value = getenv(&name[1])) != nullptr) {
|
||||
sqlite3_bind_text(
|
||||
stmt, lpc + 1, env_value, -1, SQLITE_STATIC);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_level") == 0) {
|
||||
sqlite3_bind_text(
|
||||
stmt, lpc + 1, ll->get_level_name(), -1, SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_time") == 0) {
|
||||
auto len = sql_strftime(timestamp_buffer,
|
||||
sizeof(timestamp_buffer),
|
||||
ll->get_timeval(),
|
||||
'T');
|
||||
sqlite3_bind_text(
|
||||
stmt, lpc + 1, timestamp_buffer, len, SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_time_msecs") == 0) {
|
||||
sqlite3_bind_int64(stmt, lpc + 1, ll->get_time_in_millis());
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_format") == 0) {
|
||||
const auto format_name = format->get_name();
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
format_name.get(),
|
||||
format_name.size(),
|
||||
SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_format_regex") == 0) {
|
||||
const auto pat_name = format->get_pattern_name(line_number);
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
pat_name.get(),
|
||||
pat_name.size(),
|
||||
SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_path") == 0) {
|
||||
const auto& filename = lf.get_filename();
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
filename.c_str(),
|
||||
filename.length(),
|
||||
SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_unique_path") == 0) {
|
||||
const auto& filename = lf.get_unique_path();
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
filename.c_str(),
|
||||
filename.length(),
|
||||
SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_text") == 0) {
|
||||
sqlite3_bind_text(
|
||||
stmt, lpc + 1, sbr.get_data(), sbr.length(), SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_body") == 0) {
|
||||
auto body_attr_opt = get_string_attr(sa, SA_BODY);
|
||||
if (body_attr_opt) {
|
||||
const auto& sar
|
||||
= body_attr_opt.value().saw_string_attr->sa_range;
|
||||
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
sbr.get_data_at(sar.lr_start),
|
||||
sar.length(),
|
||||
SQLITE_STATIC);
|
||||
} else {
|
||||
sqlite3_bind_null(stmt, lpc + 1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_opid") == 0) {
|
||||
auto opid_attr_opt = get_string_attr(sa, logline::L_OPID);
|
||||
if (opid_attr_opt) {
|
||||
const auto& sar
|
||||
= opid_attr_opt.value().saw_string_attr->sa_range;
|
||||
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
sbr.get_data_at(sar.lr_start),
|
||||
sar.length(),
|
||||
SQLITE_STATIC);
|
||||
} else {
|
||||
sqlite3_bind_null(stmt, lpc + 1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_raw_text") == 0) {
|
||||
auto res = lf.read_raw_message(ll);
|
||||
|
||||
if (res.isOk()) {
|
||||
raw_sbr = res.unwrap();
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
raw_sbr.get_data(),
|
||||
raw_sbr.length(),
|
||||
SQLITE_STATIC);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
auto found = false;
|
||||
for (const auto& lv : values) {
|
||||
if (lv.lv_meta.lvm_name != &name[1]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
found = true;
|
||||
switch (lv.lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
|
||||
break;
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
sqlite3_bind_double(stmt, lpc + 1, lv.lv_value.d);
|
||||
break;
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
sqlite3_bind_int64(stmt, lpc + 1, lv.lv_value.i);
|
||||
break;
|
||||
case value_kind_t::VALUE_NULL:
|
||||
sqlite3_bind_null(stmt, lpc + 1);
|
||||
break;
|
||||
default:
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
lv.text_value(),
|
||||
lv.text_length(),
|
||||
SQLITE_TRANSIENT);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (!found) {
|
||||
missing_column = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (missing_column) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto step_res = sqlite3_step(stmt);
|
||||
|
||||
switch (step_res) {
|
||||
case SQLITE_OK:
|
||||
case SQLITE_DONE:
|
||||
continue;
|
||||
case SQLITE_ROW:
|
||||
break;
|
||||
default: {
|
||||
log_error("failed to execute watch expression: %s -- %s",
|
||||
watch_pair.first.c_str(),
|
||||
sqlite3_errmsg(lnav_db));
|
||||
watch_pair.second.cwe_enabled = false;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (!timestamp_buffer[0]) {
|
||||
sql_strftime(timestamp_buffer,
|
||||
sizeof(timestamp_buffer),
|
||||
ll->get_timeval(),
|
||||
'T');
|
||||
}
|
||||
auto lmd = lnav::events::log::msg_detected{
|
||||
watch_pair.first,
|
||||
lf.get_filename(),
|
||||
lf.get_format_name().to_string(),
|
||||
timestamp_buffer,
|
||||
};
|
||||
for (const auto& lv : values) {
|
||||
switch (lv.lv_meta.lvm_kind) {
|
||||
case value_kind_t::VALUE_NULL:
|
||||
lmd.md_values[lv.lv_meta.lvm_name.to_string()]
|
||||
= json_null_t{};
|
||||
break;
|
||||
case value_kind_t::VALUE_BOOLEAN:
|
||||
lmd.md_values[lv.lv_meta.lvm_name.to_string()]
|
||||
= lv.lv_value.i ? true : false;
|
||||
break;
|
||||
case value_kind_t::VALUE_INTEGER:
|
||||
lmd.md_values[lv.lv_meta.lvm_name.to_string()]
|
||||
= lv.lv_value.i;
|
||||
break;
|
||||
case value_kind_t::VALUE_FLOAT:
|
||||
lmd.md_values[lv.lv_meta.lvm_name.to_string()]
|
||||
= lv.lv_value.d;
|
||||
break;
|
||||
default:
|
||||
lmd.md_values[lv.lv_meta.lvm_name.to_string()]
|
||||
= lv.to_string();
|
||||
break;
|
||||
}
|
||||
}
|
||||
lnav::events::publish(lnav_db, lmd);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace watch
|
||||
} // namespace log
|
||||
} // namespace lnav
|
|
@ -0,0 +1,45 @@
|
|||
/**
|
||||
* Copyright (c) 2022, Timothy Stack
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* * Neither the name of Timothy Stack nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef lnav_log_watch_hh
|
||||
#define lnav_log_watch_hh
|
||||
|
||||
#include "logfile.hh"
|
||||
|
||||
namespace lnav {
|
||||
namespace log {
|
||||
namespace watch {
|
||||
|
||||
void eval_with(logfile& lf, logfile::iterator ll);
|
||||
|
||||
}
|
||||
} // namespace log
|
||||
} // namespace lnav
|
||||
|
||||
#endif
|
|
@ -174,7 +174,7 @@ log_data_table::next(log_cursor& lc, logfile_sub_source& lss)
|
|||
}
|
||||
|
||||
void
|
||||
log_data_table::extract(std::shared_ptr<logfile> lf,
|
||||
log_data_table::extract(logfile* lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref& line,
|
||||
std::vector<logline_value>& values)
|
||||
|
|
|
@ -63,7 +63,7 @@ public:
|
|||
|
||||
bool next(log_cursor& lc, logfile_sub_source& lss) override;
|
||||
|
||||
void extract(std::shared_ptr<logfile> lf,
|
||||
void extract(logfile* lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref& line,
|
||||
std::vector<logline_value>& values) override;
|
||||
|
|
|
@ -853,12 +853,18 @@ external_log_format::scan(logfile& lf,
|
|||
}
|
||||
|
||||
if (opid_cap != nullptr && !opid_cap->empty()) {
|
||||
auto opid_str = pi.get_substr(opid_cap);
|
||||
auto opid_sf = pi.get_string_fragment(opid_cap);
|
||||
{
|
||||
auto opid_iter = sbc.sbc_opids.find(opid_str);
|
||||
auto opid_iter = sbc.sbc_opids.find(opid_sf);
|
||||
|
||||
if (opid_iter == sbc.sbc_opids.end()) {
|
||||
sbc.sbc_opids[opid_str] = opid_time_range{log_tv, log_tv};
|
||||
auto* opid_mem
|
||||
= sbc.sbc_allocator.allocate(opid_sf.length() + 1);
|
||||
memcpy(opid_mem, opid_sf.data(), opid_sf.length());
|
||||
opid_mem[opid_sf.length()] = '\0';
|
||||
auto otr = opid_time_range{log_tv, log_tv};
|
||||
sbc.sbc_opids.emplace(
|
||||
string_fragment{opid_mem, 0, opid_sf.length()}, otr);
|
||||
} else {
|
||||
opid_iter->second.otr_end = log_tv;
|
||||
}
|
||||
|
@ -1044,9 +1050,12 @@ external_log_format::annotate(uint64_t line_number,
|
|||
return;
|
||||
}
|
||||
|
||||
values.reserve(this->elf_value_defs.size());
|
||||
|
||||
int pat_index = this->pattern_index_for_line(line_number);
|
||||
pattern& pat = *this->elf_pattern_order[pat_index];
|
||||
|
||||
sa.reserve(pat.p_pcre->get_capture_count());
|
||||
if (!pat.p_pcre->match(pc, pi, PCRE_NO_UTF8_CHECK)) {
|
||||
// A continued line still needs a body.
|
||||
lr.lr_start = 0;
|
||||
|
@ -2043,7 +2052,8 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
|
|||
errors.emplace_back(
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t("invalid pattern: ")
|
||||
.append_quoted(lnav::roles::symbol(pat.p_name)))
|
||||
.append_quoted(lnav::roles::symbol(
|
||||
pat.p_name.to_string())))
|
||||
.with_reason("pattern does not match entire "
|
||||
"multiline message")
|
||||
.with_snippet(elf_sample.s_line.to_snippet())
|
||||
|
@ -2056,7 +2066,7 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
|
|||
}
|
||||
|
||||
if (!found && !this->elf_pattern_order.empty()) {
|
||||
std::vector<std::pair<ssize_t, std::string>> partial_indexes;
|
||||
std::vector<std::pair<ssize_t, intern_string_t>> partial_indexes;
|
||||
attr_line_t notes;
|
||||
size_t max_name_width = 0;
|
||||
|
||||
|
@ -2083,7 +2093,8 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
|
|||
notes.append(" ")
|
||||
.append(part_pair.first, ' ')
|
||||
.append("^ "_snippet_border)
|
||||
.append(lnav::roles::symbol(part_pair.second))
|
||||
.append(lnav::roles::symbol(
|
||||
part_pair.second.to_string()))
|
||||
.append(" matched up to here"_snippet_border)
|
||||
.append("\n");
|
||||
}
|
||||
|
@ -2435,7 +2446,7 @@ public:
|
|||
return false;
|
||||
};
|
||||
|
||||
virtual void extract(std::shared_ptr<logfile> lf,
|
||||
virtual void extract(logfile* lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref& line,
|
||||
std::vector<logline_value>& values)
|
||||
|
@ -2618,6 +2629,18 @@ external_log_format::json_append(
|
|||
}
|
||||
}
|
||||
|
||||
intern_string_t
|
||||
external_log_format::get_pattern_name(uint64_t line_number) const
|
||||
{
|
||||
if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
|
||||
static auto structured = intern_string::lookup("structured");
|
||||
|
||||
return structured;
|
||||
}
|
||||
int pat_index = this->pattern_index_for_line(line_number);
|
||||
return this->elf_pattern_order[pat_index]->p_name;
|
||||
}
|
||||
|
||||
int
|
||||
log_format::pattern_index_for_line(uint64_t line_number) const
|
||||
{
|
||||
|
@ -2636,12 +2659,22 @@ log_format::pattern_index_for_line(uint64_t line_number) const
|
|||
}
|
||||
|
||||
std::string
|
||||
log_format::get_pattern_name(uint64_t line_number) const
|
||||
log_format::get_pattern_path(uint64_t line_number) const
|
||||
{
|
||||
int pat_index = this->pattern_index_for_line(line_number);
|
||||
return fmt::format(FMT_STRING("builtin ({})"), pat_index);
|
||||
}
|
||||
|
||||
intern_string_t
|
||||
log_format::get_pattern_name(uint64_t line_number) const
|
||||
{
|
||||
char pat_str[128];
|
||||
|
||||
int pat_index = this->pattern_index_for_line(line_number);
|
||||
snprintf(pat_str, sizeof(pat_str), "builtin (%d)", pat_index);
|
||||
return intern_string::lookup(pat_str);
|
||||
}
|
||||
|
||||
std::shared_ptr<log_format>
|
||||
log_format::find_root_format(const char* name)
|
||||
{
|
||||
|
|
|
@ -428,7 +428,9 @@ public:
|
|||
exttm log_tv,
|
||||
timeval timeval1);
|
||||
|
||||
virtual std::string get_pattern_name(uint64_t line_number) const;
|
||||
virtual std::string get_pattern_path(uint64_t line_number) const;
|
||||
|
||||
virtual intern_string_t get_pattern_name(uint64_t line_number) const;
|
||||
|
||||
virtual std::string get_pattern_regex(uint64_t line_number) const
|
||||
{
|
||||
|
|
|
@ -97,7 +97,7 @@ public:
|
|||
};
|
||||
|
||||
struct pattern {
|
||||
std::string p_name;
|
||||
intern_string_t p_name;
|
||||
std::string p_config_path;
|
||||
std::shared_ptr<pcrepp_with_options<PCRE_DOTALL>> p_pcre;
|
||||
std::vector<indexed_value_def> p_value_by_index;
|
||||
|
@ -291,7 +291,7 @@ public:
|
|||
return iter != this->elf_value_defs.end();
|
||||
}
|
||||
|
||||
std::string get_pattern_name(uint64_t line_number) const
|
||||
std::string get_pattern_path(uint64_t line_number) const
|
||||
{
|
||||
if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
|
||||
return "structured";
|
||||
|
@ -300,6 +300,8 @@ public:
|
|||
return this->elf_pattern_order[pat_index]->p_config_path;
|
||||
}
|
||||
|
||||
intern_string_t get_pattern_name(uint64_t line_number) const;
|
||||
|
||||
std::string get_pattern_regex(uint64_t line_number) const
|
||||
{
|
||||
if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
|
||||
|
|
|
@ -36,6 +36,7 @@
|
|||
|
||||
#include <sys/types.h>
|
||||
|
||||
#include "ArenaAlloc/arenaalloc.h"
|
||||
#include "base/file_range.hh"
|
||||
#include "base/string_attr_type.hh"
|
||||
#include "byte_array.hh"
|
||||
|
@ -49,9 +50,15 @@ struct opid_time_range {
|
|||
struct timeval otr_end;
|
||||
};
|
||||
|
||||
using log_opid_map = std::unordered_map<std::string, opid_time_range>;
|
||||
using log_opid_map = std::unordered_map<
|
||||
string_fragment,
|
||||
opid_time_range,
|
||||
frag_hasher,
|
||||
std::equal_to<string_fragment>,
|
||||
ArenaAlloc::Alloc<std::pair<const string_fragment, opid_time_range>>>;
|
||||
|
||||
struct scan_batch_context {
|
||||
ArenaAlloc::Alloc<char>& sbc_allocator;
|
||||
log_opid_map sbc_opids;
|
||||
};
|
||||
|
||||
|
|
|
@ -121,7 +121,7 @@ pattern_provider(const yajlpp_provider_context& ypc, external_log_format* elf)
|
|||
}
|
||||
|
||||
if (pat->p_config_path.empty()) {
|
||||
pat->p_name = regex_name;
|
||||
pat->p_name = intern_string::lookup(regex_name);
|
||||
pat->p_config_path = fmt::format(
|
||||
FMT_STRING("/{}/regex/{}"), elf->get_name(), regex_name);
|
||||
}
|
||||
|
|
|
@ -137,7 +137,7 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss)
|
|||
}
|
||||
|
||||
void
|
||||
log_search_table::extract(std::shared_ptr<logfile> lf,
|
||||
log_search_table::extract(logfile* lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref& line,
|
||||
std::vector<logline_value>& values)
|
||||
|
|
|
@ -59,7 +59,7 @@ public:
|
|||
|
||||
bool next(log_cursor& lc, logfile_sub_source& lss) override;
|
||||
|
||||
void extract(std::shared_ptr<logfile> lf,
|
||||
void extract(logfile* lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref& line,
|
||||
std::vector<logline_value>& values) override;
|
||||
|
|
|
@ -63,14 +63,15 @@ static const char* LOG_COLUMNS = R"( (
|
|||
|
||||
static const char* LOG_FOOTER_COLUMNS = R"(
|
||||
-- END Format-specific fields
|
||||
log_opid TEXT HIDDEN, -- The message's OPID
|
||||
log_format TEXT HIDDEN, -- The name of the log file format
|
||||
log_time_msecs INTEGER HIDDEN, -- The adjusted timestamp for the log message as the number of milliseconds from the epoch
|
||||
log_path TEXT HIDDEN COLLATE naturalnocase, -- The path to the log file this message is from
|
||||
log_unique_path TEXT HIDDEN COLLATE naturalnocase, -- The unique portion of the path this message is from
|
||||
log_text TEXT HIDDEN, -- The full text of the log message
|
||||
log_body TEXT HIDDEN, -- The body of the log message
|
||||
log_raw_text TEXT HIDDEN -- The raw text from the log file
|
||||
log_opid TEXT HIDDEN, -- The message's OPID
|
||||
log_format TEXT HIDDEN, -- The name of the log file format
|
||||
log_format_regex TEXT HIDDEN, -- The name of the regex used to parse this log message
|
||||
log_time_msecs INTEGER HIDDEN, -- The adjusted timestamp for the log message as the number of milliseconds from the epoch
|
||||
log_path TEXT HIDDEN COLLATE naturalnocase, -- The path to the log file this message is from
|
||||
log_unique_path TEXT HIDDEN COLLATE naturalnocase, -- The unique portion of the path this message is from
|
||||
log_text TEXT HIDDEN, -- The full text of the log message
|
||||
log_body TEXT HIDDEN, -- The body of the log message
|
||||
log_raw_text TEXT HIDDEN -- The raw text from the log file
|
||||
);
|
||||
)";
|
||||
|
||||
|
@ -182,7 +183,7 @@ log_vtab_impl::get_foreign_keys(std::vector<std::string>& keys_inout) const
|
|||
}
|
||||
|
||||
void
|
||||
log_vtab_impl::extract(std::shared_ptr<logfile> lf,
|
||||
log_vtab_impl::extract(logfile* lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref& line,
|
||||
std::vector<logline_value>& values)
|
||||
|
@ -197,7 +198,7 @@ bool
|
|||
log_vtab_impl::is_valid(log_cursor& lc, logfile_sub_source& lss)
|
||||
{
|
||||
content_line_t cl(lss.at(lc.lc_curr_line));
|
||||
std::shared_ptr<logfile> lf = lss.find(cl);
|
||||
auto* lf = lss.find_file_ptr(cl);
|
||||
auto lf_iter = lf->begin() + cl;
|
||||
|
||||
if (!lf_iter->is_message()) {
|
||||
|
@ -376,7 +377,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
|
|||
content_line_t cl(vt->lss->at(vc->log_cursor.lc_curr_line));
|
||||
uint64_t line_number;
|
||||
auto ld = vt->lss->find_data(cl, line_number);
|
||||
auto lf = (*ld)->get_file();
|
||||
auto lf = (*ld)->get_file_ptr();
|
||||
auto ll = lf->begin() + line_number;
|
||||
|
||||
require(col >= 0);
|
||||
|
@ -616,24 +617,33 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
|
|||
break;
|
||||
}
|
||||
case 2: {
|
||||
sqlite3_result_int64(ctx, ll->get_time_in_millis());
|
||||
auto pat_name
|
||||
= lf->get_format()->get_pattern_name(line_number);
|
||||
sqlite3_result_text(ctx,
|
||||
pat_name.get(),
|
||||
pat_name.size(),
|
||||
SQLITE_STATIC);
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
sqlite3_result_int64(ctx, ll->get_time_in_millis());
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
const auto& fn = lf->get_filename();
|
||||
|
||||
sqlite3_result_text(
|
||||
ctx, fn.c_str(), fn.length(), SQLITE_STATIC);
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
case 5: {
|
||||
const auto& fn = lf->get_unique_path();
|
||||
|
||||
sqlite3_result_text(
|
||||
ctx, fn.c_str(), fn.length(), SQLITE_STATIC);
|
||||
break;
|
||||
}
|
||||
case 5: {
|
||||
case 6: {
|
||||
shared_buffer_ref line;
|
||||
|
||||
lf->read_full_message(ll, line);
|
||||
|
@ -643,7 +653,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
|
|||
SQLITE_TRANSIENT);
|
||||
break;
|
||||
}
|
||||
case 6: {
|
||||
case 7: {
|
||||
if (vc->line_values.empty()) {
|
||||
lf->read_full_message(ll, vc->log_msg);
|
||||
vt->vi->extract(
|
||||
|
@ -666,7 +676,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
|
|||
}
|
||||
break;
|
||||
}
|
||||
case 7: {
|
||||
case 8: {
|
||||
auto read_res = lf->read_raw_message(ll);
|
||||
|
||||
if (read_res.isErr()) {
|
||||
|
@ -988,9 +998,10 @@ vt_filter(sqlite3_vtab_cursor* p_vtc,
|
|||
if (sqlite3_value_type(argv[lpc]) != SQLITE3_TEXT) {
|
||||
continue;
|
||||
}
|
||||
const auto* opid
|
||||
const auto* opid_str
|
||||
= (const char*) sqlite3_value_text(argv[lpc]);
|
||||
auto opid_len = sqlite3_value_bytes(argv[lpc]);
|
||||
auto opid = string_fragment{opid_str, 0, opid_len};
|
||||
for (const auto& file_data : *vt->lss) {
|
||||
if (file_data->get_file_ptr() == nullptr) {
|
||||
continue;
|
||||
|
@ -1016,7 +1027,7 @@ vt_filter(sqlite3_vtab_cursor* p_vtc,
|
|||
|
||||
opid_val = log_cursor::opid_hash{
|
||||
static_cast<unsigned int>(
|
||||
hash_str(opid, opid_len))};
|
||||
hash_str(opid_str, opid_len))};
|
||||
log_debug("filter opid %d", opid_val.value().value);
|
||||
break;
|
||||
}
|
||||
|
@ -1480,7 +1491,7 @@ log_format_vtab_impl::next(log_cursor& lc, logfile_sub_source& lss)
|
|||
}
|
||||
|
||||
auto cl = content_line_t(lss.at(lc.lc_curr_line));
|
||||
auto lf = lss.find(cl);
|
||||
auto* lf = lss.find_file_ptr(cl);
|
||||
auto lf_iter = lf->begin() + cl;
|
||||
uint8_t mod_id = lf_iter->get_module_id();
|
||||
|
||||
|
|
|
@ -138,7 +138,7 @@ public:
|
|||
|
||||
virtual void get_foreign_keys(std::vector<std::string>& keys_inout) const;
|
||||
|
||||
virtual void extract(std::shared_ptr<logfile> lf,
|
||||
virtual void extract(logfile* lf,
|
||||
uint64_t line_number,
|
||||
shared_buffer_ref& line,
|
||||
std::vector<logline_value>& values);
|
||||
|
|
|
@ -47,6 +47,7 @@
|
|||
#include "base/string_util.hh"
|
||||
#include "config.h"
|
||||
#include "lnav_util.hh"
|
||||
#include "log.watch.hh"
|
||||
#include "log_format.hh"
|
||||
#include "logfile.cfg.hh"
|
||||
|
||||
|
@ -263,7 +264,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
|||
}
|
||||
|
||||
switch (found) {
|
||||
case log_format::SCAN_MATCH:
|
||||
case log_format::SCAN_MATCH: {
|
||||
if (!this->lf_index.empty()) {
|
||||
this->lf_index.back().set_valid_utf(li.li_valid_utf);
|
||||
}
|
||||
|
@ -273,8 +274,8 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
|||
retval = true;
|
||||
}
|
||||
if (prescan_size > 0 && prescan_size < this->lf_index.size()) {
|
||||
logline& second_to_last = this->lf_index[prescan_size - 1];
|
||||
logline& latest = this->lf_index[prescan_size];
|
||||
auto& second_to_last = this->lf_index[prescan_size - 1];
|
||||
auto& latest = this->lf_index[prescan_size];
|
||||
|
||||
if (!second_to_last.is_ignored() && latest < second_to_last) {
|
||||
if (this->lf_format->lf_time_ordered) {
|
||||
|
@ -282,7 +283,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
|||
for (size_t lpc = prescan_size;
|
||||
lpc < this->lf_index.size();
|
||||
lpc++) {
|
||||
logline& line_to_update = this->lf_index[lpc];
|
||||
auto& line_to_update = this->lf_index[lpc];
|
||||
|
||||
line_to_update.set_time_skew(true);
|
||||
line_to_update.set_time(second_to_last.get_time());
|
||||
|
@ -295,6 +296,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
|
|||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case log_format::SCAN_NO_MATCH: {
|
||||
log_level_t last_level = LEVEL_UNKNOWN;
|
||||
time_t last_time = this->lf_index_time;
|
||||
|
@ -460,7 +462,7 @@ logfile::rebuild_index(nonstd::optional<ui_clock::time_point> deadline)
|
|||
log_debug(
|
||||
"loading file... %s:%d", this->lf_filename.c_str(), begin_size);
|
||||
}
|
||||
scan_batch_context sbc;
|
||||
scan_batch_context sbc{this->lf_allocator};
|
||||
auto prev_range = file_range{off};
|
||||
while (limit > 0) {
|
||||
auto load_result = this->lf_line_buffer.load_next_line(prev_range);
|
||||
|
@ -560,6 +562,17 @@ logfile::rebuild_index(nonstd::optional<ui_clock::time_point> deadline)
|
|||
&& li.li_file_range.fr_offset > 16 * 1024) {
|
||||
break;
|
||||
}
|
||||
#if 0
|
||||
if (this->lf_line_buffer.is_likely_to_flush(prev_range)
|
||||
&& this->lf_index.size() - begin_size > 1)
|
||||
{
|
||||
log_debug("likely to flush, breaking");
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
if (this->lf_format && !this->back().is_continued()) {
|
||||
lnav::log::watch::eval_with(*this, this->end() - 1);
|
||||
}
|
||||
|
||||
limit -= 1;
|
||||
}
|
||||
|
|
|
@ -44,6 +44,7 @@
|
|||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
#include "ArenaAlloc/arenaalloc.h"
|
||||
#include "base/lnav_log.hh"
|
||||
#include "base/result.h"
|
||||
#include "byte_array.hh"
|
||||
|
@ -96,8 +97,8 @@ class logfile
|
|||
: public unique_path_source
|
||||
, public std::enable_shared_from_this<logfile> {
|
||||
public:
|
||||
typedef std::vector<logline>::iterator iterator;
|
||||
typedef std::vector<logline>::const_iterator const_iterator;
|
||||
using iterator = std::vector<logline>::iterator;
|
||||
using const_iterator = std::vector<logline>::const_iterator;
|
||||
|
||||
/**
|
||||
* Construct a logfile with the given arguments.
|
||||
|
@ -416,6 +417,8 @@ private:
|
|||
uint32_t lf_out_of_time_order_count{0};
|
||||
safe_notes lf_notes;
|
||||
safe_opid_map lf_opids;
|
||||
size_t lf_watch_count{0};
|
||||
ArenaAlloc::Alloc<char> lf_allocator;
|
||||
|
||||
nonstd::optional<std::pair<file_off_t, size_t>> lf_next_line_cache;
|
||||
};
|
||||
|
|
|
@ -36,12 +36,17 @@
|
|||
|
||||
#include "base/ansi_scrubber.hh"
|
||||
#include "base/humanize.time.hh"
|
||||
#include "base/injector.hh"
|
||||
#include "base/itertools.hh"
|
||||
#include "base/string_util.hh"
|
||||
#include "bound_tags.hh"
|
||||
#include "command_executor.hh"
|
||||
#include "config.h"
|
||||
#include "k_merge_tree.h"
|
||||
#include "lnav.events.hh"
|
||||
#include "log_accel.hh"
|
||||
#include "logfile_sub_source.cfg.hh"
|
||||
#include "readline_highlighters.hh"
|
||||
#include "relative_time.hh"
|
||||
#include "sql_util.hh"
|
||||
#include "yajlpp/yajlpp.hh"
|
||||
|
@ -650,7 +655,7 @@ logfile_sub_source::rebuild_index(
|
|||
bool time_left = true;
|
||||
for (const auto file_index : file_order) {
|
||||
auto& ld = *(this->lss_files[file_index]);
|
||||
auto lf = ld.get_file_ptr();
|
||||
auto* lf = ld.get_file_ptr();
|
||||
|
||||
if (lf == nullptr) {
|
||||
if (ld.ld_lines_indexed > 0) {
|
||||
|
@ -760,7 +765,7 @@ logfile_sub_source::rebuild_index(
|
|||
for (iter = this->lss_files.begin(); iter != this->lss_files.end();
|
||||
iter++) {
|
||||
logfile_data& ld = *(*iter);
|
||||
auto lf = ld.get_file_ptr();
|
||||
auto* lf = ld.get_file_ptr();
|
||||
|
||||
if (lf == nullptr) {
|
||||
continue;
|
||||
|
@ -824,7 +829,7 @@ logfile_sub_source::rebuild_index(
|
|||
logline_cmp line_cmper(*this);
|
||||
|
||||
for (auto& ld : this->lss_files) {
|
||||
auto lf = ld->get_file_ptr();
|
||||
auto* lf = ld->get_file_ptr();
|
||||
|
||||
if (lf == nullptr) {
|
||||
continue;
|
||||
|
@ -839,7 +844,7 @@ logfile_sub_source::rebuild_index(
|
|||
|
||||
if (full_sort) {
|
||||
for (auto& ld : this->lss_files) {
|
||||
auto lf = ld->get_file_ptr();
|
||||
auto* lf = ld->get_file_ptr();
|
||||
|
||||
if (lf == nullptr) {
|
||||
continue;
|
||||
|
@ -875,8 +880,8 @@ logfile_sub_source::rebuild_index(
|
|||
|
||||
for (iter = this->lss_files.begin(); iter != this->lss_files.end();
|
||||
iter++) {
|
||||
logfile_data* ld = iter->get();
|
||||
auto lf = ld->get_file_ptr();
|
||||
auto* ld = iter->get();
|
||||
auto* lf = ld->get_file_ptr();
|
||||
if (lf == nullptr) {
|
||||
continue;
|
||||
}
|
||||
|
@ -921,7 +926,7 @@ logfile_sub_source::rebuild_index(
|
|||
|
||||
for (iter = this->lss_files.begin(); iter != this->lss_files.end();
|
||||
iter++) {
|
||||
auto lf = (*iter)->get_file_ptr();
|
||||
auto* lf = (*iter)->get_file_ptr();
|
||||
|
||||
if (lf == nullptr) {
|
||||
continue;
|
||||
|
@ -951,7 +956,7 @@ logfile_sub_source::rebuild_index(
|
|||
continue;
|
||||
}
|
||||
|
||||
auto lf = (*ld)->get_file_ptr();
|
||||
auto* lf = (*ld)->get_file_ptr();
|
||||
auto line_iter = lf->begin() + line_number;
|
||||
|
||||
if (line_iter->is_ignored()) {
|
||||
|
@ -1369,21 +1374,22 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
|
|||
return Ok(false);
|
||||
}
|
||||
|
||||
auto lf = (*ld)->get_file_ptr();
|
||||
auto* lf = (*ld)->get_file_ptr();
|
||||
char timestamp_buffer[64];
|
||||
shared_buffer_ref sbr, raw_sbr;
|
||||
lf->read_full_message(ll, sbr);
|
||||
auto format = lf->get_format();
|
||||
string_attrs_t sa;
|
||||
std::vector<logline_value> values;
|
||||
format->annotate(std::distance(lf->cbegin(), ll), sbr, sa, values);
|
||||
auto line_number = std::distance(lf->cbegin(), ll);
|
||||
format->annotate(line_number, sbr, sa, values);
|
||||
|
||||
sqlite3_reset(stmt);
|
||||
sqlite3_clear_bindings(stmt);
|
||||
|
||||
auto count = sqlite3_bind_parameter_count(stmt);
|
||||
for (int lpc = 0; lpc < count; lpc++) {
|
||||
auto* name = sqlite3_bind_parameter_name(stmt, lpc + 1);
|
||||
const auto* name = sqlite3_bind_parameter_name(stmt, lpc + 1);
|
||||
|
||||
if (name[0] == '$') {
|
||||
const char* env_value;
|
||||
|
@ -1465,6 +1471,12 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
|
|||
SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_format_regex") == 0) {
|
||||
const auto pat_name = format->get_pattern_name(line_number);
|
||||
sqlite3_bind_text(
|
||||
stmt, lpc + 1, pat_name.get(), pat_name.size(), SQLITE_STATIC);
|
||||
continue;
|
||||
}
|
||||
if (strcmp(name, ":log_path") == 0) {
|
||||
const auto& filename = lf->get_filename();
|
||||
sqlite3_bind_text(stmt,
|
||||
|
@ -1491,7 +1503,8 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
|
|||
if (strcmp(name, ":log_body") == 0) {
|
||||
auto body_attr_opt = get_string_attr(sa, SA_BODY);
|
||||
if (body_attr_opt) {
|
||||
auto& sar = body_attr_opt.value().saw_string_attr->sa_range;
|
||||
const auto& sar
|
||||
= body_attr_opt.value().saw_string_attr->sa_range;
|
||||
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
|
@ -1506,7 +1519,8 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
|
|||
if (strcmp(name, ":log_opid") == 0) {
|
||||
auto opid_attr_opt = get_string_attr(sa, logline::L_OPID);
|
||||
if (opid_attr_opt) {
|
||||
auto& sar = opid_attr_opt.value().saw_string_attr->sa_range;
|
||||
const auto& sar
|
||||
= opid_attr_opt.value().saw_string_attr->sa_range;
|
||||
|
||||
sqlite3_bind_text(stmt,
|
||||
lpc + 1,
|
||||
|
@ -2174,7 +2188,7 @@ logfile_sub_source::text_crumbs_for_line(int line,
|
|||
file_data->get_file_ptr()->get_opids());
|
||||
|
||||
for (const auto& pair : *r_opid_map) {
|
||||
retval.emplace_back(pair.first);
|
||||
retval.emplace_back(pair.first.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,49 @@
|
|||
/**
|
||||
* Copyright (c) 2022, Timothy Stack
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* * Neither the name of Timothy Stack nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef lnav_logfile_sub_source_cfg_hh
|
||||
#define lnav_logfile_sub_source_cfg_hh
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
namespace logfile_sub_source_ns {
|
||||
|
||||
struct watch_expression {
|
||||
std::string we_expr;
|
||||
bool we_enabled{true};
|
||||
};
|
||||
|
||||
struct config {
|
||||
std::map<std::string, watch_expression> c_watch_exprs;
|
||||
};
|
||||
|
||||
} // namespace logfile_sub_source_ns
|
||||
|
||||
#endif
|
|
@ -47,6 +47,7 @@
|
|||
#include "bookmarks.hh"
|
||||
#include "document.sections.hh"
|
||||
#include "filter_observer.hh"
|
||||
#include "lnav_config_fwd.hh"
|
||||
#include "log_accel.hh"
|
||||
#include "log_format.hh"
|
||||
#include "logfile.hh"
|
||||
|
@ -666,6 +667,7 @@ public:
|
|||
size_t ld_file_index;
|
||||
line_filter_observer ld_filter_state;
|
||||
size_t ld_lines_indexed{0};
|
||||
size_t ld_lines_watched{0};
|
||||
bool ld_visible;
|
||||
};
|
||||
|
||||
|
|
|
@ -430,8 +430,10 @@ add_config_possibilities()
|
|||
visited.insert(named_iter->pnc_name);
|
||||
}
|
||||
|
||||
rc->add_possibility(
|
||||
ln_mode_t::COMMAND, named_iter->pnc_name, path);
|
||||
ghc::filesystem::path path_obj(path);
|
||||
rc->add_possibility(ln_mode_t::COMMAND,
|
||||
named_iter->pnc_name,
|
||||
path_obj.parent_path().filename().string());
|
||||
}
|
||||
} else {
|
||||
rc->add_possibility(ln_mode_t::COMMAND, "config-option", path);
|
||||
|
|
|
@ -45,20 +45,19 @@ typedef struct {
|
|||
} cache_entry;
|
||||
|
||||
static cache_entry*
|
||||
find_re(const char* re)
|
||||
find_re(string_fragment re)
|
||||
{
|
||||
using safe_cache = safe::Safe<std::unordered_map<std::string, cache_entry>>;
|
||||
static safe_cache CACHE;
|
||||
using re_cache_t
|
||||
= std::unordered_map<string_fragment, cache_entry, frag_hasher>;
|
||||
static thread_local re_cache_t cache;
|
||||
|
||||
safe::WriteAccess<safe_cache> wcache(CACHE);
|
||||
std::string re_str = re;
|
||||
auto iter = wcache->find(re_str);
|
||||
|
||||
if (iter == wcache->end()) {
|
||||
auto iter = cache.find(re);
|
||||
if (iter == cache.end()) {
|
||||
cache_entry c;
|
||||
|
||||
c.re2 = std::make_shared<pcrepp>(re_str);
|
||||
auto pair = wcache->insert(std::make_pair(re_str, c));
|
||||
c.re2 = std::make_shared<pcrepp>(re.to_string());
|
||||
auto pair = cache.insert(
|
||||
std::make_pair(string_fragment{c.re2->get_pattern()}, c));
|
||||
|
||||
iter = pair.first;
|
||||
}
|
||||
|
@ -67,7 +66,7 @@ find_re(const char* re)
|
|||
}
|
||||
|
||||
static bool
|
||||
regexp(const char* re, const char* str)
|
||||
regexp(string_fragment re, string_fragment str)
|
||||
{
|
||||
cache_entry* reobj = find_re(re);
|
||||
pcre_context_static<30> pc;
|
||||
|
@ -77,7 +76,7 @@ regexp(const char* re, const char* str)
|
|||
}
|
||||
|
||||
static util::variant<int64_t, double, const char*, string_fragment, json_string>
|
||||
regexp_match(const char* re, const char* str)
|
||||
regexp_match(string_fragment re, const char* str)
|
||||
{
|
||||
cache_entry* reobj = find_re(re);
|
||||
pcre_context_static<30> pc;
|
||||
|
@ -253,7 +252,7 @@ logfmt2json(string_fragment line)
|
|||
}
|
||||
|
||||
static std::string
|
||||
regexp_replace(const char* str, const char* re, const char* repl)
|
||||
regexp_replace(const char* str, string_fragment re, const char* repl)
|
||||
{
|
||||
cache_entry* reobj = find_re(re);
|
||||
|
||||
|
|
|
@ -279,6 +279,9 @@ textfile_sub_source::text_crumbs_for_line(
|
|||
}
|
||||
|
||||
auto lf = this->current_file();
|
||||
if (lf->size() == 0) {
|
||||
return;
|
||||
}
|
||||
crumbs.emplace_back(
|
||||
lf->get_unique_path(),
|
||||
attr_line_t().append(lnav::roles::identifier(lf->get_unique_path())),
|
||||
|
|
|
@ -185,11 +185,11 @@ textview_curses::reload_config(error_reporter& reporter)
|
|||
nullptr))
|
||||
== nullptr)
|
||||
{
|
||||
reporter(
|
||||
&hl_pair.second.hc_regex,
|
||||
fmt::format(FMT_STRING("invalid highlight regex: {} at {}"),
|
||||
errptr,
|
||||
eoff));
|
||||
reporter(&hl_pair.second.hc_regex,
|
||||
lnav::console::user_message::error(fmt::format(
|
||||
FMT_STRING("invalid highlight regex: {} at {}"),
|
||||
errptr,
|
||||
eoff)));
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -205,13 +205,21 @@ textview_curses::reload_config(error_reporter& reporter)
|
|||
|
||||
auto fg = styling::color_unit::from_str(fg_color).unwrapOrElse(
|
||||
[&](const auto& msg) {
|
||||
reporter(&sc.sc_color, errmsg);
|
||||
reporter(&sc.sc_color,
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t("invalid color -- ")
|
||||
.append_quoted(sc.sc_color))
|
||||
.with_reason(msg));
|
||||
invalid = true;
|
||||
return styling::color_unit::make_empty();
|
||||
});
|
||||
auto bg = styling::color_unit::from_str(bg_color).unwrapOrElse(
|
||||
[&](const auto& msg) {
|
||||
reporter(&sc.sc_background_color, errmsg);
|
||||
reporter(&sc.sc_background_color,
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t("invalid background color -- ")
|
||||
.append_quoted(sc.sc_background_color))
|
||||
.with_reason(msg));
|
||||
invalid = true;
|
||||
return styling::color_unit::make_empty();
|
||||
});
|
||||
|
|
|
@ -0,0 +1,186 @@
|
|||
// -*- c++ -*-
|
||||
/******************************************************************************
|
||||
* arenaalloc.h
|
||||
*
|
||||
* Arena allocator based on the example logic provided by Nicolai Josuttis
|
||||
* and available at http://www.josuttis.com/libbook/examples.html.
|
||||
* This enhanced work is provided under the terms of the MIT license.
|
||||
*
|
||||
*****************************************************************************/
|
||||
|
||||
#ifndef _ARENA_ALLOC_H
|
||||
#define _ARENA_ALLOC_H
|
||||
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
|
||||
#if __cplusplus >= 201103L
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#endif
|
||||
|
||||
// Define macro ARENA_ALLOC_DEBUG to enable some tracing of the allocator
|
||||
#include "arenaallocimpl.h"
|
||||
|
||||
namespace ArenaAlloc
|
||||
{
|
||||
|
||||
struct _newAllocatorImpl
|
||||
{
|
||||
// these two functions should be supported by a specialized
|
||||
// allocator for shared memory or another source of specialized
|
||||
// memory such as device mapped memory.
|
||||
void* allocate( size_t numBytes ) { return new char[ numBytes ]; }
|
||||
void deallocate( void* ptr ) { delete[]( (char*)ptr ); }
|
||||
};
|
||||
|
||||
template <class T,
|
||||
class AllocatorImpl = _newAllocatorImpl,
|
||||
class MemblockImpl = _memblockimpl<AllocatorImpl> >
|
||||
class Alloc {
|
||||
|
||||
private:
|
||||
MemblockImpl* m_impl;
|
||||
|
||||
public:
|
||||
// type definitions
|
||||
typedef T value_type;
|
||||
typedef T* pointer;
|
||||
typedef const T* const_pointer;
|
||||
typedef T& reference;
|
||||
typedef const T& const_reference;
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
|
||||
#if __cplusplus >= 201103L
|
||||
// when containers are swapped, (i.e. vector.swap)
|
||||
// swap the allocators also. This was not specified in c++98
|
||||
// thus users of this code not using c++11 must
|
||||
// exercise caution when using the swap algorithm or
|
||||
// specialized swap member function. Specifically,
|
||||
// don't swap containers not sharing the same
|
||||
// allocator internal implementation in c++98. This is ok
|
||||
// in c++11.
|
||||
typedef std::true_type propagate_on_container_swap;
|
||||
|
||||
// container moves should move the allocator also.
|
||||
typedef std::true_type propagate_on_container_move_assignment;
|
||||
#endif
|
||||
|
||||
// rebind allocator to type U
|
||||
template <class U>
|
||||
struct rebind {
|
||||
typedef Alloc<U,AllocatorImpl,MemblockImpl> other;
|
||||
};
|
||||
|
||||
// return address of values
|
||||
pointer address (reference value) const {
|
||||
return &value;
|
||||
}
|
||||
const_pointer address (const_reference value) const {
|
||||
return &value;
|
||||
}
|
||||
|
||||
Alloc( std::size_t defaultSize = 32768, AllocatorImpl allocImpl = AllocatorImpl() ) throw():
|
||||
m_impl( MemblockImpl::create( defaultSize, allocImpl ) )
|
||||
{
|
||||
}
|
||||
|
||||
Alloc(const Alloc& src) throw():
|
||||
m_impl( src.m_impl )
|
||||
{
|
||||
m_impl->incrementRefCount();
|
||||
}
|
||||
|
||||
template <class U>
|
||||
Alloc (const Alloc<U,AllocatorImpl,MemblockImpl>& src) throw():
|
||||
m_impl( 0 )
|
||||
{
|
||||
MemblockImpl::assign( src, m_impl );
|
||||
m_impl->incrementRefCount();
|
||||
}
|
||||
|
||||
~Alloc() throw()
|
||||
{
|
||||
m_impl->decrementRefCount();
|
||||
}
|
||||
|
||||
// return maximum number of elements that can be allocated
|
||||
size_type max_size () const throw()
|
||||
{
|
||||
return std::numeric_limits<std::size_t>::max() / sizeof(T);
|
||||
}
|
||||
|
||||
// allocate but don't initialize num elements of type T
|
||||
pointer allocate (size_type num, const void* = 0)
|
||||
{
|
||||
return reinterpret_cast<pointer>( m_impl->allocate(num*sizeof(T)) );
|
||||
}
|
||||
|
||||
// initialize elements of allocated storage p with value value
|
||||
#if __cplusplus >= 201103L
|
||||
|
||||
// use c++11 style forwarding to construct the object
|
||||
template< typename P, typename... Args>
|
||||
void construct( P* obj, Args&&... args )
|
||||
{
|
||||
::new((void*) obj ) P( std::forward<Args>( args )... );
|
||||
}
|
||||
|
||||
template< typename P >
|
||||
void destroy( P* obj ) { obj->~P(); }
|
||||
|
||||
#else
|
||||
void construct (pointer p, const T& value)
|
||||
{
|
||||
new((void*)p)T(value);
|
||||
}
|
||||
void destroy (pointer p) { p->~T(); }
|
||||
#endif
|
||||
|
||||
// deallocate storage p of deleted elements
|
||||
void deallocate (pointer p, size_type num)
|
||||
{
|
||||
m_impl->deallocate( p );
|
||||
}
|
||||
|
||||
bool equals( const MemblockImpl * impl ) const
|
||||
{
|
||||
return impl == m_impl;
|
||||
}
|
||||
|
||||
bool operator == ( const Alloc& t2 ) const
|
||||
{
|
||||
return m_impl == t2.m_impl;
|
||||
}
|
||||
|
||||
friend MemblockImpl;
|
||||
|
||||
template< typename Other >
|
||||
bool operator == ( const Alloc< Other, AllocatorImpl, MemblockImpl >& t2 )
|
||||
{
|
||||
return t2.equals( m_impl );
|
||||
}
|
||||
|
||||
template< typename Other >
|
||||
bool operator != ( const Alloc< Other, AllocatorImpl, MemblockImpl >& t2 )
|
||||
{
|
||||
return !t2.equals( m_impl );
|
||||
}
|
||||
|
||||
// These are extension functions not required for an stl allocator
|
||||
size_t getNumAllocations() { return m_impl->getNumAllocations(); }
|
||||
size_t getNumDeallocations() { return m_impl->getNumDeallocations(); }
|
||||
size_t getNumBytesAllocated() { return m_impl->getNumBytesAllocated(); }
|
||||
};
|
||||
|
||||
template<typename A>
|
||||
template<typename T>
|
||||
void _memblockimpl<A>::assign( const Alloc<T,A, _memblockimpl<A> >& src, _memblockimpl<A> *& dest )
|
||||
{
|
||||
dest = const_cast<_memblockimpl<A>* >(src.m_impl);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
|
@ -0,0 +1,286 @@
|
|||
// -*- c++ -*-
|
||||
/******************************************************************************
|
||||
** arenaallocimpl.h
|
||||
**
|
||||
** Internal implementation types of the arena allocator
|
||||
** MIT license
|
||||
*****************************************************************************/
|
||||
|
||||
#ifndef _ARENA_ALLOC_IMPL_H
|
||||
#define _ARENA_ALLOC_IMPL_H
|
||||
|
||||
#ifdef ARENA_ALLOC_DEBUG
|
||||
#include <stdio.h>
|
||||
#endif
|
||||
|
||||
namespace ArenaAlloc
|
||||
{
|
||||
|
||||
template< typename T, typename A, typename M >
|
||||
class Alloc;
|
||||
|
||||
// internal structure for tracking memory blocks
|
||||
template < typename AllocImpl >
|
||||
struct _memblock
|
||||
{
|
||||
// allocations are rounded up to a multiple of the size of this
|
||||
// struct to maintain proper alignment for any pointer and double
|
||||
// values stored in the allocation.
|
||||
// A future goal is to support even stricter alignment for example
|
||||
// to support cache alignment, special device dependent mappings,
|
||||
// or GPU ops.
|
||||
union _roundsize {
|
||||
double d;
|
||||
void* p;
|
||||
};
|
||||
|
||||
_memblock* m_next{nullptr}; // blocks kept link listed for cleanup at end
|
||||
std::size_t m_bufferSize; // size of the buffer
|
||||
std::size_t m_index; // index of next allocatable byte in the block
|
||||
char* m_buffer; // pointer to large block to allocate from
|
||||
|
||||
_memblock(std::size_t bufferSize, AllocImpl& allocImpl)
|
||||
: m_bufferSize(roundSize(bufferSize)), m_index(0),
|
||||
m_buffer(reinterpret_cast<char*>(allocImpl.allocate(
|
||||
bufferSize))) // this works b/c of order of decl
|
||||
{
|
||||
}
|
||||
|
||||
std::size_t roundSize( std::size_t numBytes )
|
||||
{
|
||||
// this is subject to overflow. calling logic should not permit
|
||||
// an attempt to allocate a really massive size.
|
||||
// i.e. an attempt to allocate 10s of terabytes should be an error
|
||||
return ( ( numBytes + sizeof( _roundsize ) - 1 ) /
|
||||
sizeof( _roundsize ) ) * sizeof( _roundsize );
|
||||
}
|
||||
|
||||
char * allocate( std::size_t numBytes )
|
||||
{
|
||||
std::size_t roundedSize = roundSize( numBytes );
|
||||
if( roundedSize + m_index > m_bufferSize )
|
||||
return 0;
|
||||
|
||||
char * ptrToReturn = &m_buffer[ m_index ];
|
||||
m_index += roundedSize;
|
||||
return ptrToReturn;
|
||||
}
|
||||
|
||||
void dispose( AllocImpl& impl )
|
||||
{
|
||||
impl.deallocate( m_buffer );
|
||||
}
|
||||
|
||||
~_memblock()
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
template< typename AllocatorImpl, typename Derived >
|
||||
struct _memblockimplbase
|
||||
{
|
||||
AllocatorImpl m_alloc;
|
||||
std::size_t m_refCount; // when refs -> 0 delete this
|
||||
std::size_t m_defaultSize;
|
||||
|
||||
std::size_t m_numAllocate; // number of times allocate called
|
||||
std::size_t m_numDeallocate; // number of time deallocate called
|
||||
std::size_t m_numBytesAllocated; // A good estimate of amount of space used
|
||||
|
||||
_memblock<AllocatorImpl> * m_head;
|
||||
_memblock<AllocatorImpl> * m_current;
|
||||
|
||||
// round up 2 next power of 2 if not already
|
||||
// a power of 2
|
||||
std::size_t roundpow2( std::size_t value )
|
||||
{
|
||||
// note this works because subtracting 1 is equivalent to
|
||||
// inverting the lowest set bit and complementing any
|
||||
// bits lower than that. only a power of 2
|
||||
// will yield 0 in the following check
|
||||
if( 0 == ( value & ( value - 1 ) ) )
|
||||
return value; // already a power of 2
|
||||
|
||||
// fold t over itself. This will set all bits after the highest set bit of t to 1
|
||||
// who said bit twiddling wasn't practical?
|
||||
value |= value >> 1;
|
||||
value |= value >> 2;
|
||||
value |= value >> 4;
|
||||
value |= value >> 8;
|
||||
value |= value >> 16;
|
||||
value |= value >> 32;
|
||||
|
||||
return value + 1;
|
||||
}
|
||||
|
||||
_memblockimplbase( std::size_t defaultSize, AllocatorImpl& allocator ):
|
||||
m_alloc( allocator ),
|
||||
m_refCount( 1 ),
|
||||
m_defaultSize( defaultSize ),
|
||||
m_numAllocate( 0 ),
|
||||
m_numDeallocate( 0 ),
|
||||
m_numBytesAllocated( 0 ),
|
||||
m_head( 0 ),
|
||||
m_current( 0 )
|
||||
{
|
||||
if( m_defaultSize < 256 )
|
||||
{
|
||||
m_defaultSize = 256; // anything less is academic. a more practical size is 4k or more
|
||||
}
|
||||
else if ( m_defaultSize > 1024UL*1024*1024*16 )
|
||||
{
|
||||
// when this becomes a problem, this package has succeeded beyond my wildest expectations
|
||||
m_defaultSize = 1024UL*1024*1024*16;
|
||||
}
|
||||
|
||||
// for convenience block size should be a power of 2
|
||||
// round up to next power of 2
|
||||
m_defaultSize = roundpow2( m_defaultSize );
|
||||
allocateNewBlock( m_defaultSize );
|
||||
}
|
||||
|
||||
char * allocate( std::size_t numBytes )
|
||||
{
|
||||
char * ptrToReturn = m_current->allocate( numBytes );
|
||||
if( !ptrToReturn )
|
||||
{
|
||||
allocateNewBlock( numBytes > m_defaultSize / 2 ? roundpow2( numBytes*2 ) :
|
||||
m_defaultSize );
|
||||
|
||||
ptrToReturn = m_current->allocate( numBytes );
|
||||
}
|
||||
|
||||
#ifdef ARENA_ALLOC_DEBUG
|
||||
fprintf( stdout, "_memblockimpl=%p allocated %ld bytes at address=%p\n", this, numBytes, ptrToReturn );
|
||||
#endif
|
||||
|
||||
++ m_numAllocate;
|
||||
m_numBytesAllocated += numBytes; // does not account for the small overhead in tracking the allocation
|
||||
|
||||
return ptrToReturn;
|
||||
}
|
||||
|
||||
void allocateNewBlock( std::size_t blockSize )
|
||||
{
|
||||
_memblock<AllocatorImpl> * newBlock = new ( m_alloc.allocate( sizeof( _memblock<AllocatorImpl> ) ) )
|
||||
_memblock<AllocatorImpl>( blockSize, m_alloc );
|
||||
|
||||
#ifdef ARENA_ALLOC_DEBUG
|
||||
fprintf( stdout, "_memblockimplbase=%p allocating a new block of size=%ld\n", this, blockSize );
|
||||
#endif
|
||||
|
||||
if( m_head == 0 )
|
||||
{
|
||||
m_head = m_current = newBlock;
|
||||
}
|
||||
else
|
||||
{
|
||||
m_current->m_next = newBlock;
|
||||
m_current = newBlock;
|
||||
}
|
||||
}
|
||||
|
||||
void deallocate( void * ptr )
|
||||
{
|
||||
++ m_numDeallocate;
|
||||
}
|
||||
|
||||
size_t getNumAllocations() { return m_numAllocate; }
|
||||
size_t getNumDeallocations() { return m_numDeallocate; }
|
||||
size_t getNumBytesAllocated() { return m_numBytesAllocated; }
|
||||
|
||||
void clear()
|
||||
{
|
||||
_memblock<AllocatorImpl> * block = m_head;
|
||||
while( block )
|
||||
{
|
||||
_memblock<AllocatorImpl> * curr = block;
|
||||
block = block->m_next;
|
||||
curr->dispose( m_alloc );
|
||||
curr->~_memblock<AllocatorImpl>();
|
||||
m_alloc.deallocate( curr );
|
||||
}
|
||||
}
|
||||
|
||||
// The ref counting model does not permit the sharing of
|
||||
// this object across multiple threads unless an external locking mechanism is applied
|
||||
// to ensure the atomicity of the reference count.
|
||||
void incrementRefCount()
|
||||
{
|
||||
++m_refCount;
|
||||
#ifdef ARENA_ALLOC_DEBUG
|
||||
fprintf( stdout, "ref count on _memblockimplbase=%p incremented to %ld\n", this, m_refCount );
|
||||
#endif
|
||||
}
|
||||
|
||||
void decrementRefCount()
|
||||
{
|
||||
--m_refCount;
|
||||
#ifdef ARENA_ALLOC_DEBUG
|
||||
fprintf( stdout, "ref count on _memblockimplbase=%p decremented to %ld\n", this, m_refCount );
|
||||
#endif
|
||||
|
||||
if( m_refCount == 0 )
|
||||
{
|
||||
Derived::destroy( static_cast<Derived*>(this) );
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Each allocator points to an instance of _memblockimpl which
|
||||
// contains the list of _memblock objects and other tracking info
|
||||
// including a refcount.
|
||||
// This object is instantiated in space obtained from the allocator
|
||||
// implementation. The allocator implementation is the component
|
||||
// on which allocate/deallocate are called to obtain storage from.
|
||||
template< typename AllocatorImpl >
|
||||
struct _memblockimpl : public _memblockimplbase<AllocatorImpl, _memblockimpl<AllocatorImpl> >
|
||||
{
|
||||
private:
|
||||
|
||||
typedef struct _memblockimplbase< AllocatorImpl, _memblockimpl<AllocatorImpl> > base_t;
|
||||
friend struct _memblockimplbase< AllocatorImpl, _memblockimpl<AllocatorImpl> >;
|
||||
|
||||
// to get around some sticky access issues between Alloc<T1> and Alloc<T2> when sharing
|
||||
// the implementation.
|
||||
template <typename U, typename A, typename M >
|
||||
friend class Alloc;
|
||||
|
||||
template< typename T >
|
||||
static void assign( const Alloc<T,AllocatorImpl, _memblockimpl<AllocatorImpl> >& src,
|
||||
_memblockimpl *& dest );
|
||||
|
||||
static _memblockimpl<AllocatorImpl> * create( size_t defaultSize, AllocatorImpl& alloc )
|
||||
{
|
||||
return new ( alloc.allocate( sizeof( _memblockimpl ) ) ) _memblockimpl<AllocatorImpl>( defaultSize,
|
||||
alloc );
|
||||
}
|
||||
|
||||
static void destroy( _memblockimpl<AllocatorImpl> * objToDestroy )
|
||||
{
|
||||
AllocatorImpl allocImpl = objToDestroy->m_alloc;
|
||||
objToDestroy-> ~_memblockimpl<AllocatorImpl>();
|
||||
allocImpl.deallocate( objToDestroy );
|
||||
}
|
||||
|
||||
_memblockimpl( std::size_t defaultSize, AllocatorImpl& allocImpl ):
|
||||
_memblockimplbase<AllocatorImpl, _memblockimpl<AllocatorImpl> >( defaultSize, allocImpl )
|
||||
{
|
||||
#ifdef ARENA_ALLOC_DEBUG
|
||||
fprintf( stdout, "_memblockimpl=%p constructed with default size=%ld\n", this,
|
||||
base_t::m_defaultSize );
|
||||
#endif
|
||||
}
|
||||
|
||||
~_memblockimpl( )
|
||||
{
|
||||
#ifdef ARENA_ALLOC_DEBUG
|
||||
fprintf( stdout, "~memblockimpl() called on _memblockimpl=%p\n", this );
|
||||
#endif
|
||||
base_t::clear();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#endif
|
|
@ -0,0 +1,184 @@
|
|||
// -*- c++ -*-
|
||||
/******************************************************************************
|
||||
** recyclealloc.h
|
||||
**
|
||||
** Arena allocator with some modest recycling of freed resources.
|
||||
** MIT license
|
||||
**
|
||||
*****************************************************************************/
|
||||
#ifndef _RECYCLE_ALLOC_H
|
||||
#define _RECYCLE_ALLOC_H
|
||||
|
||||
#include "arenaalloc.h"
|
||||
#include <string.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
namespace ArenaAlloc
|
||||
{
|
||||
|
||||
// todo:
|
||||
// attempt refactor of boilerplate in _memblockimpl and _recycleallocimpl
|
||||
template< typename AllocatorImpl, uint16_t StepSize = 16, uint16_t NumBuckets = 256 >
|
||||
struct _recycleallocimpl : public _memblockimplbase<AllocatorImpl, _recycleallocimpl<AllocatorImpl> >
|
||||
{
|
||||
private:
|
||||
|
||||
static_assert( ( StepSize >= 16 && NumBuckets >= 16 ), "Min step size=16, Min num buckets=16" );
|
||||
static_assert( !( StepSize & ( StepSize - 1 ) ), "Step size must be a power of 2" );
|
||||
|
||||
struct _freeEntry
|
||||
{
|
||||
// note: order of declaration matters
|
||||
std::size_t m_size;
|
||||
_freeEntry * m_next;
|
||||
};
|
||||
|
||||
_freeEntry * m_buckets[ NumBuckets ]; // m_buckets[ NumBuckets - 1 ] is the oversize bucket
|
||||
|
||||
typedef struct _memblockimplbase< AllocatorImpl, _recycleallocimpl<AllocatorImpl> > base_t;
|
||||
friend struct _memblockimplbase< AllocatorImpl, _recycleallocimpl<AllocatorImpl> >;
|
||||
|
||||
// to get around some sticky access issues between Alloc<T1> and Alloc<T2> when sharing
|
||||
// the implementation.
|
||||
template <typename U, typename A, typename M >
|
||||
friend class Alloc;
|
||||
|
||||
template< typename T >
|
||||
static void assign( const Alloc<T,AllocatorImpl, _recycleallocimpl<AllocatorImpl> >& src,
|
||||
_recycleallocimpl *& dest )
|
||||
{
|
||||
dest = const_cast< _recycleallocimpl<AllocatorImpl>* >( src.m_impl );
|
||||
}
|
||||
|
||||
static _recycleallocimpl<AllocatorImpl> * create( std::size_t defaultSize, AllocatorImpl& alloc )
|
||||
{
|
||||
return new (
|
||||
alloc.allocate( sizeof( _recycleallocimpl ) ) ) _recycleallocimpl<AllocatorImpl>( defaultSize,
|
||||
alloc );
|
||||
}
|
||||
|
||||
static void destroy( _recycleallocimpl<AllocatorImpl> * objToDestroy )
|
||||
{
|
||||
AllocatorImpl allocImpl = objToDestroy->m_alloc;
|
||||
objToDestroy-> ~_recycleallocimpl<AllocatorImpl>();
|
||||
allocImpl.deallocate( objToDestroy );
|
||||
}
|
||||
|
||||
_recycleallocimpl( std::size_t defaultSize, AllocatorImpl& allocImpl ):
|
||||
_memblockimplbase<AllocatorImpl, _recycleallocimpl<AllocatorImpl> >( defaultSize, allocImpl )
|
||||
{
|
||||
memset( m_buckets, 0, sizeof( m_buckets ) );
|
||||
|
||||
#ifdef ARENA_ALLOC_DEBUG
|
||||
fprintf( stdout, "_recycleallocimpl=%p constructed with default size=%ld\n", this,
|
||||
base_t::m_defaultSize );
|
||||
#endif
|
||||
}
|
||||
|
||||
~_recycleallocimpl( )
|
||||
{
|
||||
#ifdef ARENA_ALLOC_DEBUG
|
||||
fprintf( stdout, "~_recycleallocimpl() called on _recycleallocimpl=%p\n", this );
|
||||
#endif
|
||||
base_t::clear();
|
||||
}
|
||||
|
||||
char * allocate( std::size_t numBytes )
|
||||
{
|
||||
|
||||
numBytes = ( (numBytes + sizeof( std::size_t ) + StepSize - 1) / StepSize ) * StepSize;
|
||||
|
||||
char * returnValue = allocateInternal( numBytes );
|
||||
if( !returnValue )
|
||||
{
|
||||
char * allocValue = base_t::allocate( numBytes );
|
||||
|
||||
if( !allocValue )
|
||||
return 0; //allocation failure
|
||||
|
||||
*((std::size_t*)allocValue ) = numBytes; // that includes the header
|
||||
return allocValue + sizeof( std::size_t );
|
||||
}
|
||||
|
||||
return returnValue;
|
||||
}
|
||||
|
||||
void deallocate( void * ptr )
|
||||
{
|
||||
deallocateInternal( reinterpret_cast<char*>(ptr) );
|
||||
base_t::deallocate( ptr ); // this is called b/c it is known this just updates stats
|
||||
}
|
||||
|
||||
char * allocateInternal( std::size_t numBytes )
|
||||
{
|
||||
// numBytes must already be rounded to a multiple of stepsize and have an
|
||||
// extra sizeof( std::size_t ) bytes tacked on for the header
|
||||
// pointer returned points sizeof( std::size_t ) bytes into the allocation
|
||||
// bucket 0 is always null in this scheme.
|
||||
|
||||
uint16_t bucketNumber = numBytes / StepSize;
|
||||
|
||||
if( bucketNumber > NumBuckets - 1 )
|
||||
bucketNumber = NumBuckets - 1; // oversize alloc
|
||||
|
||||
// search max 3 consecutive buckets for an item large enough.
|
||||
// in the oversize bucket and only in the oversize bucket,
|
||||
// search upto 3 items into the linked list for an entry
|
||||
// large enough for the specified size
|
||||
for( uint16_t bkt = bucketNumber, i = 0; i < 3 && bkt < NumBuckets; ++i, ++bkt )
|
||||
{
|
||||
if( m_buckets[ bkt ] )
|
||||
return allocateFrom( numBytes, m_buckets[ bkt ] );
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
char * allocateFrom( std::size_t numBytes, _freeEntry *& head )
|
||||
{
|
||||
_freeEntry * current = head;
|
||||
_freeEntry * prev = 0;
|
||||
|
||||
int count = 0;
|
||||
|
||||
while( current && count < 3 )
|
||||
{
|
||||
if( current->m_size >= numBytes )
|
||||
{
|
||||
if( prev == 0 )
|
||||
head = current->m_next;
|
||||
else
|
||||
prev->m_next = current->m_next;
|
||||
|
||||
return reinterpret_cast<char*>(¤t->m_next);
|
||||
}
|
||||
|
||||
++count;
|
||||
prev = current;
|
||||
current = current->m_next;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void deallocateInternal( char * ptr )
|
||||
{
|
||||
_freeEntry * v = reinterpret_cast< _freeEntry* >( ptr - sizeof( std::size_t ) );
|
||||
uint16_t bucketNumber = v->m_size / StepSize;
|
||||
|
||||
if( bucketNumber > NumBuckets - 1 )
|
||||
bucketNumber = NumBuckets - 1;
|
||||
|
||||
_freeEntry * next = m_buckets[ bucketNumber ];
|
||||
v->m_next = next;
|
||||
m_buckets[ bucketNumber ] = v;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
template< typename T, typename Allocator = _newAllocatorImpl >
|
||||
using RecycleAlloc = Alloc< T, Allocator, _recycleallocimpl<Allocator> >;
|
||||
|
||||
}
|
||||
|
||||
#endif
|
|
@ -0,0 +1,770 @@
|
|||
/*
|
||||
* xxHash - Extremely Fast Hash algorithm
|
||||
* Copyright (C) 2020-2021 Yann Collet
|
||||
*
|
||||
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are
|
||||
* met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following disclaimer
|
||||
* in the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - xxHash homepage: https://www.xxhash.com
|
||||
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
|
||||
/*!
|
||||
* @file xxh_x86dispatch.c
|
||||
*
|
||||
* Automatic dispatcher code for the @ref XXH3_family on x86-based targets.
|
||||
*
|
||||
* Optional add-on.
|
||||
*
|
||||
* **Compile this file with the default flags for your target.** Do not compile
|
||||
* with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`, there will be
|
||||
* an error. See @ref XXH_X86DISPATCH_ALLOW_AVX for details.
|
||||
*
|
||||
* @defgroup dispatch x86 Dispatcher
|
||||
* @{
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64))
|
||||
# error "Dispatching is currently only supported on x86 and x86_64."
|
||||
#endif
|
||||
|
||||
/*!
|
||||
* @def XXH_X86DISPATCH_ALLOW_AVX
|
||||
* @brief Disables the AVX sanity check.
|
||||
*
|
||||
* Don't compile xxh_x86dispatch.c with options like `-mavx*`, `-march=native`,
|
||||
* or `/arch:AVX*`. It is intended to be compiled for the minimum target, and
|
||||
* it selectively enables SSE2, AVX2, and AVX512 when it is needed.
|
||||
*
|
||||
* Using this option _globally_ allows this feature, and therefore makes it
|
||||
* undefined behavior to execute on any CPU without said feature.
|
||||
*
|
||||
* Even if the source code isn't directly using AVX intrinsics in a function,
|
||||
* the compiler can still generate AVX code from autovectorization and by
|
||||
* "upgrading" SSE2 intrinsics to use the VEX prefixes (a.k.a. AVX128).
|
||||
*
|
||||
* Use the same flags that you use to compile the rest of the program; this
|
||||
* file will safely generate SSE2, AVX2, and AVX512 without these flags.
|
||||
*
|
||||
* Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check, and feel free to open
|
||||
* an issue if there is a target in the future where AVX is a default feature.
|
||||
*/
|
||||
#ifdef XXH_DOXYGEN
|
||||
# define XXH_X86DISPATCH_ALLOW_AVX
|
||||
#endif
|
||||
|
||||
#if defined(__AVX__) && !defined(XXH_X86DISPATCH_ALLOW_AVX)
|
||||
# error "Do not compile xxh_x86dispatch.c with AVX enabled! See the comment above."
|
||||
#endif
|
||||
|
||||
#ifdef __has_include
|
||||
# define XXH_HAS_INCLUDE(header) __has_include(header)
|
||||
#else
|
||||
# define XXH_HAS_INCLUDE(header) 0
|
||||
#endif
|
||||
|
||||
/*!
|
||||
* @def XXH_DISPATCH_SCALAR
|
||||
* @brief Enables/dispatching the scalar code path.
|
||||
*
|
||||
* If this is defined to 0, SSE2 support is assumed. This reduces code size
|
||||
* when the scalar path is not needed.
|
||||
*
|
||||
* This is automatically defined to 0 when...
|
||||
* - SSE2 support is enabled in the compiler
|
||||
* - Targeting x86_64
|
||||
* - Targeting Android x86
|
||||
* - Targeting macOS
|
||||
*/
|
||||
#ifndef XXH_DISPATCH_SCALAR
|
||||
# if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \
|
||||
|| defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \
|
||||
|| defined(__ANDROID__) || defined(__APPLEv__) /* Android or macOS */
|
||||
# define XXH_DISPATCH_SCALAR 0 /* disable */
|
||||
# else
|
||||
# define XXH_DISPATCH_SCALAR 1
|
||||
# endif
|
||||
#endif
|
||||
/*!
|
||||
* @def XXH_DISPATCH_AVX2
|
||||
* @brief Enables/disables dispatching for AVX2.
|
||||
*
|
||||
* This is automatically detected if it is not defined.
|
||||
* - GCC 4.7 and later are known to support AVX2, but >4.9 is required for
|
||||
* to get the AVX2 intrinsics and typedefs without -mavx -mavx2.
|
||||
* - Visual Studio 2013 Update 2 and later are known to support AVX2.
|
||||
* - The GCC/Clang internal header `<avx2intrin.h>` is detected. While this is
|
||||
* not allowed to be included directly, it still appears in the builtin
|
||||
* include path and is detectable with `__has_include`.
|
||||
*
|
||||
* @see XXH_AVX2
|
||||
*/
|
||||
#ifndef XXH_DISPATCH_AVX2
|
||||
# if (defined(__GNUC__) && (__GNUC__ > 4)) /* GCC 5.0+ */ \
|
||||
|| (defined(_MSC_VER) && _MSC_VER >= 1900) /* VS 2015+ */ \
|
||||
|| (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 180030501) /* VS 2013 Update 2 */ \
|
||||
|| XXH_HAS_INCLUDE(<avx2intrin.h>) /* GCC/Clang internal header */
|
||||
# define XXH_DISPATCH_AVX2 1 /* enable dispatch towards AVX2 */
|
||||
# else
|
||||
# define XXH_DISPATCH_AVX2 0
|
||||
# endif
|
||||
#endif /* XXH_DISPATCH_AVX2 */
|
||||
|
||||
/*!
|
||||
* @def XXH_DISPATCH_AVX512
|
||||
* @brief Enables/disables dispatching for AVX512.
|
||||
*
|
||||
* Automatically detected if one of the following conditions is met:
|
||||
* - GCC 4.9 and later are known to support AVX512.
|
||||
* - Visual Studio 2017 and later are known to support AVX2.
|
||||
* - The GCC/Clang internal header `<avx512fintrin.h>` is detected. While this
|
||||
* is not allowed to be included directly, it still appears in the builtin
|
||||
* include path and is detectable with `__has_include`.
|
||||
*
|
||||
* @see XXH_AVX512
|
||||
*/
|
||||
#ifndef XXH_DISPATCH_AVX512
|
||||
# if (defined(__GNUC__) \
|
||||
&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9))) /* GCC 4.9+ */ \
|
||||
|| (defined(_MSC_VER) && _MSC_VER >= 1910) /* VS 2017+ */ \
|
||||
|| XXH_HAS_INCLUDE(<avx512fintrin.h>) /* GCC/Clang internal header */
|
||||
# define XXH_DISPATCH_AVX512 1 /* enable dispatch towards AVX512 */
|
||||
# else
|
||||
# define XXH_DISPATCH_AVX512 0
|
||||
# endif
|
||||
#endif /* XXH_DISPATCH_AVX512 */
|
||||
|
||||
/*!
|
||||
* @def XXH_TARGET_SSE2
|
||||
* @brief Allows a function to be compiled with SSE2 intrinsics.
|
||||
*
|
||||
* Uses `__attribute__((__target__("sse2")))` on GCC to allow SSE2 to be used
|
||||
* even with `-mno-sse2`.
|
||||
*
|
||||
* @def XXH_TARGET_AVX2
|
||||
* @brief Like @ref XXH_TARGET_SSE2, but for AVX2.
|
||||
*
|
||||
* @def XXH_TARGET_AVX512
|
||||
* @brief Like @ref XXH_TARGET_SSE2, but for AVX512.
|
||||
*/
|
||||
#if defined(__GNUC__)
|
||||
# include <emmintrin.h> /* SSE2 */
|
||||
# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
||||
# include <immintrin.h> /* AVX2, AVX512F */
|
||||
# endif
|
||||
# define XXH_TARGET_SSE2 __attribute__((__target__("sse2")))
|
||||
# define XXH_TARGET_AVX2 __attribute__((__target__("avx2")))
|
||||
# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f")))
|
||||
#elif defined(_MSC_VER)
|
||||
# include <intrin.h>
|
||||
# define XXH_TARGET_SSE2
|
||||
# define XXH_TARGET_AVX2
|
||||
# define XXH_TARGET_AVX512
|
||||
#else
|
||||
# error "Dispatching is currently not supported for your compiler."
|
||||
#endif
|
||||
|
||||
#ifdef XXH_DISPATCH_DEBUG
|
||||
/* debug logging */
|
||||
# include <stdio.h>
|
||||
# define XXH_debugPrint(str) { fprintf(stderr, "DEBUG: xxHash dispatch: %s \n", str); fflush(NULL); }
|
||||
#else
|
||||
# define XXH_debugPrint(str) ((void)0)
|
||||
# undef NDEBUG /* avoid redefinition */
|
||||
# define NDEBUG
|
||||
#endif
|
||||
#include <assert.h>
|
||||
|
||||
#define XXH_INLINE_ALL
|
||||
#define XXH_X86DISPATCH
|
||||
#include "xxhash.h"
|
||||
|
||||
/*
|
||||
* Support both AT&T and Intel dialects
|
||||
*
|
||||
* GCC doesn't convert AT&T syntax to Intel syntax, and will error out if
|
||||
* compiled with -masm=intel. Instead, it supports dialect switching with
|
||||
* curly braces: { AT&T syntax | Intel syntax }
|
||||
*
|
||||
* Clang's integrated assembler automatically converts AT&T syntax to Intel if
|
||||
* needed, making the dialect switching useless (it isn't even supported).
|
||||
*
|
||||
* Note: Comments are written in the inline assembly itself.
|
||||
*/
|
||||
#ifdef __clang__
|
||||
# define XXH_I_ATT(intel, att) att "\n\t"
|
||||
#else
|
||||
# define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t"
|
||||
#endif
|
||||
|
||||
/*!
|
||||
* @internal
|
||||
* @brief Runs CPUID.
|
||||
*
|
||||
* @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively.
|
||||
* @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }`
|
||||
*/
|
||||
static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd)
|
||||
{
|
||||
#if defined(_MSC_VER)
|
||||
__cpuidex(abcd, eax, ecx);
|
||||
#else
|
||||
xxh_u32 ebx, edx;
|
||||
# if defined(__i386__) && defined(__PIC__)
|
||||
__asm__(
|
||||
"# Call CPUID\n\t"
|
||||
"#\n\t"
|
||||
"# On 32-bit x86 with PIC enabled, we are not allowed to overwrite\n\t"
|
||||
"# EBX, so we use EDI instead.\n\t"
|
||||
XXH_I_ATT("mov edi, ebx", "movl %%ebx, %%edi")
|
||||
XXH_I_ATT("cpuid", "cpuid" )
|
||||
XXH_I_ATT("xchg edi, ebx", "xchgl %%ebx, %%edi")
|
||||
: "=D" (ebx),
|
||||
# else
|
||||
__asm__(
|
||||
"# Call CPUID\n\t"
|
||||
XXH_I_ATT("cpuid", "cpuid")
|
||||
: "=b" (ebx),
|
||||
# endif
|
||||
"+a" (eax), "+c" (ecx), "=d" (edx));
|
||||
abcd[0] = eax;
|
||||
abcd[1] = ebx;
|
||||
abcd[2] = ecx;
|
||||
abcd[3] = edx;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Modified version of Intel's guide
|
||||
* https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family
|
||||
*/
|
||||
|
||||
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
||||
/*!
|
||||
* @internal
|
||||
* @brief Runs `XGETBV`.
|
||||
*
|
||||
* While the CPU may support AVX2, the operating system might not properly save
|
||||
* the full YMM/ZMM registers.
|
||||
*
|
||||
* xgetbv is used for detecting this: Any compliant operating system will define
|
||||
* a set of flags in the xcr0 register indicating how it saves the AVX registers.
|
||||
*
|
||||
* You can manually disable this flag on Windows by running, as admin:
|
||||
*
|
||||
* bcdedit.exe /set xsavedisable 1
|
||||
*
|
||||
* and rebooting. Run the same command with 0 to re-enable it.
|
||||
*/
|
||||
static xxh_u64 XXH_xgetbv(void)
|
||||
{
|
||||
#if defined(_MSC_VER)
|
||||
return _xgetbv(0); /* min VS2010 SP1 compiler is required */
|
||||
#else
|
||||
xxh_u32 xcr0_lo, xcr0_hi;
|
||||
__asm__(
|
||||
"# Call XGETBV\n\t"
|
||||
"#\n\t"
|
||||
"# Older assemblers (e.g. macOS's ancient GAS version) don't support\n\t"
|
||||
"# the XGETBV opcode, so we encode it by hand instead.\n\t"
|
||||
"# See <https://github.com/asmjit/asmjit/issues/78> for details.\n\t"
|
||||
".byte 0x0f, 0x01, 0xd0\n\t"
|
||||
: "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0));
|
||||
return xcr0_lo | ((xxh_u64)xcr0_hi << 32);
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
#define XXH_SSE2_CPUID_MASK (1 << 26)
|
||||
#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27))
|
||||
#define XXH_AVX2_CPUID_MASK (1 << 5)
|
||||
#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1))
|
||||
#define XXH_AVX512F_CPUID_MASK (1 << 16)
|
||||
#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1))
|
||||
|
||||
/*!
|
||||
* @internal
|
||||
* @brief Returns the best XXH3 implementation.
|
||||
*
|
||||
* Runs various CPUID/XGETBV tests to try and determine the best implementation.
|
||||
*
|
||||
* @return The best @ref XXH_VECTOR implementation.
|
||||
* @see XXH_VECTOR_TYPES
|
||||
*/
|
||||
static int XXH_featureTest(void)
|
||||
{
|
||||
xxh_u32 abcd[4];
|
||||
xxh_u32 max_leaves;
|
||||
int best = XXH_SCALAR;
|
||||
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
||||
xxh_u64 xgetbv_val;
|
||||
#endif
|
||||
#if defined(__GNUC__) && defined(__i386__)
|
||||
xxh_u32 cpuid_supported;
|
||||
__asm__(
|
||||
"# For the sake of ruthless backwards compatibility, check if CPUID\n\t"
|
||||
"# is supported in the EFLAGS on i386.\n\t"
|
||||
"# This is not necessary on x86_64 - CPUID is mandatory.\n\t"
|
||||
"# The ID flag (bit 21) in the EFLAGS register indicates support\n\t"
|
||||
"# for the CPUID instruction. If a software procedure can set and\n\t"
|
||||
"# clear this flag, the processor executing the procedure supports\n\t"
|
||||
"# the CPUID instruction.\n\t"
|
||||
"# <https://c9x.me/x86/html/file_module_x86_id_45.html>\n\t"
|
||||
"#\n\t"
|
||||
"# Routine is from <https://wiki.osdev.org/CPUID>.\n\t"
|
||||
|
||||
"# Save EFLAGS\n\t"
|
||||
XXH_I_ATT("pushfd", "pushfl" )
|
||||
"# Store EFLAGS\n\t"
|
||||
XXH_I_ATT("pushfd", "pushfl" )
|
||||
"# Invert the ID bit in stored EFLAGS\n\t"
|
||||
XXH_I_ATT("xor dword ptr[esp], 0x200000", "xorl $0x200000, (%%esp)")
|
||||
"# Load stored EFLAGS (with ID bit inverted)\n\t"
|
||||
XXH_I_ATT("popfd", "popfl" )
|
||||
"# Store EFLAGS again (ID bit may or not be inverted)\n\t"
|
||||
XXH_I_ATT("pushfd", "pushfl" )
|
||||
"# eax = modified EFLAGS (ID bit may or may not be inverted)\n\t"
|
||||
XXH_I_ATT("pop eax", "popl %%eax" )
|
||||
"# eax = whichever bits were changed\n\t"
|
||||
XXH_I_ATT("xor eax, dword ptr[esp]", "xorl (%%esp), %%eax" )
|
||||
"# Restore original EFLAGS\n\t"
|
||||
XXH_I_ATT("popfd", "popfl" )
|
||||
"# eax = zero if ID bit can't be changed, else non-zero\n\t"
|
||||
XXH_I_ATT("and eax, 0x200000", "andl $0x200000, %%eax" )
|
||||
: "=a" (cpuid_supported) :: "cc");
|
||||
|
||||
if (XXH_unlikely(!cpuid_supported)) {
|
||||
XXH_debugPrint("CPUID support is not detected!");
|
||||
return best;
|
||||
}
|
||||
|
||||
#endif
|
||||
/* Check how many CPUID pages we have */
|
||||
XXH_cpuid(0, 0, abcd);
|
||||
max_leaves = abcd[0];
|
||||
|
||||
/* Shouldn't happen on hardware, but happens on some QEMU configs. */
|
||||
if (XXH_unlikely(max_leaves == 0)) {
|
||||
XXH_debugPrint("Max CPUID leaves == 0!");
|
||||
return best;
|
||||
}
|
||||
|
||||
/* Check for SSE2, OSXSAVE and xgetbv */
|
||||
XXH_cpuid(1, 0, abcd);
|
||||
|
||||
/*
|
||||
* Test for SSE2. The check is redundant on x86_64, but it doesn't hurt.
|
||||
*/
|
||||
if (XXH_unlikely((abcd[3] & XXH_SSE2_CPUID_MASK) != XXH_SSE2_CPUID_MASK))
|
||||
return best;
|
||||
|
||||
XXH_debugPrint("SSE2 support detected.");
|
||||
|
||||
best = XXH_SSE2;
|
||||
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
||||
/* Make sure we have enough leaves */
|
||||
if (XXH_unlikely(max_leaves < 7))
|
||||
return best;
|
||||
|
||||
/* Test for OSXSAVE and XGETBV */
|
||||
if ((abcd[2] & XXH_OSXSAVE_CPUID_MASK) != XXH_OSXSAVE_CPUID_MASK)
|
||||
return best;
|
||||
|
||||
/* CPUID check for AVX features */
|
||||
XXH_cpuid(7, 0, abcd);
|
||||
|
||||
xgetbv_val = XXH_xgetbv();
|
||||
#if XXH_DISPATCH_AVX2
|
||||
/* Validate that AVX2 is supported by the CPU */
|
||||
if ((abcd[1] & XXH_AVX2_CPUID_MASK) != XXH_AVX2_CPUID_MASK)
|
||||
return best;
|
||||
|
||||
/* Validate that the OS supports YMM registers */
|
||||
if ((xgetbv_val & XXH_AVX2_XGETBV_MASK) != XXH_AVX2_XGETBV_MASK) {
|
||||
XXH_debugPrint("AVX2 supported by the CPU, but not the OS.");
|
||||
return best;
|
||||
}
|
||||
|
||||
/* AVX2 supported */
|
||||
XXH_debugPrint("AVX2 support detected.");
|
||||
best = XXH_AVX2;
|
||||
#endif
|
||||
#if XXH_DISPATCH_AVX512
|
||||
/* Check if AVX512F is supported by the CPU */
|
||||
if ((abcd[1] & XXH_AVX512F_CPUID_MASK) != XXH_AVX512F_CPUID_MASK) {
|
||||
XXH_debugPrint("AVX512F not supported by CPU");
|
||||
return best;
|
||||
}
|
||||
|
||||
/* Validate that the OS supports ZMM registers */
|
||||
if ((xgetbv_val & XXH_AVX512F_XGETBV_MASK) != XXH_AVX512F_XGETBV_MASK) {
|
||||
XXH_debugPrint("AVX512F supported by the CPU, but not the OS.");
|
||||
return best;
|
||||
}
|
||||
|
||||
/* AVX512F supported */
|
||||
XXH_debugPrint("AVX512F support detected.");
|
||||
best = XXH_AVX512;
|
||||
#endif
|
||||
#endif
|
||||
return best;
|
||||
}
|
||||
|
||||
|
||||
/* === Vector implementations === */
|
||||
|
||||
/*!
|
||||
* @internal
|
||||
* @brief Defines the various dispatch functions.
|
||||
*
|
||||
* TODO: Consolidate?
|
||||
*
|
||||
* @param suffix The suffix for the functions, e.g. sse2 or scalar
|
||||
* @param target XXH_TARGET_* or empty.
|
||||
*/
|
||||
#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target) \
|
||||
\
|
||||
/* === XXH3, default variants === */ \
|
||||
\
|
||||
XXH_NO_INLINE target XXH64_hash_t \
|
||||
XXHL64_default_##suffix(const void* XXH_RESTRICT input, size_t len) \
|
||||
{ \
|
||||
return XXH3_hashLong_64b_internal( \
|
||||
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
|
||||
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \
|
||||
); \
|
||||
} \
|
||||
\
|
||||
/* === XXH3, Seeded variants === */ \
|
||||
\
|
||||
XXH_NO_INLINE target XXH64_hash_t \
|
||||
XXHL64_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \
|
||||
XXH64_hash_t seed) \
|
||||
{ \
|
||||
return XXH3_hashLong_64b_withSeed_internal( \
|
||||
input, len, seed, XXH3_accumulate_512_##suffix, \
|
||||
XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \
|
||||
); \
|
||||
} \
|
||||
\
|
||||
/* === XXH3, Secret variants === */ \
|
||||
\
|
||||
XXH_NO_INLINE target XXH64_hash_t \
|
||||
XXHL64_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \
|
||||
const void* secret, size_t secretLen) \
|
||||
{ \
|
||||
return XXH3_hashLong_64b_internal( \
|
||||
input, len, secret, secretLen, \
|
||||
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \
|
||||
); \
|
||||
} \
|
||||
\
|
||||
/* === XXH3 update variants === */ \
|
||||
\
|
||||
XXH_NO_INLINE target XXH_errorcode \
|
||||
XXH3_update_##suffix(XXH3_state_t* state, const void* input, size_t len) \
|
||||
{ \
|
||||
return XXH3_update(state, (const xxh_u8*)input, len, \
|
||||
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \
|
||||
} \
|
||||
\
|
||||
/* === XXH128 default variants === */ \
|
||||
\
|
||||
XXH_NO_INLINE target XXH128_hash_t \
|
||||
XXHL128_default_##suffix(const void* XXH_RESTRICT input, size_t len) \
|
||||
{ \
|
||||
return XXH3_hashLong_128b_internal( \
|
||||
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
|
||||
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \
|
||||
); \
|
||||
} \
|
||||
\
|
||||
/* === XXH128 Secret variants === */ \
|
||||
\
|
||||
XXH_NO_INLINE target XXH128_hash_t \
|
||||
XXHL128_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \
|
||||
const void* XXH_RESTRICT secret, size_t secretLen) \
|
||||
{ \
|
||||
return XXH3_hashLong_128b_internal( \
|
||||
input, len, (const xxh_u8*)secret, secretLen, \
|
||||
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \
|
||||
} \
|
||||
\
|
||||
/* === XXH128 Seeded variants === */ \
|
||||
\
|
||||
XXH_NO_INLINE target XXH128_hash_t \
|
||||
XXHL128_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \
|
||||
XXH64_hash_t seed) \
|
||||
{ \
|
||||
return XXH3_hashLong_128b_withSeed_internal(input, len, seed, \
|
||||
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix, \
|
||||
XXH3_initCustomSecret_##suffix); \
|
||||
}
|
||||
|
||||
/* End XXH_DEFINE_DISPATCH_FUNCS */
|
||||
|
||||
#if XXH_DISPATCH_SCALAR
|
||||
XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */)
|
||||
#endif
|
||||
XXH_DEFINE_DISPATCH_FUNCS(sse2, XXH_TARGET_SSE2)
|
||||
#if XXH_DISPATCH_AVX2
|
||||
XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2)
|
||||
#endif
|
||||
#if XXH_DISPATCH_AVX512
|
||||
XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512)
|
||||
#endif
|
||||
#undef XXH_DEFINE_DISPATCH_FUNCS
|
||||
|
||||
/* ==== Dispatchers ==== */
|
||||
|
||||
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(const void* XXH_RESTRICT, size_t);
|
||||
|
||||
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t);
|
||||
|
||||
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t);
|
||||
|
||||
typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH3_state_t*, const void*, size_t);
|
||||
|
||||
typedef struct {
|
||||
XXH3_dispatchx86_hashLong64_default hashLong64_default;
|
||||
XXH3_dispatchx86_hashLong64_withSeed hashLong64_seed;
|
||||
XXH3_dispatchx86_hashLong64_withSecret hashLong64_secret;
|
||||
XXH3_dispatchx86_update update;
|
||||
} XXH_dispatchFunctions_s;
|
||||
|
||||
#define XXH_NB_DISPATCHES 4
|
||||
|
||||
/*!
|
||||
* @internal
|
||||
* @brief Table of dispatchers for @ref XXH3_64bits().
|
||||
*
|
||||
* @pre The indices must match @ref XXH_VECTOR_TYPE.
|
||||
*/
|
||||
static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = {
|
||||
#if XXH_DISPATCH_SCALAR
|
||||
/* Scalar */ { XXHL64_default_scalar, XXHL64_seed_scalar, XXHL64_secret_scalar, XXH3_update_scalar },
|
||||
#else
|
||||
/* Scalar */ { NULL, NULL, NULL, NULL },
|
||||
#endif
|
||||
/* SSE2 */ { XXHL64_default_sse2, XXHL64_seed_sse2, XXHL64_secret_sse2, XXH3_update_sse2 },
|
||||
#if XXH_DISPATCH_AVX2
|
||||
/* AVX2 */ { XXHL64_default_avx2, XXHL64_seed_avx2, XXHL64_secret_avx2, XXH3_update_avx2 },
|
||||
#else
|
||||
/* AVX2 */ { NULL, NULL, NULL, NULL },
|
||||
#endif
|
||||
#if XXH_DISPATCH_AVX512
|
||||
/* AVX512 */ { XXHL64_default_avx512, XXHL64_seed_avx512, XXHL64_secret_avx512, XXH3_update_avx512 }
|
||||
#else
|
||||
/* AVX512 */ { NULL, NULL, NULL, NULL }
|
||||
#endif
|
||||
};
|
||||
/*!
|
||||
* @internal
|
||||
* @brief The selected dispatch table for @ref XXH3_64bits().
|
||||
*/
|
||||
static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL };
|
||||
|
||||
|
||||
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(const void* XXH_RESTRICT, size_t);
|
||||
|
||||
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t);
|
||||
|
||||
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t);
|
||||
|
||||
typedef struct {
|
||||
XXH3_dispatchx86_hashLong128_default hashLong128_default;
|
||||
XXH3_dispatchx86_hashLong128_withSeed hashLong128_seed;
|
||||
XXH3_dispatchx86_hashLong128_withSecret hashLong128_secret;
|
||||
XXH3_dispatchx86_update update;
|
||||
} XXH_dispatch128Functions_s;
|
||||
|
||||
|
||||
/*!
|
||||
* @internal
|
||||
* @brief Table of dispatchers for @ref XXH3_128bits().
|
||||
*
|
||||
* @pre The indices must match @ref XXH_VECTOR_TYPE.
|
||||
*/
|
||||
static const XXH_dispatch128Functions_s XXH_kDispatch128[XXH_NB_DISPATCHES] = {
|
||||
#if XXH_DISPATCH_SCALAR
|
||||
/* Scalar */ { XXHL128_default_scalar, XXHL128_seed_scalar, XXHL128_secret_scalar, XXH3_update_scalar },
|
||||
#else
|
||||
/* Scalar */ { NULL, NULL, NULL, NULL },
|
||||
#endif
|
||||
/* SSE2 */ { XXHL128_default_sse2, XXHL128_seed_sse2, XXHL128_secret_sse2, XXH3_update_sse2 },
|
||||
#if XXH_DISPATCH_AVX2
|
||||
/* AVX2 */ { XXHL128_default_avx2, XXHL128_seed_avx2, XXHL128_secret_avx2, XXH3_update_avx2 },
|
||||
#else
|
||||
/* AVX2 */ { NULL, NULL, NULL, NULL },
|
||||
#endif
|
||||
#if XXH_DISPATCH_AVX512
|
||||
/* AVX512 */ { XXHL128_default_avx512, XXHL128_seed_avx512, XXHL128_secret_avx512, XXH3_update_avx512 }
|
||||
#else
|
||||
/* AVX512 */ { NULL, NULL, NULL, NULL }
|
||||
#endif
|
||||
};
|
||||
|
||||
/*!
|
||||
* @internal
|
||||
* @brief The selected dispatch table for @ref XXH3_64bits().
|
||||
*/
|
||||
static XXH_dispatch128Functions_s XXH_g_dispatch128 = { NULL, NULL, NULL, NULL };
|
||||
|
||||
/*!
|
||||
* @internal
|
||||
* @brief Runs a CPUID check and sets the correct dispatch tables.
|
||||
*/
|
||||
static void XXH_setDispatch(void)
|
||||
{
|
||||
int vecID = XXH_featureTest();
|
||||
XXH_STATIC_ASSERT(XXH_AVX512 == XXH_NB_DISPATCHES-1);
|
||||
assert(XXH_SCALAR <= vecID && vecID <= XXH_AVX512);
|
||||
#if !XXH_DISPATCH_SCALAR
|
||||
assert(vecID != XXH_SCALAR);
|
||||
#endif
|
||||
#if !XXH_DISPATCH_AVX512
|
||||
assert(vecID != XXH_AVX512);
|
||||
#endif
|
||||
#if !XXH_DISPATCH_AVX2
|
||||
assert(vecID != XXH_AVX2);
|
||||
#endif
|
||||
XXH_g_dispatch = XXH_kDispatch[vecID];
|
||||
XXH_g_dispatch128 = XXH_kDispatch128[vecID];
|
||||
}
|
||||
|
||||
|
||||
/* ==== XXH3 public functions ==== */
|
||||
|
||||
static XXH64_hash_t
|
||||
XXH3_hashLong_64b_defaultSecret_selection(const void* input, size_t len,
|
||||
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
|
||||
{
|
||||
(void)seed64; (void)secret; (void)secretLen;
|
||||
if (XXH_g_dispatch.hashLong64_default == NULL) XXH_setDispatch();
|
||||
return XXH_g_dispatch.hashLong64_default(input, len);
|
||||
}
|
||||
|
||||
XXH64_hash_t XXH3_64bits_dispatch(const void* input, size_t len)
|
||||
{
|
||||
return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_defaultSecret_selection);
|
||||
}
|
||||
|
||||
static XXH64_hash_t
|
||||
XXH3_hashLong_64b_withSeed_selection(const void* input, size_t len,
|
||||
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
|
||||
{
|
||||
(void)secret; (void)secretLen;
|
||||
if (XXH_g_dispatch.hashLong64_seed == NULL) XXH_setDispatch();
|
||||
return XXH_g_dispatch.hashLong64_seed(input, len, seed64);
|
||||
}
|
||||
|
||||
XXH64_hash_t XXH3_64bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed)
|
||||
{
|
||||
return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed_selection);
|
||||
}
|
||||
|
||||
static XXH64_hash_t
|
||||
XXH3_hashLong_64b_withSecret_selection(const void* input, size_t len,
|
||||
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
|
||||
{
|
||||
(void)seed64;
|
||||
if (XXH_g_dispatch.hashLong64_secret == NULL) XXH_setDispatch();
|
||||
return XXH_g_dispatch.hashLong64_secret(input, len, secret, secretLen);
|
||||
}
|
||||
|
||||
XXH64_hash_t XXH3_64bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen)
|
||||
{
|
||||
return XXH3_64bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_64b_withSecret_selection);
|
||||
}
|
||||
|
||||
XXH_errorcode
|
||||
XXH3_64bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len)
|
||||
{
|
||||
if (XXH_g_dispatch.update == NULL) XXH_setDispatch();
|
||||
return XXH_g_dispatch.update(state, (const xxh_u8*)input, len);
|
||||
}
|
||||
|
||||
|
||||
/* ==== XXH128 public functions ==== */
|
||||
|
||||
static XXH128_hash_t
|
||||
XXH3_hashLong_128b_defaultSecret_selection(const void* input, size_t len,
|
||||
XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
||||
{
|
||||
(void)seed64; (void)secret; (void)secretLen;
|
||||
if (XXH_g_dispatch128.hashLong128_default == NULL) XXH_setDispatch();
|
||||
return XXH_g_dispatch128.hashLong128_default(input, len);
|
||||
}
|
||||
|
||||
XXH128_hash_t XXH3_128bits_dispatch(const void* input, size_t len)
|
||||
{
|
||||
return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_defaultSecret_selection);
|
||||
}
|
||||
|
||||
static XXH128_hash_t
|
||||
XXH3_hashLong_128b_withSeed_selection(const void* input, size_t len,
|
||||
XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
||||
{
|
||||
(void)secret; (void)secretLen;
|
||||
if (XXH_g_dispatch128.hashLong128_seed == NULL) XXH_setDispatch();
|
||||
return XXH_g_dispatch128.hashLong128_seed(input, len, seed64);
|
||||
}
|
||||
|
||||
XXH128_hash_t XXH3_128bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed)
|
||||
{
|
||||
return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed_selection);
|
||||
}
|
||||
|
||||
static XXH128_hash_t
|
||||
XXH3_hashLong_128b_withSecret_selection(const void* input, size_t len,
|
||||
XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
||||
{
|
||||
(void)seed64;
|
||||
if (XXH_g_dispatch128.hashLong128_secret == NULL) XXH_setDispatch();
|
||||
return XXH_g_dispatch128.hashLong128_secret(input, len, secret, secretLen);
|
||||
}
|
||||
|
||||
XXH128_hash_t XXH3_128bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen)
|
||||
{
|
||||
return XXH3_128bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_128b_withSecret_selection);
|
||||
}
|
||||
|
||||
XXH_errorcode
|
||||
XXH3_128bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len)
|
||||
{
|
||||
if (XXH_g_dispatch128.update == NULL) XXH_setDispatch();
|
||||
return XXH_g_dispatch128.update(state, (const xxh_u8*)input, len);
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
/*! @} */
|
|
@ -0,0 +1,85 @@
|
|||
/*
|
||||
* xxHash - XXH3 Dispatcher for x86-based targets
|
||||
* Copyright (C) 2020-2021 Yann Collet
|
||||
*
|
||||
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are
|
||||
* met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following disclaimer
|
||||
* in the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - xxHash homepage: https://www.xxhash.com
|
||||
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
#ifndef XXH_X86DISPATCH_H_13563687684
|
||||
#define XXH_X86DISPATCH_H_13563687684
|
||||
|
||||
#include "xxhash.h" /* XXH64_hash_t, XXH3_state_t */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_dispatch(const void* input, size_t len);
|
||||
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed);
|
||||
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen);
|
||||
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len);
|
||||
|
||||
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_dispatch(const void* input, size_t len);
|
||||
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed);
|
||||
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen);
|
||||
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/* automatic replacement of XXH3 functions.
|
||||
* can be disabled by setting XXH_DISPATCH_DISABLE_REPLACE */
|
||||
#ifndef XXH_DISPATCH_DISABLE_REPLACE
|
||||
|
||||
# undef XXH3_64bits
|
||||
# define XXH3_64bits XXH3_64bits_dispatch
|
||||
# undef XXH3_64bits_withSeed
|
||||
# define XXH3_64bits_withSeed XXH3_64bits_withSeed_dispatch
|
||||
# undef XXH3_64bits_withSecret
|
||||
# define XXH3_64bits_withSecret XXH3_64bits_withSecret_dispatch
|
||||
# undef XXH3_64bits_update
|
||||
# define XXH3_64bits_update XXH3_64bits_update_dispatch
|
||||
|
||||
# undef XXH128
|
||||
# define XXH128 XXH3_128bits_withSeed_dispatch
|
||||
# undef XXH3_128bits
|
||||
# define XXH3_128bits XXH3_128bits_dispatch
|
||||
# undef XXH3_128bits_withSeed
|
||||
# define XXH3_128bits_withSeed XXH3_128bits_withSeed_dispatch
|
||||
# undef XXH3_128bits_withSecret
|
||||
# define XXH3_128bits_withSecret XXH3_128bits_withSecret_dispatch
|
||||
# undef XXH3_128bits_update
|
||||
# define XXH3_128bits_update XXH3_128bits_update_dispatch
|
||||
|
||||
#endif /* XXH_DISPATCH_DISABLE_REPLACE */
|
||||
|
||||
#endif /* XXH_X86DISPATCH_H_13563687684 */
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* xxHash - Extremely Fast Hash algorithm
|
||||
* Copyright (C) 2012-2021 Yann Collet
|
||||
*
|
||||
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are
|
||||
* met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following disclaimer
|
||||
* in the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - xxHash homepage: https://www.xxhash.com
|
||||
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* xxhash.c instantiates functions defined in xxhash.h
|
||||
*/
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
|
||||
#define XXH_IMPLEMENTATION /* access definitions */
|
||||
|
||||
#include "xxhash.h"
|
File diff suppressed because it is too large
Load Diff
|
@ -39,6 +39,7 @@
|
|||
|
||||
#include "base/ansi_scrubber.hh"
|
||||
#include "base/attr_line.hh"
|
||||
#include "base/itertools.hh"
|
||||
#include "base/lnav_log.hh"
|
||||
#include "config.h"
|
||||
#include "lnav_config.hh"
|
||||
|
@ -486,8 +487,15 @@ public:
|
|||
auto iter = lnav_config.lc_ui_theme_defs.find(lnav_config.lc_ui_theme);
|
||||
|
||||
if (iter == lnav_config.lc_ui_theme_defs.end()) {
|
||||
auto theme_names
|
||||
= lnav_config.lc_ui_theme_defs | lnav::itertools::first();
|
||||
|
||||
reporter(&lnav_config.lc_ui_theme,
|
||||
"unknown theme -- " + lnav_config.lc_ui_theme);
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t("unknown theme -- ")
|
||||
.append_quoted(lnav_config.lc_ui_theme))
|
||||
.with_help(attr_line_t("The available themes are: ")
|
||||
.join(theme_names, ", ")));
|
||||
|
||||
vc.init_roles(lnav_config.lc_ui_theme_defs["default"], reporter);
|
||||
return;
|
||||
|
@ -558,7 +566,7 @@ view_colors::init()
|
|||
initialized = true;
|
||||
|
||||
{
|
||||
auto reporter = [](const void*, const std::string&) {
|
||||
auto reporter = [](const void*, const lnav::console::user_message&) {
|
||||
|
||||
};
|
||||
|
||||
|
@ -631,12 +639,20 @@ view_colors::to_attrs(int& pair_base,
|
|||
|
||||
auto fg = styling::color_unit::from_str(fg_color).unwrapOrElse(
|
||||
[&](const auto& msg) {
|
||||
reporter(&sc.sc_color, msg);
|
||||
reporter(
|
||||
&sc.sc_color,
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t("invalid color -- ").append_quoted(sc.sc_color))
|
||||
.with_reason(msg));
|
||||
return styling::color_unit::make_empty();
|
||||
});
|
||||
auto bg = styling::color_unit::from_str(bg_color).unwrapOrElse(
|
||||
[&](const auto& msg) {
|
||||
reporter(&sc.sc_background_color, msg);
|
||||
reporter(&sc.sc_background_color,
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t("invalid background color -- ")
|
||||
.append_quoted(sc.sc_background_color))
|
||||
.with_reason(msg));
|
||||
return styling::color_unit::make_empty();
|
||||
});
|
||||
|
||||
|
@ -674,7 +690,12 @@ view_colors::init_roles(const lnav_theme& lt,
|
|||
shlex(ident_sc.sc_background_color).eval(bg_color, lt.lt_vars);
|
||||
auto rgb_bg = rgb_color::from_str(bg_color).unwrapOrElse(
|
||||
[&](const auto& msg) {
|
||||
reporter(&ident_sc.sc_background_color, msg);
|
||||
reporter(
|
||||
&ident_sc.sc_background_color,
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t("invalid background color -- ")
|
||||
.append_quoted(ident_sc.sc_background_color))
|
||||
.with_reason(msg));
|
||||
return rgb_color{};
|
||||
});
|
||||
ident_bg = vc_active_palette->match_color(lab_color(rgb_bg));
|
||||
|
@ -710,12 +731,20 @@ view_colors::init_roles(const lnav_theme& lt,
|
|||
|
||||
auto rgb_fg = rgb_color::from_str(fg_str).unwrapOrElse(
|
||||
[&](const auto& msg) {
|
||||
reporter(&fg_str, msg);
|
||||
reporter(&fg_str,
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t("invalid color -- ")
|
||||
.append_quoted(fg_str))
|
||||
.with_reason(msg));
|
||||
return rgb_color{};
|
||||
});
|
||||
auto rgb_bg = rgb_color::from_str(bg_str).unwrapOrElse(
|
||||
[&](const auto& msg) {
|
||||
reporter(&fg_str, msg);
|
||||
reporter(&bg_str,
|
||||
lnav::console::user_message::error(
|
||||
attr_line_t("invalid background color -- ")
|
||||
.append_quoted(bg_str))
|
||||
.with_reason(msg));
|
||||
return rgb_color{};
|
||||
});
|
||||
|
||||
|
|
|
@ -129,4 +129,35 @@ main(int argc, char* argv[])
|
|||
|
||||
assert(FOO_COUNT == 1);
|
||||
}
|
||||
|
||||
{
|
||||
const char* TEST_INPUT = R"({
|
||||
"msg": "Hello, World!",
|
||||
"parent1": {
|
||||
"child": {}
|
||||
},
|
||||
"parent2": {
|
||||
"child": {"name": "steve"}
|
||||
},
|
||||
"parent3": {
|
||||
"child": {},
|
||||
"sibling": {"name": "mongoose"}
|
||||
}
|
||||
})";
|
||||
const std::string EXPECTED_OUTPUT
|
||||
= "{\"msg\":\"Hello, "
|
||||
"World!\",\"parent2\":{\"child\":{\"name\":\"steve\"}},"
|
||||
"\"parent3\":{\"sibling\":{\"name\":\"mongoose\"}}}";
|
||||
|
||||
char errbuf[1024];
|
||||
|
||||
auto tree = yajl_tree_parse(TEST_INPUT, errbuf, sizeof(errbuf));
|
||||
yajl_cleanup_tree(tree);
|
||||
|
||||
yajlpp_gen gen;
|
||||
|
||||
yajl_gen_tree(gen, tree);
|
||||
auto actual = gen.to_string_fragment().to_string();
|
||||
assert(EXPECTED_OUTPUT == actual);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -45,6 +45,101 @@
|
|||
const json_path_handler_base::enum_value_t
|
||||
json_path_handler_base::ENUM_TERMINATOR((const char*) nullptr, 0);
|
||||
|
||||
yajl_gen_status
|
||||
yajl_gen_tree(yajl_gen hand, yajl_val val)
|
||||
{
|
||||
switch (val->type) {
|
||||
case yajl_t_string: {
|
||||
return yajl_gen_string(hand, YAJL_GET_STRING(val));
|
||||
}
|
||||
case yajl_t_number: {
|
||||
if (YAJL_IS_INTEGER(val)) {
|
||||
return yajl_gen_integer(hand, YAJL_GET_INTEGER(val));
|
||||
}
|
||||
if (YAJL_IS_DOUBLE(val)) {
|
||||
return yajl_gen_double(hand, YAJL_GET_DOUBLE(val));
|
||||
}
|
||||
return yajl_gen_number(
|
||||
hand, YAJL_GET_NUMBER(val), strlen(YAJL_GET_NUMBER(val)));
|
||||
}
|
||||
case yajl_t_object: {
|
||||
auto rc = yajl_gen_map_open(hand);
|
||||
if (rc != yajl_gen_status_ok) {
|
||||
return rc;
|
||||
}
|
||||
for (size_t lpc = 0; lpc < YAJL_GET_OBJECT(val)->len; lpc++) {
|
||||
rc = yajl_gen_string(hand, YAJL_GET_OBJECT(val)->keys[lpc]);
|
||||
if (rc != yajl_gen_status_ok) {
|
||||
return rc;
|
||||
}
|
||||
rc = yajl_gen_tree(hand, YAJL_GET_OBJECT(val)->values[lpc]);
|
||||
if (rc != yajl_gen_status_ok) {
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
rc = yajl_gen_map_close(hand);
|
||||
if (rc != yajl_gen_status_ok) {
|
||||
return rc;
|
||||
}
|
||||
return yajl_gen_status_ok;
|
||||
}
|
||||
case yajl_t_array: {
|
||||
auto rc = yajl_gen_array_open(hand);
|
||||
if (rc != yajl_gen_status_ok) {
|
||||
return rc;
|
||||
}
|
||||
for (size_t lpc = 0; lpc < YAJL_GET_ARRAY(val)->len; lpc++) {
|
||||
rc = yajl_gen_tree(hand, YAJL_GET_ARRAY(val)->values[lpc]);
|
||||
if (rc != yajl_gen_status_ok) {
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
rc = yajl_gen_array_close(hand);
|
||||
if (rc != yajl_gen_status_ok) {
|
||||
return rc;
|
||||
}
|
||||
return yajl_gen_status_ok;
|
||||
}
|
||||
case yajl_t_true: {
|
||||
return yajl_gen_bool(hand, true);
|
||||
}
|
||||
case yajl_t_false: {
|
||||
return yajl_gen_bool(hand, false);
|
||||
}
|
||||
case yajl_t_null: {
|
||||
return yajl_gen_null(hand);
|
||||
}
|
||||
default:
|
||||
return yajl_gen_status_ok;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
yajl_cleanup_tree(yajl_val val)
|
||||
{
|
||||
if (YAJL_IS_OBJECT(val)) {
|
||||
auto* val_as_obj = YAJL_GET_OBJECT(val);
|
||||
|
||||
for (size_t lpc = 0; lpc < val_as_obj->len;) {
|
||||
auto* child_val = val_as_obj->values[lpc];
|
||||
|
||||
yajl_cleanup_tree(child_val);
|
||||
if (YAJL_IS_OBJECT(child_val)
|
||||
&& YAJL_GET_OBJECT(child_val)->len == 0) {
|
||||
free((char*) val_as_obj->keys[lpc]);
|
||||
yajl_tree_free(val_as_obj->values[lpc]);
|
||||
val_as_obj->len -= 1;
|
||||
for (auto lpc2 = lpc; lpc2 < val_as_obj->len; lpc2++) {
|
||||
val_as_obj->keys[lpc2] = val_as_obj->keys[lpc2 + 1];
|
||||
val_as_obj->values[lpc2] = val_as_obj->values[lpc2 + 1];
|
||||
}
|
||||
} else {
|
||||
lpc++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
json_path_handler_base::json_path_handler_base(const std::string& property)
|
||||
: jph_property(property.back() == '#'
|
||||
? property.substr(0, property.size() - 1)
|
||||
|
@ -372,7 +467,16 @@ json_path_handler_base::walk(
|
|||
this->jph_path_provider(root, local_paths);
|
||||
|
||||
for (auto& lpath : local_paths) {
|
||||
cb(*this, lpath, nullptr);
|
||||
cb(*this,
|
||||
fmt::format(FMT_STRING("{}{}{}"),
|
||||
base,
|
||||
lpath,
|
||||
this->jph_children ? "/" : ""),
|
||||
nullptr);
|
||||
}
|
||||
if (this->jph_obj_deleter) {
|
||||
local_paths.clear();
|
||||
this->jph_path_provider(root, local_paths);
|
||||
}
|
||||
} else {
|
||||
local_paths.emplace_back(this->jph_property);
|
||||
|
@ -386,7 +490,7 @@ json_path_handler_base::walk(
|
|||
|
||||
if (this->jph_children) {
|
||||
for (const auto& lpath : local_paths) {
|
||||
for (auto& jph : this->jph_children->jpc_children) {
|
||||
for (const auto& jph : this->jph_children->jpc_children) {
|
||||
static const auto POSS_SRC
|
||||
= intern_string::lookup("possibilities");
|
||||
|
||||
|
@ -669,8 +773,6 @@ yajlpp_parse_context::update_callbacks(const json_path_container* orig_handlers,
|
|||
return;
|
||||
}
|
||||
}
|
||||
|
||||
this->ypc_handler_stack.emplace_back(nullptr);
|
||||
}
|
||||
|
||||
int
|
||||
|
@ -837,8 +939,32 @@ yajlpp_parse_context::handle_unused(void* ctx)
|
|||
return 1;
|
||||
}
|
||||
|
||||
int
|
||||
yajlpp_parse_context::handle_unused_or_delete(void* ctx)
|
||||
{
|
||||
yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx;
|
||||
|
||||
if (!ypc->ypc_handler_stack.empty()
|
||||
&& ypc->ypc_handler_stack.back()->jph_obj_deleter)
|
||||
{
|
||||
pcre_context_static<30> pc;
|
||||
auto key_start = ypc->ypc_path_index_stack.back();
|
||||
pcre_input pi(&ypc->ypc_path[key_start + 1],
|
||||
0,
|
||||
ypc->ypc_path.size() - key_start - 2);
|
||||
yajlpp_provider_context provider_ctx{{pc, pi}, static_cast<size_t>(-1)};
|
||||
ypc->ypc_handler_stack.back()->jph_regex->match(pc, pi);
|
||||
|
||||
ypc->ypc_handler_stack.back()->jph_obj_deleter(
|
||||
provider_ctx, ypc->ypc_obj_stack.top());
|
||||
return 1;
|
||||
}
|
||||
|
||||
return handle_unused(ctx);
|
||||
}
|
||||
|
||||
const yajl_callbacks yajlpp_parse_context::DEFAULT_CALLBACKS = {
|
||||
yajlpp_parse_context::handle_unused,
|
||||
yajlpp_parse_context::handle_unused_or_delete,
|
||||
(int (*)(void*, int)) yajlpp_parse_context::handle_unused,
|
||||
(int (*)(void*, long long)) yajlpp_parse_context::handle_unused,
|
||||
(int (*)(void*, double)) yajlpp_parse_context::handle_unused,
|
||||
|
@ -1194,13 +1320,13 @@ json_path_handler_base::report_pattern_error(yajlpp_parse_context* ypc,
|
|||
}
|
||||
|
||||
attr_line_t
|
||||
json_path_handler_base::get_help_text(yajlpp_parse_context* ypc) const
|
||||
json_path_handler_base::get_help_text(const std::string& full_path) const
|
||||
{
|
||||
attr_line_t retval;
|
||||
|
||||
retval.append(lnav::roles::h2("Property Synopsis"))
|
||||
.append("\n ")
|
||||
.append(lnav::roles::symbol(ypc->get_full_path().to_string()))
|
||||
.append(lnav::roles::symbol(full_path))
|
||||
.append(" ")
|
||||
.append(lnav::roles::variable(this->jph_synopsis))
|
||||
.append("\n")
|
||||
|
@ -1234,6 +1360,12 @@ json_path_handler_base::get_help_text(yajlpp_parse_context* ypc) const
|
|||
return retval;
|
||||
}
|
||||
|
||||
attr_line_t
|
||||
json_path_handler_base::get_help_text(yajlpp_parse_context* ypc) const
|
||||
{
|
||||
return this->get_help_text(ypc->get_full_path().to_string());
|
||||
}
|
||||
|
||||
void
|
||||
json_path_handler_base::report_min_value_error(yajlpp_parse_context* ypc,
|
||||
long long value) const
|
||||
|
|
|
@ -72,6 +72,10 @@ yajl_gen_string(yajl_gen hand, const std::string& str)
|
|||
hand, (const unsigned char*) str.c_str(), str.length());
|
||||
}
|
||||
|
||||
yajl_gen_status yajl_gen_tree(yajl_gen hand, yajl_val val);
|
||||
|
||||
void yajl_cleanup_tree(yajl_val val);
|
||||
|
||||
template<typename T>
|
||||
struct positioned_property {
|
||||
intern_string_t pp_path;
|
||||
|
@ -206,6 +210,8 @@ struct json_path_handler_base {
|
|||
jph_obj_provider;
|
||||
std::function<void(void* root, std::vector<std::string>& paths_out)>
|
||||
jph_path_provider;
|
||||
std::function<void(const yajlpp_provider_context& pe, void* root)>
|
||||
jph_obj_deleter;
|
||||
std::function<size_t(void* root)> jph_size_provider;
|
||||
const char* jph_synopsis{""};
|
||||
const char* jph_description{""};
|
||||
|
@ -241,6 +247,7 @@ struct json_path_handler_base {
|
|||
const std::string& value_str,
|
||||
const pcrepp::compile_error& ce) const;
|
||||
|
||||
attr_line_t get_help_text(const std::string& full_path) const;
|
||||
attr_line_t get_help_text(yajlpp_parse_context* ypc) const;
|
||||
};
|
||||
|
||||
|
@ -425,6 +432,7 @@ private:
|
|||
static int array_start(void* ctx);
|
||||
static int array_end(void* ctx);
|
||||
static int handle_unused(void* ctx);
|
||||
static int handle_unused_or_delete(void* ctx);
|
||||
};
|
||||
|
||||
class yajlpp_generator {
|
||||
|
|
|
@ -35,6 +35,7 @@
|
|||
#include <chrono>
|
||||
|
||||
#include "config.h"
|
||||
#include "mapbox/variant.hpp"
|
||||
#include "relative_time.hh"
|
||||
#include "view_curses.hh"
|
||||
#include "yajlpp.hh"
|
||||
|
@ -66,6 +67,13 @@ assign(Container<std::string>& lhs, const string_fragment& rhs)
|
|||
return lhs;
|
||||
}
|
||||
|
||||
struct json_null_t {
|
||||
bool operator==(const json_null_t& other) const { return true; }
|
||||
};
|
||||
|
||||
using json_any_t
|
||||
= mapbox::util::variant<json_null_t, bool, int64_t, double, std::string>;
|
||||
|
||||
struct json_path_container;
|
||||
|
||||
struct json_path_handler : public json_path_handler_base {
|
||||
|
@ -230,6 +238,17 @@ struct json_path_handler : public json_path_handler_base {
|
|||
return *this;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
json_path_handler& with_obj_deleter(
|
||||
void (*provider)(const yajlpp_provider_context& pc, T* root))
|
||||
{
|
||||
this->jph_obj_deleter
|
||||
= [provider](const yajlpp_provider_context& ypc, void* root) {
|
||||
provider(ypc, (T*) root);
|
||||
};
|
||||
return *this;
|
||||
}
|
||||
|
||||
template<typename T, typename MEM_T, MEM_T T::*MEM>
|
||||
static void* get_field_lvalue_cb(void* root,
|
||||
nonstd::optional<std::string> name)
|
||||
|
@ -692,6 +711,77 @@ struct json_path_handler : public json_path_handler_base {
|
|||
return *this;
|
||||
}
|
||||
|
||||
template<typename... Args,
|
||||
std::enable_if_t<
|
||||
LastIs<std::map<std::string, json_any_t>, Args...>::value,
|
||||
bool> = true>
|
||||
json_path_handler& for_field(Args... args)
|
||||
{
|
||||
this->add_cb(bool_field_cb);
|
||||
this->jph_bool_cb = [args...](yajlpp_parse_context* ypc, int val) {
|
||||
auto* obj = ypc->ypc_obj_stack.top();
|
||||
auto key = ypc->get_path_fragment(-1);
|
||||
|
||||
json_path_handler::get_field(obj, args...)[key] = val ? true
|
||||
: false;
|
||||
|
||||
return 1;
|
||||
};
|
||||
this->add_cb(int_field_cb);
|
||||
this->jph_integer_cb
|
||||
= [args...](yajlpp_parse_context* ypc, long long val) {
|
||||
auto* obj = ypc->ypc_obj_stack.top();
|
||||
auto key = ypc->get_path_fragment(-1);
|
||||
|
||||
json_path_handler::get_field(obj, args...)[key] = val;
|
||||
|
||||
return 1;
|
||||
};
|
||||
this->add_cb(str_field_cb2);
|
||||
this->jph_str_cb = [args...](yajlpp_parse_context* ypc,
|
||||
const unsigned char* str,
|
||||
size_t len) {
|
||||
auto* obj = ypc->ypc_obj_stack.top();
|
||||
auto key = ypc->get_path_fragment(-1);
|
||||
|
||||
json_path_handler::get_field(obj, args...)[key]
|
||||
= std::string((const char*) str, len);
|
||||
|
||||
return 1;
|
||||
};
|
||||
this->jph_gen_callback = [args...](yajlpp_gen_context& ygc,
|
||||
const json_path_handler_base& jph,
|
||||
yajl_gen handle) {
|
||||
const auto& field = json_path_handler::get_field(
|
||||
ygc.ygc_obj_stack.top(), args...);
|
||||
|
||||
if (!ygc.ygc_default_stack.empty()) {
|
||||
const auto& field_def = json_path_handler::get_field(
|
||||
ygc.ygc_default_stack.top(), args...);
|
||||
|
||||
if (field == field_def) {
|
||||
return yajl_gen_status_ok;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
yajlpp_generator gen(handle);
|
||||
|
||||
for (const auto& pair : field) {
|
||||
gen(pair.first);
|
||||
pair.second.match([&gen](json_null_t v) { gen(); },
|
||||
[&gen](bool v) { gen(v); },
|
||||
[&gen](int64_t v) { gen(v); },
|
||||
[&gen](double v) { gen(v); },
|
||||
[&gen](const std::string& v) { gen(v); });
|
||||
}
|
||||
}
|
||||
|
||||
return yajl_gen_status_ok;
|
||||
};
|
||||
return *this;
|
||||
}
|
||||
|
||||
template<typename... Args,
|
||||
std::enable_if_t<LastIs<std::string, Args...>::value, bool> = true>
|
||||
json_path_handler& for_field(Args... args)
|
||||
|
|
|
@ -100,6 +100,7 @@ LDADD = \
|
|||
$(CONFIG_OBJS) \
|
||||
$(TEXT2C_OBJS) \
|
||||
$(DUMMY_OBJS) \
|
||||
../src/lnav.events.$(OBJEXT) \
|
||||
$(top_builddir)/src/libdiag.a \
|
||||
$(top_builddir)/src/libdatascanner.a \
|
||||
$(top_builddir)/src/formats/logfmt/liblogfmt.a \
|
||||
|
@ -464,6 +465,7 @@ distclean-local:
|
|||
$(RM_V)rm -rf test-config
|
||||
$(RM_V)rm -rf .lnav
|
||||
$(RM_V)rm -rf regex101-home
|
||||
$(RM_V)rm -rf events-home
|
||||
$(RM_V)rm -rf ../installer-test-home
|
||||
|
||||
expected:
|
||||
|
|
|
@ -157,7 +157,8 @@ main(int argc, char* argv[])
|
|||
logfile_open_options loo;
|
||||
auto open_res = logfile::open(argv[lpc], loo);
|
||||
auto lf = open_res.unwrap();
|
||||
scan_batch_context sbc;
|
||||
ArenaAlloc::Alloc<char> allocator;
|
||||
scan_batch_context sbc{allocator};
|
||||
for (iter = root_formats.begin();
|
||||
iter != root_formats.end() && !found;
|
||||
++iter) {
|
||||
|
|
|
@ -206,6 +206,8 @@ EXPECTED_FILES = \
|
|||
$(srcdir)/%reldir%/test_config.sh_2765ea0d4c037b8c935840604edb0ae796c97a04.out \
|
||||
$(srcdir)/%reldir%/test_config.sh_5105c29004e297521310ca0bd0fd560b01c2c549.err \
|
||||
$(srcdir)/%reldir%/test_config.sh_5105c29004e297521310ca0bd0fd560b01c2c549.out \
|
||||
$(srcdir)/%reldir%/test_config.sh_5fd9fbccc35e9b06abdd913da0c16bdb306b926e.err \
|
||||
$(srcdir)/%reldir%/test_config.sh_5fd9fbccc35e9b06abdd913da0c16bdb306b926e.out \
|
||||
$(srcdir)/%reldir%/test_config.sh_b08f7523659d1c12f0e59920cd40d17d4a83b72f.err \
|
||||
$(srcdir)/%reldir%/test_config.sh_b08f7523659d1c12f0e59920cd40d17d4a83b72f.out \
|
||||
$(srcdir)/%reldir%/test_config.sh_d622658dc98327b1b2fd346802d24bc633e34ac7.err \
|
||||
|
@ -214,8 +216,20 @@ EXPECTED_FILES = \
|
|||
$(srcdir)/%reldir%/test_config.sh_d708b6fd32d83ce0ee00ca5383388308ba5a06e1.out \
|
||||
$(srcdir)/%reldir%/test_config.sh_eec3768ebc201ca63bca1411270965f78db1abfc.err \
|
||||
$(srcdir)/%reldir%/test_config.sh_eec3768ebc201ca63bca1411270965f78db1abfc.out \
|
||||
$(srcdir)/%reldir%/test_events.sh_09ba47d70bfca88e89faf29598c1095292cad435.err \
|
||||
$(srcdir)/%reldir%/test_events.sh_09ba47d70bfca88e89faf29598c1095292cad435.out \
|
||||
$(srcdir)/%reldir%/test_events.sh_153e221f3cb50f4d3e4581be0bf311e62489c42d.err \
|
||||
$(srcdir)/%reldir%/test_events.sh_153e221f3cb50f4d3e4581be0bf311e62489c42d.out \
|
||||
$(srcdir)/%reldir%/test_events.sh_3dae146ef3bf201c43656344803694a34a3dbfec.err \
|
||||
$(srcdir)/%reldir%/test_events.sh_3dae146ef3bf201c43656344803694a34a3dbfec.out \
|
||||
$(srcdir)/%reldir%/test_events.sh_6f9523d43f174397829b6a7fe6ee0090d97df5f9.err \
|
||||
$(srcdir)/%reldir%/test_events.sh_6f9523d43f174397829b6a7fe6ee0090d97df5f9.out \
|
||||
$(srcdir)/%reldir%/test_events.sh_729f77b8e7136d64d22a6610a80ba6b584a2d896.err \
|
||||
$(srcdir)/%reldir%/test_events.sh_729f77b8e7136d64d22a6610a80ba6b584a2d896.out \
|
||||
$(srcdir)/%reldir%/test_events.sh_d9c7907f907b2335e1328b23fdc46d0968a608d9.err \
|
||||
$(srcdir)/%reldir%/test_events.sh_d9c7907f907b2335e1328b23fdc46d0968a608d9.out \
|
||||
$(srcdir)/%reldir%/test_events.sh_ed8dc44add223341c03ccb7b3e18371bdb42b710.err \
|
||||
$(srcdir)/%reldir%/test_events.sh_ed8dc44add223341c03ccb7b3e18371bdb42b710.out \
|
||||
$(srcdir)/%reldir%/test_format_loader.sh_15e861d2327512a721fd42ae51dc5427689e0bb6.err \
|
||||
$(srcdir)/%reldir%/test_format_loader.sh_15e861d2327512a721fd42ae51dc5427689e0bb6.out \
|
||||
$(srcdir)/%reldir%/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err \
|
||||
|
|
|
@ -1,2 +1,3 @@
|
|||
192.168.202.254 - - [20/Jul/2009:22:59:26 +0000] "GET /vmw/cgi/tramp HTTP/1.0" 200 134 "-" "gPXE/0.9.7"
|
||||
[31m192.168.202.254[0m[31m - [0m[31m-[0m[31m [[0m[31m20/Jul/2009:22:59:29 +0000[0m[31m] "[0m[31mGET[0m[31m [0m[31m/vmw/vSphere/default/vmkboot.gz[0m[31m [0m[31mHTTP/1.0[0m[31m" 404 46210 "[0m[31m-[0m[31m" "[0m[31mgPXE/0.9.7[0m[31m"[0m
|
||||
192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
|
||||
|
|
|
@ -1,2 +1,3 @@
|
|||
192.168.202.254 - - [20/Jul/2009:22:59:26 +0000] "GET /vmw/cgi/tramp HTTP/1.0" 200 134 "-" "gPXE/0.9.7"
|
||||
[31m192.168.202.254[0m[31m - [0m[31m-[0m[31m [[0m[31m20/Jul/2009:22:59:29 +0000[0m[31m] "[0m[31mGET[0m[31m [0m[31m/vmw/vSphere/default/vmkboot.gz[0m[31m [0m[31mHTTP/1.0[0m[31m" 404 46210 "[0m[31m-[0m[31m" "[0m[31mgPXE/0.9.7[0m[31m"[0m
|
||||
192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
[1m$schema[0m [4m<schema-uri>[0m
|
||||
[1mtuning[0m[1m/[0m
|
||||
[1mui[0m[1m/[0m
|
||||
[1mlog[0m[1m/[0m
|
||||
[1mglobal[0m[1m/[0m
|
||||
[31m✘ error[0m: invalid JSON
|
||||
[36m --> [0m[1m{test_dir}/bad-config2/formats/invalid-config/config.malformed.json[0m:3
|
||||
|
@ -30,6 +31,7 @@
|
|||
[1m$schema[0m [4m<schema-uri>[0m
|
||||
[1mtuning[0m[1m/[0m
|
||||
[1mui[0m[1m/[0m
|
||||
[1mlog[0m[1m/[0m
|
||||
[1mglobal[0m[1m/[0m
|
||||
[31m✘ error[0m: invalid JSON
|
||||
[31mreason[0m: parse error: premature EOF
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
[31m✘ error[0m: unknown configuration option -- /bad/path
|
||||
[36m --> [0m[1mcommand-option[0m:1
|
||||
[36m | [0m[37m[40m:[0m[1m[36m[40mreset-config[0m[37m[40m /bad/path [0m
|
||||
[36m =[0m [36mhelp[0m: [4m:[0m[1m[4mreset-config[0m[4m [0m[4moption[0m
|
||||
══════════════════════════════════════════════════════════════════════
|
||||
Reset the configuration option to its default value
|
|
@ -1,3 +1,12 @@
|
|||
[31m✘ error[0m: invalid value for property “[1m/ui/theme-defs/default/styles/text/color[0m”
|
||||
[31mreason[0m: Could not parse color: #f
|
||||
[31mreason[0m: invalid color -- “#f”
|
||||
[31m | [0m [31mreason[0m: Could not parse color: #f
|
||||
[36m --> [0m[1mcommand-option[0m:1
|
||||
[36m =[0m [36mhelp[0m: [1mProperty Synopsis[0m
|
||||
[1m/ui/theme-defs/default/styles/text/color[0m [4m#hex|color_name[0m
|
||||
[1mDescription[0m
|
||||
The foreground color value for this style. The value can be the name of an xterm color, the hexadecimal value, or a theme variable reference.
|
||||
[1mExamples[0m
|
||||
#fff
|
||||
Green
|
||||
$black
|
||||
|
|
|
@ -1,3 +1,8 @@
|
|||
[31m✘ error[0m: invalid value for property “[1m/ui/theme[0m”
|
||||
[31mreason[0m: unknown theme -- baddy
|
||||
[31mreason[0m: unknown theme -- “baddy”
|
||||
[31m | [0m [36m =[0m [36mhelp[0m: The available themes are: default, eldar, grayscale, monocai, night-owl, solarized-dark, solarized-light
|
||||
[36m --> [0m[1mcommand-option[0m:1
|
||||
[36m =[0m [36mhelp[0m: [1mProperty Synopsis[0m
|
||||
[1m/ui/theme[0m [4mtheme_name[0m
|
||||
[1mDescription[0m
|
||||
The name of the theme to use.
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
/log/watch-expressions = {
|
||||
"http-errors": {
|
||||
"expr": ":sc_status >= 400",
|
||||
"enabled": true
|
||||
}
|
||||
}
|
|
@ -0,0 +1,3 @@
|
|||
/log/watch-expressions = {
|
||||
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
[31m✘ error[0m: invalid value for property “[1m/log/watch-expressions/http-errors/expr[0m”
|
||||
[31mreason[0m: SQL expression is invalid
|
||||
[31m | [0m [31mreason[0m: no such column: sc_status
|
||||
[31m | [0m [36m --> [0m[1m/log/watch-expressions/http-errors/expr[0m
|
||||
[31m | [0m [36m | [0m[37m[40msc_status[0m[37m[40m [0m[1m[37m[40m>[0m[1m[37m[40m=[0m[37m[40m 400 [0m[1m[36m[40mAND[0m[37m[40m [0m[37m[40mbad[0m[37m[40m [0m
|
||||
[36m --> [0m[1mcommand-option[0m:1
|
||||
[36m =[0m [36mhelp[0m: [1mProperty Synopsis[0m
|
||||
[1m/log/watch-expressions/http-errors/expr[0m [4m<SQL-expression>[0m
|
||||
[1mDescription[0m
|
||||
The SQL expression to execute for each input line. If expression evaluates to true, a 'log message detected' event will be published.
|
|
@ -0,0 +1,3 @@
|
|||
{"content":{"$schema":"https://lnav.org/event-file-open-v1.schema.json","filename":"{test_dir}/logfile_access_log.0"}}
|
||||
{"content":{"$schema":"https://lnav.org/event-file-format-detected-v1.schema.json","filename":"{test_dir}/logfile_access_log.0","format":"access_log"}}
|
||||
{"content":{"$schema":"https://lnav.org/event-log-msg-detected-v1.schema.json","watch-name":"http-errors","filename":"{test_dir}/logfile_access_log.0","format":"access_log","timestamp":"2009-07-20T22:59:29.000","values":{"body":"","c_ip":"192.168.202.254","cs_method":"GET","cs_referer":"-","cs_uri_query":null,"cs_uri_stem":"/vmw/vSphere/default/vmkboot.gz","cs_user_agent":"gPXE/0.9.7","cs_username":"-","cs_version":"HTTP/1.0","sc_bytes":46210,"sc_status":404,"timestamp":"20/Jul/2009:22:59:29 +0000"}}}
|
|
@ -34,3 +34,6 @@ run_cap_test ${lnav_test} -n \
|
|||
run_cap_test ${lnav_test} -n \
|
||||
-I ${test_dir}/bad-config2 \
|
||||
${test_dir}/logfile_access_log.0
|
||||
|
||||
run_cap_test ${lnav_test} -nN \
|
||||
-c ":reset-config /bad/path"
|
||||
|
|
|
@ -1,6 +1,31 @@
|
|||
#! /bin/bash
|
||||
|
||||
rm -rf events-home
|
||||
mkdir -p events-home
|
||||
export HOME=events-home
|
||||
export YES_COLOR=1
|
||||
|
||||
run_cap_test ${lnav_test} -n \
|
||||
-c ';SELECT json(content) as content FROM lnav_events' \
|
||||
-c ':write-jsonlines-to -' \
|
||||
${test_dir}/logfile_access_log.0
|
||||
|
||||
run_cap_test ${lnav_test} -nN \
|
||||
-c ':config /log/watch-expressions/http-errors/expr sc_status >= 400 AND bad'
|
||||
|
||||
run_cap_test ${lnav_test} -nN \
|
||||
-c ':config /log/watch-expressions/http-errors/expr :sc_status >= 400'
|
||||
|
||||
run_cap_test env TEST_COMMENT="watch expression generate detect event" ${lnav_test} -n \
|
||||
-c ';SELECT json(content) as content FROM lnav_events' \
|
||||
-c ':write-jsonlines-to -' \
|
||||
${test_dir}/logfile_access_log.0
|
||||
|
||||
run_cap_test env TEST_COMMENT="show the configuration" ${lnav_test} -nN \
|
||||
-c ':config /log/watch-expressions'
|
||||
|
||||
run_cap_test env TEST_COMMENT="delete the configuration" ${lnav_test} -nN \
|
||||
-c ':reset-config /log/watch-expressions/http-errors/'
|
||||
|
||||
run_cap_test env TEST_COMMENT="config should be gone now" ${lnav_test} -nN \
|
||||
-c ':config /log/watch-expressions'
|
||||
|
|
Loading…
Reference in New Issue