diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dcb519b3..4344903a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,6 +6,7 @@ set(diag_STAT_SRCS command_executor.cc curl_looper.cc db_sub_source.cc + elem_to_json.cc environ_vtab.cc extension-functions.c field_overlay_source.cc @@ -88,6 +89,7 @@ set(diag_STAT_SRCS concise_index.hh column_namer.hh curl_looper.hh + elem_to_json.hh field_overlay_source.hh filter_observer.hh format-text-files.hh diff --git a/src/Makefile.am b/src/Makefile.am index 54d9a0bb..3d0151be 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -123,6 +123,7 @@ noinst_HEADERS = \ data_parser.hh \ default-log-formats-json.hh \ db_sub_source.hh \ + elem_to_json.hh \ environ_vtab.hh \ field_overlay_source.hh \ filter_observer.hh \ @@ -223,6 +224,7 @@ libdiag_a_SOURCES = \ command_executor.cc \ curl_looper.cc \ db_sub_source.cc \ + elem_to_json.cc \ environ_vtab.cc \ extension-functions.c \ field_overlay_source.cc \ diff --git a/src/data_parser.cc b/src/data_parser.cc index d9d4acc2..8a803ec3 100644 --- a/src/data_parser.cc +++ b/src/data_parser.cc @@ -175,6 +175,10 @@ data_format_state_t dfs_comma_next(data_format_state_t state, switch (next_token) { case DT_COLON: case DT_EQUALS: + case DT_LPAREN: + case DT_LCURLY: + case DT_LSQUARE: + case DT_LANGLE: retval = DFS_VALUE; break; diff --git a/src/data_parser.hh b/src/data_parser.hh index 82e7f0ff..4f46eaaa 100644 --- a/src/data_parser.hh +++ b/src/data_parser.hh @@ -47,12 +47,13 @@ #include "byte_array.hh" #include "data_scanner.hh" -#define ELEMENT_LIST_T(var) var("" #var, __FILE__, __LINE__) +#define ELEMENT_LIST_T(var) var("" #var, __FILE__, __LINE__, group_depth) #define PUSH_FRONT(elem) push_front(elem, __FILE__, __LINE__) #define PUSH_BACK(elem) push_back(elem, __FILE__, __LINE__) #define POP_FRONT(elem) pop_front(__FILE__, __LINE__) #define POP_BACK(elem) pop_back(__FILE__, __LINE__) #define CLEAR(elem) clear2(__FILE__, __LINE__) +#define SWAP(other) swap(other, __FILE__, __LINE__) #define SPLICE(pos, other, first, last) splice(pos, other, first, last, \ __FILE__, __LINE__) @@ -102,16 +103,17 @@ data_format_state_t dfs_semi_next(data_format_state_t state, data_format_state_t dfs_comma_next(data_format_state_t state, data_token_t next_token); -#define LIST_INIT_TRACE \ - do { \ - if (TRACE_FILE != NULL) { \ - fprintf(TRACE_FILE, \ - "%p %s:%d %s %s\n", \ - this, \ - fn, line, \ - __func__, \ - varname); \ - } \ +#define LIST_INIT_TRACE \ + do { \ + if (TRACE_FILE != NULL) { \ + fprintf(TRACE_FILE, \ + "%p %s:%d %s %s %d\n", \ + this, \ + fn, line, \ + __func__, \ + varname, \ + group_depth); \ + } \ } while (false) #define LIST_DEINIT_TRACE \ @@ -165,6 +167,18 @@ data_format_state_t dfs_comma_next(data_format_state_t state, } \ } while (false); +#define SWAP_TRACE(other) \ + do { \ + if (TRACE_FILE != NULL) { \ + fprintf(TRACE_FILE, \ + "%p %s:%d %s %p\n", \ + this, \ + fn, line, \ + __func__, \ + &other); \ + } \ + } while (false); + #define POINT_TRACE(name) \ do { \ if (TRACE_FILE) { \ @@ -175,6 +189,33 @@ data_format_state_t dfs_comma_next(data_format_state_t state, } \ } while (false); +#define FORMAT_TRACE(elist) \ + do { \ + if (TRACE_FILE) { \ + const data_format &df = elist.el_format; \ + fprintf(TRACE_FILE, \ + "%p %s:%d format %d %s %s %s %s %s\n", \ + &elist, \ + __FILE__, __LINE__, \ + group_depth, \ + data_scanner::token2name(df.df_appender), \ + data_scanner::token2name(df.df_terminator), \ + data_scanner::token2name(df.df_qualifier), \ + data_scanner::token2name(df.df_separator), \ + data_scanner::token2name(df.df_prefix_terminator)); \ + } \ + } while (false); + +#define CONSUMED_TRACE(elist) \ + do { \ + if (TRACE_FILE) { \ + fprintf(TRACE_FILE, \ + "%p %s:%d consumed\n", \ + &elist, \ + __FILE__, __LINE__); \ + } \ + } while (false); + class data_parser { public: static data_format FORMAT_SEMI; @@ -190,7 +231,7 @@ public: class element_list_t : public std::list { public: - element_list_t(const char *varname, const char *fn, int line) + element_list_t(const char *varname, const char *fn, int line, int group_depth = -1) { LIST_INIT_TRACE; } @@ -200,6 +241,7 @@ public: const char *varname = "_anon2_"; const char *fn = __FILE__; int line = __LINE__; + int group_depth = -1; LIST_INIT_TRACE; }; @@ -251,6 +293,12 @@ public: this->std::list::clear(); }; + void swap(element_list_t &other, const char *fn, int line) { + SWAP_TRACE(other); + + this->std::list::swap(other); + } + void splice(iterator pos, element_list_t &other, iterator first, @@ -310,14 +358,14 @@ public: return *this; }; - void assign_elements(element_list_t &subs) + void assign_elements(element_list_t &subs) { if (this->e_sub_elements == NULL) { this->e_sub_elements = new element_list_t("_sub_", __FILE__, __LINE__); this->e_sub_elements->el_format = subs.el_format; } - this->e_sub_elements->swap(subs); + this->e_sub_elements->SWAP(subs); this->update_capture(); }; @@ -331,14 +379,14 @@ public: } }; - const element & get_pair_value(void) const + const element &get_pair_value(void) const { require(this->e_token == DNT_PAIR); return this->e_sub_elements->back(); }; - data_token_t value_token(void) const + data_token_t value_token(void) const { data_token_t retval = DT_INVALID; @@ -357,6 +405,23 @@ public: return retval; }; + const element &get_value_elem() const { + if (this->e_token == DNT_VALUE) { + if (this->e_sub_elements != NULL && + this->e_sub_elements->size() == 1) { + return this->e_sub_elements->front(); + } + } + return *this; + }; + + const element &get_pair_elem() const { + if (this->e_token == DNT_VALUE) { + return this->e_sub_elements->front(); + } + return *this; + } + void print(FILE *out, pcre_input &pi, int offset = 0) const { @@ -503,7 +568,7 @@ private: }; void pairup(schema_id_t *schema, element_list_t &pairs_out, - element_list_t &in_list) + element_list_t &in_list, int group_depth = 0) { element_list_t ELEMENT_LIST_T(el_stack), ELEMENT_LIST_T(free_row), ELEMENT_LIST_T(key_comps), ELEMENT_LIST_T(value), @@ -514,13 +579,15 @@ private: POINT_TRACE("pairup_start"); + FORMAT_TRACE(in_list); + for (element_list_t::iterator iter = in_list.begin(); iter != in_list.end(); ++iter) { if (iter->e_token == DNT_GROUP) { element_list_t ELEMENT_LIST_T(group_pairs); - this->pairup(NULL, group_pairs, *iter->e_sub_elements); + this->pairup(NULL, group_pairs, *iter->e_sub_elements, group_depth + 1); if (!group_pairs.empty()) { iter->assign_elements(group_pairs); } @@ -535,7 +602,7 @@ private: } } else if (iter->e_token == in_list.el_format.df_terminator) { - this->end_of_value(el_stack, key_comps, value, in_list); + this->end_of_value(el_stack, key_comps, value, in_list, group_depth); key_comps.PUSH_BACK(*iter); } @@ -639,12 +706,16 @@ private: POINT_TRACE("pairup_eol"); - if (el_stack.empty()) { + CONSUMED_TRACE(in_list); + + // Only perform the free-row logic at the top level, if we're in a group + // assume it is a list. + if (group_depth < 1 && el_stack.empty()) { free_row.SPLICE(free_row.begin(), key_comps, key_comps.begin(), key_comps.end()); } else { - this->end_of_value(el_stack, key_comps, value, in_list); + this->end_of_value(el_stack, key_comps, value, in_list, group_depth); } POINT_TRACE("pairup_stack"); @@ -764,6 +835,10 @@ private: } } + if (group_depth >= 1 && pairs_out.empty() && !free_row.empty()) { + pairs_out.SWAP(free_row); + } + if (pairs_out.empty() && !free_row.empty()) { while (!free_row.empty()) { switch (free_row.front().e_token) { @@ -957,7 +1032,8 @@ private: void end_of_value(element_list_t &el_stack, element_list_t &key_comps, element_list_t &value, - const element_list_t &in_list) { + const element_list_t &in_list, + int group_depth) { key_comps.remove_if(element_if(in_list.el_format.df_terminator)); key_comps.remove_if(element_if(DT_COMMA)); value.remove_if(element_if(in_list.el_format.df_terminator)); diff --git a/src/elem_to_json.cc b/src/elem_to_json.cc new file mode 100644 index 00000000..6c5bf519 --- /dev/null +++ b/src/elem_to_json.cc @@ -0,0 +1,210 @@ +/** + * Copyright (c) 2016, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "elem_to_json.hh" + +using namespace std; + +static +void element_to_json(yajl_gen gen, data_parser &dp, const data_parser::element &elem) +{ + size_t value_len; + const char *value_str = dp.get_element_string(elem, value_len); + + switch (elem.value_token()) { + case DT_NUMBER: { + yajl_gen_number(gen, value_str, value_len); + break; + } + case DNT_GROUP: { + elements_to_json(gen, dp, elem.get_value_elem().e_sub_elements, false); + break; + } + case DNT_PAIR: { + const data_parser::element &pair_elem = elem.get_pair_elem(); + string key_str = dp.get_element_string(pair_elem.e_sub_elements->front()); + + if (!key_str.empty()) { + yajlpp_map singleton_map(gen); + + singleton_map.gen(key_str); + element_to_json(gen, dp, pair_elem.get_pair_value()); + } + else { + element_to_json(gen, dp, pair_elem.get_pair_value()); + } + break; + } + case DT_CONSTANT: { + if (strncasecmp("true", value_str, value_len) == 0) { + yajl_gen_bool(gen, true); + } + else if (strncasecmp("false", value_str, value_len) == 0) { + yajl_gen_bool(gen, false); + } + else { + yajl_gen_null(gen); + } + break; + } + default: + yajl_gen_pstring(gen, value_str, value_len); + break; + } +} + +static +void map_elements_to_json2(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el) +{ + yajlpp_map root_map(gen); + int col = 0; + + for (data_parser::element_list_t::iterator iter = el->begin(); + iter != el->end(); + ++iter) { + const data_parser::element &pvalue = iter->get_pair_value(); + + if (pvalue.value_token() == DT_INVALID) { + log_debug("invalid!!"); + // continue; + } + + std::string key_str = dp.get_element_string( + iter->e_sub_elements->front()); + + if (key_str.empty()) { + char buffer[32]; + + snprintf(buffer, sizeof(buffer), "col_%d", col); + key_str = buffer; + col += 1; + } + root_map.gen(key_str); + element_to_json(gen, dp, pvalue); + } +} + +static +void list_body_elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el) +{ + for (data_parser::element_list_t::iterator iter = el->begin(); + iter != el->end(); + ++iter) { + element_to_json(gen, dp, *iter); + } +} + +static +void list_elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el) +{ + yajlpp_array root_array(gen); + + list_body_elements_to_json(gen, dp, el); +} + +static +void map_elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el) +{ + bool unique_names = el->size() > 1; + vector names; + + for (data_parser::element_list_t::iterator iter = el->begin(); + iter != el->end(); + ++iter) { + const data_parser::element &pvalue = iter->get_pair_value(); + + if (pvalue.value_token() == DT_INVALID) { + log_debug("invalid!!"); + // continue; + } + + std::string key_str = dp.get_element_string( + iter->e_sub_elements->front()); + if (key_str.empty()) { + continue; + } + if (find(names.begin(), names.end(), key_str) != names.end()) { + unique_names = false; + break; + } + else { + names.push_back(key_str); + } + } + + names.clear(); + + if (unique_names) { + map_elements_to_json2(gen, dp, el); + } + else { + list_elements_to_json(gen, dp, el); + } +} + +void elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el, bool root) +{ + if (el->empty()) { + yajl_gen_null(gen); + } + else { + switch (el->front().e_token) { + case DNT_PAIR: { + if (root && el->size() == 1) { + const data_parser::element &pair_elem = el->front().get_pair_elem(); + std::string key_str = dp.get_element_string( + pair_elem.e_sub_elements->front()); + + if (key_str.empty() && + el->front().get_pair_value().value_token() == DNT_GROUP) { + element_to_json(gen, dp, el->front().get_pair_value()); + } + else { + yajlpp_map singleton_map(gen); + + if (key_str.empty()) { + key_str = "col_0"; + } + singleton_map.gen(key_str); + element_to_json(gen, dp, pair_elem.get_pair_value()); + } + } + else { + map_elements_to_json(gen, dp, el); + } + break; + } + default: + list_elements_to_json(gen, dp, el); + break; + } + } +} diff --git a/src/elem_to_json.hh b/src/elem_to_json.hh new file mode 100644 index 00000000..e84c50dd --- /dev/null +++ b/src/elem_to_json.hh @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2016, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __elem_to_json_hh +#define __elem_to_json_hh + +#include "yajl/api/yajl_gen.h" +#include "data_parser.hh" + +void elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el, bool root = true); + +#endif diff --git a/src/log_data_table.hh b/src/log_data_table.hh index 8e1a91fb..a71b17e1 100644 --- a/src/log_data_table.hh +++ b/src/log_data_table.hh @@ -175,7 +175,7 @@ public: } this->ldt_pairs.clear(); - this->ldt_pairs.swap(dp.dp_pairs); + this->ldt_pairs.swap(dp.dp_pairs, __FILE__, __LINE__); this->ldt_instance += 1; return true; diff --git a/src/string-extension-functions.cc b/src/string-extension-functions.cc index 9e3f4496..f0b95651 100644 --- a/src/string-extension-functions.cc +++ b/src/string-extension-functions.cc @@ -23,6 +23,7 @@ #include "sqlite-extension-func.h" #include "data_scanner.hh" #include "data_parser.hh" +#include "elem_to_json.hh" typedef struct { char * s; @@ -36,6 +37,8 @@ typedef struct { #define JSON_SUBTYPE 74 /* Ascii for "J" */ +using namespace std; + static cache_entry *find_re(sqlite3_context *ctx, const char *re) { @@ -244,105 +247,6 @@ void regexp_match(sqlite3_context *ctx, int argc, sqlite3_value **argv) #endif } -static -void elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el); - -static -void element_to_json(yajl_gen gen, data_parser &dp, const data_parser::element &elem) -{ - size_t value_len; - const char *value_str = dp.get_element_string(elem, value_len); - - switch (elem.value_token()) { - case DT_NUMBER: { - yajl_gen_number(gen, value_str, value_len); - break; - } - case DNT_GROUP: { - elements_to_json(gen, dp, elem.e_sub_elements); - break; - } - case DNT_PAIR: { - const data_parser::element &pair_elem = elem.e_sub_elements->front(); - yajlpp_map singleton_map(gen); - - singleton_map.gen(dp.get_element_string(pair_elem.e_sub_elements->front())); - element_to_json(gen, dp, pair_elem.get_pair_value()); - break; - } - case DT_CONSTANT: { - if (strncasecmp("true", value_str, value_len) == 0) { - yajl_gen_bool(gen, true); - } - else if (strncasecmp("false", value_str, value_len) == 0) { - yajl_gen_bool(gen, false); - } - else { - yajl_gen_null(gen); - } - break; - } - default: - yajl_gen_pstring(gen, value_str, value_len); - break; - } -} - -static -void map_elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el) -{ - yajlpp_map root_map(gen); - column_namer cn; - - for (data_parser::element_list_t::iterator iter = el->begin(); - iter != el->end(); - ++iter) { - const data_parser::element &pvalue = iter->get_pair_value(); - - if (pvalue.value_token() == DT_INVALID) { - log_debug("invalid!!"); - // continue; - } - - std::string key_str = dp.get_element_string( - iter->e_sub_elements->front()); - string colname = cn.add_column(key_str); - - root_map.gen(colname); - element_to_json(gen, dp, pvalue); - } -} - -static -void list_elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el) -{ - yajlpp_array root_array(gen); - - for (data_parser::element_list_t::iterator iter = el->begin(); - iter != el->end(); - ++iter) { - element_to_json(gen, dp, *iter); - } -} - -static -void elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el) -{ - if (el->empty()) { - yajl_gen_null(gen); - } - else { - switch (el->front().e_token) { - case DNT_PAIR: - map_elements_to_json(gen, dp, el); - break; - default: - list_elements_to_json(gen, dp, el); - break; - } - } -} - static void extract(sqlite3_context *ctx, int argc, sqlite3_value **argv) { diff --git a/test/Makefile.am b/test/Makefile.am index 62143d7f..bee304ea 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -237,6 +237,13 @@ dist_noinst_DATA = \ datafile_simple.11 \ datafile_simple.12 \ datafile_simple.13 \ + datafile_simple.14 \ + datafile_simple.15 \ + datafile_simple.16 \ + datafile_simple.17 \ + datafile_simple.18 \ + datafile_simple.19 \ + datafile_simple.20 \ datafile_xml.0 \ listview_output.0 \ listview_output.1 \ diff --git a/test/datafile_simple.0 b/test/datafile_simple.0 index d2cfedd0..459716a4 100644 --- a/test/datafile_simple.0 +++ b/test/datafile_simple.0 @@ -17,3 +17,9 @@ pair 8:11 ^-^ c=3 pair 12:13 ^ 4 msg :a=1 b=2 c=3,4 format :a=# b=# c=#,# +{ + "a": 1, + "b": 2, + "c": 3, + "col_0": 4 +} diff --git a/test/datafile_simple.1 b/test/datafile_simple.1 index 9cd0db3c..48bb61ac 100644 --- a/test/datafile_simple.1 +++ b/test/datafile_simple.1 @@ -7,3 +7,7 @@ pair 0:13 ^-----------^ current speed pair 15:17 ^^ 38 msg :current speed: 38 mph format :#: # mph +{ + "col_0": "current speed", + "col_1": 38 +} diff --git a/test/datafile_simple.10 b/test/datafile_simple.10 index 339def08..928f183d 100644 --- a/test/datafile_simple.10 +++ b/test/datafile_simple.10 @@ -4,3 +4,6 @@ quot 16:26 ^--------^ pm/runtime pair 16:26 ^--------^ pm/runtime msg :quoted string u'pm/runtime' format :quoted string # +{ + "col_0": "pm/runtime" +} diff --git a/test/datafile_simple.11 b/test/datafile_simple.11 index 3fa15bd7..f12969f4 100644 --- a/test/datafile_simple.11 +++ b/test/datafile_simple.11 @@ -7,3 +7,7 @@ vers 25:31 ^----^ 1.2-a1 pair 25:31 ^----^ 1.2-a1 msg :version numbers 0.6.16 1.2-a1 format :version numbers # # +{ + "col_0": "0.6.16", + "col_1": "1.2-a1" +} diff --git a/test/datafile_simple.12 b/test/datafile_simple.12 index 09837908..e19007ef 100644 --- a/test/datafile_simple.12 +++ b/test/datafile_simple.12 @@ -5,3 +5,6 @@ pair 0:34 ^--------------------------------^ kickoff_duration=4.41074371338e-05 msg :kickoff_duration=4.41074371338e-05 format :kickoff_duration=# +{ + "kickoff_duration": 4.41074371338e-05 +} diff --git a/test/datafile_simple.13 b/test/datafile_simple.13 index ab3795f2..80210104 100644 --- a/test/datafile_simple.13 +++ b/test/datafile_simple.13 @@ -5,3 +5,6 @@ pair 0:34 ^--------------------------------^ kickoff_duration=4.41074371338E-05 msg :kickoff_duration=4.41074371338E-05 format :kickoff_duration=# +{ + "kickoff_duration": 4.41074371338E-05 +} diff --git a/test/datafile_simple.14 b/test/datafile_simple.14 index 8f2cc275..4c548b90 100644 --- a/test/datafile_simple.14 +++ b/test/datafile_simple.14 @@ -18,10 +18,7 @@ wspc 55:56 ^ val 49:60 ^---------^ Fanime 2015 pair 44:60 ^--------------^ name=Fanime 2015 key 62:67 ^---^ route - key 69:69 ^ - sym 69:79 ^--------^ CloudEntry -pair 69:79 ^--------^ CloudEntry - key 80:80 ^ + key 69:79 ^--------^ CloudEntry key 80:86 ^----^ doc_id sym 87:131 ^------------------------------------------^ 1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg val 87:131 ^------------------------------------------^ 1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg @@ -33,7 +30,8 @@ word 146:151 val 141:151 ^--------^ Baby Names pair 132:151 ^-----------------^ filename=Baby Names grp 80:151 ^---------------------------------------------------------------------^ doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names -pair 80:151 ^---------------------------------------------------------------------^ doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names + val 80:151 ^---------------------------------------------------------------------^ doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names +pair 69:151 ^--------------------------------------------------------------------------------^ CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names grp 69:151 ^--------------------------------------------------------------------------------^ CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names val 69:151 ^--------------------------------------------------------------------------------^ CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names pair 62:151 ^---------------------------------------------------------------------------------------^ route=[CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names @@ -41,3 +39,19 @@ pair 62:151 ^---- pair 9:151 ^--------------------------------------------------------------------------------------------------------------------------------------------^ Direction.DOWNLOAD, Action.CREATE, name=Fanime 2015, route=[CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names msg :FSChange(Direction.DOWNLOAD, Action.CREATE, name=Fanime 2015, route=[CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names)]) format :#(#)]) +{ + "col_0": "FSChange", + "col_1": { + "col_0": "Direction.DOWNLOAD", + "col_1": "Action.CREATE", + "name": "Fanime 2015", + "route": [ + { + "CloudEntry": { + "doc_id": "1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg", + "filename": "Baby Names" + } + } + ] + } +} diff --git a/test/datafile_simple.15 b/test/datafile_simple.15 index 89a99fbf..f0e053d7 100644 --- a/test/datafile_simple.15 +++ b/test/datafile_simple.15 @@ -1,9 +1,6 @@ Worker successfully completed [ImmutableChange(Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False)] key 31:31 ^ - key 31:31 ^ - sym 31:46 ^-------------^ ImmutableChange -pair 31:46 ^-------------^ ImmutableChange - key 47:47 ^ + key 31:46 ^-------------^ ImmutableChange key 47:47 ^ sym 47:63 ^--------------^ Direction.UPLOAD val 47:63 ^--------------^ Direction.UPLOAD @@ -45,8 +42,34 @@ cnst 216:221 val 216:221 ^---^ False pair 206:221 ^-------------^ is_folder=False grp 47:221 ^----------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False -pair 47:221 ^----------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False + val 47:221 ^----------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False +pair 31:221 ^--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ ImmutableChange(Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False grp 31:221 ^--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ ImmutableChange(Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False pair 31:221 ^--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ ImmutableChange(Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False msg :Worker successfully completed [ImmutableChange(Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False)] format :Worker successfully completed [#)] +[ + { + "ImmutableChange": { + "col_0": "Direction.UPLOAD", + "col_1": "Action.CREATE", + "ino": { + "LocalID": [ + { + "inode": 5567236 + } + ] + }, + "path": "/Users/stack/Google Drive", + "name": "pyjsonpath1.patch", + "parent_ino": { + "LocalID": [ + { + "inode": 46166734 + } + ] + }, + "is_folder": false + } + } +] diff --git a/test/datafile_simple.16 b/test/datafile_simple.16 index a1c935d5..f549fc72 100644 --- a/test/datafile_simple.16 +++ b/test/datafile_simple.16 @@ -4,3 +4,6 @@ pair 0:4 ^--^ test msg :test: c'est la vie charlie's format :#: c'est la vie charlie's +{ + "col_0": "test" +} diff --git a/test/datafile_simple.17 b/test/datafile_simple.17 index 933ab9a9..132eda6c 100644 --- a/test/datafile_simple.17 +++ b/test/datafile_simple.17 @@ -4,16 +4,19 @@ val 5:5 ^ pair 0:5 ^---^ foo=[ key 8:11 ^-^ bar - key 14:14 ^ quot 14:15 ^ a val 14:15 ^ a -pair 14:15 ^ a - key 19:19 ^ quot 19:20 ^ b val 19:20 ^ b -pair 19:20 ^ b grp 14:20 ^----^ a', 'b val 14:20 ^----^ a', 'b pair 8:20 ^----------^ bar=['a', 'b msg :foo=[]; bar=['a', 'b'] format :foo=[#]; bar=[#] +{ + "foo": null, + "bar": [ + "a", + "b" + ] +} diff --git a/test/datafile_simple.18 b/test/datafile_simple.18 new file mode 100644 index 00000000..55f125b8 --- /dev/null +++ b/test/datafile_simple.18 @@ -0,0 +1,23 @@ + list [foo(bar=1)] + key 6:6 ^ + key 6:9 ^-^ foo + key 10:13 ^-^ bar + num 14:15 ^ 1 + val 14:15 ^ 1 +pair 10:15 ^---^ bar=1 + grp 10:15 ^---^ bar=1 + val 10:15 ^---^ bar=1 +pair 6:15 ^-------^ foo(bar=1 + grp 6:15 ^-------^ foo(bar=1 +pair 6:15 ^-------^ foo(bar=1 +msg :list [foo(bar=1)] +format :list [#)] +[ + { + "foo": [ + { + "bar": 1 + } + ] + } +] diff --git a/test/datafile_simple.19 b/test/datafile_simple.19 new file mode 100644 index 00000000..ce58fc99 --- /dev/null +++ b/test/datafile_simple.19 @@ -0,0 +1,53 @@ + list [foo(bar=1), foo(bar=2), foo(bar=3)] + key 6:6 ^ + key 6:9 ^-^ foo + key 10:13 ^-^ bar + num 14:15 ^ 1 + val 14:15 ^ 1 +pair 10:15 ^---^ bar=1 + grp 10:15 ^---^ bar=1 + val 10:15 ^---^ bar=1 +pair 6:15 ^-------^ foo(bar=1 + key 18:21 ^-^ foo + key 22:25 ^-^ bar + num 26:27 ^ 2 + val 26:27 ^ 2 +pair 22:27 ^---^ bar=2 + grp 22:27 ^---^ bar=2 + val 22:27 ^---^ bar=2 +pair 18:27 ^-------^ foo(bar=2 + key 30:33 ^-^ foo + key 34:37 ^-^ bar + num 38:39 ^ 3 + val 38:39 ^ 3 +pair 34:39 ^---^ bar=3 + grp 34:39 ^---^ bar=3 + val 34:39 ^---^ bar=3 +pair 30:39 ^-------^ foo(bar=3 + grp 6:39 ^-------------------------------^ foo(bar=1), foo(bar=2), foo(bar=3 +pair 6:39 ^-------------------------------^ foo(bar=1), foo(bar=2), foo(bar=3 +msg :list [foo(bar=1), foo(bar=2), foo(bar=3)] +format :list [#)] +[ + { + "foo": [ + { + "bar": 1 + } + ] + }, + { + "foo": [ + { + "bar": 2 + } + ] + }, + { + "foo": [ + { + "bar": 3 + } + ] + } +] diff --git a/test/datafile_simple.2 b/test/datafile_simple.2 index df25cf74..c1415448 100644 --- a/test/datafile_simple.2 +++ b/test/datafile_simple.2 @@ -29,3 +29,12 @@ pair 13:16 ^-^ six pair 17:18 ^ 7 msg :1,2,3,4,five,six,7 format :#,#,#,#,#,#,# +{ + "col_0": 1, + "col_1": 2, + "col_2": 3, + "col_3": 4, + "col_4": "five", + "col_5": "six", + "col_6": 7 +} diff --git a/test/datafile_simple.20 b/test/datafile_simple.20 new file mode 100644 index 00000000..ea81659a --- /dev/null +++ b/test/datafile_simple.20 @@ -0,0 +1,17 @@ + list ["abc", "def", "ghi"] + key 7:7 ^ +quot 7:10 ^-^ abc + val 7:10 ^-^ abc +quot 14:17 ^-^ def + val 14:17 ^-^ def +quot 21:24 ^-^ ghi + val 21:24 ^-^ ghi + grp 7:24 ^---------------^ abc", "def", "ghi +pair 7:24 ^---------------^ abc", "def", "ghi +msg :list ["abc", "def", "ghi"] +format :list [#] +[ + "abc", + "def", + "ghi" +] diff --git a/test/datafile_simple.3 b/test/datafile_simple.3 index 1e9c1d49..63126837 100644 --- a/test/datafile_simple.3 +++ b/test/datafile_simple.3 @@ -16,3 +16,10 @@ pair 6:7 ^ 4 pair 17:18 ^ 7 msg :1 2 3 4 five six 7 format :# # # # five six # +{ + "col_0": 1, + "col_1": 2, + "col_2": 3, + "col_3": 4, + "col_4": 7 +} diff --git a/test/datafile_simple.4 b/test/datafile_simple.4 index aa78727c..339aa41d 100644 --- a/test/datafile_simple.4 +++ b/test/datafile_simple.4 @@ -5,3 +5,6 @@ quot 12:25 ^-----------^ Hello, World! pair 0:25 ^-----------------------^ the-value: "Hello, World! msg :the-value: "Hello, World!" format :the-value: # +{ + "the-value": "Hello, World!" +} diff --git a/test/datafile_simple.5 b/test/datafile_simple.5 index c7ac79cb..606a4988 100644 --- a/test/datafile_simple.5 +++ b/test/datafile_simple.5 @@ -5,3 +5,6 @@ pair 0:45 ^-------------------------------------------^ this is a url: http://www.example.com/foo-bar msg :this is a url: http://www.example.com/foo-bar format :this is a url: # +{ + "this is a url": "http://www.example.com/foo-bar" +} diff --git a/test/datafile_simple.6 b/test/datafile_simple.6 index df67f68c..15cfab1d 100644 --- a/test/datafile_simple.6 +++ b/test/datafile_simple.6 @@ -17,3 +17,9 @@ pair 16:21 ^---^ foo=1 pair 22:27 ^---^ bar=2 msg :qualified:name: foo=1 bar=2 format :#:#: foo=# bar=# +{ + "col_0": "qualified", + "col_1": "name", + "foo": 1, + "bar": 2 +} diff --git a/test/datafile_simple.7 b/test/datafile_simple.7 index 625443d8..3c0810f4 100644 --- a/test/datafile_simple.7 +++ b/test/datafile_simple.7 @@ -12,3 +12,7 @@ pair 15:22 ^-----^ arg2="b pair 5:22 ^---------------^ arg1="a", arg2="b msg :func(arg1="a", arg2="b") format :func(#) +{ + "arg1": "a", + "arg2": "b" +} diff --git a/test/datafile_simple.8 b/test/datafile_simple.8 index e9e31f54..c6363c34 100644 --- a/test/datafile_simple.8 +++ b/test/datafile_simple.8 @@ -5,31 +5,40 @@ pair 29:60 ^-----------------------------^ key 73:73 ^ quot 73:95 ^--------------------^ /usr/libexec/taskgated pair 73:95 ^--------------------^ /usr/libexec/taskgated - key 98:98 ^ key 98:98 ^ num 98:103 ^---^ 76339 -pair 98:103 ^---^ 76339 + val 98:103 ^---^ 76339 grp 98:103 ^---^ 76339 pair 98:103 ^---^ 76339 key 135:135 ^ quot 135:157 ^--------------------^ /usr/libexec/taskgated pair 135:157 ^--------------------^ /usr/libexec/taskgated - key 160:160 ^ key 160:160 ^ num 160:165 ^---^ 77395 -pair 160:165 ^---^ 77395 + val 160:165 ^---^ 77395 grp 160:165 ^---^ 77395 pair 160:165 ^---^ 77395 - key 168:168 ^ key 168:168 ^ num 168:174 ^----^ 100003 val 168:174 ^----^ 100003 -pair 168:174 ^----^ 100003 - key 175:175 ^ num 175:176 ^ 1 val 175:176 ^ 1 -pair 175:176 ^ 1 grp 168:176 ^------^ 100003,1 pair 168:176 ^------^ 100003,1 msg :Succeeded authorizing right 'system.privilege.taskport.debug' by client '/usr/libexec/taskgated' [76339] for authorization created by '/usr/libexec/taskgated' [77395] (100003,1) format :Succeeded authorizing right # by client # [#] for authorization created by # [#] (#) +{ + "col_0": "system.privilege.taskport.debug", + "col_1": "/usr/libexec/taskgated", + "col_2": [ + 76339 + ], + "col_3": "/usr/libexec/taskgated", + "col_4": [ + 77395 + ], + "col_5": [ + 100003, + 1 + ] +} diff --git a/test/datafile_simple.9 b/test/datafile_simple.9 index 63eb20e3..85513b9c 100644 --- a/test/datafile_simple.9 +++ b/test/datafile_simple.9 @@ -10,3 +10,8 @@ time 17:29 ^----------^ 12:01:22.123 pair 17:29 ^----------^ 12:01:22.123 msg :12:01 12:01:22 12:01:22.123 format :# # # +{ + "col_0": "12:01", + "col_1": "12:01:22", + "col_2": "12:01:22.123" +} diff --git a/test/datafile_xml.0 b/test/datafile_xml.0 index 09578d86..9b5a1025 100644 --- a/test/datafile_xml.0 +++ b/test/datafile_xml.0 @@ -18,3 +18,9 @@ format :# # # # +{ + "col_0": "", + "col_1": "", + "col_2": "", + "col_3": "" +} diff --git a/test/drive_data_scanner.cc b/test/drive_data_scanner.cc index 9d9d3c25..c0f2763b 100644 --- a/test/drive_data_scanner.cc +++ b/test/drive_data_scanner.cc @@ -43,6 +43,7 @@ #include "log_format_loader.hh" #include "pretty_printer.hh" #include "shared_buffer.hh" +#include "elem_to_json.hh" #include "../src/data_parser.hh" #include "../src/view_curses.hh" @@ -178,6 +179,20 @@ int main(int argc, char *argv[]) string pretty_out = pp.print(); fprintf(out, "\n--\n%s", pretty_out.c_str()); } + + auto_mem gen(yajl_gen_free); + + gen = yajl_gen_alloc(NULL); + yajl_gen_config(gen.in(), yajl_gen_beautify, true); + + elements_to_json(gen, dp, &dp.dp_pairs); + + const unsigned char *buf; + size_t len; + + yajl_gen_get_buf(gen, &buf, &len); + fwrite(buf, 1, len, out); + fclose(out); sprintf(cmd, "diff -u %s %s", argv[lpc], TMP_NAME); @@ -186,6 +201,7 @@ int main(int argc, char *argv[]) if (prompt) { char resp[4]; + printf("\nOriginal line:\n%s\n", sub_line.c_str() + body.lr_start); printf("Would you like to update the original file? (y/N) "); fflush(stdout); log_perror(scanf("%3s", resp)); diff --git a/test/log-samples/sample-057d6c669632ef9d07b6adec605f6bdeae19af27.txt b/test/log-samples/sample-057d6c669632ef9d07b6adec605f6bdeae19af27.txt index 04087d8d..c1d9b88b 100644 --- a/test/log-samples/sample-057d6c669632ef9d07b6adec605f6bdeae19af27.txt +++ b/test/log-samples/sample-057d6c669632ef9d07b6adec605f6bdeae19af27.txt @@ -7,3 +7,7 @@ pair 29:40 ^---------^ com pair 58:62 ^--^ 8099 msg ::com.twisted:Site starting on 8099 format ::#:Site starting on # +{ + "col_0": "com.twisted", + "col_1": 8099 +} diff --git a/test/log-samples/sample-06aaa6f48a801f592558575d886864d6c3ab9ed4.txt b/test/log-samples/sample-06aaa6f48a801f592558575d886864d6c3ab9ed4.txt index b309e28c..bd09d45d 100644 --- a/test/log-samples/sample-06aaa6f48a801f592558575d886864d6c3ab9ed4.txt +++ b/test/log-samples/sample-06aaa6f48a801f592558575d886864d6c3ab9ed4.txt @@ -2,9 +2,8 @@ key 40:46 ^----^ Device path 48:56 ^------^ /dev/sda wspc 56:57 ^ - key 58:58 ^ sym 58:61 ^-^ SAT -pair 58:61 ^-^ SAT + val 58:61 ^-^ SAT grp 58:61 ^-^ SAT val 48:61 ^-----------^ /dev/sda [SAT pair 40:61 ^-------------------^ Device: /dev/sda [SAT @@ -32,3 +31,10 @@ wspc 116:117 pair 112:119 ^-----^ 17.1 GB msg : Device: /dev/sda [SAT], VBOX HARDDISK, S/N:VBc8882b62-a0263a39, FW:1.0, 17.1 GB format : Device: #], #, #, FW:#, # +{ + "Device": "/dev/sda [SAT", + "col_0": "VBOX HARDDISK", + "col_1": "S/N:VBc8882b62-a0263a39", + "FW": 1.0, + "col_2": "17.1 GB" +} diff --git a/test/log-samples/sample-1aeb47c0a97d19bb7418f0172480e05e49c6e53e.txt b/test/log-samples/sample-1aeb47c0a97d19bb7418f0172480e05e49c6e53e.txt index 71e6bb94..ac174221 100644 --- a/test/log-samples/sample-1aeb47c0a97d19bb7418f0172480e05e49c6e53e.txt +++ b/test/log-samples/sample-1aeb47c0a97d19bb7418f0172480e05e49c6e53e.txt @@ -10,3 +10,8 @@ pair 61:63 ^^ pair 75:80 ^---^ 55327 msg : bound to 10.1.10.62 -- renewal in 55327 seconds. format : bound to # # renewal in # seconds +{ + "col_0": "10.1.10.62", + "col_1": "--", + "col_2": 55327 +} diff --git a/test/log-samples/sample-27353a72ba4025448f261dcfa6ea16e474187795.txt b/test/log-samples/sample-27353a72ba4025448f261dcfa6ea16e474187795.txt index 9a2d2fd2..314d9a63 100644 --- a/test/log-samples/sample-27353a72ba4025448f261dcfa6ea16e474187795.txt +++ b/test/log-samples/sample-27353a72ba4025448f261dcfa6ea16e474187795.txt @@ -21,3 +21,10 @@ path 103:110 pair 95:110 ^-------------^ COMMAND=/bin/ls msg : stack : TTY=ttys002 ; PWD=/ ; USER=root ; COMMAND=/bin/ls format : # : TTY=# ; PWD=# ; USER=# ; COMMAND=# +{ + "col_0": "stack", + "TTY": "ttys002", + "PWD": "/", + "USER": "root", + "COMMAND": "/bin/ls" +} diff --git a/test/log-samples/sample-3856ad0f551a04fde41a020158d6b33ef97c870a.txt b/test/log-samples/sample-3856ad0f551a04fde41a020158d6b33ef97c870a.txt index 9aafa81c..95e3e9ce 100644 --- a/test/log-samples/sample-3856ad0f551a04fde41a020158d6b33ef97c870a.txt +++ b/test/log-samples/sample-3856ad0f551a04fde41a020158d6b33ef97c870a.txt @@ -10,3 +10,8 @@ ipv4 116:126 pair 116:126 ^--------^ 10.1.10.62 msg : Leaving mDNS multicast group on interface eth0.IPv4 with address 10.1.10.62. format : Leaving # multicast group on interface # with address # +{ + "col_0": "mDNS", + "col_1": "eth0.IPv4", + "col_2": "10.1.10.62" +} diff --git a/test/log-samples/sample-45364b3fd51af92a4ad8a309b5f4fd88.txt b/test/log-samples/sample-45364b3fd51af92a4ad8a309b5f4fd88.txt index f198ad64..cc1a173f 100644 --- a/test/log-samples/sample-45364b3fd51af92a4ad8a309b5f4fd88.txt +++ b/test/log-samples/sample-45364b3fd51af92a4ad8a309b5f4fd88.txt @@ -29,3 +29,12 @@ pair 162:177 pair 179:199 ^------------------^ compressor_pages 144 msg : hibernate_teardown: wired_pages 518290, free_pages 5699523, active_pages 40010, inactive_pages 0, speculative_pages 0, cleaned_pages 0, compressor_pages 144 format : hibernate_teardown: wired_pages #, free_pages #, active_pages #, inactive_pages #, speculative_pages #, cleaned_pages #, compressor_pages # +{ + "wired_pages": 518290, + "free_pages": 5699523, + "active_pages": 40010, + "inactive_pages": 0, + "speculative_pages": 0, + "cleaned_pages": 0, + "compressor_pages": 144 +} diff --git a/test/log-samples/sample-500c9e492e04f5f58862c8086ca301de0dd976ce.txt b/test/log-samples/sample-500c9e492e04f5f58862c8086ca301de0dd976ce.txt index 79570264..abe71806 100644 --- a/test/log-samples/sample-500c9e492e04f5f58862c8086ca301de0dd976ce.txt +++ b/test/log-samples/sample-500c9e492e04f5f58862c8086ca301de0dd976ce.txt @@ -7,3 +7,7 @@ pair 74:83 pair 88:92 ^--^ mDNS msg : New relevant interface eth0.IPv4 for mDNS. format : New relevant interface # for # +{ + "col_0": "eth0.IPv4", + "col_1": "mDNS" +} diff --git a/test/log-samples/sample-55ac97afae4b0650ccb62e2dbc8d89bb.txt b/test/log-samples/sample-55ac97afae4b0650ccb62e2dbc8d89bb.txt index 7430d6b6..5e3e6a09 100644 --- a/test/log-samples/sample-55ac97afae4b0650ccb62e2dbc8d89bb.txt +++ b/test/log-samples/sample-55ac97afae4b0650ccb62e2dbc8d89bb.txt @@ -9,3 +9,7 @@ pair 51:90 ^--------------- pair 92:106 ^------------^ mux-device:509 msg : Entered:__thr_AMMuxedDeviceDisconnected, mux-device:509 format : Entered:#, mux-device:# +{ + "Entered": "__thr_AMMuxedDeviceDisconnected", + "mux-device": 509 +} diff --git a/test/log-samples/sample-6049d4309f26eefb1a3406d937a9ba8a0df592a7.txt b/test/log-samples/sample-6049d4309f26eefb1a3406d937a9ba8a0df592a7.txt index 701e2760..c22238d9 100644 --- a/test/log-samples/sample-6049d4309f26eefb1a3406d937a9ba8a0df592a7.txt +++ b/test/log-samples/sample-6049d4309f26eefb1a3406d937a9ba8a0df592a7.txt @@ -7,3 +7,7 @@ pair 82:92 pair 96:100 ^--^ eth0 msg : Withdrawing address record for 10.1.10.62 on eth0. format : Withdrawing address record for # on # +{ + "col_0": "10.1.10.62", + "col_1": "eth0" +} diff --git a/test/log-samples/sample-62315d884afdc4155b35f905415c74bfcfd39fc2.txt b/test/log-samples/sample-62315d884afdc4155b35f905415c74bfcfd39fc2.txt index beb0c9a8..02739e96 100644 --- a/test/log-samples/sample-62315d884afdc4155b35f905415c74bfcfd39fc2.txt +++ b/test/log-samples/sample-62315d884afdc4155b35f905415c74bfcfd39fc2.txt @@ -10,3 +10,8 @@ ipv4 116:126 pair 116:126 ^--------^ 10.1.10.62 msg : Joining mDNS multicast group on interface eth0.IPv4 with address 10.1.10.62. format : Joining # multicast group on interface # with address # +{ + "col_0": "mDNS", + "col_1": "eth0.IPv4", + "col_2": "10.1.10.62" +} diff --git a/test/log-samples/sample-70c906b3c1a1cf03f15bde92ee78edfa6f9b7960.txt b/test/log-samples/sample-70c906b3c1a1cf03f15bde92ee78edfa6f9b7960.txt index bbab0333..a4a6fba9 100644 --- a/test/log-samples/sample-70c906b3c1a1cf03f15bde92ee78edfa6f9b7960.txt +++ b/test/log-samples/sample-70c906b3c1a1cf03f15bde92ee78edfa6f9b7960.txt @@ -26,3 +26,10 @@ word 125:127 pair 95:127 ^------------------------------^ COMMAND=/usr/bin/env VAR1=foo ls msg : stack : TTY=ttys002 ; PWD=/ ; USER=root ; COMMAND=/usr/bin/env VAR1=foo ls format : # : TTY=# ; PWD=# ; USER=# ; COMMAND=# +{ + "col_0": "stack", + "TTY": "ttys002", + "PWD": "/", + "USER": "root", + "COMMAND": "/usr/bin/env VAR1=foo ls" +} diff --git a/test/log-samples/sample-9cf7fbb3546c676c686fac0ed096d026f46c875f.txt b/test/log-samples/sample-9cf7fbb3546c676c686fac0ed096d026f46c875f.txt index 6a7972f9..d1bf5d45 100644 --- a/test/log-samples/sample-9cf7fbb3546c676c686fac0ed096d026f46c875f.txt +++ b/test/log-samples/sample-9cf7fbb3546c676c686fac0ed096d026f46c875f.txt @@ -7,3 +7,7 @@ quot 169:199 pair 169:199 ^----------------------------^ com.json.components.JSONReader msg : Doing prepare for resource name "Json_Reader", component "com.json.components.JSONReader" format : Doing prepare for resource name #, component # +{ + "col_0": "Json_Reader", + "col_1": "com.json.components.JSONReader" +} diff --git a/test/log-samples/sample-a74570613c082c7fe283672031e18e54e8887ffb.txt b/test/log-samples/sample-a74570613c082c7fe283672031e18e54e8887ffb.txt index 619a619e..bc72196d 100644 --- a/test/log-samples/sample-a74570613c082c7fe283672031e18e54e8887ffb.txt +++ b/test/log-samples/sample-a74570613c082c7fe283672031e18e54e8887ffb.txt @@ -7,3 +7,7 @@ pair 61:70 ^----- pair 94:98 ^--^ mDNS msg : Interface eth0.IPv4 no longer relevant for mDNS. format : Interface # no longer relevant for # +{ + "col_0": "eth0.IPv4", + "col_1": "mDNS" +} diff --git a/test/log-samples/sample-aca2878a2e50779c6697c0747ab1f60e4b368dcb.txt b/test/log-samples/sample-aca2878a2e50779c6697c0747ab1f60e4b368dcb.txt index c71015fb..46107ac1 100644 --- a/test/log-samples/sample-aca2878a2e50779c6697c0747ab1f60e4b368dcb.txt +++ b/test/log-samples/sample-aca2878a2e50779c6697c0747ab1f60e4b368dcb.txt @@ -9,3 +9,7 @@ path 75:91 pair 67:91 ^----------------------^ updated /etc/resolv.conf msg : /sbin/dhclient-script : updated /etc/resolv.conf format : # : updated # +{ + "col_0": "/sbin/dhclient-script", + "updated": "/etc/resolv.conf" +} diff --git a/test/log-samples/sample-ad31f12d2adabd07e3ddda3ad5b0dbf6b49c4c99.txt b/test/log-samples/sample-ad31f12d2adabd07e3ddda3ad5b0dbf6b49c4c99.txt index 8338f412..6f16014a 100644 --- a/test/log-samples/sample-ad31f12d2adabd07e3ddda3ad5b0dbf6b49c4c99.txt +++ b/test/log-samples/sample-ad31f12d2adabd07e3ddda3ad5b0dbf6b49c4c99.txt @@ -15,3 +15,7 @@ wspc 86:87 pair 50:105 ^-----------------------------------------------------^ VNetUserIf_Create: created userIf at 0xffffff802644f400 msg : vmnet: VNetUserIf_Create: created userIf at 0xffffff802644f400. format : vmnet:# VNetUserIf_Create: # +{ + "vmnet": "", + "VNetUserIf_Create": "created userIf at 0xffffff802644f400" +} diff --git a/test/log-samples/sample-bc6f6cf689fa5455616b4d9fbe121a48d3c9de59.txt b/test/log-samples/sample-bc6f6cf689fa5455616b4d9fbe121a48d3c9de59.txt index cd134257..0bbe08e4 100644 --- a/test/log-samples/sample-bc6f6cf689fa5455616b4d9fbe121a48d3c9de59.txt +++ b/test/log-samples/sample-bc6f6cf689fa5455616b4d9fbe121a48d3c9de59.txt @@ -14,3 +14,12 @@ pair 65:79 ^- pair 65:79 ^------------^ xid=0x4e17f141 msg : DHCPNAK from 10.1.10.1 (xid=0x4e17f141) format : # from # (#) +{ + "col_0": "DHCPNAK", + "col_1": "10.1.10.1", + "col_2": [ + { + "xid": "0x4e17f141" + } + ] +} diff --git a/test/log-samples/sample-c15acd32844669d23d0cbc88ec548129ed2c592e.txt b/test/log-samples/sample-c15acd32844669d23d0cbc88ec548129ed2c592e.txt index 99b3ebfa..3a7d5bcc 100644 --- a/test/log-samples/sample-c15acd32844669d23d0cbc88ec548129ed2c592e.txt +++ b/test/log-samples/sample-c15acd32844669d23d0cbc88ec548129ed2c592e.txt @@ -67,3 +67,21 @@ pair 246:258 pair 259:265 ^----^ URGP=0 msg : [31809412.513897] [UFW BLOCK] IN=eth0 OUT= MAC=40:40:2e:9a:ad:92:c4:71:fe:f1:b9:7f:08:00 SRC=69.60.116.202 DST=173.203.237.224 LEN=44 TOS=0x00 PREC=0x00 TTL=29 ID=15852 PROTO=TCP SPT=43998 DPT=3389 WINDOW=3072 RES=0x00 SYN URGP=0 format : [31809412.513897] [UFW BLOCK] IN=# OUT=# MAC=# SRC=# DST=# LEN=# TOS=# PREC=# TTL=# ID=# PROTO=# SPT=# DPT=# WINDOW=# RES=# URGP=# +{ + "31809412.513897] [UFW BLOCK] IN": "eth0", + "OUT": "", + "MAC": "40:40:2e:9a:ad:92:c4:71:fe:f1:b9:7f:08:00", + "SRC": "69.60.116.202", + "DST": "173.203.237.224", + "LEN": 44, + "TOS": "0x00", + "PREC": "0x00", + "TTL": 29, + "ID": 15852, + "PROTO": "TCP", + "SPT": 43998, + "DPT": 3389, + "WINDOW": 3072, + "RES": "0x00 SYN", + "URGP": 0 +} diff --git a/test/log-samples/sample-c23f22c1b932b904203e018f78dead95fb89b15d.txt b/test/log-samples/sample-c23f22c1b932b904203e018f78dead95fb89b15d.txt index 02c24491..51a66742 100644 --- a/test/log-samples/sample-c23f22c1b932b904203e018f78dead95fb89b15d.txt +++ b/test/log-samples/sample-c23f22c1b932b904203e018f78dead95fb89b15d.txt @@ -23,3 +23,15 @@ pair 101:114 pair 101:114 ^-----------^ xid=0xd16b79d msg : DHCPDISCOVER on eth0 to 255.255.255.255 port 67 interval 5 (xid=0xd16b79d) format : # on # to # port # interval # (#) +{ + "col_0": "DHCPDISCOVER", + "col_1": "eth0", + "col_2": "255.255.255.255", + "col_3": 67, + "col_4": 5, + "col_5": [ + { + "xid": "0xd16b79d" + } + ] +} diff --git a/test/log-samples/sample-d0d6b3fc6766caac5ac3fac4a3754ceaab785eb8.txt b/test/log-samples/sample-d0d6b3fc6766caac5ac3fac4a3754ceaab785eb8.txt index 1d8c5844..95255e9f 100644 --- a/test/log-samples/sample-d0d6b3fc6766caac5ac3fac4a3754ceaab785eb8.txt +++ b/test/log-samples/sample-d0d6b3fc6766caac5ac3fac4a3754ceaab785eb8.txt @@ -20,3 +20,14 @@ pair 83:96 pair 83:96 ^-----------^ xid=0xd16b79d msg : DHCPREQUEST on eth0 to 10.1.10.1 port 67 (xid=0xd16b79d) format : # on # to # port # (#) +{ + "col_0": "DHCPREQUEST", + "col_1": "eth0", + "col_2": "10.1.10.1", + "col_3": 67, + "col_4": [ + { + "xid": "0xd16b79d" + } + ] +} diff --git a/test/log-samples/sample-d4a0aedc8350f64b22403eeef4eca71fbf749d2b.txt b/test/log-samples/sample-d4a0aedc8350f64b22403eeef4eca71fbf749d2b.txt index 70ef8e0f..a82a9e21 100644 --- a/test/log-samples/sample-d4a0aedc8350f64b22403eeef4eca71fbf749d2b.txt +++ b/test/log-samples/sample-d4a0aedc8350f64b22403eeef4eca71fbf749d2b.txt @@ -4,3 +4,6 @@ ipv6 85:110 pair 85:110 ^-----------------------^ fe80::22c9:d0ff:fe15:1b7c msg : Invalid response packet from host fe80::22c9:d0ff:fe15:1b7c. format : Invalid response packet from host # +{ + "col_0": "fe80::22c9:d0ff:fe15:1b7c" +} diff --git a/test/log-samples/sample-d714b5e8cd354321f376ed1c0a70ec9a2f58076d.txt b/test/log-samples/sample-d714b5e8cd354321f376ed1c0a70ec9a2f58076d.txt index e8ab3391..4bc4d321 100644 --- a/test/log-samples/sample-d714b5e8cd354321f376ed1c0a70ec9a2f58076d.txt +++ b/test/log-samples/sample-d714b5e8cd354321f376ed1c0a70ec9a2f58076d.txt @@ -4,3 +4,6 @@ ipv4 85:95 pair 85:95 ^--------^ 10.1.10.10 msg : Invalid response packet from host 10.1.10.10. format : Invalid response packet from host # +{ + "col_0": "10.1.10.10" +} diff --git a/test/log-samples/sample-dd7d406352ec6a11d966b6f015a9482b060f2b29.txt b/test/log-samples/sample-dd7d406352ec6a11d966b6f015a9482b060f2b29.txt index 973e5d19..bcf39085 100644 --- a/test/log-samples/sample-dd7d406352ec6a11d966b6f015a9482b060f2b29.txt +++ b/test/log-samples/sample-dd7d406352ec6a11d966b6f015a9482b060f2b29.txt @@ -2,14 +2,22 @@ key 29:29 ^ key 29:40 ^---------^ com.twisted pair 29:40 ^---------^ com.twisted - key 59:59 ^ key 59:59 ^ sym 59:82 ^---------------------^ twisted.web.server.Site -pair 59:82 ^---------------------^ twisted.web.server.Site - key 95:95 ^ +wspc 82:83 ^ +word 83:91 ^------^ instance +wspc 91:92 ^ +word 92:94 ^^ at +wspc 94:95 ^ hex 95:104 ^-------^ 0x1de9290 -pair 95:104 ^-------^ 0x1de9290 + val 59:104 ^-------------------------------------------^ twisted.web.server.Site instance at 0x1de9290 grp 59:104 ^-------------------------------------------^ twisted.web.server.Site instance at 0x1de9290 pair 59:104 ^-------------------------------------------^ twisted.web.server.Site instance at 0x1de9290 msg ::com.twisted:Starting factory format ::#:Starting factory <#> +{ + "col_0": "com.twisted", + "col_1": [ + "twisted.web.server.Site instance at 0x1de9290" + ] +} diff --git a/test/log-samples/sample-e779d1771e34f5203ae73e85802e78002be63db6.txt b/test/log-samples/sample-e779d1771e34f5203ae73e85802e78002be63db6.txt index 2698fede..51c8e1b3 100644 --- a/test/log-samples/sample-e779d1771e34f5203ae73e85802e78002be63db6.txt +++ b/test/log-samples/sample-e779d1771e34f5203ae73e85802e78002be63db6.txt @@ -14,3 +14,12 @@ pair 65:78 ^- pair 65:78 ^-----------^ xid=0xd16b79d msg : DHCPACK from 10.1.10.1 (xid=0xd16b79d) format : # from # (#) +{ + "col_0": "DHCPACK", + "col_1": "10.1.10.1", + "col_2": [ + { + "xid": "0xd16b79d" + } + ] +} diff --git a/test/log-samples/sample-f5afbee90a8c054061c4e9ffe673293cce7761de.txt b/test/log-samples/sample-f5afbee90a8c054061c4e9ffe673293cce7761de.txt index c6e923cc..29005598 100644 --- a/test/log-samples/sample-f5afbee90a8c054061c4e9ffe673293cce7761de.txt +++ b/test/log-samples/sample-f5afbee90a8c054061c4e9ffe673293cce7761de.txt @@ -7,3 +7,7 @@ ipv4 56:65 ^-------^ pair 56:65 ^-------^ 10.1.10.1 msg : DHCPOFFER from 10.1.10.1 format : # from # +{ + "col_0": "DHCPOFFER", + "col_1": "10.1.10.1" +} diff --git a/test/log-samples/sample-fc8923633e57bacd641d80dde3ff878212230552.txt b/test/log-samples/sample-fc8923633e57bacd641d80dde3ff878212230552.txt index e081f67d..42593c3f 100644 --- a/test/log-samples/sample-fc8923633e57bacd641d80dde3ff878212230552.txt +++ b/test/log-samples/sample-fc8923633e57bacd641d80dde3ff878212230552.txt @@ -7,3 +7,7 @@ pair 86:96 pair 100:104 ^--^ eth0 msg : Registering new address record for 10.1.10.62 on eth0. format : Registering new address record for # on # +{ + "col_0": "10.1.10.62", + "col_1": "eth0" +} diff --git a/test/parser_debugger.py b/test/parser_debugger.py index 65416cf1..476a2de5 100755 --- a/test/parser_debugger.py +++ b/test/parser_debugger.py @@ -41,6 +41,8 @@ SRC_DIR = os.path.join(ROOT_DIR, "src") addr_to_name = {} name_to_addr = {} element_lists = collections.defaultdict(list) +list_depth = {} +list_format = {} breakpoints = set() def completer(text, state): @@ -74,13 +76,20 @@ def getstr(capture): def printlist(name_or_addr): if name_or_addr in name_to_addr: - print "(%s) %s" % (name_or_addr, element_lists[name_to_addr[name_or_addr]]) + addr = name_to_addr[name_or_addr] + print "% 3d (%s:%s) %s" % (list_depth.get(addr, -1), name_or_addr, addr, element_lists[addr]) elif name_or_addr in element_lists: - print "(%s) %s" % (addr_to_name.get(name_or_addr, name_or_addr), - element_lists[name_or_addr]) + addr = name_or_addr + print "% 3d (%s:%s) %s" % (list_depth.get(name_or_addr, -1), + addr_to_name.get(name_or_addr, name_or_addr), + name_or_addr, + element_lists[name_or_addr]) else: print "error: unknown list --", name_or_addr + if addr in list_format: + print " format -- appender(%s) term(%s) qual(%s) sep(%s) prefix_term(%s)" % tuple(list_format[addr]) + def handleop(fields): addr = fields[0] loc = fields[1].split(':') @@ -95,8 +104,14 @@ def handleop(fields): if method_name == 'element_list_t': addr_to_name[addr] = method_args[0] name_to_addr[method_args[0]] = addr + list_depth[addr] = int(method_args[1]) elif method_name == '~element_list_t': - pass + del element_lists[addr] + elif method_name == 'format': + list_depth[addr] = int(method_args[0]) + list_format[addr] = method_args[1:] + elif method_name == 'consumed': + list_depth[addr] = -1 elif method_name == 'push_back': el.append((method_args[0], getstr(method_args[1]))) elif method_name == 'pop_front': @@ -113,6 +128,10 @@ def handleop(fields): sub_list = other[start:end] del other[start:end] el[pos:pos] = sub_list + elif method_name == 'swap': + other = element_lists[method_args[0]] + element_lists[method_args[0]] = el + element_lists[addr] = other elif method_name == 'point': breakpoints.add(method_args[0]) else: @@ -122,6 +141,7 @@ def playupto(length): addr_to_name.clear() name_to_addr.clear() element_lists.clear() + list_depth.clear() for index in range(length): handleop(ops[index]) @@ -147,6 +167,13 @@ def find_next_point(start, name): return start + 1 return orig_start + 1 +def printall(): + print input_line + sorted_lists = [(list_depth.get(addr, -1), addr) for addr in element_lists] + sorted_lists.sort() + for _depth, addr in sorted_lists: + printlist(addr) + index = len(ops) last_cmd = [''] watch_list = set() @@ -209,9 +236,7 @@ while True: if len(cmd) > 1: printlist(cmd[1]) else: - print input_line - for addr in element_lists: - printlist(addr) + printall() elif cmd[0] == 'w': watch_list.add(cmd[1]) elif cmd[0] == 'u': @@ -220,4 +245,6 @@ while True: else: print "error: unknown command --", cmd + printall() + last_cmd = cmd diff --git a/test/update_parser_output.sh b/test/update_parser_output.sh new file mode 100755 index 00000000..334892bb --- /dev/null +++ b/test/update_parser_output.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +test_dir=`dirname $0` + +for fn in ${test_dir}/datafile_simple.*; do + echo "Checking $fn" + ./drive_data_scanner -p $fn +done + +for fn in ${test_dir}/log-samples/sample-*; do + echo "Checking $fn" + ./drive_data_scanner -p -l $fn +done