From a813482a543a48c5949cefc519477b86d5dcfc94 Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Fri, 2 Sep 2022 23:06:03 -0700 Subject: [PATCH] [log_formats] JSON number time to local time zone Fixes #1038 --- src/log_format.cc | 18 ++++++++++++++++++ test/expected/expected.am | 12 ++++++++---- ...315a3d6124c14cbe3c474b6dbf4cc8720a9859f.err | 0 ...315a3d6124c14cbe3c474b6dbf4cc8720a9859f.out | 3 +++ ...0ec34389274affb70a5a76ba4789d51fd60f602.err | 0 ...0ec34389274affb70a5a76ba4789d51fd60f602.out | 4 ++++ ...7362cffc8335c2fe6b6527315de59bd6f5dcc7f.err | 0 ...7362cffc8335c2fe6b6527315de59bd6f5dcc7f.out | 3 +++ ...fff27a651650a04d93de9a06ab5480e94ce3a79.err | 0 ...fff27a651650a04d93de9a06ab5480e94ce3a79.out | 4 ++++ test/formats/jsontest3/format.json | 11 ++++++----- test/logfile_json3.json | 2 +- test/test_json_format.sh | 15 +++++++++++++-- 13 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 test/expected/test_json_format.sh_4315a3d6124c14cbe3c474b6dbf4cc8720a9859f.err create mode 100644 test/expected/test_json_format.sh_4315a3d6124c14cbe3c474b6dbf4cc8720a9859f.out create mode 100644 test/expected/test_json_format.sh_d0ec34389274affb70a5a76ba4789d51fd60f602.err create mode 100644 test/expected/test_json_format.sh_d0ec34389274affb70a5a76ba4789d51fd60f602.out create mode 100644 test/expected/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.err create mode 100644 test/expected/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.out create mode 100644 test/expected/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.err create mode 100644 test/expected/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.out diff --git a/src/log_format.cc b/src/log_format.cc index 43191660..31de1bd6 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -479,6 +479,15 @@ read_json_int(yajlpp_parse_context* ypc, long long val) tv.tv_sec = val / divisor; tv.tv_usec = (val % divisor) * (1000000.0 / divisor); + if (jlu->jlu_format->lf_date_time.dts_local_time) { + struct tm ltm; + localtime_r(&tv.tv_sec, <m); +#ifdef HAVE_STRUCT_TM_TM_ZONE + ltm.tm_zone = nullptr; +#endif + ltm.tm_isdst = 0; + tv.tv_sec = tm2sec(<m); + } jlu->jlu_base_line->set_time(tv); } else if (jlu->jlu_format->elf_level_field == field_name) { if (jlu->jlu_format->elf_level_pairs.empty()) { @@ -524,6 +533,15 @@ read_json_double(yajlpp_parse_context* ypc, double val) tv.tv_sec = val / divisor; tv.tv_usec = fmod(val, divisor) * (1000000.0 / divisor); + if (jlu->jlu_format->lf_date_time.dts_local_time) { + struct tm ltm; + localtime_r(&tv.tv_sec, <m); +#ifdef HAVE_STRUCT_TM_TM_ZONE + ltm.tm_zone = nullptr; +#endif + ltm.tm_isdst = 0; + tv.tv_sec = tm2sec(<m); + } jlu->jlu_base_line->set_time(tv); } diff --git a/test/expected/expected.am b/test/expected/expected.am index d4749fcf..b66d41ae 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -248,6 +248,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_json_format.sh_1bb0fd243e916546aea22029245ac590dae17a86.out \ $(srcdir)/%reldir%/test_json_format.sh_40223ac4742883f883ccc61044bfffd6e102cca6.err \ $(srcdir)/%reldir%/test_json_format.sh_40223ac4742883f883ccc61044bfffd6e102cca6.out \ + $(srcdir)/%reldir%/test_json_format.sh_4315a3d6124c14cbe3c474b6dbf4cc8720a9859f.err \ + $(srcdir)/%reldir%/test_json_format.sh_4315a3d6124c14cbe3c474b6dbf4cc8720a9859f.out \ $(srcdir)/%reldir%/test_json_format.sh_469f005b0708d629bc95f0c48a5e390f440c1fef.err \ $(srcdir)/%reldir%/test_json_format.sh_469f005b0708d629bc95f0c48a5e390f440c1fef.out \ $(srcdir)/%reldir%/test_json_format.sh_6fbe20faa161ab9fa77df7568fff84bf3e47e920.err \ @@ -264,18 +266,20 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_json_format.sh_84a71e94dc34661a70bb9015b67ba00e93e9cfb5.out \ $(srcdir)/%reldir%/test_json_format.sh_85d03b1b41a7f819af135d2521a8f2c59418e907.err \ $(srcdir)/%reldir%/test_json_format.sh_85d03b1b41a7f819af135d2521a8f2c59418e907.out \ - $(srcdir)/%reldir%/test_json_format.sh_90a037c7d9d70ac4ca97158271ea242787313377.err \ - $(srcdir)/%reldir%/test_json_format.sh_90a037c7d9d70ac4ca97158271ea242787313377.out \ $(srcdir)/%reldir%/test_json_format.sh_952297a90e312d2184fe3e4df795ddc731b096c9.err \ $(srcdir)/%reldir%/test_json_format.sh_952297a90e312d2184fe3e4df795ddc731b096c9.out \ $(srcdir)/%reldir%/test_json_format.sh_989e52d167582648b73c5d025cc0e814c642b3c8.err \ $(srcdir)/%reldir%/test_json_format.sh_989e52d167582648b73c5d025cc0e814c642b3c8.out \ $(srcdir)/%reldir%/test_json_format.sh_a06b3cdd46b387e72d6faa4cce648b8b11ae870b.err \ $(srcdir)/%reldir%/test_json_format.sh_a06b3cdd46b387e72d6faa4cce648b8b11ae870b.out \ - $(srcdir)/%reldir%/test_json_format.sh_a6be47f1311ed92feaf303142fcb103deb80f456.err \ - $(srcdir)/%reldir%/test_json_format.sh_a6be47f1311ed92feaf303142fcb103deb80f456.out \ $(srcdir)/%reldir%/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.err \ $(srcdir)/%reldir%/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.out \ + $(srcdir)/%reldir%/test_json_format.sh_d0ec34389274affb70a5a76ba4789d51fd60f602.err \ + $(srcdir)/%reldir%/test_json_format.sh_d0ec34389274affb70a5a76ba4789d51fd60f602.out \ + $(srcdir)/%reldir%/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.err \ + $(srcdir)/%reldir%/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.out \ + $(srcdir)/%reldir%/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.err \ + $(srcdir)/%reldir%/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.out \ $(srcdir)/%reldir%/test_json_format.sh_fe19b7ebd349cd689b3f5c22618eab5ce995e68e.err \ $(srcdir)/%reldir%/test_json_format.sh_fe19b7ebd349cd689b3f5c22618eab5ce995e68e.out \ $(srcdir)/%reldir%/test_logfile.sh_08d731a04c877a34819b35de185e30a74c9fd497.err \ diff --git a/test/expected/test_json_format.sh_4315a3d6124c14cbe3c474b6dbf4cc8720a9859f.err b/test/expected/test_json_format.sh_4315a3d6124c14cbe3c474b6dbf4cc8720a9859f.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_json_format.sh_4315a3d6124c14cbe3c474b6dbf4cc8720a9859f.out b/test/expected/test_json_format.sh_4315a3d6124c14cbe3c474b6dbf4cc8720a9859f.out new file mode 100644 index 00000000..30ddacc5 --- /dev/null +++ b/test/expected/test_json_format.sh_4315a3d6124c14cbe3c474b6dbf4cc8720a9859f.out @@ -0,0 +1,3 @@ +2017-03-24T20:06:26.240 1.1.1.1 GET 200 /example/uri/5 +2017-03-24T20:12:47.764 1.1.1.1 GET 500 /example/uri/5 +2017-03-24T20:15:31.694 1.1.1.1 GET 400 /example/uri/5 diff --git a/test/expected/test_json_format.sh_d0ec34389274affb70a5a76ba4789d51fd60f602.err b/test/expected/test_json_format.sh_d0ec34389274affb70a5a76ba4789d51fd60f602.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_json_format.sh_d0ec34389274affb70a5a76ba4789d51fd60f602.out b/test/expected/test_json_format.sh_d0ec34389274affb70a5a76ba4789d51fd60f602.out new file mode 100644 index 00000000..b3568987 --- /dev/null +++ b/test/expected/test_json_format.sh_d0ec34389274affb70a5a76ba4789d51fd60f602.out @@ -0,0 +1,4 @@ +log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,client_ip,request/method,request/uri,request/size,response/status,details1,details2,details3 +0,,2017-03-24 20:06:26.240,0,info,0,,,,1.1.1.1,GET,/example/uri/5,166,200,,, +1,,2017-03-24 20:12:47.764,381524,critical,0,,,,1.1.1.1,GET,/example/uri/5,166,500,,, +2,,2017-03-24 20:15:31.694,163930,warning,0,,,,1.1.1.1,GET,/example/uri/5,166,400,"{""foo"": ""bar""}","{""foo"": ""bar""}","{""foo"": ""bar""}" diff --git a/test/expected/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.err b/test/expected/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.out b/test/expected/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.out new file mode 100644 index 00000000..951a3890 --- /dev/null +++ b/test/expected/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.out @@ -0,0 +1,3 @@ +2017-03-24T16:06:26.240 1.1.1.1 GET 200 /example/uri/5 +2017-03-24T16:12:47.764 1.1.1.1 GET 500 /example/uri/5 +2017-03-24T16:15:31.694 1.1.1.1 GET 400 /example/uri/5 diff --git a/test/expected/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.err b/test/expected/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.out b/test/expected/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.out new file mode 100644 index 00000000..c3d0110f --- /dev/null +++ b/test/expected/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.out @@ -0,0 +1,4 @@ +log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,log_comment,log_tags,log_filters,client_ip,request/method,request/uri,request/size,response/status,details1,details2,details3 +0,,2017-03-24 16:06:26.240,0,info,0,,,,1.1.1.1,GET,/example/uri/5,166,200,,, +1,,2017-03-24 16:12:47.764,381524,critical,0,,,,1.1.1.1,GET,/example/uri/5,166,500,,, +2,,2017-03-24 16:15:31.694,163930,warning,0,,,,1.1.1.1,GET,/example/uri/5,166,400,"{""foo"": ""bar""}","{""foo"": ""bar""}","{""foo"": ""bar""}" diff --git a/test/formats/jsontest3/format.json b/test/formats/jsontest3/format.json index efa984c6..519cc407 100644 --- a/test/formats/jsontest3/format.json +++ b/test/formats/jsontest3/format.json @@ -6,6 +6,7 @@ "file-pattern": "logfile_json3\\.json", "json": true, "hide-extra": true, + "convert-to-local-time": true, "line-format": [ { "field": "__timestamp__" @@ -52,18 +53,18 @@ "response/status": { "kind": "string" }, - "details1" : { + "details1": { "hidden": true }, - "details2" : { + "details2": { "hidden": true }, - "details3" : { + "details3": { "hidden": true } }, - "timestamp-field" : "started_at", - "timestamp-divisor" : 1000, + "timestamp-field": "started_at", + "timestamp-divisor": 1000, "level-field": "response/status", "level": { "info": "2\\d+", diff --git a/test/logfile_json3.json b/test/logfile_json3.json index 03a09a72..5d232791 100644 --- a/test/logfile_json3.json +++ b/test/logfile_json3.json @@ -1,3 +1,3 @@ { "started_at": 1490385986240, "response": { "status": 200, "size": "443", "headers": { "server": "nginx\/1.11.10", "content-type": "application\/json", "connection": "close", "cache-control": "max-age=0, must-revalidate, no-cache, no-store, private" } }, "request": { "method": "GET", "uri": "\/example\/uri\/5", "size": "166", "querystring": {}, "headers": { "host": "example.com" } }, "client_ip": "1.1.1.1" } -{ "started_at": 1490386367764, "response": { "status": 500, "size": "443", "headers": { "server": "nginx\/1.11.10", "content-type": "application\/json", "connection": "close", "cache-control": "max-age=0, must-revalidate, no-cache, no-store, private" } }, "request": { "method": "GET", "uri": "\/example\/uri\/5", "size": "166", "querystring": {}, "headers": { "host": "example.com" } }, "client_ip": "1.1.1.1" } +{ "started_at": 1490386367764.0, "response": { "status": 500, "size": "443", "headers": { "server": "nginx\/1.11.10", "content-type": "application\/json", "connection": "close", "cache-control": "max-age=0, must-revalidate, no-cache, no-store, private" } }, "request": { "method": "GET", "uri": "\/example\/uri\/5", "size": "166", "querystring": {}, "headers": { "host": "example.com" } }, "client_ip": "1.1.1.1" } { "started_at": 1490386531694, "response": { "status": 400, "size": "443", "headers": { "server": "nginx\/1.11.10", "content-type": "application\/json", "connection": "close", "cache-control": "max-age=0, must-revalidate, no-cache, no-store, private" } }, "request": { "method": "GET", "uri": "\/example\/uri\/5", "size": "166", "querystring": {}, "headers": { "host": "example.com" } }, "client_ip": "1.1.1.1", "details1": {"foo": "bar"}, "details2": {"foo": "bar"}, "details3": {"foo": "bar"} } diff --git a/test/test_json_format.sh b/test/test_json_format.sh index 20e6f9dc..06051832 100644 --- a/test/test_json_format.sh +++ b/test/test_json_format.sh @@ -98,12 +98,23 @@ run_cap_test ${lnav_test} -n \ ${test_dir}/logfile_nested_json.json # json log3 format is not working" -run_cap_test ${lnav_test} -n \ +run_cap_test env TZ=UTC ${lnav_test} -n \ -I ${test_dir} \ ${test_dir}/logfile_json3.json # json log3 format is not working" -run_cap_test ${lnav_test} -n \ +run_cap_test env TZ=UTC ${lnav_test} -n \ + -I ${test_dir} \ + -c ';select * from json_log3' \ + -c ':write-csv-to -' \ + ${test_dir}/logfile_json3.json + +run_cap_test env TZ=America/New_York ${lnav_test} -n \ + -I ${test_dir} \ + ${test_dir}/logfile_json3.json + +# json log3 format is not working" +run_cap_test env TZ=America/New_York ${lnav_test} -n \ -I ${test_dir} \ -c ';select * from json_log3' \ -c ':write-csv-to -' \