Release/1.2.0 (#195)

* Resolve error with failure to upgrade example.conf * Automatically resolve correct time zone for near real time streaming sources * Support specific time zones by host/ip/subnet for sources with high latency
CSVD · Nov 18, 2019 · 8660924 · 8660924
1 parent eef0d59
commit 8660924
Show file tree

Hide file tree

Showing 11 changed files with 102 additions and 8 deletions.
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
@@ -16,7 +16,7 @@ log {
 
     parser {
         kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}"));
-        date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}));
+        date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
     };
 
     rewrite {

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
@@ -15,6 +15,7 @@ log {
 
     rewrite {
         set("cisco_ios", value("fields.sc4s_vendor_product"));
+        guess-time-zone();
         r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"))
     };
     parser {

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl
@@ -15,6 +15,7 @@ log {
 
     rewrite {
         set("cisco_nxos", value("fields.sc4s_vendor_product"));
+        guess-time-zone();
         r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))
     };
 

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl
@@ -16,7 +16,7 @@ log {
 
     parser {
         kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}"));
-        date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}));
+        date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
     };
 
     rewrite {

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
@@ -38,7 +38,8 @@ log {
         date-parser(
                 format("%Y/%m/%d %H:%M:%S")
                 template("${.pan.GeneratedTime}")
-                time-zone("Universal")
+                time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})
+                flags(guess-timezone)
         );
     };
 

diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf
@@ -39,4 +39,13 @@ filter f_proofpoint_pps_sendmail {
 filter f_ubiquiti_unifi_fw {
         host("usg-*" type(glob)) or
         netmask(192.168.6.0/24)
+};
+filter f_tzfixhst {
+        host("tzfhst-*" type(glob)) or
+        netmask(192.168.6.0/24)
+};
+
+filter f_tzfixny {
+        host("tzfny-*" type(glob)) or
+        netmask(192.168.6.0/24)
 };
diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv
@@ -7,4 +7,6 @@ f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen"
 f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os"
 f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail"
 f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter"
-f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw"
+f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw"
+f_tzfixhst,sc4s_time_zone,"Pacific/Honolulu"
+f_tzfixny,sc4s_time_zone,"America/New_York"
diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t
@@ -104,7 +104,7 @@ source s_{{ .port_id}} {
             rewrite(set_rfc5424_epochtime);
         } else {
             parser {
-                syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message));
+                syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone));
             };
             rewrite(set_rfc3164);
         };
@@ -114,6 +114,16 @@ source s_{{ .port_id}} {
         parser {
             vendor_product_by_source();
         };
+
+        if {
+            filter { match("." value("fields.sc4s_time_zone") ) };
+            rewrite {
+                fix-time-zone("${fields.sc4s_time_zone}");
+                unset(value("fields.sc4s_time_zone"));
+            };
+        };
+
+
    };
 };
 {{- end }}

diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh
@@ -12,7 +12,7 @@ gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --
 mkdir -p /opt/syslog-ng/etc/conf.d/local/context/
 mkdir -p /opt/syslog-ng/etc/conf.d/local/config/
 cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/
-cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/
+cp --verbose -R /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/
 
 echo syslog-ng starting
 exec /opt/syslog-ng/sbin/syslog-ng $@
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -11,3 +11,4 @@ jinja2
 jinja2-time
 splunk-sdk
 flake8
+pytz
diff --git a/tests/test_common.py b/tests/test_common.py
@@ -3,9 +3,11 @@
 # Use of this source code is governed by a BSD-2-clause-style
 # license that can be found in the LICENSE-BSD2 file or at
 # https://opensource.org/licenses/BSD-2-Clause
+import datetime
 import random
+import pytz
 
-from jinja2 import Environment
+from jinja2 import Environment, environment
 
 from .sendmessage import *
 from .splunkutils import *
@@ -79,4 +81,71 @@ def test_metrics(record_property, setup_wordlist, setup_splunk):
 
     record_property("resultCount", resultCount)
 
-    assert resultCount == 1
+    assert resultCount == 1
+
+def test_tz_guess(record_property, setup_wordlist, setup_splunk):
+
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }} {% now 'America/Los_Angeles', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n")
+    message = mt.render(mark="<111>", host=host)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+
+def test_tz_fix_hst(record_property, setup_wordlist, setup_splunk):
+
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    dt = datetime.datetime.utcnow() - datetime.timedelta(hours=10, minutes=10)
+    mt = env.from_string(
+        "{{ mark }} {{ dt }} tzfhst-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n")
+    message = mt.render(mark="<111>", host=host, dt=dt.strftime('%b %d %H:%M:%S'))
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netfw host=\"tzfhst-{{ host }}\" sourcetype=\"cisco:asa\"")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk):
+
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    tz_NY = pytz.timezone('America/New_York')
+    dt = datetime.datetime.now(tz_NY) - datetime.timedelta(minutes=10)
+    mt = env.from_string(
+        "{{ mark }} {{ dt }} tzfny-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n")
+    message = mt.render(mark="<111>", host=host, dt=dt.strftime('%b %d %H:%M:%S'))
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netfw host=\"tzfny-{{ host }}\" sourcetype=\"cisco:asa\"")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1