From 866092488703dce6522fa3f6af2800a0ef1c6979 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Sun, 17 Nov 2019 19:41:21 -0500 Subject: [PATCH] Release/1.2.0 (#195) * Resolve error with failure to upgrade example.conf * Automatically resolve correct time zone for near real time streaming sources * Support specific time zones by host/ip/subnet for sources with high latency --- .../p_rfc3164-checkpoint_splunk.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 1 + .../log_paths/p_rfc3164-cisco_nxos.conf.tmpl | 1 + .../p_rfc3164-fortinet_fortios.conf.tmpl | 2 +- .../p_rfc3164-paloalto_panos.conf.tmpl | 3 +- .../vendor_product_by_source.conf | 9 +++ .../vendor_product_by_source.csv | 4 +- package/etc/go_templates/source_network.t | 12 ++- package/sbin/entrypoint.sh | 2 +- tests/requirements.txt | 1 + tests/test_common.py | 73 ++++++++++++++++++- 11 files changed, 102 insertions(+), 8 deletions(-) diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index ee81c82..cd99a70 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -16,7 +16,7 @@ log { parser { kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); - date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); + date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite { diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index 3c51cf6..b9d997c 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -15,6 +15,7 @@ log { rewrite { set("cisco_ios", value("fields.sc4s_vendor_product")); + guess-time-zone(); r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops")) }; parser { diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index f2a4478..e351441 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -15,6 +15,7 @@ log { rewrite { set("cisco_nxos", value("fields.sc4s_vendor_product")); + guess-time-zone(); r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg")) }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 39d2bc9..16b35f4 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -16,7 +16,7 @@ log { parser { kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); - date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); + date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite { diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 60acc74..73dc1d0 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -38,7 +38,8 @@ log { date-parser( format("%Y/%m/%d %H:%M:%S") template("${.pan.GeneratedTime}") - time-zone("Universal") + time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) + flags(guess-timezone) ); }; diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index ec20e82..b0609de 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -39,4 +39,13 @@ filter f_proofpoint_pps_sendmail { filter f_ubiquiti_unifi_fw { host("usg-*" type(glob)) or netmask(192.168.6.0/24) +}; +filter f_tzfixhst { + host("tzfhst-*" type(glob)) or + netmask(192.168.6.0/24) +}; + +filter f_tzfixny { + host("tzfny-*" type(glob)) or + netmask(192.168.6.0/24) }; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 698f672..975d558 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -7,4 +7,6 @@ f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" -f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" \ No newline at end of file +f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" +f_tzfixhst,sc4s_time_zone,"Pacific/Honolulu" +f_tzfixny,sc4s_time_zone,"America/New_York" \ No newline at end of file diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index badcbbb..63fb6d2 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -104,7 +104,7 @@ source s_{{ .port_id}} { rewrite(set_rfc5424_epochtime); } else { parser { - syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone)); }; rewrite(set_rfc3164); }; @@ -114,6 +114,16 @@ source s_{{ .port_id}} { parser { vendor_product_by_source(); }; + + if { + filter { match("." value("fields.sc4s_time_zone") ) }; + rewrite { + fix-time-zone("${fields.sc4s_time_zone}"); + unset(value("fields.sc4s_time_zone")); + }; + }; + + }; }; {{- end }} diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index a93736d..9abad88 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -12,7 +12,7 @@ gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl -- mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ -cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ +cp --verbose -R /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index 758c531..6d908dd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,3 +11,4 @@ jinja2 jinja2-time splunk-sdk flake8 +pytz \ No newline at end of file diff --git a/tests/test_common.py b/tests/test_common.py index 8325b52..ce279a6 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -3,9 +3,11 @@ # Use of this source code is governed by a BSD-2-clause-style # license that can be found in the LICENSE-BSD2 file or at # https://opensource.org/licenses/BSD-2-Clause +import datetime import random +import pytz -from jinja2 import Environment +from jinja2 import Environment, environment from .sendmessage import * from .splunkutils import * @@ -79,4 +81,71 @@ def test_metrics(record_property, setup_wordlist, setup_splunk): record_property("resultCount", resultCount) - assert resultCount == 1 \ No newline at end of file + assert resultCount == 1 + +def test_tz_guess(record_property, setup_wordlist, setup_splunk): + + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'America/Los_Angeles', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +def test_tz_fix_hst(record_property, setup_wordlist, setup_splunk): + + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.utcnow() - datetime.timedelta(hours=10, minutes=10) + mt = env.from_string( + "{{ mark }} {{ dt }} tzfhst-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") + message = mt.render(mark="<111>", host=host, dt=dt.strftime('%b %d %H:%M:%S')) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"tzfhst-{{ host }}\" sourcetype=\"cisco:asa\"") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk): + + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + tz_NY = pytz.timezone('America/New_York') + dt = datetime.datetime.now(tz_NY) - datetime.timedelta(minutes=10) + mt = env.from_string( + "{{ mark }} {{ dt }} tzfny-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") + message = mt.render(mark="<111>", host=host, dt=dt.strftime('%b %d %H:%M:%S')) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"tzfny-{{ host }}\" sourcetype=\"cisco:asa\"") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1