From a784099891562f9c4d2dda0766e0b696a59bf4b5 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 6 Jun 2020 18:20:59 -0400 Subject: [PATCH] Support . as indicator time is out of sync --- .../conf.d/filters/cisco/cisco_syslog.conf | 27 ++++-- tests/test_cisco_ios.py | 94 ++++++++++++++++--- 2 files changed, 100 insertions(+), 21 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 44f8d5c..e6b9084 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -13,7 +13,7 @@ parser cisco-parser-ex{ if { #Cisco IOS-XR devices with node-id format filter { - message('^<\d*>(?:(\d+)\: )?(RP\/\d*\/RSP\d*\/CPU\d*:)?(?:([^\: ]+):)(?:(\*)?(\w\w\w {1,2}\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}\.\d+))? : ?([^\[]{1,30}\[\d*\]: ?\%[^\: ]+\:? ?.*)' flags(store-matches)); + message('^<\d*>(?:(\d+)\: )?(RP\/\d*\/RSP\d*\/CPU\d*:)?(?:([^\: ]+):)(?:(\*|\.)?(\w\w\w {1,2}\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}\.\d+))? : ?([^\[]{1,30}\[\d*\]: ?\%[^\: ]+\:? ?.*)' flags(store-matches)); }; parser { date-parser-nofilter(format( @@ -45,7 +45,7 @@ parser cisco-parser-ex{ } elif { #Cisco IOS Other filter { - message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Za-z]{3,3} )?)? ?((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' + message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*|\.)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Za-z]{3,3} )?)? ?((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); }; @@ -73,14 +73,21 @@ parser cisco-parser-ex{ condition(not host(".")) ); }; - parser { date-parser-nofilter(format( - '%b %d %H:%M:%S.%f', - '%b %d %H:%M:%S', - '%b %d %I:%M:%S %p.%f', - '%b %d %I:%M:%S %p', - '%b %d %Y %H:%M:%S.%f', - '%b %d %Y %H:%M:%S') - template("$8")); + if { + filter { + match('^(\*|\.)$' value("7")); + }; + rewrite { set("cisco reported time error : ${8}" value("fields.sc4s_error")); }; + } else { + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("$8")); + }; }; } else { #Cisco AireOS format diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index 2de6739..40cfda9 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -11,6 +11,7 @@ from .timeutils import * import pytest + env = Environment() @@ -29,6 +30,20 @@ # *Mar 1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) # <132>xxxxx: *spamApTask1: May 26 18:52:01.958: %CAPWAP-4-DISC_INTF_ERR2: [PA]capwap_ac_sm.c:2053 Ignoring Primary discovery request received on a wrong VLAN (202) on interface (8) from AP 00:b7:00:00:00:00 testdata = [ + "{{ mark }}{{ seq }}: {{ host }}: 6340004: {{ bsd }}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet", + "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated {{ bsd }}.{{ millisec }}", + "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", + "{{ mark }}{{ host }}: {{ bsd }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.", + "{{ mark }}{{ seq }}: {{ host }}: 6340004: {{ bsd }}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet", + "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", + "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ millisec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", + "{{ mark }}{{ host }}: {{ bsd }}.{{ millisec }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure. {{ bsd }}.{{ millisec }}", + "{{ mark }}{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}", + "{{ mark }}84027: {{ bsd }}.{{ millisec }} dst: %SYS-5-CONFIG_I: Configured from console by username on vty0 ({{ host }})", + "{{ mark }}{{ host }}: *spamApTask1: {{ bsd }}.{{ millisec }}: %CAPWAP-4-DISC_INTF_ERR2: [PA]capwap_ac_sm.c:2053 Ignoring Primary discovery request received on a wrong VLAN (202) on interface (8) from AP 00:b7:00:00:00:00", + "{{ mark }}22191: {{ host }}: 022546: {{ bsd }}.{{ millisec }} CDT: %PARSER-5-CFGLOG_LOGGEDCMD: User:dfa_service_admin logged command:!exec: enable", +] +testdata_badtime = [ "{{ mark }}{{ seq }}: {{ host }}: 6340004: *{{ bsd }}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet", "{{ mark }}{{ seq }}: {{ host }}: *{{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated {{ bsd }}.{{ millisec }}", "{{ mark }}{{ seq }}: {{ host }}: *{{ bsd }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", @@ -39,7 +54,8 @@ "{{ mark }}{{ host }}: {{ bsd }}.{{ millisec }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure. {{ bsd }}.{{ millisec }}", "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}", "{{ mark }}84027: {{ bsd }}.{{ millisec }} dst: %SYS-5-CONFIG_I: Configured from console by username on vty0 ({{ host }})", - "{{ mark }}{{ host }}: *spamApTask1: {{ bsd }}.{{ millisec }}: %CAPWAP-4-DISC_INTF_ERR2: [PA]capwap_ac_sm.c:2053 Ignoring Primary discovery request received on a wrong VLAN (202) on interface (8) from AP 00:b7:00:00:00:00" + "{{ mark }}{{ host }}: *spamApTask1: {{ bsd }}.{{ millisec }}: %CAPWAP-4-DISC_INTF_ERR2: [PA]capwap_ac_sm.c:2053 Ignoring Primary discovery request received on a wrong VLAN (202) on interface (8) from AP 00:b7:00:00:00:00", + "{{ mark }}22191: {{ host }}: 022546: .{{ bsd }}.{{ millisec }} CDT: %PARSER-5-CFGLOG_LOGGEDCMD: User:dfa_service_admin logged command:!exec: enable", ] testdata_uptime = [ @@ -48,12 +64,57 @@ "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}{{ seq }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ seq }}: {{ host }}: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.", - "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}" + "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}", ] @pytest.mark.parametrize("event", testdata) -def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): +def test_cisco_ios( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + time = time[:-7] + millisec = iso[20:23] + microsec = iso[20:26] + + mt = env.from_string(event + "\n") + message = mt.render( + mark="<166>", + seq=20, + bsd=bsd, + time=time, + millisec=millisec, + microsec=microsec, + tzname=tzname, + host=host, + ) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + ) + search = st.render(epoch=epoch, millisec=millisec, microsec=microsec, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +@pytest.mark.parametrize("event", testdata_badtime) +def test_cisco_ios_badtime( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): host = get_host_key dt = datetime.datetime.now() @@ -66,15 +127,23 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, - millisec=millisec, microsec=microsec, tzname=tzname, host=host) + message = mt.render( + mark="<166>", + seq=20, + bsd=bsd, + time=time, + millisec=millisec, + microsec=microsec, + tzname=tzname, + host=host, + ) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") - search = st.render(epoch=epoch, millisec=millisec, - microsec=microsec, host=host) + 'search index=netops earliest=-1m@m latest=+1m@m sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + ) + search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -86,7 +155,9 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, @pytest.mark.parametrize("event", testdata_uptime) -def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): +def test_cisco_ios_uptime( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): host = get_host_key mt = env.from_string(event + "\n") @@ -95,7 +166,8 @@ def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_s sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops earliest=-1m@m latest=+1m@m sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") + 'search index=netops earliest=-1m@m latest=+1m@m sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + ) search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -104,4 +176,4 @@ def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_s record_property("resultCount", resultCount) record_property("message", message) - assert resultCount == 1 \ No newline at end of file + assert resultCount == 1