From 070a15007e9be58736de95c3cdc105199f61512b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 16 May 2020 14:55:18 -0400 Subject: [PATCH 1/5] Cisco IOS XR Enhancement --- .../conf.d/filters/cisco/cisco_syslog.conf | 44 +++++++++---------- tests/test_cisco_ios.py | 41 +++++++++++------ 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 3938476..6766d1a 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -11,38 +11,38 @@ filter f_is_cisco_syslog{ parser cisco-parser-ex{ channel { filter { - message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); - }; - - parser { date-parser-nofilter(format( - '%b %d %H:%M:%S.%f', - '%b %d %H:%M:%S', - '%b %d %I:%M:%S %p.%f', - '%b %d %I:%M:%S %p', - '%b %d %Y %H:%M:%S.%f', - '%b %d %Y %H:%M:%S') - template("$8")); + message( + '^<\d*> ?(?:\d+\: )?(?RP\/\d*\/RSP\d*\/CPU\d*:)?(?:(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])): ?)?(?:\d+: )?(?:(?:\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?(?(?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?(?: [AP]M)?)(?: [A-Z]{3,3})?)? ?(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: (?(?:(?[^\[]{1,30})\[(?\d*)\]: ?)?(?\%[^\: ]+)\:? ?.*)' + flags(store-matches) + ); }; rewrite { set( - "${4}", + "${H1}", value("HOST") - condition(not match('^\d+$', value('4')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('4'))) + condition(not match('^\d+$', value('H1')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H1'))) ); set( - "${13}", + "${H2}", value("HOST") - condition(not match('^\d+$', value('13')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) + condition(not match('^\d+$', value('H2')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H2'))) ); set( - "${15}", - value("PROGRAM") - ); - set( - "${14}", + "${CISCOMESSAGE}", value("MESSAGE") - ); + ); + }; + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("${CISCOTS}")); + }; + }; -}; +}; \ No newline at end of file diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index 108bec2..30ea68d 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -11,6 +11,7 @@ from .timeutils import * import pytest + env = Environment() @@ -27,7 +28,7 @@ # foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # 101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # *Mar 1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) - +# <189>357492: RP/0/RSP0/CPU0:May 14 16:44:40.145 : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576 testdata = [ "{{ mark }}{{ seq }}: {{ host }}: 6340004: *{{ bsd }}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet", "{{ mark }}{{ seq }}: {{ host }}: *{{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated {{ bsd }}.{{ millisec }}", @@ -37,21 +38,24 @@ "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ millisec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ host }}: {{ bsd }}.{{ millisec }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure. {{ bsd }}.{{ millisec }}", - "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}" + "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}", + "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from {{ host }} has reached 792340, max 1048576", + "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ host }}:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576", ] - testdata_uptime = [ "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}{{ seq }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ seq }}: {{ host }}: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.", - "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}" + "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}", ] @pytest.mark.parametrize("event", testdata) -def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): +def test_cisco_ios( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): host = get_host_key dt = datetime.datetime.now() @@ -64,15 +68,23 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, - millisec=millisec, microsec=microsec, tzname=tzname, host=host) + message = mt.render( + mark="<166>", + seq=20, + bsd=bsd, + time=time, + millisec=millisec, + microsec=microsec, + tzname=tzname, + host=host, + ) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") - search = st.render(epoch=epoch, millisec=millisec, - microsec=microsec, host=host) + 'search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + ) + search = st.render(epoch=epoch, millisec=millisec, microsec=microsec, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -84,7 +96,9 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, @pytest.mark.parametrize("event", testdata_uptime) -def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): +def test_cisco_ios_uptime( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): host = get_host_key mt = env.from_string(event + "\n") @@ -93,7 +107,8 @@ def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_s sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops earliest=-1m@m latest=+1m@m sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") + 'search index=netops earliest=-1m@m latest=+1m@m sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + ) search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -102,4 +117,4 @@ def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_s record_property("resultCount", resultCount) record_property("message", message) - assert resultCount == 1 + assert resultCount == 1 \ No newline at end of file From 44397431b1e2be8e28ee455bd76f929cdc055f41 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 08:46:15 -0400 Subject: [PATCH 2/5] Update for cisco IOS-XR --- .../conf.d/filters/cisco/cisco_syslog.conf | 93 ++++++++++++------- .../conf.d/log_paths/lp-cisco_z_ios.conf.tmpl | 2 +- 2 files changed, 62 insertions(+), 33 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 6766d1a..e9acf58 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -10,39 +10,68 @@ filter f_is_cisco_syslog{ parser cisco-parser-ex{ channel { - filter { - message( - '^<\d*> ?(?:\d+\: )?(?RP\/\d*\/RSP\d*\/CPU\d*:)?(?:(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])): ?)?(?:\d+: )?(?:(?:\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?(?(?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?(?: [AP]M)?)(?: [A-Z]{3,3})?)? ?(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: (?(?:(?[^\[]{1,30})\[(?\d*)\]: ?)?(?\%[^\: ]+)\:? ?.*)' - flags(store-matches) - ); - }; + if { + #Cisco IOS-XR devices with node-id format + filter { + message('^<\d*>(?:(\d+)\: )?(RP\/\d*\/RSP\d*\/CPU\d*:)?(?:([^\: ]+):)(?:(\*)?(\w\w\w {1,2}\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}\.\d+))? : ?([^\[]{1,30}\[\d*\]: ?\%[^\: ]+\:? ?.*)' flags(store-matches)); + }; - rewrite { - set( - "${H1}", - value("HOST") - condition(not match('^\d+$', value('H1')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H1'))) - ); - set( - "${H2}", - value("HOST") - condition(not match('^\d+$', value('H2')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H2'))) - ); - set( - "${CISCOMESSAGE}", - value("MESSAGE") - ); - - }; - parser { date-parser-nofilter(format( - '%b %d %H:%M:%S.%f', - '%b %d %H:%M:%S', - '%b %d %I:%M:%S %p.%f', - '%b %d %I:%M:%S %p', - '%b %d %Y %H:%M:%S.%f', - '%b %d %Y %H:%M:%S') - template("${CISCOTS}")); - }; + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("$5")); + }; + rewrite { + set( + "${3}", + value("HOST") + condition(not match('^\d+$', value('3')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('3'))) + ); + set( + "${6}", + value("MESSAGE") + ); + }; + } else { + # All other cisco syslog + filter { + message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); + }; + + rewrite { + set( + "${4}", + value("HOST") + condition(not match('^\d+$', value('4')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('4'))) + ); + set( + "${13}", + value("HOST") + condition(not match('^\d+$', value('13')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) + ); + set( + "${15}", + value("PROGRAM") + ); + set( + "${14}", + value("MESSAGE") + ); + }; + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("$8")); + }; + }; }; }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl index b4c6eea..f3f06c1 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl @@ -46,4 +46,4 @@ log { {{- end }} flags(flow-control,final); -}; +}; \ No newline at end of file From 547f313a2c6bcb0c586ed260de4fc250e42786c2 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 09:37:20 -0400 Subject: [PATCH 3/5] Additional type of ACL log from APIC --- package/etc/conf.d/filters/cisco/apic.conf | 4 ++-- tests/test_cisco_apic.py | 27 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/apic.conf b/package/etc/conf.d/filters/cisco/apic.conf index ea6660d..22478a6 100644 --- a/package/etc/conf.d/filters/cisco/apic.conf +++ b/package/etc/conf.d/filters/cisco/apic.conf @@ -1,6 +1,6 @@ filter f_cisco_apic { program('^%LOG_LOCAL\d-\d-'); - or - program('^%ACLLOG-\d-ACLLOG_PKTLOG'); + or program('^%LOG_-\d-'); + or program('^%ACLLOG-\d-ACLLOG_PKTLOG'); }; \ No newline at end of file diff --git a/tests/test_cisco_apic.py b/tests/test_cisco_apic.py index 5f59fea..0de42d7 100644 --- a/tests/test_cisco_apic.py +++ b/tests/test_cisco_apic.py @@ -14,7 +14,7 @@ env = Environment() #<11>July 22 22:45:28 apic1 %LOG_LOCAL0-2-SYSTEM_MSG [F0110][soaking][node-failed][critical][topology/pod-1/node-102/fault-F0110] Node 102 not reachable. unknown -def test_cisco_aci(record_property, setup_wordlist, setup_splunk, setup_sc4s): +def test_cisco_aci_loglocal(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -39,6 +39,31 @@ def test_cisco_aci(record_property, setup_wordlist, setup_splunk, setup_sc4s): assert resultCount == 1 +def test_cisco_aci_log(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions for Cisco APIC + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} %LOG_-2-SYSTEM_MSG [F0110][soaking][node-failed][critical][topology/pod-1/node-102/fault-F0110]\n") + message = mt.render(mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"cisco:apic:events\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + #%ACLLOG-5-ACLLOG_PKTLOG def test_cisco_aci_acl(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) From 563e9e70adf1d5ffea23c7b88f5a2fb390f3cd04 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 09:53:25 -0400 Subject: [PATCH 4/5] Revert "Cisco IOS XR Improvements" This reverts commit 0f7c2d09472727e61fb8513c9fe0d995ccb142db. --- .../conf.d/filters/cisco/cisco_syslog.conf | 42 +++++++++---------- tests/test_cisco_ios.py | 39 ++++++----------- 2 files changed, 33 insertions(+), 48 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 82cd4db..3938476 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -11,38 +11,38 @@ filter f_is_cisco_syslog{ parser cisco-parser-ex{ channel { filter { - message( - '^<\d*> ?(?:\d+\: )?(?RP\/\d*\/RSP\d*\/CPU\d*:)?(?:(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])): ?)?(?:\d+: )?(?:(?:\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?(?(?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?(?: [AP]M)?)(?: [A-Z]{3,3})?)? ?(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: (?(?:(?[^\[]{1,30})\[(?\d*)\]: ?)?(?\%[^\: ]+)\:? ?.*)' - flags(store-matches) - ); + message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); + }; + + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("$8")); }; rewrite { set( - "${H1}", + "${4}", value("HOST") - condition(not match('^\d+$', value('H1')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H1'))) + condition(not match('^\d+$', value('4')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('4'))) ); set( - "${H2}", + "${13}", value("HOST") - condition(not match('^\d+$', value('H2')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H2'))) + condition(not match('^\d+$', value('13')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) ); set( - "${CISCOMESSAGE}", + "${15}", + value("PROGRAM") + ); + set( + "${14}", value("MESSAGE") - ); - + ); }; - parser { date-parser-nofilter(format( - '%b %d %H:%M:%S.%f', - '%b %d %H:%M:%S', - '%b %d %I:%M:%S %p.%f', - '%b %d %I:%M:%S %p', - '%b %d %Y %H:%M:%S.%f', - '%b %d %Y %H:%M:%S') - template("${CISCOTS}")); - }; - }; }; diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index ab24128..108bec2 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -11,7 +11,6 @@ from .timeutils import * import pytest - env = Environment() @@ -28,7 +27,7 @@ # foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # 101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # *Mar 1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) -# <189>357492: RP/0/RSP0/CPU0:May 14 16:44:40.145 : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576 + testdata = [ "{{ mark }}{{ seq }}: {{ host }}: 6340004: *{{ bsd }}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet", "{{ mark }}{{ seq }}: {{ host }}: *{{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated {{ bsd }}.{{ millisec }}", @@ -38,24 +37,21 @@ "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ millisec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ host }}: {{ bsd }}.{{ millisec }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure. {{ bsd }}.{{ millisec }}", - "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}", - "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from {{ host }} has reached 792340, max 1048576", - "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ host }}:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576", + "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}" ] + testdata_uptime = [ "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}{{ seq }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ seq }}: {{ host }}: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.", - "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}", + "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}" ] @pytest.mark.parametrize("event", testdata) -def test_cisco_ios( - record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event -): +def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): host = get_host_key dt = datetime.datetime.now() @@ -68,23 +64,15 @@ def test_cisco_ios( microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render( - mark="<166>", - seq=20, - bsd=bsd, - time=time, - millisec=millisec, - microsec=microsec, - tzname=tzname, - host=host, - ) + message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, + millisec=millisec, microsec=microsec, tzname=tzname, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' - ) - search = st.render(epoch=epoch, millisec=millisec, microsec=microsec, host=host) + "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") + search = st.render(epoch=epoch, millisec=millisec, + microsec=microsec, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -96,9 +84,7 @@ def test_cisco_ios( @pytest.mark.parametrize("event", testdata_uptime) -def test_cisco_ios_uptime( - record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event -): +def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): host = get_host_key mt = env.from_string(event + "\n") @@ -107,8 +93,7 @@ def test_cisco_ios_uptime( sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=netops earliest=-1m@m latest=+1m@m sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' - ) + "search index=netops earliest=-1m@m latest=+1m@m sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) From f066cd1367866297c91506151bda1a59c397c40e Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 17:15:26 -0400 Subject: [PATCH 5/5] Support cisco firepower unified logs --- docs/sources/Cisco/index.md | 9 ++++-- .../conf.d/log_paths/lp-cisco_asa.conf.tmpl | 28 +++++++++++++---- tests/test_cisco_asa.py | 30 +++++++++++++++++++ 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index 9fccf8d..fd58a34 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -95,13 +95,14 @@ Use the following search to validate events are present index= sourcetype=cisco:apic:* ``` -Verify timestamp, and host values match as expected +Verify timestamp, and host values match as expected ## Product - ASA AND FTD (Firepower) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | https://splunkbase.splunk.com/app/1620/ | +| Splunk Add-on for ASA | https://splunkbase.splunk.com/app/1620/ | +| Cisco eStreamer for Splunk | https://splunkbase.splunk.com/app/1629/ | | Product Manual | https://www.cisco.com/c/en/us/td/docs/security/asa/asa82/configuration/guide/config/monitor_syslog.html | @@ -109,13 +110,15 @@ Verify timestamp, and host values match as expected | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| cisco:asa | cisco FTD Firepower will also use this source type | +| cisco:asa | cisco FTD Firepower will also use this source type except those noted below | +| cisco:firepower:syslog | FTD Unified events see https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide.pdf | ### Sourcetype and Index Configuration | key | sourcetype | index | notes | |----------------|----------------|----------------|----------------| | cisco_asa | cisco:asa | netfw | none | +| cisco_ftd | cisco:firepower:syslog | netfw | none | ### Filter type diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl index 76c8a8b..b60f1d6 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl @@ -21,13 +21,29 @@ log { }; }; - rewrite { - set("cisco_asa", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw")) + if { + filter { + message('^%FTD-\d+-43000\d: ') or + match('^%FTD-\d+-43000\d:', value("LEGACY_MSGHDR")); + }; + rewrite { + set("cisco_ftd", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:firepower:syslog"), index("netfw")) + }; + parser {p_add_context_splunk(key("cisco_ftd")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + + } else { + rewrite { + set("cisco_asa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw")) + }; + parser {p_add_context_splunk(key("cisco_asa")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + }; - parser {p_add_context_splunk(key("cisco_asa")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_ASA_HEC" "no")) }} destination(d_hec); diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index 77a934f..6657876 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -100,3 +100,33 @@ def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk, setup_ record_property("message", message) assert resultCount == 1 + +#<118>2020-02-04T11:00:54Z %FTD-6-430003: DeviceUUID: 90e14378-2081-11e8-a7fa-d34972ba379f, AccessControlRuleAction: Allow, SrcIP: 75.150.94.75, DstIP: 172.30.0.2, SrcPort: 59698, DstPort: 8027, Protocol: tcp, IngressInterface: Outside2, EgressInterface: DMZ, IngressZone: Outside, EgressZone: DMZ, ACPolicy: Rapid7 5525X, AccessControlRuleName: Allow MDM - Out to DMZ, Prefilter Policy: Default Prefilter Policy, User: No Authentication Required, ConnectionDuration: 600, InitiatorPackets: 0, ResponderPackets: 0, InitiatorBytes: 31, ResponderBytes: 0, NAPPolicy: Balanced Security and Connectivity +def test_cisco_ftd(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + +# Get UTC-based 'dt' time structure + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + # iso from included timeutils is from local timezone; need to keep iso as UTC + iso = dt.isoformat()[0:19] + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ iso }}Z {{ host }} : %FTD-6-430003: DeviceUUID: 90e14378-2081-11e8-a7fa-d34972ba379f, AccessControlRuleAction: Allow, SrcIP: 75.150.94.75, DstIP: 172.30.0.2, SrcPort: 59698, DstPort: 8027, Protocol: tcp, IngressInterface: Outside2, EgressInterface: DMZ, IngressZone: Outside, EgressZone: DMZ, ACPolicy: Rapid7 5525X, AccessControlRuleName: Allow MDM - Out to DMZ, Prefilter Policy: Default Prefilter Policy, User: No Authentication Required, ConnectionDuration: 600, InitiatorPackets: 0, ResponderPackets: 0, InitiatorBytes: 31, ResponderBytes: 0, NAPPolicy: Balanced Security and Connectivity\n") + message = mt.render(mark="<166>", iso=iso, epoch=epoch, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cisco:firepower:syslog\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file