From f32c9a1b23fd9a7b3571b98f106496bf7fb1ff86 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 31 Jan 2020 09:32:51 -0800 Subject: [PATCH 1/4] Update ASA log path to account for events with no hostname * Update ASA log path and add filter to detect events with no hostname included --- package/etc/conf.d/filters/cisco/asa.conf | 9 +++++++-- .../etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl | 6 +++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/asa.conf b/package/etc/conf.d/filters/cisco/asa.conf index bbd3157..a7ac9b7 100644 --- a/package/etc/conf.d/filters/cisco/asa.conf +++ b/package/etc/conf.d/filters/cisco/asa.conf @@ -1,3 +1,8 @@ filter f_cisco_asa { - message('^%ASA-\d+-\d{1,10}: '); -}; \ No newline at end of file + message('^%ASA-\d+-\d{1,10}: ') or + match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR")); +}; + +filter f_cisco_asa_nohost { + match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR")); +}; diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl index 0749e68..a8bf9c5 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl @@ -27,7 +27,11 @@ log { }; parser {p_add_context_splunk(key("cisco_asa")); }; parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + if (filter (f_cisco_asa_nohost)) { + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + } else { + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_ASA_HEC" "no")) }} destination(d_hec); From d5f68b1e6795be91dc05b8236aa0dc9db3abbe27 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 31 Jan 2020 10:16:11 -0800 Subject: [PATCH 2/4] Add Cisco ASA "nohost" test * Add Cisco ASA test for events with no hostname set --- tests/test_cisco_asa.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index 1399ac8..7c1e5bc 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -36,6 +36,27 @@ def test_cisco_asa_traditional(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 +# <164>Jan 31 2020 17:24:03: %ASA-4-402119: IPSEC: Received an ESP packet (SPI= 0x0C190BF9, sequence number= 0x598243) from 192.0.0.1 (user= 192.0.0.1) to 192.0.0.2 that failed anti-replay checking. +def test_cisco_asa_traditional_nohost(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %}: %ASA-4-402119: IPSEC: Received an ESP packet (SPI= 0x0C190BF9, sequence number= 0x598243) from 192.0.0.1 (user= 192.0.0.1) to 192.0.0.2 that failed anti-replay checking.\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw sourcetype=\"cisco:asa\" \"%ASA-4-402119\" | head 1") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + # <166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk): From 4e639d9b6b9a15e659efe4566cef079b3551ccd7 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 31 Jan 2020 10:44:22 -0800 Subject: [PATCH 3/4] Update test_cisco_asa.py --- tests/test_cisco_asa.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index 7c1e5bc..6bcd920 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -41,12 +41,12 @@ def test_cisco_asa_traditional_nohost(record_property, setup_wordlist, setup_spl host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %}: %ASA-4-402119: IPSEC: Received an ESP packet (SPI= 0x0C190BF9, sequence number= 0x598243) from 192.0.0.1 (user= 192.0.0.1) to 192.0.0.2 that failed anti-replay checking.\n") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %}: %ASA-4-402119: IPSEC: Received an ESP packet (SPI= 0x0C190BF9, sequence number= 0x598243) from {host} (user= 192.0.0.1) to 192.0.0.2 that failed anti-replay checking.\n") message = mt.render(mark="<111>", host=host) sendsingle(message) - st = env.from_string("search index=netfw sourcetype=\"cisco:asa\" \"%ASA-4-402119\" | head 1") + st = env.from_string("search index=netfw sourcetype=\"cisco:asa\" \"%ASA-4-402119\" \"{host}\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) From 334886800c2771c600059b1b02666c20e6b54a88 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sat, 1 Feb 2020 21:43:16 -0800 Subject: [PATCH 4/4] Add RAWMSG storage for debugging * Add an environment variable `SC4S_SOURCE_STORE_RAWMSG` to enable the collection/storage of the "on the wire" message for use in the 3164 fallback template. * Useful for development and debugging of log paths and filters. NOT FOR USE IN PRODUCTION due to memory/disk overhead. --- docker-compose.yml | 1 + docs/configuration.md | 2 +- package/etc/conf.d/conflib/_common/templates.conf | 1 + package/etc/go_templates/source_network.t | 8 ++++---- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 323da58..a2b7ae4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,7 @@ services: - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 - SC4S_ARCHIVE_GLOBAL=yes +# - SC4S_SOURCE_STORE_RAWMSG=yes volumes: - ./tls:/opt/syslog-ng/tls splunk: diff --git a/docs/configuration.md b/docs/configuration.md index f19488b..75b7e82 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -73,7 +73,7 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_TCP_IW_SIZE | 20000000 | Initial Window size | | SC4S_SOURCE_TCP_FETCH_LIMIT | 2000 | Number of events to fetch from server buffer at once | | SC4S_SOURCE_UDP_SO_RCVBUFF | 425984 | UDP server buffer size in bytes | - +| SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | ## Syslog Source TLS Certificate Configuration diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index 46b63b2..5746a1c 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -61,6 +61,7 @@ template t_JSON_3164 { template('$(format-json --scope rfc3164 --pair PRI="<$PRI>" --key LEGACY_MSGHDR + --key RAWMSG --exclude FACILITY --exclude PRIORITY )'); diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 7b201eb..a49e89b 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -60,14 +60,14 @@ source s_{{ .port_id }} { }; {{ if eq .parser "rfc3164" }} parser { - syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})); }; rewrite(set_rfc3164); {{ else if eq .parser "rfc3164_version" }} # filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})); }; rewrite(set_rfc3164_version); {{ else if eq .parser "rfc5424_strict" }} @@ -104,7 +104,7 @@ source s_{{ .port_id }} { filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})); }; rewrite(set_rfc3164_version); } elif { @@ -118,7 +118,7 @@ source s_{{ .port_id }} { rewrite(set_cisco_ios); } else { parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})); }; rewrite(set_rfc3164); };