From c817b86ee598f463abd2ec59569786f2164ecd9d Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sun, 19 Apr 2020 13:53:43 -0700 Subject: [PATCH 01/10] Refine rawmsg destination * Refine rawmsg destination to not fire if `RAWMSG` macro is not set or null --- package/etc/conf.d/destinations/rawmsg_file.conf | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/package/etc/conf.d/destinations/rawmsg_file.conf b/package/etc/conf.d/destinations/rawmsg_file.conf index f5faf88..7f63cf5 100644 --- a/package/etc/conf.d/destinations/rawmsg_file.conf +++ b/package/etc/conf.d/destinations/rawmsg_file.conf @@ -1,5 +1,12 @@ destination d_rawmsg { - file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" - template("${RAWMSG}\n") - ); -}; + channel { + # Test for length of RAWMSG macro contents, minimum (meaning not set) appears to be "2" + filter { "$(length (${RAWMSG}))" != "2" }; + destination { + file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" + template("${RAWMSG}\n") +# template("Length of RAWMSG is: $(length (${RAWMSG})), RAWMSG is: <${RAWMSG}>\n") + ); + }; + }; +}; \ No newline at end of file From 865415019f9933619abf8b880efbe037efba2433 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 20 Apr 2020 11:10:46 -0700 Subject: [PATCH 02/10] Refine rawmsg destination * Refine rawmsg destination test based on Bazsi's input --- package/etc/conf.d/destinations/rawmsg_file.conf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/package/etc/conf.d/destinations/rawmsg_file.conf b/package/etc/conf.d/destinations/rawmsg_file.conf index 7f63cf5..8ca2f79 100644 --- a/package/etc/conf.d/destinations/rawmsg_file.conf +++ b/package/etc/conf.d/destinations/rawmsg_file.conf @@ -1,12 +1,11 @@ destination d_rawmsg { channel { - # Test for length of RAWMSG macro contents, minimum (meaning not set) appears to be "2" - filter { "$(length (${RAWMSG}))" != "2" }; - destination { - file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" - template("${RAWMSG}\n") -# template("Length of RAWMSG is: $(length (${RAWMSG})), RAWMSG is: <${RAWMSG}>\n") - ); + if ("${RAWMSG}" ne "") { + destination { + file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" + template("${RAWMSG}\n") + ); + }; }; }; }; \ No newline at end of file From e37447745acc25dd82f6289f7e5b3b602ce9db84 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Wed, 22 Apr 2020 08:58:14 -0700 Subject: [PATCH 03/10] zscaler lss log path cleanup * Clean up extraneous parser/rewrites in `lp-zscaler_lss.conf.tmpl` * Previous code is functionally OK; this is a cleanup only --- .../etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl index ff95eea..63cb036 100644 --- a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl @@ -47,8 +47,6 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-app"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } elif { filter { match('.' value('.json.Exporter')) @@ -57,8 +55,6 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-bba"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } elif { filter { match('.' value('.json.Connector')) @@ -67,8 +63,6 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-connector"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } elif { filter { match('.' value('.json.SAMLAttributes')) @@ -76,8 +70,6 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netauth"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } else { rewrite { set("zscaler_lss_rogue_message", value("fields.sc4s_vendor_product")); @@ -85,11 +77,13 @@ log { r_set_splunk_dest_default(sourcetype("zscalerlss:rogue"), index("netproxy")) }; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); + # Rogue message needs a different template than valid LSS events. Final rewrite (further below) will be a + # no-op in this case. rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; }; - + # Parser for all valid LSS events. Rogue events, having previously loaded $MSG with the entire payload, + # will be unaffected by the rewrite here. parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; From 18fbe70151a949fdeb05b623e541b300646afee0 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Wed, 22 Apr 2020 13:46:12 -0400 Subject: [PATCH 04/10] Alpha Support for this protocol --- .../log_paths/lp-bbb-ietf_syslog.conf.tmpl | 41 +++++++++++++++++++ .../log_paths/lp-zzz-fallback.conf.tmpl | 1 + package/etc/conf.d/sources/rfc5687.conf.tmpl | 25 +++++++++++ 3 files changed, 67 insertions(+) create mode 100644 package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl create mode 100644 package/etc/conf.d/sources/rfc5687.conf.tmpl diff --git a/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl new file mode 100644 index 0000000..85a083d --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl @@ -0,0 +1,41 @@ +# Linux/Unix OS system logs +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "IETF_SYSLOG" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log { + junction { + channel { + # Listen on the default port (typically 514) for IETF_SYSLOG traffic + source (s_ietf); + flags(final); + }; + }; + + rewrite { + set("IETF_SYSLOG", value("fields.sc4s_vendor_product")); + }; + + rewrite { r_set_splunk_dest_default(sourcetype("ietf:syslog"), index("main"), source("${APP}:${PROGRAM}")) }; + parser { p_add_context_splunk(key("IETF_SYSLOG")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_IETF_SYSLOG_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_IETF_SYSLOG" "no")) }} + destination(d_archive); +{{- end}} + +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES")) }} + {{ getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + + flags(flow-control,final); +}; diff --git a/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl b/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl index a3a33d9..d8bbd88 100644 --- a/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl @@ -1,6 +1,7 @@ # Fallback for un-parsed sources log { + source(s_ietf); source(s_DEFAULT); rewrite { set("SC4S_fallback", value("fields.sc4s_vendor_product")); }; diff --git a/package/etc/conf.d/sources/rfc5687.conf.tmpl b/package/etc/conf.d/sources/rfc5687.conf.tmpl new file mode 100644 index 0000000..b5044b9 --- /dev/null +++ b/package/etc/conf.d/sources/rfc5687.conf.tmpl @@ -0,0 +1,25 @@ +source s_ietf { + channel { + source { + syslog ( + transport("tcp") + port(601) + ip-protocol(4) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(validate-utf8, syslog-protocol) + ); + }; + + if { + parser { app-parser(topic(syslog)); }; + }; + rewrite(set_rfc5424_strict); + parser { + vendor_product_by_source(); + }; + }; +}; From 01fa2f8811953c092be69a8fd701ce585202f2f4 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Wed, 22 Apr 2020 13:49:54 -0400 Subject: [PATCH 05/10] Update templates.conf --- package/etc/conf.d/conflib/_common/templates.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index c86400f..4d62bdb 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -83,6 +83,7 @@ template t_JSON_5424 { --exclude DATE --exclude FACILITY --exclude PRIORITY + --exclude HOST )'); }; From fbd71628cabf50710604273fc31931b6058295a3 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Wed, 22 Apr 2020 16:05:01 -0700 Subject: [PATCH 06/10] Update IETF syslog log path * Remove unique port settings for IETF syslog * Fix commenting * Consider removing junction/channel for initial source parsing; unnecessarily complicated --- package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl index 85a083d..35366a8 100644 --- a/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl @@ -1,12 +1,9 @@ -# Linux/Unix OS system logs -{{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "IETF_SYSLOG" "parser" "rfc3164" }} -{{- tmpl.Exec "t/source_network.t" $context }} +# IETF Syslog log { junction { channel { - # Listen on the default port (typically 514) for IETF_SYSLOG traffic + # Listen on the default port (typically 601) for IETF_SYSLOG traffic source (s_ietf); flags(final); }; From 9dcf4699eaa9b9eddec0aa21b5c220ac985b39f8 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 23 Apr 2020 14:54:14 -0700 Subject: [PATCH 07/10] Add new template for 5424 SDATA only * Add template to output only the structured data (SDATA) portion of the 5424 event --- .../etc/conf.d/conflib/_common/templates.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index c86400f..7d582c5 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -86,6 +86,22 @@ template t_JSON_5424 { )'); }; +# =============================================================================================== +# JSON_5424_SDATA; for JSON pretty-printing (for RFC5424 messages with duplicate data in MESSAGE) +# =============================================================================================== + +template t_JSON_5424_SDATA { + template('$(format-json --scope rfc5424 + --pair PRI="<$PRI>" + --key ISODATE + --exclude DATE + --exclude HOST + --exclude FACILITY + --exclude PRIORITY + --exclude MESSAGE + )'); + }; + template t_snmp_trap { template('$(format-json .snmp.* --rekey .snmp.* --shift-levels 2)'); From 82c0de2ecdafad85dd7df0ccdb7af055411c6ab3 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 23 Apr 2020 16:13:12 -0700 Subject: [PATCH 08/10] Add context entry for mcafee * Add mcafee entry to `splunk_index.csv` * Clean up log path --- package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl | 8 +------- package/etc/context_templates/splunk_index.csv.example | 1 + 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl b/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl index 671fc5e..36419fb 100644 --- a/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl @@ -24,17 +24,11 @@ log { rewrite { set("mcafee_epo", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav")) }; - rewrite { r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav")) }; parser {p_add_context_splunk(key("mcafee_epo")); }; - parser (compliance_meta_by_source); - - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_MCAFEE_EPO_STRUCTURED_HEC" "no")) }} diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 8a77f3d..6339988 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -52,6 +52,7 @@ #juniper_nsm,index,netfw #juniper_nsm_idp,index,netids #juniper_legacy,index,netops +#mcafee_epo,index,epav #nix_syslog,index,osnix #pan_traffic,index,netfw #pan_threat,index,netproxy From 39a19a2c83a1a2d3dd06486d8213177707e60289 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Tue, 28 Apr 2020 08:38:55 -0700 Subject: [PATCH 09/10] Update metadata keys for Meraki and Zscaler * Fix incorrect Meraki and Zscaler keys in docs * Change zscaler LSS `zpa-auth` logs to use `netproxy` index (to match all other LSS events). --- docs/sources/Cisco/index.md | 2 +- docs/sources/Zscaler/index.md | 27 ++++++++++--------- .../conf.d/log_paths/lp-zscaler_lss.conf.tmpl | 2 +- .../splunk_index.csv.example | 9 ++++++- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index 9cf0707..9467fa2 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -291,7 +291,7 @@ Verify timestamp, and host values match as expected | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| merkai | None | +| meraki | None | ### Sourcetype and Index Configuration diff --git a/docs/sources/Zscaler/index.md b/docs/sources/Zscaler/index.md index d922ef7..c087e3b 100644 --- a/docs/sources/Zscaler/index.md +++ b/docs/sources/Zscaler/index.md @@ -25,13 +25,14 @@ the IP or host name of the SC4S instance and port 514 ### Sourcetype and Index Configuration -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| zscalernss_alerts | zscalernss-alerts | main | none | -| zscalernss_dns | zscalernss-dns | netdns | none | -| zscalernss_fw | zscalernss-fw | netfw | none | -| zscalernss_web | zscalernss-web | netproxy | none | - +| key | sourcetype | index | notes | +|---------------------|------------------------|----------|---------| +| zscaler_alerts | zscalernss-alerts | main | none | +| zscaler_dns | zscalernss-dns | netdns | none | +| zscaler_fw | zscalernss-fw | netfw | none | +| zscaler_web | zscalernss-web | netproxy | none | +| zscaler_zia_audit | zscalernss-zia-audit | netops | none | +| zscaler_zia_sandbox | zscalernss-zia-sandbox | main | none | ### Filter type @@ -87,12 +88,12 @@ the IP or host name of the SC4S instance and port 514 ### Sourcetype and Index Configuration -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| zscalernss-zpa-app | zscalerlss_zpa-app | netproxy | none | -| zscalernss-zpa-auth | zscalerlss_zpa_auth | netauth | none | -| zscalernss-zpa-bba | zscalerlss_zpa_auth | netproxy | none | -| zscalernss-zpa-connector | zscalerlss_zpa_connector | netproxy | none | +| key | sourcetype | index | notes | +|----------------|--------------------------|------------|---------| +| zscaler_lss | zscalerlss_zpa-app | netproxy | none | +| zscaler_lss | zscalerlss_zpa_auth | netproxy | none | +| zscaler_lss | zscalerlss_zpa_auth | netproxy | none | +| zscaler_lss | zscalerlss_zpa_connector | netproxy | none | ### Filter type diff --git a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl index ff95eea..602df4e 100644 --- a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl @@ -74,7 +74,7 @@ log { match('.' value('.json.SAMLAttributes')) and match('.' value('.json.Customer')) }; - rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netauth"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 8a77f3d..2755202 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -69,4 +69,11 @@ #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics #symanrtec_ep,index,epav -#vmware_nsx,index,main \ No newline at end of file +#vmware_nsx,index,main +#zscaler_alerts,index,main +#zscaler_dns,index,netdns +#zscaler_fw,index,netfw +#zscaler_web,index,netproxy +#zscaler_zia_audit,index,netops +#zscaler_zia_sandbox,index,main +#zscaler_lss,index,netproxy \ No newline at end of file From 2db657a954a4e74d3172c6736fe71224b10b434f Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Tue, 28 Apr 2020 08:53:53 -0700 Subject: [PATCH 10/10] Update `test_zscaler_lss_zpa_auth` test * Update `test_zscaler_lss_zpa_auth` test to use `netproxy` index --- tests/test_zscaler_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py index 389069e..b8cbe50 100644 --- a/tests/test_zscaler_proxy.py +++ b/tests/test_zscaler_proxy.py @@ -247,7 +247,7 @@ def test_zscaler_lss_zpa_auth(record_property, setup_wordlist, setup_splunk, set message = mt.render(mark="<134>", lss_time=lss_time, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netauth sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"") + st = env.from_string("search _time={{ epoch }} index=netproxy sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"") search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search)