diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index 9cf0707..9467fa2 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -291,7 +291,7 @@ Verify timestamp, and host values match as expected | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| merkai | None | +| meraki | None | ### Sourcetype and Index Configuration diff --git a/docs/sources/Zscaler/index.md b/docs/sources/Zscaler/index.md index d922ef7..c087e3b 100644 --- a/docs/sources/Zscaler/index.md +++ b/docs/sources/Zscaler/index.md @@ -25,13 +25,14 @@ the IP or host name of the SC4S instance and port 514 ### Sourcetype and Index Configuration -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| zscalernss_alerts | zscalernss-alerts | main | none | -| zscalernss_dns | zscalernss-dns | netdns | none | -| zscalernss_fw | zscalernss-fw | netfw | none | -| zscalernss_web | zscalernss-web | netproxy | none | - +| key | sourcetype | index | notes | +|---------------------|------------------------|----------|---------| +| zscaler_alerts | zscalernss-alerts | main | none | +| zscaler_dns | zscalernss-dns | netdns | none | +| zscaler_fw | zscalernss-fw | netfw | none | +| zscaler_web | zscalernss-web | netproxy | none | +| zscaler_zia_audit | zscalernss-zia-audit | netops | none | +| zscaler_zia_sandbox | zscalernss-zia-sandbox | main | none | ### Filter type @@ -87,12 +88,12 @@ the IP or host name of the SC4S instance and port 514 ### Sourcetype and Index Configuration -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| zscalernss-zpa-app | zscalerlss_zpa-app | netproxy | none | -| zscalernss-zpa-auth | zscalerlss_zpa_auth | netauth | none | -| zscalernss-zpa-bba | zscalerlss_zpa_auth | netproxy | none | -| zscalernss-zpa-connector | zscalerlss_zpa_connector | netproxy | none | +| key | sourcetype | index | notes | +|----------------|--------------------------|------------|---------| +| zscaler_lss | zscalerlss_zpa-app | netproxy | none | +| zscaler_lss | zscalerlss_zpa_auth | netproxy | none | +| zscaler_lss | zscalerlss_zpa_auth | netproxy | none | +| zscaler_lss | zscalerlss_zpa_connector | netproxy | none | ### Filter type diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index c86400f..fc0c7a0 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -83,6 +83,23 @@ template t_JSON_5424 { --exclude DATE --exclude FACILITY --exclude PRIORITY + --exclude HOST + )'); + }; + +# =============================================================================================== +# JSON_5424_SDATA; for JSON pretty-printing (for RFC5424 messages with duplicate data in MESSAGE) +# =============================================================================================== + +template t_JSON_5424_SDATA { + template('$(format-json --scope rfc5424 + --pair PRI="<$PRI>" + --key ISODATE + --exclude DATE + --exclude HOST + --exclude FACILITY + --exclude PRIORITY + --exclude MESSAGE )'); }; diff --git a/package/etc/conf.d/destinations/rawmsg_file.conf b/package/etc/conf.d/destinations/rawmsg_file.conf index f5faf88..8ca2f79 100644 --- a/package/etc/conf.d/destinations/rawmsg_file.conf +++ b/package/etc/conf.d/destinations/rawmsg_file.conf @@ -1,5 +1,11 @@ destination d_rawmsg { - file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" - template("${RAWMSG}\n") - ); -}; + channel { + if ("${RAWMSG}" ne "") { + destination { + file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" + template("${RAWMSG}\n") + ); + }; + }; + }; +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl new file mode 100644 index 0000000..35366a8 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl @@ -0,0 +1,38 @@ +# IETF Syslog + +log { + junction { + channel { + # Listen on the default port (typically 601) for IETF_SYSLOG traffic + source (s_ietf); + flags(final); + }; + }; + + rewrite { + set("IETF_SYSLOG", value("fields.sc4s_vendor_product")); + }; + + rewrite { r_set_splunk_dest_default(sourcetype("ietf:syslog"), index("main"), source("${APP}:${PROGRAM}")) }; + parser { p_add_context_splunk(key("IETF_SYSLOG")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_IETF_SYSLOG_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_IETF_SYSLOG" "no")) }} + destination(d_archive); +{{- end}} + +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES")) }} + {{ getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + + flags(flow-control,final); +}; diff --git a/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl b/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl index 671fc5e..36419fb 100644 --- a/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl @@ -24,17 +24,11 @@ log { rewrite { set("mcafee_epo", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav")) }; - rewrite { r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav")) }; parser {p_add_context_splunk(key("mcafee_epo")); }; - parser (compliance_meta_by_source); - - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_MCAFEE_EPO_STRUCTURED_HEC" "no")) }} diff --git a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl index ff95eea..7d5344b 100644 --- a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl @@ -47,8 +47,6 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-app"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } elif { filter { match('.' value('.json.Exporter')) @@ -57,8 +55,6 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-bba"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } elif { filter { match('.' value('.json.Connector')) @@ -67,17 +63,13 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-connector"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } elif { filter { match('.' value('.json.SAMLAttributes')) and match('.' value('.json.Customer')) }; - rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netauth"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } else { rewrite { set("zscaler_lss_rogue_message", value("fields.sc4s_vendor_product")); @@ -85,11 +77,13 @@ log { r_set_splunk_dest_default(sourcetype("zscalerlss:rogue"), index("netproxy")) }; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); + # Rogue message needs a different template than valid LSS events. Final rewrite (further below) will be a + # no-op in this case. rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; }; - + # Parser for all valid LSS events. Rogue events, having previously loaded $MSG with the entire payload, + # will be unaffected by the rewrite here. parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; diff --git a/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl b/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl index a3a33d9..d8bbd88 100644 --- a/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl @@ -1,6 +1,7 @@ # Fallback for un-parsed sources log { + source(s_ietf); source(s_DEFAULT); rewrite { set("SC4S_fallback", value("fields.sc4s_vendor_product")); }; diff --git a/package/etc/conf.d/sources/rfc5687.conf.tmpl b/package/etc/conf.d/sources/rfc5687.conf.tmpl new file mode 100644 index 0000000..b5044b9 --- /dev/null +++ b/package/etc/conf.d/sources/rfc5687.conf.tmpl @@ -0,0 +1,25 @@ +source s_ietf { + channel { + source { + syslog ( + transport("tcp") + port(601) + ip-protocol(4) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(validate-utf8, syslog-protocol) + ); + }; + + if { + parser { app-parser(topic(syslog)); }; + }; + rewrite(set_rfc5424_strict); + parser { + vendor_product_by_source(); + }; + }; +}; diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 8a77f3d..d7edbdb 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -52,6 +52,7 @@ #juniper_nsm,index,netfw #juniper_nsm_idp,index,netids #juniper_legacy,index,netops +#mcafee_epo,index,epav #nix_syslog,index,osnix #pan_traffic,index,netfw #pan_threat,index,netproxy @@ -69,4 +70,11 @@ #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics #symanrtec_ep,index,epav -#vmware_nsx,index,main \ No newline at end of file +#vmware_nsx,index,main +#zscaler_alerts,index,main +#zscaler_dns,index,netdns +#zscaler_fw,index,netfw +#zscaler_web,index,netproxy +#zscaler_zia_audit,index,netops +#zscaler_zia_sandbox,index,main +#zscaler_lss,index,netproxy \ No newline at end of file diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py index 389069e..b8cbe50 100644 --- a/tests/test_zscaler_proxy.py +++ b/tests/test_zscaler_proxy.py @@ -247,7 +247,7 @@ def test_zscaler_lss_zpa_auth(record_property, setup_wordlist, setup_splunk, set message = mt.render(mark="<134>", lss_time=lss_time, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netauth sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"") + st = env.from_string("search _time={{ epoch }} index=netproxy sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"") search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search)