diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 1aeea8d..47f6493 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -159,7 +159,6 @@ sudo bash /opt/sc4s/bin/preconfig.sh SYSLOGNG_OPTS=-f /etc/syslog-ng/syslog-ng.conf SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index e216dc3..dea3f12 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -107,7 +107,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index d1c795d..8c1d6f1 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -115,7 +115,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 4cc1a49..f7fde28 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -113,7 +113,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 3ebdade..7bb6ee6 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -1,8 +1,37 @@ -# WARNING: Do _not_ use Podman with RHEL/CentOS 7.x or earlier! +# UPDATE: Podman/RHEL UDP data block issue: netfilter connection table -There have been cases where UDP packet loss is noted when Podman is used with RHEL/CentOS 7.x versions. Stay tuned; the cause is -currently unkown. +We have determined the root cause for the issue with UDP data blocking and Podman/RHEL. The crux of the issue is that the netfilter +connection tables are _not_ udpdated when a new container starts _and_ there is a constant stream of UDP traffic from a given IP destined +for a given port. The table is _only_ updated if the trafffic pauses for the length of the connection table timeout (30 seconds by default). + +Therefore, if you attempt to start up sc4s on a server to which, for example, a firewall is sending a steady stream of UDP events, the kernel +will mistakenly keep trying to route the packets to the server itself rather than through the virtual network created by the new container. +Until the firewall pauses its output stream (unlikely) _or_ the workaround provided below is applied, traffic from that particular firewall +will never been seen by the container (and hence sc4s). + +## WORKAROUND + +There is a utility called `conntrack` that allows you to view/manipulate the netfilter connection tables in real time. Follow the steps below +to install and run it each time sc4s starts. It should be available in all RHEL 7/8 subscriptions. + +``` + install conntrack +``` + +After this is done, add the following entry to the unit file (and/or use the command when starting sc4s manually): + +``` +ExecStartPost=sleep 2; conntrack -D -p udp +``` + +This command will delete the old (stale) UDP entries two seconds after the container starts and allow the system to build a new table that +will properly route to the container when it sees UDP traffic. Note that this command resets the table for _all_ UDP +ports; for a purpose-built sc4s server this should not cause issues. If for any reason more granular control over _which_ UPD ports are +reset is desired, there are additional arguments to `conntrack` that can be used to select the specific UDP ports that are deleted in the +table. See the man page for `conntrack` for more information. + +The unit file entry above has been added to the example below for completeness. # Install podman @@ -53,6 +82,7 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ "$SC4S_TLS_DIR" \ --name SC4S \ --rm $SC4S_IMAGE +ExecStartPost=sleep 2; conntrack -D -p udp ``` * Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication @@ -100,7 +130,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index 9cf0707..9467fa2 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -291,7 +291,7 @@ Verify timestamp, and host values match as expected | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| merkai | None | +| meraki | None | ### Sourcetype and Index Configuration diff --git a/docs/sources/Zscaler/index.md b/docs/sources/Zscaler/index.md index d922ef7..c087e3b 100644 --- a/docs/sources/Zscaler/index.md +++ b/docs/sources/Zscaler/index.md @@ -25,13 +25,14 @@ the IP or host name of the SC4S instance and port 514 ### Sourcetype and Index Configuration -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| zscalernss_alerts | zscalernss-alerts | main | none | -| zscalernss_dns | zscalernss-dns | netdns | none | -| zscalernss_fw | zscalernss-fw | netfw | none | -| zscalernss_web | zscalernss-web | netproxy | none | - +| key | sourcetype | index | notes | +|---------------------|------------------------|----------|---------| +| zscaler_alerts | zscalernss-alerts | main | none | +| zscaler_dns | zscalernss-dns | netdns | none | +| zscaler_fw | zscalernss-fw | netfw | none | +| zscaler_web | zscalernss-web | netproxy | none | +| zscaler_zia_audit | zscalernss-zia-audit | netops | none | +| zscaler_zia_sandbox | zscalernss-zia-sandbox | main | none | ### Filter type @@ -87,12 +88,12 @@ the IP or host name of the SC4S instance and port 514 ### Sourcetype and Index Configuration -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| zscalernss-zpa-app | zscalerlss_zpa-app | netproxy | none | -| zscalernss-zpa-auth | zscalerlss_zpa_auth | netauth | none | -| zscalernss-zpa-bba | zscalerlss_zpa_auth | netproxy | none | -| zscalernss-zpa-connector | zscalerlss_zpa_connector | netproxy | none | +| key | sourcetype | index | notes | +|----------------|--------------------------|------------|---------| +| zscaler_lss | zscalerlss_zpa-app | netproxy | none | +| zscaler_lss | zscalerlss_zpa_auth | netproxy | none | +| zscaler_lss | zscalerlss_zpa_auth | netproxy | none | +| zscaler_lss | zscalerlss_zpa_connector | netproxy | none | ### Filter type diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index c86400f..fc0c7a0 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -83,6 +83,23 @@ template t_JSON_5424 { --exclude DATE --exclude FACILITY --exclude PRIORITY + --exclude HOST + )'); + }; + +# =============================================================================================== +# JSON_5424_SDATA; for JSON pretty-printing (for RFC5424 messages with duplicate data in MESSAGE) +# =============================================================================================== + +template t_JSON_5424_SDATA { + template('$(format-json --scope rfc5424 + --pair PRI="<$PRI>" + --key ISODATE + --exclude DATE + --exclude HOST + --exclude FACILITY + --exclude PRIORITY + --exclude MESSAGE )'); }; diff --git a/package/etc/conf.d/destinations/rawmsg_file.conf b/package/etc/conf.d/destinations/rawmsg_file.conf index f5faf88..8ca2f79 100644 --- a/package/etc/conf.d/destinations/rawmsg_file.conf +++ b/package/etc/conf.d/destinations/rawmsg_file.conf @@ -1,5 +1,11 @@ destination d_rawmsg { - file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" - template("${RAWMSG}\n") - ); -}; + channel { + if ("${RAWMSG}" ne "") { + destination { + file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" + template("${RAWMSG}\n") + ); + }; + }; + }; +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl new file mode 100644 index 0000000..35366a8 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl @@ -0,0 +1,38 @@ +# IETF Syslog + +log { + junction { + channel { + # Listen on the default port (typically 601) for IETF_SYSLOG traffic + source (s_ietf); + flags(final); + }; + }; + + rewrite { + set("IETF_SYSLOG", value("fields.sc4s_vendor_product")); + }; + + rewrite { r_set_splunk_dest_default(sourcetype("ietf:syslog"), index("main"), source("${APP}:${PROGRAM}")) }; + parser { p_add_context_splunk(key("IETF_SYSLOG")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_IETF_SYSLOG_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_IETF_SYSLOG" "no")) }} + destination(d_archive); +{{- end}} + +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES")) }} + {{ getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + + flags(flow-control,final); +}; diff --git a/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl b/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl index 671fc5e..36419fb 100644 --- a/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl @@ -24,17 +24,11 @@ log { rewrite { set("mcafee_epo", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav")) }; - rewrite { r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav")) }; parser {p_add_context_splunk(key("mcafee_epo")); }; - parser (compliance_meta_by_source); - - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_MCAFEE_EPO_STRUCTURED_HEC" "no")) }} diff --git a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl index baf6edc..7d5344b 100644 --- a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl @@ -10,6 +10,14 @@ log { channel { # Listen on the specified dedicated port(s) for ZSCALER_LSS traffic source (s_ZSCALER_LSS); + parser { + #.jsonLog.Timestamp Mar 04 20:37:53 2020 + date-parser-nofilter( + format('%a %b %d %H:%M:%S %Y', + '%a %b %d %k:%M:%S %Y') + template("${.json.LogTimestamp}") + ); + }; flags (final); }; {{- end}} @@ -17,17 +25,20 @@ log { # Listen on the default port (typically 514) for ZSCALER_LSS traffic source (s_DEFAULT); filter(f_msg_is_tcp_json); + parser { + #.jsonLog.Timestamp Mar 04 20:37:53 2020 + date-parser( + format('%a %b %d %H:%M:%S %Y', + '%a %b %d %k:%M:%S %Y') + template("${.json.LogTimestamp}") + time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) + flags(guess-timezone) + ); + }; flags(final); }; }; - parser { - #.jsonLog.Timestamp Mar 04 20:37:53 2020 - date-parser-nofilter( - format('%a %b %d %H:%M:%S %Y', - '%a %b %d %k:%M:%S %Y') - template("${.json.LogTimestamp}") - ); - }; + if { filter { match('.' value('.json.ClientZEN')) @@ -36,8 +47,6 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-app"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } elif { filter { match('.' value('.json.Exporter')) @@ -46,8 +55,6 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-bba"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } elif { filter { match('.' value('.json.Connector')) @@ -56,20 +63,27 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-connector"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } elif { filter { match('.' value('.json.SAMLAttributes')) and match('.' value('.json.Customer')) }; - rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netauth"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_lss")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + } else { + rewrite { + set("zscaler_lss_rogue_message", value("fields.sc4s_vendor_product")); + set("Possible rogue message on zscaler_lss unique port", value("fields.sc4s_error")); + r_set_splunk_dest_default(sourcetype("zscalerlss:rogue"), index("netproxy")) + }; + parser { p_add_context_splunk(key("zscaler_lss")); }; + # Rogue message needs a different template than valid LSS events. Final rewrite (further below) will be a + # no-op in this case. + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; }; - + # Parser for all valid LSS events. Rogue events, having previously loaded $MSG with the entire payload, + # will be unaffected by the rewrite here. parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; diff --git a/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl b/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl index a3a33d9..d8bbd88 100644 --- a/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl @@ -1,6 +1,7 @@ # Fallback for un-parsed sources log { + source(s_ietf); source(s_DEFAULT); rewrite { set("SC4S_fallback", value("fields.sc4s_vendor_product")); }; diff --git a/package/etc/conf.d/sources/rfc5687.conf.tmpl b/package/etc/conf.d/sources/rfc5687.conf.tmpl new file mode 100644 index 0000000..b5044b9 --- /dev/null +++ b/package/etc/conf.d/sources/rfc5687.conf.tmpl @@ -0,0 +1,25 @@ +source s_ietf { + channel { + source { + syslog ( + transport("tcp") + port(601) + ip-protocol(4) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(validate-utf8, syslog-protocol) + ); + }; + + if { + parser { app-parser(topic(syslog)); }; + }; + rewrite(set_rfc5424_strict); + parser { + vendor_product_by_source(); + }; + }; +}; diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 361ed9d..694dec6 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -53,6 +53,7 @@ #juniper_nsm,index,netfw #juniper_nsm_idp,index,netids #juniper_legacy,index,netops +#mcafee_epo,index,epav #nix_syslog,index,osnix #pan_traffic,index,netfw #pan_threat,index,netproxy @@ -70,4 +71,11 @@ #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics #symanrtec_ep,index,epav -#vmware_nsx,index,main \ No newline at end of file +#vmware_nsx,index,main +#zscaler_alerts,index,main +#zscaler_dns,index,netdns +#zscaler_fw,index,netfw +#zscaler_web,index,netproxy +#zscaler_zia_audit,index,netops +#zscaler_zia_sandbox,index,main +#zscaler_lss,index,netproxy \ No newline at end of file diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py index 389069e..b8cbe50 100644 --- a/tests/test_zscaler_proxy.py +++ b/tests/test_zscaler_proxy.py @@ -247,7 +247,7 @@ def test_zscaler_lss_zpa_auth(record_property, setup_wordlist, setup_splunk, set message = mt.render(mark="<134>", lss_time=lss_time, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netauth sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"") + st = env.from_string("search _time={{ epoch }} index=netproxy sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"") search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search)