diff --git a/docs/sources.md b/docs/sources.md index 16c7f0c..f7e85cb 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -896,3 +896,71 @@ An active proxy will generate frequent events. Use the following search to valid ``` index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` + + +# Vendor - Zscaler + +## Product - All Products + +The ZScaler product manual includes and extensive section of configuration for multiple Splunk TCP input ports around page +26. When using SC4S these ports are not required and should not be used. Simply configure all outputs from the NSS to utilize +the IP or host name of the SC4S instance and port 514 + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3865/ | +| Product Manual | https://community.zscaler.com/t/zscaler-splunk-app-design-and-installation-documentation/4728 | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| zscalernss-alerts | Requires format customization add ``\tvendor=Zscaler\tproduct=alerts`` immediately prior to the ``\n`` in the NSS Alert Web format. See Zscaler manual for more info. | +| zscalernss-dns | Requires format customization add ``\tvendor=Zscaler\tproduct=dns`` immediately prior to the ``\n`` in the NSS DNS format. See Zscaler manual for more info. | +| zscalernss-web | None | +| zscalernss-zpa-app | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | +| zscalernss-zpa-auth | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | +| zscalernss-zpa-connector | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth_connector`` immediately prior to the ``\n`` in the LSS Connector format. See Zscaler manual for more info. | +| zscalernss-fw | Requires format customization add ``\tvendor=Zscaler\tproduct=fw`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| zscalernss_alerts | zscalernss-alerts | main | none | +| zscalernss_dns | zscalernss-dns | netdns | none | +| zscalernss_fw | zscalernss-fw | netfw | none | +| zscalernss_web | zscalernss-web | netproxy | none | +| zscalernss-zpa-app | zscalernss_zpa-app | netids | none | +| zscalernss-zpa-auth | zscalernss_zpa_auth | netauth | none | +| zscalernss-zpa-connector | zscalernss_zpa_connector | netops | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_ZSCALER_NSS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=zscalernss-* | stats count by host +``` diff --git a/package/etc/conf.d/filters/zscaler/nss.conf b/package/etc/conf.d/filters/zscaler/nss.conf new file mode 100644 index 0000000..9ee4e1a --- /dev/null +++ b/package/etc/conf.d/filters/zscaler/nss.conf @@ -0,0 +1,3 @@ +filter f_zscaler_nss { + message('\tvendor=Zscaler\t'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl new file mode 100644 index 0000000..7bd86fc --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -0,0 +1,75 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_zscaler_nss); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_ZSCALER_NSS); +{{- end }} + + rewrite { + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + }; + parser { + #basic parsing + kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}")); + }; + + if (match("alerts" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_alerts")); }; + } elif (match("dns" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_dns")); }; + } elif (match("fw" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_fw")); }; + } elif (match("NSS" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_web")); }; + } elif (match("audit" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zia_audit")); }; + } elif (match("sandbox" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zia_sandbox")); }; + } elif (match("zpa" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa")); }; + } elif (match("zpa_auth" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_auth")); }; + } elif (match("zpa_auth_connector" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_connector")); }; + } elif (match("zpa_bba" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_bba")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("zscaler_nss")); + }; + }; + + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for ZSCALER_NSS traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for ZSCALER_NSS traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/syslog-ng b/package/syslog-ng index f219fbb..26c0fe2 160000 --- a/package/syslog-ng +++ b/package/syslog-ng @@ -1 +1 @@ -Subproject commit f219fbbb12dad0b0b6cb458a0bb4415b2cd94d8a +Subproject commit 26c0fe20a7169a2007de561ff3c4bc7df93ca86b diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py new file mode 100644 index 0000000..e0f7fb1 --- /dev/null +++ b/tests/test_zscaler_proxy.py @@ -0,0 +1,55 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +#Note the long white space is a \t +#2019-10-16 15:44:36 reason=Allowed event_id=6748427317914894361 protocol=HTTPS action=Allowed transactionsize=663 responsesize=65 requestsize=598 urlcategory=UK_ALLOW_Pharmacies serverip=216.58.204.70 clienttranstime=0 requestmethod=CONNECT refererURL=None useragent=Windows Windows 10 Enterprise ZTunnel/1.0 product=NSS location=UK_Wynyard_VPN->other ClientIP=192.168.0.38 status=200 user=first.last@example.com url=4171764.fls.doubleclick.net:443 vendor=Zscaler hostname=4171764.fls.doubleclick.net clientpublicIP=213.86.221.94 threatcategory=None threatname=None filetype=None appname=DoubleClick pagerisk=0 department=Procurement, Generics urlsupercategory=User-defined appclass=Sales and Marketing dlpengine=None urlclass=Bandwidth Loss threatclass=None dlpdictionaries=None fileclass=None bwthrottle=NO servertranstime=0 md5=None +def test_zscaler_proxy(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894361\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# +def test_zscaler_proxy_pri(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894362\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#