Fixes #144 Add ZScaler support (#159)

CSVD · Oct 16, 2019 · 3a5e588 · 3a5e588
1 parent d9394a8
commit 3a5e588
Show file tree

Hide file tree

Showing 5 changed files with 202 additions and 1 deletion.
diff --git a/docs/sources.md b/docs/sources.md
@@ -896,3 +896,71 @@ An active proxy will generate frequent events. Use the following search to valid
 ```
 index=<asconfigured> sourcetype=bluecoat:proxysg:access:kv | stats count by host
 ```
+
+
+# Vendor - Zscaler
+
+## Product - All Products
+
+The ZScaler product manual includes and extensive section of configuration for multiple Splunk TCP input ports around page
+26. When using SC4S these ports are not required and should not be used. Simply configure all outputs from the NSS to utilize
+the IP or host name of the SC4S instance and port 514
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/3865/                                                                 |
+| Product Manual | https://community.zscaler.com/t/zscaler-splunk-app-design-and-installation-documentation/4728                                                      |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| zscalernss-alerts  | Requires format customization add ``\tvendor=Zscaler\tproduct=alerts`` immediately prior to the ``\n`` in the NSS Alert Web format. See Zscaler manual for more info. |
+| zscalernss-dns  | Requires format customization  add ``\tvendor=Zscaler\tproduct=dns`` immediately prior to the ``\n`` in the NSS DNS format. See Zscaler manual for more info. |
+| zscalernss-web  | None    |
+| zscalernss-zpa-app  | Requires format customization  add ``\tvendor=Zscaler\tproduct=zpa`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. |
+| zscalernss-zpa-auth  | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. |
+| zscalernss-zpa-connector  | Requires format customization  add ``\tvendor=Zscaler\tproduct=zpa_auth_connector`` immediately prior to the ``\n`` in the LSS Connector format. See Zscaler manual for more info. |
+| zscalernss-fw  | Requires format customization add ``\tvendor=Zscaler\tproduct=fw`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. |
+
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| zscalernss_alerts      | zscalernss-alerts       | main          | none          |
+| zscalernss_dns      | zscalernss-dns     | netdns          | none          |
+| zscalernss_fw      | zscalernss-fw       | netfw          | none          |
+| zscalernss_web      | zscalernss-web       | netproxy          | none          |
+| zscalernss-zpa-app      | zscalernss_zpa-app       | netids          | none          |
+| zscalernss-zpa-auth      | zscalernss_zpa_auth       | netauth          | none          |
+| zscalernss-zpa-connector    | zscalernss_zpa_connector       | netops          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_ZSCALER_NSS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=zscalernss-* | stats count by host
+```
diff --git a/package/etc/conf.d/filters/zscaler/nss.conf b/package/etc/conf.d/filters/zscaler/nss.conf
@@ -0,0 +1,3 @@
+filter f_zscaler_nss {
+    message('\tvendor=Zscaler\t');
+};
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl
@@ -0,0 +1,75 @@
+# Proofpoint
+{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }}
+{{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+{{- end -}}
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_default-ports);
+    filter(f_zscaler_nss);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_dedicated_port_ZSCALER_NSS);
+{{- end }}
+
+    rewrite {
+        subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+    };
+    parser {
+        #basic parsing
+        kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}"));
+    };
+
+    if (match("alerts" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_alerts")); };
+    } elif (match("dns" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_dns")); };
+    } elif (match("fw" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_fw")); };
+    } elif (match("NSS" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_web")); };
+    } elif (match("audit" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_zia_audit")); };
+    } elif (match("sandbox" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_zia_sandbox")); };
+    } elif (match("zpa" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_zpa")); };
+    } elif (match("zpa_auth" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_zpa_auth")); };
+    } elif (match("zpa_auth_connector" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_zpa_connector")); };
+    } elif (match("zpa_bba" value(".kv.product"))) {
+       rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"), template("t_msg_only"))};
+       parser { p_add_context_splunk(key("zscaler_zpa_bba")); };
+    } else {
+        rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"), template("t_msg_only"))};
+        parser {
+            p_add_context_splunk(key("zscaler_nss"));
+        };
+    };
+
+
+    parser (compliance_meta_by_source);
+
+    destination(d_hec);  #--HEC--
+
+    flags(flow-control);
+};
+{{- end}}
+{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }}
+# Listen on the specified dedicated port(s) for ZSCALER_NSS traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for ZSCALER_NSS traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/package/syslog-ng b/package/syslog-ng
diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py
@@ -0,0 +1,55 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import random
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+#Note the long white space is a \t
+#2019-10-16 15:44:36    reason=Allowed    event_id=6748427317914894361    protocol=HTTPS    action=Allowed    transactionsize=663    responsesize=65    requestsize=598    urlcategory=UK_ALLOW_Pharmacies    serverip=216.58.204.70    clienttranstime=0    requestmethod=CONNECT    refererURL=None    useragent=Windows Windows 10 Enterprise ZTunnel/1.0    product=NSS    location=UK_Wynyard_VPN->other    ClientIP=192.168.0.38    status=200    user=first.last@example.com    url=4171764.fls.doubleclick.net:443    vendor=Zscaler    hostname=4171764.fls.doubleclick.net    clientpublicIP=213.86.221.94    threatcategory=None    threatname=None    filetype=None    appname=DoubleClick    pagerisk=0    department=Procurement, Generics    urlsupercategory=User-defined    appclass=Sales and Marketing    dlpengine=None    urlclass=Bandwidth Loss    threatclass=None    dlpdictionaries=None    fileclass=None    bwthrottle=NO    servertranstime=0    md5=None
+def test_zscaler_proxy(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894361\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None")
+    message = mt.render(mark="<134>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net  | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#
+def test_zscaler_proxy_pri(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{mark}}{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894362\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None")
+    message = mt.render(mark="<134>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#