Merge branch 'develop' into imperva-waf-test-cases-424

CSVD · May 4, 2020 · d1072fe · d1072fe
2 parents 5229a3b + 1a06a81
commit d1072fe
Show file tree

Hide file tree

Showing 11 changed files with 122 additions and 38 deletions.
diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md
@@ -291,7 +291,7 @@ Verify timestamp, and host values match as expected
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| merkai     | None                                                                                                    |
+| meraki     | None                                                                                                    |
 
 ### Sourcetype and Index Configuration
 

diff --git a/docs/sources/Zscaler/index.md b/docs/sources/Zscaler/index.md
@@ -25,13 +25,14 @@ the IP or host name of the SC4S instance and port 514
 
 ### Sourcetype and Index Configuration
 
-| key            | sourcetype     | index          | notes          |
-|----------------|----------------|----------------|----------------|
-| zscalernss_alerts      | zscalernss-alerts       | main          | none          |
-| zscalernss_dns      | zscalernss-dns     | netdns          | none          |
-| zscalernss_fw      | zscalernss-fw       | netfw          | none          |
-| zscalernss_web      | zscalernss-web       | netproxy          | none          |
-
+| key                 | sourcetype             | index    | notes   |
+|---------------------|------------------------|----------|---------|
+| zscaler_alerts      | zscalernss-alerts      | main     | none    |
+| zscaler_dns         | zscalernss-dns         | netdns   | none    |
+| zscaler_fw          | zscalernss-fw          | netfw    | none    |
+| zscaler_web         | zscalernss-web         | netproxy | none    |
+| zscaler_zia_audit   | zscalernss-zia-audit   | netops   | none    |
+| zscaler_zia_sandbox | zscalernss-zia-sandbox | main     | none    |
 
 ### Filter type
 
@@ -87,12 +88,12 @@ the IP or host name of the SC4S instance and port 514
 
 ### Sourcetype and Index Configuration
 
-| key            | sourcetype     | index          | notes          |
-|----------------|----------------|----------------|----------------|
-| zscalernss-zpa-app      | zscalerlss_zpa-app       | netproxy          | none          |
-| zscalernss-zpa-auth      | zscalerlss_zpa_auth       | netauth          | none          |
-| zscalernss-zpa-bba      | zscalerlss_zpa_auth       | netproxy          | none          |
-| zscalernss-zpa-connector    | zscalerlss_zpa_connector       | netproxy          | none          |
+| key            | sourcetype               | index      | notes   |
+|----------------|--------------------------|------------|---------|
+| zscaler_lss    | zscalerlss_zpa-app       | netproxy   | none    |
+| zscaler_lss    | zscalerlss_zpa_auth      | netproxy   | none    |
+| zscaler_lss    | zscalerlss_zpa_auth      | netproxy   | none    |
+| zscaler_lss    | zscalerlss_zpa_connector | netproxy   | none    |
 
 
 ### Filter type

diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf
@@ -83,6 +83,23 @@ template t_JSON_5424 {
                 --exclude DATE
                 --exclude FACILITY
                 --exclude PRIORITY
+                --exclude HOST
+                )');
+    };
+
+# ===============================================================================================
+# JSON_5424_SDATA; for JSON pretty-printing (for RFC5424 messages with duplicate data in MESSAGE)
+# ===============================================================================================
+
+template t_JSON_5424_SDATA {
+    template('$(format-json --scope rfc5424
+                --pair PRI="<$PRI>"
+                --key ISODATE
+                --exclude DATE
+                --exclude HOST
+                --exclude FACILITY
+                --exclude PRIORITY
+                --exclude MESSAGE
                 )');
     };
 

diff --git a/package/etc/conf.d/destinations/rawmsg_file.conf b/package/etc/conf.d/destinations/rawmsg_file.conf
@@ -1,5 +1,11 @@
 destination d_rawmsg {
-    file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log"
-         template("${RAWMSG}\n")
-     );
-};
+    channel {
+        if ("${RAWMSG}" ne "") {
+            destination {
+                file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log"
+                template("${RAWMSG}\n")
+                );
+            };
+        };
+    };
+};
diff --git a/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl
@@ -0,0 +1,38 @@
+# IETF Syslog
+
+log {
+    junction {
+        channel {
+        # Listen on the default port (typically 601) for IETF_SYSLOG traffic
+            source (s_ietf);
+            flags(final);
+        };
+    };
+
+    rewrite {
+        set("IETF_SYSLOG", value("fields.sc4s_vendor_product"));
+    };
+
+    rewrite { r_set_splunk_dest_default(sourcetype("ietf:syslog"), index("main"), source("${APP}:${PROGRAM}")) };
+    parser { p_add_context_splunk(key("IETF_SYSLOG")); };
+    parser (compliance_meta_by_source);
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); };
+
+{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_IETF_SYSLOG_HEC" "no")) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_IETF_SYSLOG" "no")) }}
+    destination(d_archive);
+{{- end}}
+
+{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
+    {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+{{- end }}
+
+{{- if (print (getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES")) }}
+    {{ getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+{{- end }}
+
+    flags(flow-control,final);
+};
diff --git a/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl b/package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl
@@ -24,17 +24,11 @@ log {
 
     rewrite {
         set("mcafee_epo", value("fields.sc4s_vendor_product"));
+        r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav"))
     };
-    rewrite { r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav")) };
     parser {p_add_context_splunk(key("mcafee_epo")); };
 
-
     parser (compliance_meta_by_source);
-
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
     rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
 
 {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_MCAFEE_EPO_STRUCTURED_HEC" "no")) }}

diff --git a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl
@@ -47,8 +47,6 @@ log {
         };
         rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-app"), index("netproxy"))};
         parser { p_add_context_splunk(key("zscaler_lss")); };
-        parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
     } elif {
         filter {
             match('.' value('.json.Exporter'))
@@ -57,8 +55,6 @@ log {
         };
         rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-bba"), index("netproxy"))};
         parser { p_add_context_splunk(key("zscaler_lss")); };
-        parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };    
     } elif {
         filter {
             match('.' value('.json.Connector'))
@@ -67,29 +63,27 @@ log {
         };
         rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-connector"), index("netproxy"))};
         parser { p_add_context_splunk(key("zscaler_lss")); };
-        parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };    
     } elif {
         filter {
             match('.' value('.json.SAMLAttributes'))
             and match('.' value('.json.Customer'))
         };
-        rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netauth"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netproxy"))};
         parser { p_add_context_splunk(key("zscaler_lss")); };
-        parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };    
     } else {
         rewrite {
             set("zscaler_lss_rogue_message", value("fields.sc4s_vendor_product"));
             set("Possible rogue message on zscaler_lss unique port", value("fields.sc4s_error"));
             r_set_splunk_dest_default(sourcetype("zscalerlss:rogue"), index("netproxy"))
         };
         parser { p_add_context_splunk(key("zscaler_lss")); };
-        parser (compliance_meta_by_source);
+        # Rogue message needs a different template than valid LSS events.  Final rewrite (further below) will be a
+        # no-op in this case.
         rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };
     };
 
-
+    # Parser for all valid LSS events.  Rogue events, having previously loaded $MSG with the entire payload,
+    # will be unaffected by the rewrite here.
     parser (compliance_meta_by_source);
     rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
 

diff --git a/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl b/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl
@@ -1,6 +1,7 @@
 # Fallback for un-parsed sources
 
 log {
+    source(s_ietf);
     source(s_DEFAULT);
 
     rewrite { set("SC4S_fallback", value("fields.sc4s_vendor_product")); };

diff --git a/package/etc/conf.d/sources/rfc5687.conf.tmpl b/package/etc/conf.d/sources/rfc5687.conf.tmpl
@@ -0,0 +1,25 @@
+source s_ietf {
+    channel {
+        source {
+            syslog (
+                transport("tcp")
+                port(601)
+                ip-protocol(4)
+                keep-hostname(yes)
+                keep-timestamp(yes)
+                use-dns(no)
+                use-fqdn(no)
+                chain-hostnames(off)
+                flags(validate-utf8, syslog-protocol)
+            );    
+        };
+
+        if {
+            parser { app-parser(topic(syslog)); };
+        };
+        rewrite(set_rfc5424_strict);
+        parser {
+                vendor_product_by_source();
+        };
+    };
+};
diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example
@@ -52,6 +52,7 @@
 #juniper_nsm,index,netfw
 #juniper_nsm_idp,index,netids
 #juniper_legacy,index,netops
+#mcafee_epo,index,epav
 #nix_syslog,index,osnix
 #pan_traffic,index,netfw
 #pan_threat,index,netproxy
@@ -69,4 +70,11 @@
 #sc4s_fallback,index,main
 #sc4s_metrics,index,em_metrics
 #symanrtec_ep,index,epav
-#vmware_nsx,index,main
+#vmware_nsx,index,main
+#zscaler_alerts,index,main
+#zscaler_dns,index,netdns
+#zscaler_fw,index,netfw
+#zscaler_web,index,netproxy
+#zscaler_zia_audit,index,netops
+#zscaler_zia_sandbox,index,main
+#zscaler_lss,index,netproxy
diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py
@@ -247,7 +247,7 @@ def test_zscaler_lss_zpa_auth(record_property, setup_wordlist, setup_splunk, set
     message = mt.render(mark="<134>", lss_time=lss_time, host=host)
     sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
 
-    st = env.from_string("search _time={{ epoch }} index=netauth sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"")
+    st = env.from_string("search _time={{ epoch }} index=netproxy sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"")
     search = st.render(epoch=epoch, host=host)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)