Merge branch 'develop' into f5_filters_for_irule_and_asm

docs/gettingstarted/byoe-rhel7.md

@@ -159,7 +159,6 @@ sudo bash /opt/sc4s/bin/preconfig.sh
 SYSLOGNG_OPTS=-f /etc/syslog-ng/syslog-ng.conf 
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

docs/gettingstarted/docker-swarm-general.md

@@ -107,7 +107,6 @@ SC4S is almost entirely controlled through environment variables, which are read
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

docs/gettingstarted/docker-swarm-rhel7.md

@@ -115,7 +115,6 @@ SC4S is almost entirely controlled through environment variables, which are read
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

docs/gettingstarted/docker-systemd-general.md

@@ -113,7 +113,6 @@ SC4S is almost entirely controlled through environment variables, which are read
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

docs/gettingstarted/podman-systemd-general.md

@@ -1,8 +1,37 @@
 
-# WARNING:  Do _not_ use Podman with RHEL/CentOS 7.x or earlier!
+# UPDATE:  Podman/RHEL UDP data block issue: netfilter connection table
 
-There have been cases where UDP packet loss is noted when Podman is used with RHEL/CentOS 7.x versions.  Stay tuned; the cause is
-currently unkown.
+We have determined the root cause for the issue with UDP data blocking and Podman/RHEL.  The crux of the issue is that the netfilter
+connection tables are _not_ udpdated when a new container starts _and_ there is a constant stream of UDP traffic from a given IP destined
+for a given port.  The table is _only_ updated if the trafffic pauses for the length of the connection table timeout (30 seconds by default).  
+
+Therefore, if you attempt to start up sc4s on a server to which, for example, a firewall is sending a steady stream of UDP events, the kernel
+will mistakenly keep trying to route the packets to the server itself rather than through the virtual network created by the new container.
+Until the firewall pauses its output stream (unlikely) _or_ the workaround provided below is applied, traffic from that particular firewall
+will never been seen by the container (and hence sc4s).
+
+## WORKAROUND
+
+There is a utility called `conntrack` that allows you to view/manipulate the netfilter connection tables in real time. Follow the steps below
+to install and run it each time sc4s starts.  It should be available in all RHEL 7/8 subscriptions.
+
+```
+<dnf or yum> install conntrack
+```
+
+After this is done, add the following entry to the unit file (and/or use the command when starting sc4s manually):
+
+```
+ExecStartPost=sleep 2; conntrack -D -p udp
+```
+
+This command will delete the old (stale) UDP entries two seconds after the container starts and allow the system to build a new table that
+will properly route to the container when it sees UDP traffic.  Note that this command resets the table for _all_ UDP
+ports; for a purpose-built sc4s server this should not cause issues.  If for any reason more granular control over _which_ UPD ports are
+reset is desired, there are additional arguments to `conntrack` that can be used to select the specific UDP ports that are deleted in the
+table.  See the man page for `conntrack` for more information.
+
+The unit file entry above has been added to the example below for completeness.
 
 # Install podman
 
@@ -53,6 +82,7 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \
         "$SC4S_TLS_DIR" \
         --name SC4S \
         --rm $SC4S_IMAGE
+ExecStartPost=sleep 2; conntrack -D -p udp
 ```
 
 * Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication
@@ -100,7 +130,6 @@ SC4S is almost entirely controlled through environment variables, which are read
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

docs/sources/Cisco/index.md

@@ -291,7 +291,7 @@ Verify timestamp, and host values match as expected
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| merkai     | None                                                                                                    |
+| meraki     | None                                                                                                    |
 
 ### Sourcetype and Index Configuration
 

docs/sources/Zscaler/index.md

@@ -25,13 +25,14 @@ the IP or host name of the SC4S instance and port 514
 
 ### Sourcetype and Index Configuration
 
-| key            | sourcetype     | index          | notes          |
-|----------------|----------------|----------------|----------------|
-| zscalernss_alerts      | zscalernss-alerts       | main          | none          |
-| zscalernss_dns      | zscalernss-dns     | netdns          | none          |
-| zscalernss_fw      | zscalernss-fw       | netfw          | none          |
-| zscalernss_web      | zscalernss-web       | netproxy          | none          |
-
+| key                 | sourcetype             | index    | notes   |
+|---------------------|------------------------|----------|---------|
+| zscaler_alerts      | zscalernss-alerts      | main     | none    |
+| zscaler_dns         | zscalernss-dns         | netdns   | none    |
+| zscaler_fw          | zscalernss-fw          | netfw    | none    |
+| zscaler_web         | zscalernss-web         | netproxy | none    |
+| zscaler_zia_audit   | zscalernss-zia-audit   | netops   | none    |
+| zscaler_zia_sandbox | zscalernss-zia-sandbox | main     | none    |
 
 ### Filter type
 
@@ -87,12 +88,12 @@ the IP or host name of the SC4S instance and port 514
 
 ### Sourcetype and Index Configuration
 
-| key            | sourcetype     | index          | notes          |
-|----------------|----------------|----------------|----------------|
-| zscalernss-zpa-app      | zscalerlss_zpa-app       | netproxy          | none          |
-| zscalernss-zpa-auth      | zscalerlss_zpa_auth       | netauth          | none          |
-| zscalernss-zpa-bba      | zscalerlss_zpa_auth       | netproxy          | none          |
-| zscalernss-zpa-connector    | zscalerlss_zpa_connector       | netproxy          | none          |
+| key            | sourcetype               | index      | notes   |
+|----------------|--------------------------|------------|---------|
+| zscaler_lss    | zscalerlss_zpa-app       | netproxy   | none    |
+| zscaler_lss    | zscalerlss_zpa_auth      | netproxy   | none    |
+| zscaler_lss    | zscalerlss_zpa_auth      | netproxy   | none    |
+| zscaler_lss    | zscalerlss_zpa_connector | netproxy   | none    |
 
 
 ### Filter type

package/etc/conf.d/conflib/_common/templates.conf

@@ -83,6 +83,23 @@ template t_JSON_5424 {
                 --exclude DATE
                 --exclude FACILITY
                 --exclude PRIORITY
+                --exclude HOST
+                )');
+    };
+
+# ===============================================================================================
+# JSON_5424_SDATA; for JSON pretty-printing (for RFC5424 messages with duplicate data in MESSAGE)
+# ===============================================================================================
+
+template t_JSON_5424_SDATA {
+    template('$(format-json --scope rfc5424
+                --pair PRI="<$PRI>"
+                --key ISODATE
+                --exclude DATE
+                --exclude HOST
+                --exclude FACILITY
+                --exclude PRIORITY
+                --exclude MESSAGE
                 )');
     };
 

package/etc/conf.d/destinations/rawmsg_file.conf

@@ -1,5 +1,11 @@
 destination d_rawmsg {
-    file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log"
-         template("${RAWMSG}\n")
-     );
-};
+    channel {
+        if ("${RAWMSG}" ne "") {
+            destination {
+                file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log"
+                template("${RAWMSG}\n")
+                );
+            };
+        };
+    };
+};

package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl

@@ -0,0 +1,38 @@
+# IETF Syslog
+
+log {
+    junction {
+        channel {
+        # Listen on the default port (typically 601) for IETF_SYSLOG traffic
+            source (s_ietf);
+            flags(final);
+        };
+    };
+
+    rewrite {
+        set("IETF_SYSLOG", value("fields.sc4s_vendor_product"));
+    };
+
+    rewrite { r_set_splunk_dest_default(sourcetype("ietf:syslog"), index("main"), source("${APP}:${PROGRAM}")) };
+    parser { p_add_context_splunk(key("IETF_SYSLOG")); };
+    parser (compliance_meta_by_source);
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); };
+
+{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_IETF_SYSLOG_HEC" "no")) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_IETF_SYSLOG" "no")) }}
+    destination(d_archive);
+{{- end}}
+
+{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
+    {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+{{- end }}
+
+{{- if (print (getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES")) }}
+    {{ getenv "SC4S_DEST_IETF_SYSLOG_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+{{- end }}
+
+    flags(flow-control,final);
+};

package/etc/conf.d/log_paths/lp-mcafee_epo.conf.tmpl

@@ -24,17 +24,11 @@ log {
 
     rewrite {
         set("mcafee_epo", value("fields.sc4s_vendor_product"));
+        r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav"))
     };
-    rewrite { r_set_splunk_dest_default(sourcetype("mcafee:epo:syslog"), index("epav")) };
     parser {p_add_context_splunk(key("mcafee_epo")); };
 
-
     parser (compliance_meta_by_source);
-
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
     rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
 
 {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_MCAFEE_EPO_STRUCTURED_HEC" "no")) }}

package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl

@@ -10,24 +10,35 @@ log {
         channel {
         # Listen on the specified dedicated port(s) for ZSCALER_LSS traffic
             source (s_ZSCALER_LSS);
+            parser {
+                #.jsonLog.Timestamp Mar 04 20:37:53 2020
+                date-parser-nofilter(
+                format('%a %b %d %H:%M:%S %Y',
+                       '%a %b %d %k:%M:%S %Y')
+                template("${.json.LogTimestamp}")
+                );
+            };
             flags (final);
 	    };
 {{- end}}
         channel {
         # Listen on the default port (typically 514) for ZSCALER_LSS traffic
             source (s_DEFAULT);
             filter(f_msg_is_tcp_json);
+            parser {
+                #.jsonLog.Timestamp Mar 04 20:37:53 2020
+                date-parser(
+                format('%a %b %d %H:%M:%S %Y',
+                       '%a %b %d %k:%M:%S %Y')
+                template("${.json.LogTimestamp}")
+                time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})
+                flags(guess-timezone)
+                );
+            };
             flags(final);
         };
     };
-    parser {
-        #.jsonLog.Timestamp Mar 04 20:37:53 2020
-        date-parser-nofilter(
-        format('%a %b %d %H:%M:%S %Y',
-               '%a %b %d %k:%M:%S %Y')
-               template("${.json.LogTimestamp}")
-               );
-    };
+
     if {
         filter {
             match('.' value('.json.ClientZEN'))
@@ -36,8 +47,6 @@ log {
         };
         rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-app"), index("netproxy"))};
         parser { p_add_context_splunk(key("zscaler_lss")); };
-        parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
     } elif {
         filter {
             match('.' value('.json.Exporter'))
@@ -46,8 +55,6 @@ log {
         };
         rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-bba"), index("netproxy"))};
         parser { p_add_context_splunk(key("zscaler_lss")); };
-        parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };    
     } elif {
         filter {
             match('.' value('.json.Connector'))
@@ -56,20 +63,27 @@ log {
         };
         rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-connector"), index("netproxy"))};
         parser { p_add_context_splunk(key("zscaler_lss")); };
-        parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };    
     } elif {
         filter {
             match('.' value('.json.SAMLAttributes'))
             and match('.' value('.json.Customer'))
         };
-        rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netauth"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netproxy"))};
         parser { p_add_context_splunk(key("zscaler_lss")); };
-        parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };    
+    } else {
+        rewrite {
+            set("zscaler_lss_rogue_message", value("fields.sc4s_vendor_product"));
+            set("Possible rogue message on zscaler_lss unique port", value("fields.sc4s_error"));
+            r_set_splunk_dest_default(sourcetype("zscalerlss:rogue"), index("netproxy"))
+        };
+        parser { p_add_context_splunk(key("zscaler_lss")); };
+        # Rogue message needs a different template than valid LSS events.  Final rewrite (further below) will be a
+        # no-op in this case.
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };
     };
 
-
+    # Parser for all valid LSS events.  Rogue events, having previously loaded $MSG with the entire payload,
+    # will be unaffected by the rewrite here.
     parser (compliance_meta_by_source);
     rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
 

package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl

@@ -1,6 +1,7 @@
 # Fallback for un-parsed sources
 
 log {
+    source(s_ietf);
     source(s_DEFAULT);
 
     rewrite { set("SC4S_fallback", value("fields.sc4s_vendor_product")); };

package/etc/conf.d/sources/rfc5687.conf.tmpl

@@ -0,0 +1,25 @@
+source s_ietf {
+    channel {
+        source {
+            syslog (
+                transport("tcp")
+                port(601)
+                ip-protocol(4)
+                keep-hostname(yes)
+                keep-timestamp(yes)
+                use-dns(no)
+                use-fqdn(no)
+                chain-hostnames(off)
+                flags(validate-utf8, syslog-protocol)
+            );    
+        };
+
+        if {
+            parser { app-parser(topic(syslog)); };
+        };
+        rewrite(set_rfc5424_strict);
+        parser {
+                vendor_product_by_source();
+        };
+    };
+};

package/etc/context_templates/splunk_index.csv.example

@@ -53,6 +53,7 @@
 #juniper_nsm,index,netfw
 #juniper_nsm_idp,index,netids
 #juniper_legacy,index,netops
+#mcafee_epo,index,epav
 #nix_syslog,index,osnix
 #pan_traffic,index,netfw
 #pan_threat,index,netproxy
@@ -70,4 +71,11 @@
 #sc4s_fallback,index,main
 #sc4s_metrics,index,em_metrics
 #symanrtec_ep,index,epav
-#vmware_nsx,index,main
+#vmware_nsx,index,main
+#zscaler_alerts,index,main
+#zscaler_dns,index,netdns
+#zscaler_fw,index,netfw
+#zscaler_web,index,netproxy
+#zscaler_zia_audit,index,netops
+#zscaler_zia_sandbox,index,main
+#zscaler_lss,index,netproxy

tests/test_zscaler_proxy.py

@@ -247,7 +247,7 @@ def test_zscaler_lss_zpa_auth(record_property, setup_wordlist, setup_splunk, set
     message = mt.render(mark="<134>", lss_time=lss_time, host=host)
     sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
 
-    st = env.from_string("search _time={{ epoch }} index=netauth sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"")
+    st = env.from_string("search _time={{ epoch }} index=netproxy sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\"")
     search = st.render(epoch=epoch, host=host)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)