WIP Nix support

.gitignore

@@ -384,3 +384,4 @@ fabric.properties
 tests/test_plugin_*.py
 # package/etc/conf.d/local/
 !package/etc/conf.d/local
+replay

docker-compose.yml

@@ -30,7 +30,7 @@ services:
         RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
 #When this is enabled test_common will fail
-#    command: -det
+    command: -det
     ports:
       - "514:514"
       - "601:601"
@@ -72,6 +72,19 @@ services:
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
     volumes:
       - splunk-etc:/opt/splunk/etc
+  udpreplay:
+    build:
+      context: ./utility/udpreplay
+      args:
+        RH_ORG: ${RH_ORG}
+        RH_ACTIVATION: ${RH_ACTIVATION}
+    entrypoint: tail -f /dev/null
+    links:
+      - splunk
+      - sc4s
+    volumes:
+      - ./replay:/work
+
 volumes:
   sc4s-results:
     external: true

package/etc/conf.d/conflib/_common/templates.conf

@@ -42,6 +42,9 @@ template t_hdr_msg {
         template("${MSGHDR}${MESSAGE}");
         };
 
+template t_legacy_hdr_msg {
+        template("${LEGACY_MSGHDR}${MESSAGE}");
+        };
 # ===============================================================================================
 # Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper
 # ===============================================================================================

package/etc/conf.d/filters/nix/syslog.conf

@@ -0,0 +1,5 @@
+filter f_nix_syslog {
+    program("[a-zA-Z0-9\/]+")
+    and
+    match('[a-zA-Z\]]: $' value("LEGACY_MSGHDR"))
+};

package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl

@@ -78,7 +78,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 {{- if or (or (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }}

package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl

@@ -41,7 +41,7 @@ log {
 {{- end}}
 
 
-    flags(flow-control);
+    flags(flow-control,final);
 
 };
 {{- end}}

package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl

@@ -43,7 +43,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl

@@ -92,7 +92,7 @@ log {
         destination(d_archive);
     {{- end}}
 
-        flags(flow-control);
+        flags(flow-control,final);
     };
 
 

package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl

@@ -44,7 +44,7 @@ log {
 {{- end}}
 
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl

@@ -41,7 +41,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl

@@ -60,7 +60,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl

@@ -45,7 +45,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl

@@ -58,7 +58,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl

@@ -42,7 +42,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl

@@ -43,7 +43,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl

@@ -40,7 +40,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl

@@ -88,7 +88,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 
 };
 {{- end}}

package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl

@@ -92,7 +92,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl

@@ -50,7 +50,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl

@@ -39,7 +39,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl

@@ -126,7 +126,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 
 };
 {{- end}}

package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl

@@ -81,7 +81,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl

@@ -39,7 +39,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl

@@ -42,7 +42,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl

@@ -61,6 +61,8 @@ log {
     destination(d_archive);
 {{- end}}
 
+
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl

@@ -39,7 +39,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl

@@ -0,0 +1,58 @@
+# Proofpoint
+{{ $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_DEFAULT);
+    filter(f_nix_syslog);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_NIX_SYSLOG);
+{{- end }}
+
+    rewrite {
+        set("zscaler_nss", value("fields.sc4s_vendor_product"));
+        subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+    };
+    parser {
+        #basic parsing
+        kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}"));
+    };
+
+    rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"))};
+    parser { p_add_context_splunk(key("nix_syslog")); };
+
+    parser (compliance_meta_by_source);
+
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+        set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+        groupunset(values(".kv.*"));
+    };
+
+{{- if ((getenv "SC4S_NIX_SYSLOG_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_NIX_SYSLOG_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_NIX_SYSLOG") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};
+{{- end}}
+
+{{- if or (or (getenv  (print "SC4S_LISTEN_NIX_SYSLOG_TCP_PORT")) (getenv  (print "SC4S_LISTEN_NIX_SYSLOG_UDP_PORT"))) (getenv  (print "SC4S_NIX_SYSLOG_NSS_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for NIX_SYSLOG traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for NIX_SYSLOG traffic
+{{ tmpl.Exec "log_path" "yes" }}

package/etc/conf.d/log_paths/zfallback.conf.tmpl → package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl

@@ -16,7 +16,7 @@ log {
 
     #in fallback archive only write rawmsg as msg
     rewrite {
-        set("value(RAWMSG)" value("MSG"));
+        set("$RAWMSG" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
@@ -27,5 +27,5 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control,fallback);
+    flags(flow-control,fallback,final);
 };

package/etc/syslog-ng.conf

@@ -52,11 +52,14 @@ options {
 @include "conf.d/conflib/blocks/*/*.conf"
 
 @include "conf.d/filters/*/*.conf"
-@include "conf.d/sources/*.conf"
-@include "conf.d/destinations/*.conf"
-@include "conf.d/log_paths/*.conf"
-
 @include "conf.d/local/config/filters/*.conf"
+
+@include "conf.d/sources/*.conf"
 @include "conf.d/local/config/sources/*.conf"
+@include "conf.d/destinations/*.conf"
 @include "conf.d/local/config/destinations/*.conf"
+
 @include "conf.d/local/config/log_paths/*.conf"
+@include "conf.d/log_paths/*.conf"
+
+

tests/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
 addopts =
-    --force-flaky --max-runs=3 --min-passes=1
+#    --force-flaky --max-runs=3 --min-passes=1
 filterwarnings =
     ignore::DeprecationWarning

tests/test_common.py

@@ -17,7 +17,7 @@
 def test_defaultroute(record_property, setup_wordlist, setup_splunk):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
 
-    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4sdefault[0]: test\n")
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} test something else\n")
     message = mt.render(mark="<111>", host=host)
 
     sendsingle(message)
@@ -55,7 +55,7 @@ def test_internal(record_property, setup_wordlist, setup_splunk):
 def test_tag(record_property, setup_wordlist, setup_splunk):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
 
-    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} sc4sdefault[0]: test\n")
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} test\n")
     message = mt.render(mark="<111>", host=host)
 
     sendsingle(message)