From 258455e7036efb3290ae58b2cd2ffaa6ffb873cd Mon Sep 17 00:00:00 2001
From: Mark Bonsack <mbonsack@splunk.com>
Date: Mon, 9 Dec 2019 16:08:29 -0800
Subject: [PATCH 01/47] Nonroutable addresses for vendor_product_by_source.conf

Update vendor_product_by_source.conf to use nonroutable addresses to prevent confict with real installations
---
 .../vendor_product_by_source.conf             | 33 +++++++++----------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf
index b0609de..4c8127f 100644
--- a/package/etc/context_templates/vendor_product_by_source.conf
+++ b/package/etc/context_templates/vendor_product_by_source.conf
@@ -2,50 +2,49 @@
 
 filter f_test_test {
     host("testvp-*" type(glob)) or
-    netmask(192.168.100.1/24)
+    netmask(169.254.1.0/24)
 };
 filter f_juniper_idp {
     host("jnpidp-*" type(glob)) or
-    netmask(192.168.3.0/24)
+    netmask(169.254.2.0/24)
 };
 filter f_juniper_netscreen {
     host("jnpns-*" type(glob)) or
-    netmask(192.168.4.0/24)
+    netmask(169.254.3.0/24)
 };
 filter f_juniper_nsm {
     host("jnpnsm-*" type(glob)) or
-    netmask(192.168.1.0/24)
+    netmask(169.254.4.0/24)
 };
 filter f_juniper_nsm_idp {
     host("jnpnsmidp-*" type(glob)) or
-    netmask(192.168.2.0/24)
+    netmask(169.254.5.0/24)
 };
 filter f_cisco_meraki {
     host("testcm-*" type(glob)) or
-    netmask(192.168.4.0/24)
+    netmask(169.254.6.0/24)
 };
 filter f_cisco_nx_os {
     host("csconx-*" type(glob)) or
-    netmask(192.168.5.0/24)
+    netmask(169.254.7.0/24)
 };
 filter f_proofpoint_pps_filter {
     host("pps-*" type(glob)) or
-    netmask(192.168.7.0/24)
+    netmask(169.254.8.0/24)
 };
 filter f_proofpoint_pps_sendmail {
     host("pps-*" type(glob)) or
-    netmask(192.168.6.0/24)
+    netmask(169.254.9.0/24)
 };
 filter f_ubiquiti_unifi_fw {
-        host("usg-*" type(glob)) or
-        netmask(192.168.6.0/24)
+    host("usg-*" type(glob)) or
+    netmask(169.254.10.0/24)
 };
 filter f_tzfixhst {
-        host("tzfhst-*" type(glob)) or
-        netmask(192.168.6.0/24)
+    host("tzfhst-*" type(glob)) or
+    netmask(169.254.11.0/24)
 };
-
 filter f_tzfixny {
-        host("tzfny-*" type(glob)) or
-        netmask(192.168.6.0/24)
-};
\ No newline at end of file
+    host("tzfny-*" type(glob)) or
+    netmask(169.254.12.0/24)
+};

From 492b81ce84bc7a691dfe8eb6a911ca2d7c4b4b5d Mon Sep 17 00:00:00 2001
From: Mark Bonsack <mbonsack@splunk.com>
Date: Mon, 9 Dec 2019 17:08:24 -0800
Subject: [PATCH 02/47] fix/startup_warnings

Fix startup warnings at startup caused by fake comments in csv file
---
 .../etc/context_templates/compliance_meta_by_source.conf | 9 +++++----
 .../etc/context_templates/compliance_meta_by_source.csv  | 4 ++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf
index 322b938..90b67f4 100644
--- a/package/etc/context_templates/compliance_meta_by_source.conf
+++ b/package/etc/context_templates/compliance_meta_by_source.conf
@@ -1,5 +1,6 @@
 @version: 3.24
-#filter f_test_test {
-#    host("something-*" type(glob)) or
-#    netmask(192.168.100.1/24)
-#};
+filter f_test_test {
+#   host("something-*" type(glob)) or
+#   netmask(169.254.100.0/24)
+    host("cannot_ever_happen")
+};
diff --git a/package/etc/context_templates/compliance_meta_by_source.csv b/package/etc/context_templates/compliance_meta_by_source.csv
index 6608db0..4ecf53a 100644
--- a/package/etc/context_templates/compliance_meta_by_source.csv
+++ b/package/etc/context_templates/compliance_meta_by_source.csv
@@ -1,2 +1,2 @@
-#f_test_test,.splunk.index,"badindex"
-#f_test_test,fields.compliance,"pci"
+f_test_test,.splunk.index,"will_never_happen_index"
+f_test_test,fields.compliance,"pci"

From 2b933323c8321b1f7e9686e5280ad892ac5cd610 Mon Sep 17 00:00:00 2001
From: Mark Bonsack <mbonsack@splunk.com>
Date: Tue, 10 Dec 2019 20:17:10 -0800
Subject: [PATCH 03/47] Remove extraneous proofpoint log path

Remove extra proofpoint log path; causes event to be written twice
---
 .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 52 -------------------
 1 file changed, 52 deletions(-)
 delete mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl
deleted file mode 100644
index 09dd410..0000000
--- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl
+++ /dev/null
@@ -1,52 +0,0 @@
-# Proofpoint
-{{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }}
-{{ tmpl.Exec "t/source_network.t" $context }}
-# The following is an inline template; we will use this to generate the actual log path
-{{ define "log_path" }}
-log {
-{{- if eq (.) "yes" }}
-    source(s_DEFAULT);
-    filter(f_proofpoint_pps_filter);
-{{- end }}
-{{- if eq (.) "no" }}
-    source (s_PROOFPOINT_PPS_FILTER);
-{{- end }}
-
-    rewrite {
-        set("proofpoint_pps_filter", value("fields.sc4s_vendor_product"));
-        r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"))};
-    parser {
-        p_add_context_splunk(key("proofpoint_pps_filter"));
-    };
-
-    parser (compliance_meta_by_source);
-
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-        groupunset(values(".kv.*"));
-    };
-
-    destination(d_hec);
-
-{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS") }}
-    destination(d_archive);
-{{- end}}
-
-    flags(flow-control);
-};
-{{- end}}
-
-{{- if or (or (getenv  (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT")) (getenv  (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT")) }}
-# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic
-    {{ tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for PROOFPOINT_PPS_FILTER traffic
-{{ tmpl.Exec "log_path" "yes" }}

From 62eda439c8e2f47c9d8294600d88d0d849f05f29 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Wed, 11 Dec 2019 14:12:15 -0500
Subject: [PATCH 04/47] WIP Nix support

---
 .gitignore                                    |  1 +
 docker-compose.yml                            | 15 ++++-
 .../etc/conf.d/conflib/_common/templates.conf |  3 +
 package/etc/conf.d/filters/nix/syslog.conf    |  5 ++
 .../p_rfc3164-checkpoint_splunk.conf.tmpl     |  2 +-
 .../log_paths/p_rfc3164-cisco_asa.conf.tmpl   |  2 +-
 .../log_paths/p_rfc3164-cisco_ios.conf.tmpl   |  2 +-
 .../log_paths/p_rfc3164-cisco_ise.conf.tmpl   |  2 +-
 .../log_paths/p_rfc3164-cisco_nxos.conf.tmpl  |  2 +-
 .../p_rfc3164-forcepoint_webprotect.conf.tmpl |  2 +-
 .../p_rfc3164-fortinet_fortios.conf.tmpl      |  2 +-
 .../log_paths/p_rfc3164-juniper_idp.conf.tmpl |  2 +-
 .../p_rfc3164-juniper_junos.conf.tmpl         |  2 +-
 .../p_rfc3164-juniper_netscreen.conf.tmpl     |  2 +-
 .../log_paths/p_rfc3164-juniper_nsm.conf.tmpl |  2 +-
 .../p_rfc3164-juniper_nsm_idp.conf.tmpl       |  2 +-
 .../p_rfc3164-microfocus_arcsight.conf.tmpl   |  2 +-
 .../p_rfc3164-paloalto_panos.conf.tmpl        |  2 +-
 .../p_rfc3164-proofpoint_pps.conf.tmpl        |  2 +-
 .../p_rfc3164-proofpoint_pps_filter.conf.tmpl |  2 +-
 .../p_rfc3164-ubiquiti_unifi.conf.tmpl        |  2 +-
 .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl |  2 +-
 .../p_rfc5424-noversion_cisco_asa.conf.tmpl   |  2 +-
 ...rfc5424-noversion_symantec_proxy.conf.tmpl |  2 +-
 .../p_rfc5424-strict_juniper_junos.conf.tmpl  |  2 +
 .../p_rfc5424_epoch-cisco_meraki.conf.tmpl    |  2 +-
 .../log_paths/p_za_nix_syslog.conf.tmpl       | 58 +++++++++++++++++++
 ...back.conf.tmpl => p_zz_fallback.conf.tmpl} |  4 +-
 package/etc/syslog-ng.conf                    | 11 ++--
 tests/pytest.ini                              |  2 +-
 tests/test_common.py                          |  4 +-
 tests/test_linux_syslog.py                    | 57 ++++++++++++++++++
 utility/udpreplay/Dockerfile                  | 40 +++++++++++++
 33 files changed, 213 insertions(+), 31 deletions(-)
 create mode 100644 package/etc/conf.d/filters/nix/syslog.conf
 create mode 100644 package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
 rename package/etc/conf.d/log_paths/{zfallback.conf.tmpl => p_zz_fallback.conf.tmpl} (91%)
 create mode 100644 tests/test_linux_syslog.py
 create mode 100644 utility/udpreplay/Dockerfile

diff --git a/.gitignore b/.gitignore
index 4d40b52..921f415 100644
--- a/.gitignore
+++ b/.gitignore
@@ -384,3 +384,4 @@ fabric.properties
 tests/test_plugin_*.py
 # package/etc/conf.d/local/
 !package/etc/conf.d/local
+replay
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index d1e1558..2a37a59 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -30,7 +30,7 @@ services:
         RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
 #When this is enabled test_common will fail
-#    command: -det
+    command: -det
     ports:
       - "514:514"
       - "601:601"
@@ -72,6 +72,19 @@ services:
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
     volumes:
       - splunk-etc:/opt/splunk/etc
+  udpreplay:
+    build:
+      context: ./utility/udpreplay
+      args:
+        RH_ORG: ${RH_ORG}
+        RH_ACTIVATION: ${RH_ACTIVATION}
+    entrypoint: tail -f /dev/null
+    links:
+      - splunk
+      - sc4s
+    volumes:
+      - ./replay:/work
+
 volumes:
   sc4s-results:
     external: true
diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf
index a08f36d..729e5e2 100644
--- a/package/etc/conf.d/conflib/_common/templates.conf
+++ b/package/etc/conf.d/conflib/_common/templates.conf
@@ -42,6 +42,9 @@ template t_hdr_msg {
         template("${MSGHDR}${MESSAGE}");
         };
 
+template t_legacy_hdr_msg {
+        template("${LEGACY_MSGHDR}${MESSAGE}");
+        };
 # ===============================================================================================
 # Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper
 # ===============================================================================================
diff --git a/package/etc/conf.d/filters/nix/syslog.conf b/package/etc/conf.d/filters/nix/syslog.conf
new file mode 100644
index 0000000..bfeeb2a
--- /dev/null
+++ b/package/etc/conf.d/filters/nix/syslog.conf
@@ -0,0 +1,5 @@
+filter f_nix_syslog {
+    program("[a-zA-Z0-9\/]+")
+    and
+    match('[a-zA-Z\]]: $' value("LEGACY_MSGHDR"))
+};
\ No newline at end of file
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
index 1f5c620..316bd6d 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
@@ -78,7 +78,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 {{- if or (or (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }}
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl
index a07d6a1..26dfaea 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl
@@ -41,7 +41,7 @@ log {
 {{- end}}
 
 
-    flags(flow-control);
+    flags(flow-control,final);
 
 };
 {{- end}}
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
index 554277b..d03081a 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
@@ -43,7 +43,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl
index cca9afd..bdd92d4 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl
@@ -92,7 +92,7 @@ log {
         destination(d_archive);
     {{- end}}
 
-        flags(flow-control);
+        flags(flow-control,final);
     };
 
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl
index fd7a8b4..0d28a3a 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl
@@ -44,7 +44,7 @@ log {
 {{- end}}
 
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl
index b35e857..d86957d 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl
@@ -41,7 +41,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl
index 1be63f1..02be55a 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl
@@ -60,7 +60,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl
index 761a8c0..b9d1ca1 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl
@@ -45,7 +45,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl
index d461e5e..0ecaee2 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl
@@ -58,7 +58,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl
index b735a73..ca717e8 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl
@@ -42,7 +42,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl
index 5571ba0..94bbba5 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl
@@ -43,7 +43,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl
index bb717e0..e571083 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl
@@ -40,7 +40,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl
index db6cd6a..0114932 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl
@@ -88,7 +88,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 
 };
 {{- end}}
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
index 05c45ad..6ee0ad0 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
@@ -92,7 +92,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl
index 78f234c..4b383e8 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl
@@ -50,7 +50,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl
index 09dd410..2325290 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl
@@ -39,7 +39,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl
index 55bd6a7..7ca3176 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl
@@ -126,7 +126,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 
 };
 {{- end}}
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl
index 47a77c1..21ffa8b 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl
@@ -81,7 +81,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl
index a7f069c..0e803a7 100644
--- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl
@@ -39,7 +39,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl
index 855d390..2c3c547 100644
--- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl
@@ -42,7 +42,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl
index 40bce61..766bbff 100644
--- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl
@@ -61,6 +61,8 @@ log {
     destination(d_archive);
 {{- end}}
 
+
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl
index 18a37b4..949fcce 100644
--- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl
@@ -39,7 +39,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 
diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
new file mode 100644
index 0000000..1d3aef3
--- /dev/null
+++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
@@ -0,0 +1,58 @@
+# Proofpoint
+{{ $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_DEFAULT);
+    filter(f_nix_syslog);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_NIX_SYSLOG);
+{{- end }}
+
+    rewrite {
+        set("zscaler_nss", value("fields.sc4s_vendor_product"));
+        subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+    };
+    parser {
+        #basic parsing
+        kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}"));
+    };
+
+    rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"))};
+    parser { p_add_context_splunk(key("nix_syslog")); };
+
+    parser (compliance_meta_by_source);
+
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+        set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+        groupunset(values(".kv.*"));
+    };
+
+{{- if ((getenv "SC4S_NIX_SYSLOG_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_NIX_SYSLOG_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_NIX_SYSLOG") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};
+{{- end}}
+
+{{- if or (or (getenv  (print "SC4S_LISTEN_NIX_SYSLOG_TCP_PORT")) (getenv  (print "SC4S_LISTEN_NIX_SYSLOG_UDP_PORT"))) (getenv  (print "SC4S_NIX_SYSLOG_NSS_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for NIX_SYSLOG traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for NIX_SYSLOG traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/package/etc/conf.d/log_paths/zfallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
similarity index 91%
rename from package/etc/conf.d/log_paths/zfallback.conf.tmpl
rename to package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
index 0f36b1f..15c3931 100644
--- a/package/etc/conf.d/log_paths/zfallback.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
@@ -16,7 +16,7 @@ log {
 
     #in fallback archive only write rawmsg as msg
     rewrite {
-        set("value(RAWMSG)" value("MSG"));
+        set("$RAWMSG" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
@@ -27,5 +27,5 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control,fallback);
+    flags(flow-control,fallback,final);
 };
diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf
index 4f19083..84c177b 100644
--- a/package/etc/syslog-ng.conf
+++ b/package/etc/syslog-ng.conf
@@ -52,11 +52,14 @@ options {
 @include "conf.d/conflib/blocks/*/*.conf"
 
 @include "conf.d/filters/*/*.conf"
-@include "conf.d/sources/*.conf"
-@include "conf.d/destinations/*.conf"
-@include "conf.d/log_paths/*.conf"
-
 @include "conf.d/local/config/filters/*.conf"
+
+@include "conf.d/sources/*.conf"
 @include "conf.d/local/config/sources/*.conf"
+@include "conf.d/destinations/*.conf"
 @include "conf.d/local/config/destinations/*.conf"
+
 @include "conf.d/local/config/log_paths/*.conf"
+@include "conf.d/log_paths/*.conf"
+
+
diff --git a/tests/pytest.ini b/tests/pytest.ini
index 00b64d9..ce9083e 100644
--- a/tests/pytest.ini
+++ b/tests/pytest.ini
@@ -1,5 +1,5 @@
 [pytest]
 addopts =
-    --force-flaky --max-runs=3 --min-passes=1
+#    --force-flaky --max-runs=3 --min-passes=1
 filterwarnings =
     ignore::DeprecationWarning
diff --git a/tests/test_common.py b/tests/test_common.py
index ce279a6..9855cc9 100644
--- a/tests/test_common.py
+++ b/tests/test_common.py
@@ -17,7 +17,7 @@
 def test_defaultroute(record_property, setup_wordlist, setup_splunk):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
 
-    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4sdefault[0]: test\n")
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} test something else\n")
     message = mt.render(mark="<111>", host=host)
 
     sendsingle(message)
@@ -55,7 +55,7 @@ def test_internal(record_property, setup_wordlist, setup_splunk):
 def test_tag(record_property, setup_wordlist, setup_splunk):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
 
-    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} sc4sdefault[0]: test\n")
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} test\n")
     message = mt.render(mark="<111>", host=host)
 
     sendsingle(message)
diff --git a/tests/test_linux_syslog.py b/tests/test_linux_syslog.py
new file mode 100644
index 0000000..3731db0
--- /dev/null
+++ b/tests/test_linux_syslog.py
@@ -0,0 +1,57 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import datetime
+import random
+import pytz
+
+from jinja2 import Environment, environment
+
+from .sendmessage import *
+from .splunkutils import *
+import random
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+#<78>Oct 25 09:10:00 /usr/sbin/cron[54928]: (root) CMD (/usr/libexec/atrun)
+def test_linux_program_as_path(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} /usr/sbin/cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n")
+    message = mt.render(mark="<111>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+def test_linux_program_conforms(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n")
+    message = mt.render(mark="<111>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
\ No newline at end of file
diff --git a/utility/udpreplay/Dockerfile b/utility/udpreplay/Dockerfile
new file mode 100644
index 0000000..2dbd80f
--- /dev/null
+++ b/utility/udpreplay/Dockerfile
@@ -0,0 +1,40 @@
+#To the extent possible under law, the person who associated CC0 with
+#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights
+#to Splunk Connect for Syslog (SC4S).
+#
+#You should have received a copy of the CC0 legalcode along with this
+#work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+FROM registry.access.redhat.com/rhel7/rhel
+
+ARG RH_ORG
+ARG RH_ACTIVATION
+
+RUN subscription-manager register --org=$RH_ORG --activationkey=$RH_ACTIVATION --force
+RUN subscription-manager repos --enable=rhel-7-server-extras-rpms --enable=rhel-7-server-optional-rpms --enable=rhel-server-rhscl-7-rpms
+
+RUN yum -y install libpcap-devel gcc-c++ git findutils autoconf \
+    autoconf-archive automake ca-certificates git libtool pkgconfig bison byacc file \
+    flex pcre-devel glib2-devel openssl-devel librdkafka-devel libcurl-devel \
+    rh-python36 rh-python36-python-tools rh-python36-scldevel\
+    net-snmp-devel \
+    libuuid-devel make libxslt docbook-style-xsl gcc-c++ tzdata libxml2 sqlite \
+    json-c-devel gnupg wget curl which bzip2 doxygen libsecret ivykis-devel -y
+
+RUN cd /tmp ;\
+    wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm ;\
+    yum install epel-release-latest-7.noarch.rpm -y ;\
+    rm epel-release-latest-7.noarch.rpm ;\
+    yum install -y cmake3 boost-devel
+
+
+
+RUN cd ~ ; git clone https://github.com/ska-sa/udpreplay.git
+RUN cd ~/udpreplay && \
+  ./bootstrap.sh && \
+  ./configure && \
+  make && \
+  make install  && \
+  cd ~ ; rm -Rf udpreplay
+
+RUN subscription-manager unregister
+ENTRYPOINT ["/bin/tail", "-f", "/dev/null"]

From b48c445c8e4b04f7687e51c92f2d31fa26b14996 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Wed, 11 Dec 2019 19:22:19 -0500
Subject: [PATCH 05/47] Update p_za_nix_syslog.conf.tmpl

---
 package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
index 1d3aef3..b809caf 100644
--- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
@@ -13,13 +13,9 @@ log {
 {{- end }}
 
     rewrite {
-        set("zscaler_nss", value("fields.sc4s_vendor_product"));
+        set("nix_syslog", value("fields.sc4s_vendor_product"));
         subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
     };
-    parser {
-        #basic parsing
-        kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}"));
-    };
 
     rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"))};
     parser { p_add_context_splunk(key("nix_syslog")); };
@@ -34,7 +30,6 @@ log {
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
-        groupunset(values(".kv.*"));
     };
 
 {{- if ((getenv "SC4S_NIX_SYSLOG_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_NIX_SYSLOG_HEC" "no") | conv.ToBool) }}

From 85a59d857a6f85d79b36c6e2e21282d49042b822 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 10:38:34 -0500
Subject: [PATCH 06/47] Update replay tool

---
 tests/test_symantec_ep.py       | 76 +++++++++++++++++++++++++++++++++
 utility/udpreplay/Dockerfile    | 30 +++----------
 utility/udpreplay/entrypoint.sh |  3 ++
 3 files changed, 86 insertions(+), 23 deletions(-)
 create mode 100644 tests/test_symantec_ep.py
 create mode 100755 utility/udpreplay/entrypoint.sh

diff --git a/tests/test_symantec_ep.py b/tests/test_symantec_ep.py
new file mode 100644
index 0000000..b3c906e
--- /dev/null
+++ b/tests/test_symantec_ep.py
@@ -0,0 +1,76 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import datetime
+import random
+import pytz
+
+from jinja2 import Environment, environment
+
+from .sendmessage import *
+from .splunkutils import *
+import random
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+def test_symantec_ep_msg_1(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: WORK1-PC,Local Host: 0.0.0.0,Local Port: 29555,Local Host MAC: FFFFFFFFFFFF,Remote Host IP: 0.0.0.0,Remote Host Name: ,Remote Port: 0,Remote Host MAC: WORK2-PC,7,Inbound,Begin: 2019-10-25 00:06:22,End: 2019-10-25 00:06:22,Occurrences: 1,Application: ,Rule: B-ALL-B,Location: Untrusted,User: johndoe,Domain: AD-ENT,Action: Blocked,SHA-256: ,MD-5:\n")
+    message = mt.render(mark="<111>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+def test_symantec_ep_msg_two(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: WORK1-PC,Local Host: 192.168.1.85,Local Port: 59929,Local Host MAC: D4D252E652BA,Remote Host IP: 10.217.138.110,Remote Host Name: host.example.com,Remote Port: 9000,Remote Host MAC: D4B17A775938,TCP,Outbound,Begin: 2019-10-25 00:06:09,End: 2019-10-25 00:06:18,Occurrences: 3,Application: C:/Program Files/Preton/PretonSaver/PretonService.exe,Rule: B-ALL-B,Location: Untrusted,User: SYSTEM,Domain: NT AUTHORITY,Action: Blocked,SHA-256: ba532f64bd6a31cf5f1938820f458d31fed8faa01733c9de3a1d313198b0dd9c,MD-5: 1AE7578A3CF3EABE492463C2AB7D7318\n")
+    message = mt.render(mark="<111>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+def test_symantec_ep_msg_two(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: Site: WORK-A,Server: FOOFOO,Domain: Desktop,The client has downloaded the content package successfully,FOOFO,USERNAME,ENT.EXAMPLE.CORP\n")
+    message = mt.render(mark="<111>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
\ No newline at end of file
diff --git a/utility/udpreplay/Dockerfile b/utility/udpreplay/Dockerfile
index 2dbd80f..62a5397 100644
--- a/utility/udpreplay/Dockerfile
+++ b/utility/udpreplay/Dockerfile
@@ -4,28 +4,9 @@
 #
 #You should have received a copy of the CC0 legalcode along with this
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
-FROM registry.access.redhat.com/rhel7/rhel
-
-ARG RH_ORG
-ARG RH_ACTIVATION
-
-RUN subscription-manager register --org=$RH_ORG --activationkey=$RH_ACTIVATION --force
-RUN subscription-manager repos --enable=rhel-7-server-extras-rpms --enable=rhel-7-server-optional-rpms --enable=rhel-server-rhscl-7-rpms
-
-RUN yum -y install libpcap-devel gcc-c++ git findutils autoconf \
-    autoconf-archive automake ca-certificates git libtool pkgconfig bison byacc file \
-    flex pcre-devel glib2-devel openssl-devel librdkafka-devel libcurl-devel \
-    rh-python36 rh-python36-python-tools rh-python36-scldevel\
-    net-snmp-devel \
-    libuuid-devel make libxslt docbook-style-xsl gcc-c++ tzdata libxml2 sqlite \
-    json-c-devel gnupg wget curl which bzip2 doxygen libsecret ivykis-devel -y
-
-RUN cd /tmp ;\
-    wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm ;\
-    yum install epel-release-latest-7.noarch.rpm -y ;\
-    rm epel-release-latest-7.noarch.rpm ;\
-    yum install -y cmake3 boost-devel
+FROM ubuntu:latest
 
+RUN apt-get update ;apt-get install -y build-essential autoconf wget git libboost-dev libpcap-dev libtool autogen
 
 
 RUN cd ~ ; git clone https://github.com/ska-sa/udpreplay.git
@@ -36,5 +17,8 @@ RUN cd ~/udpreplay && \
   make install  && \
   cd ~ ; rm -Rf udpreplay
 
-RUN subscription-manager unregister
-ENTRYPOINT ["/bin/tail", "-f", "/dev/null"]
+RUN cd ~ ; git clone https://github.com/GabrielGanne/tcpreplay.git
+RUN cd ~/tcpreplay && \
+    ./autogen.sh; ./configure; make install
+COPY entrypoint.sh /
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/utility/udpreplay/entrypoint.sh b/utility/udpreplay/entrypoint.sh
new file mode 100755
index 0000000..fb099fc
--- /dev/null
+++ b/utility/udpreplay/entrypoint.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+tail -f /dev/null

From 3ebe4ddbbb4190a0d753d0577ca4854885918332 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 11:21:20 -0500
Subject: [PATCH 07/47] update utility

---
 docker-compose.yml                              | 7 ++-----
 utility/{udpreplay => pcapreplay}/Dockerfile    | 0
 utility/{udpreplay => pcapreplay}/entrypoint.sh | 0
 3 files changed, 2 insertions(+), 5 deletions(-)
 rename utility/{udpreplay => pcapreplay}/Dockerfile (100%)
 rename utility/{udpreplay => pcapreplay}/entrypoint.sh (100%)

diff --git a/docker-compose.yml b/docker-compose.yml
index 2a37a59..d24be63 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -72,12 +72,9 @@ services:
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
     volumes:
       - splunk-etc:/opt/splunk/etc
-  udpreplay:
+  pcapreplay:
     build:
-      context: ./utility/udpreplay
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
+      context: ./utility/pcapreplay
     entrypoint: tail -f /dev/null
     links:
       - splunk
diff --git a/utility/udpreplay/Dockerfile b/utility/pcapreplay/Dockerfile
similarity index 100%
rename from utility/udpreplay/Dockerfile
rename to utility/pcapreplay/Dockerfile
diff --git a/utility/udpreplay/entrypoint.sh b/utility/pcapreplay/entrypoint.sh
similarity index 100%
rename from utility/udpreplay/entrypoint.sh
rename to utility/pcapreplay/entrypoint.sh

From df88021b95ab7a811573633951bf67419158616e Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 11:41:22 -0500
Subject: [PATCH 08/47] Delete test_symantec_ep.py

---
 tests/test_symantec_ep.py | 76 ---------------------------------------
 1 file changed, 76 deletions(-)
 delete mode 100644 tests/test_symantec_ep.py

diff --git a/tests/test_symantec_ep.py b/tests/test_symantec_ep.py
deleted file mode 100644
index b3c906e..0000000
--- a/tests/test_symantec_ep.py
+++ /dev/null
@@ -1,76 +0,0 @@
-# Copyright 2019 Splunk, Inc.
-#
-# Use of this source code is governed by a BSD-2-clause-style
-# license that can be found in the LICENSE-BSD2 file or at
-# https://opensource.org/licenses/BSD-2-Clause
-import datetime
-import random
-import pytz
-
-from jinja2 import Environment, environment
-
-from .sendmessage import *
-from .splunkutils import *
-import random
-
-env = Environment(extensions=['jinja2_time.TimeExtension'])
-
-def test_symantec_ep_msg_1(record_property, setup_wordlist, setup_splunk):
-    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
-    pid = random.randint(1000, 32000)
-
-    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: WORK1-PC,Local Host: 0.0.0.0,Local Port: 29555,Local Host MAC: FFFFFFFFFFFF,Remote Host IP: 0.0.0.0,Remote Host Name: ,Remote Port: 0,Remote Host MAC: WORK2-PC,7,Inbound,Begin: 2019-10-25 00:06:22,End: 2019-10-25 00:06:22,Occurrences: 1,Application: ,Rule: B-ALL-B,Location: Untrusted,User: johndoe,Domain: AD-ENT,Action: Blocked,SHA-256: ,MD-5:\n")
-    message = mt.render(mark="<111>", host=host, pid=pid)
-
-    sendsingle(message)
-
-    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
-    search = st.render(host=host, pid=pid)
-
-    resultCount, eventCount = splunk_single(setup_splunk, search)
-
-    record_property("host", host)
-    record_property("resultCount", resultCount)
-    record_property("message", message)
-
-    assert resultCount == 1
-
-def test_symantec_ep_msg_two(record_property, setup_wordlist, setup_splunk):
-    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
-    pid = random.randint(1000, 32000)
-
-    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: WORK1-PC,Local Host: 192.168.1.85,Local Port: 59929,Local Host MAC: D4D252E652BA,Remote Host IP: 10.217.138.110,Remote Host Name: host.example.com,Remote Port: 9000,Remote Host MAC: D4B17A775938,TCP,Outbound,Begin: 2019-10-25 00:06:09,End: 2019-10-25 00:06:18,Occurrences: 3,Application: C:/Program Files/Preton/PretonSaver/PretonService.exe,Rule: B-ALL-B,Location: Untrusted,User: SYSTEM,Domain: NT AUTHORITY,Action: Blocked,SHA-256: ba532f64bd6a31cf5f1938820f458d31fed8faa01733c9de3a1d313198b0dd9c,MD-5: 1AE7578A3CF3EABE492463C2AB7D7318\n")
-    message = mt.render(mark="<111>", host=host, pid=pid)
-
-    sendsingle(message)
-
-    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
-    search = st.render(host=host, pid=pid)
-
-    resultCount, eventCount = splunk_single(setup_splunk, search)
-
-    record_property("host", host)
-    record_property("resultCount", resultCount)
-    record_property("message", message)
-
-    assert resultCount == 1
-
-def test_symantec_ep_msg_two(record_property, setup_wordlist, setup_splunk):
-    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
-    pid = random.randint(1000, 32000)
-
-    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: Site: WORK-A,Server: FOOFOO,Domain: Desktop,The client has downloaded the content package successfully,FOOFO,USERNAME,ENT.EXAMPLE.CORP\n")
-    message = mt.render(mark="<111>", host=host, pid=pid)
-
-    sendsingle(message)
-
-    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
-    search = st.render(host=host, pid=pid)
-
-    resultCount, eventCount = splunk_single(setup_splunk, search)
-
-    record_property("host", host)
-    record_property("resultCount", resultCount)
-    record_property("message", message)
-
-    assert resultCount == 1
\ No newline at end of file

From 1233911d5a6f7b2bd2dfee8b9fb6c25a599a9256 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 12:29:45 -0500
Subject: [PATCH 09/47] final

---
 docs/sources/nix/index.md                     | 50 +++++++++++++++++++
 mkdocs.yml                                    |  1 +
 .../log_paths/p_za_nix_syslog.conf.tmpl       |  8 ++-
 .../etc/context_templates/splunk_index.csv    |  1 +
 .../apps/SA-syslog-ng/default/indexes.conf    |  5 ++
 tests/test_linux_syslog.py                    | 44 +++++++++++++++-
 6 files changed, 106 insertions(+), 3 deletions(-)
 create mode 100644 docs/sources/nix/index.md

diff --git a/docs/sources/nix/index.md b/docs/sources/nix/index.md
new file mode 100644
index 0000000..64e9e4b
--- /dev/null
+++ b/docs/sources/nix/index.md
@@ -0,0 +1,50 @@
+# Vendor - Nix Generic
+
+## Product - All Products
+
+Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution when configured to provide
+syslog output these devices can be monitored using the common Splunk Nix TA
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/833/                                                                 |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| nix:syslog  | None |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| nix_syslog      | nix:syslog       | osnix          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_ARCHIVE_NIX_SYSLOG | no | Enable archive to disk for this specific source |
+| SC4S_DEST_NIX_SYSLOG_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=osnix sourcetype=nix:syslog | stats count by host
+```
diff --git a/mkdocs.yml b/mkdocs.yml
index 6b82a92..3848849 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -17,6 +17,7 @@ nav:
       - Fortinet: sources/Fortinet/index.md
       - Imperva: sources/Imperva/index.md
       - Juniper: sources/Juniper/index.md
+      - Nix: sources/nix/index.md
       - Microfocus: sources/Microfocus/index.md
       - 'Paloalto Networks': sources/PaloaltoNetworks/index.md
       - Proofpoint: sources/Proofpoint/index.md
diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
index b809caf..9e2547e 100644
--- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
@@ -15,9 +15,15 @@ log {
     rewrite {
         set("nix_syslog", value("fields.sc4s_vendor_product"));
         subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+        set("${PROGRAM}", value(".PROGRAM"));
+        subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+    };
+
+    rewrite {
+        r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"), source("program:${.PROGRAM}") )
+
     };
 
-    rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"))};
     parser { p_add_context_splunk(key("nix_syslog")); };
 
     parser (compliance_meta_by_source);
diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv
index d4c05b9..eaf846d 100644
--- a/package/etc/context_templates/splunk_index.csv
+++ b/package/etc/context_templates/splunk_index.csv
@@ -35,6 +35,7 @@
 #juniper_nsm,index,netfw
 #juniper_nsm_idp,index,netids
 #juniper_legacy,index,netops
+#nix_syslog,index,osnix
 #pan_traffic,index,netfw
 #pan_threat,index,netproxy
 #pan_system,index,netops
diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
index 04c2df6..9c64a84 100644
--- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
+++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
@@ -26,6 +26,11 @@ homePath   = $SPLUNK_DB/test2/db
 coldPath   = $SPLUNK_DB/test2/colddb
 thawedPath = $SPLUNK_DB/test2/thaweddb
 
+[osnix]
+homePath   = $SPLUNK_DB/osnix/db
+coldPath   = $SPLUNK_DB/osnix/colddb
+thawedPath = $SPLUNK_DB/osnix/thaweddb
+
 [oswin]
 homePath   = $SPLUNK_DB/oswin/db
 coldPath   = $SPLUNK_DB/oswin/colddb
diff --git a/tests/test_linux_syslog.py b/tests/test_linux_syslog.py
index 3731db0..914dea6 100644
--- a/tests/test_linux_syslog.py
+++ b/tests/test_linux_syslog.py
@@ -16,7 +16,7 @@
 env = Environment(extensions=['jinja2_time.TimeExtension'])
 
 #<78>Oct 25 09:10:00 /usr/sbin/cron[54928]: (root) CMD (/usr/libexec/atrun)
-def test_linux_program_as_path(record_property, setup_wordlist, setup_splunk):
+def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_splunk):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
     pid = random.randint(1000, 32000)
 
@@ -36,7 +36,27 @@ def test_linux_program_as_path(record_property, setup_wordlist, setup_splunk):
 
     assert resultCount == 1
 
-def test_linux_program_conforms(record_property, setup_wordlist, setup_splunk):
+def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} /usr/sbin/cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n")
+    message = mt.render(mark="<111>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_splunk):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
     pid = random.randint(1000, 32000)
 
@@ -54,4 +74,24 @@ def test_linux_program_conforms(record_property, setup_wordlist, setup_splunk):
     record_property("resultCount", resultCount)
     record_property("message", message)
 
+    assert resultCount == 1
+
+def test_linux__host_program_conforms(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n")
+    message = mt.render(mark="<111>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
     assert resultCount == 1
\ No newline at end of file

From 5fff7a0eb07c05ebcb9b4f92ddd92486d375904f Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 13:02:24 -0500
Subject: [PATCH 10/47] Update index.md

---
 docs/sources/nix/index.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/sources/nix/index.md b/docs/sources/nix/index.md
index 64e9e4b..2dc0416 100644
--- a/docs/sources/nix/index.md
+++ b/docs/sources/nix/index.md
@@ -3,7 +3,11 @@
 ## Product - All Products
 
 Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution when configured to provide
-syslog output these devices can be monitored using the common Splunk Nix TA
+syslog output these devices can be monitored using the common Splunk Nix TA.
+
+Note: This is not a replacement or alternative for use of the Splunk Universal forwarder on Linux and Unix. For server applications
+the syslog only prevents full collection of events and metrics appropriate for security and operations use cases.
+
 
 
 | Ref            | Link                                                                                                    |

From d9c2bc56c92b7437c8e73c58a29808dcd6eb568b Mon Sep 17 00:00:00 2001
From: mbonsack <mbonsack@splunk.com>
Date: Thu, 12 Dec 2019 10:14:17 -0800
Subject: [PATCH 11/47] Update index.md

---
 docs/sources/nix/index.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docs/sources/nix/index.md b/docs/sources/nix/index.md
index 2dc0416..7be21b0 100644
--- a/docs/sources/nix/index.md
+++ b/docs/sources/nix/index.md
@@ -2,11 +2,12 @@
 
 ## Product - All Products
 
-Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution when configured to provide
-syslog output these devices can be monitored using the common Splunk Nix TA.
+Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution. When configured to log via
+syslog, these devices' OS logs (from a security perspective) can be monitored using the common Splunk Nix TA.
 
-Note: This is not a replacement or alternative for use of the Splunk Universal forwarder on Linux and Unix. For server applications
-the syslog only prevents full collection of events and metrics appropriate for security and operations use cases.
+Note: This is NOT a replacement for or alternative to the Splunk Universal forwarder on Linux and Unix. For general-purpose
+server applications, the Universal Forwarder offers more comprehensive collection of events and metrics appropriate for both
+security and operations use cases.
 
 
 

From f82fa5365fe8dd74a724a6bd1a2f0346f4d19c2f Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 14:54:24 -0500
Subject: [PATCH 12/47] Update Dockerfile

---
 utility/pcapreplay/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utility/pcapreplay/Dockerfile b/utility/pcapreplay/Dockerfile
index 62a5397..322eb6a 100644
--- a/utility/pcapreplay/Dockerfile
+++ b/utility/pcapreplay/Dockerfile
@@ -6,7 +6,7 @@
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
 FROM ubuntu:latest
 
-RUN apt-get update ;apt-get install -y build-essential autoconf wget git libboost-dev libpcap-dev libtool autogen
+RUN apt-get update ;apt-get install -y build-essential autoconf wget git libboost-all-dev libpcap-dev libtool autogen
 
 
 RUN cd ~ ; git clone https://github.com/ska-sa/udpreplay.git

From a2108393998ac30a97377902aa709c2b6752366c Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 15:05:00 -0500
Subject: [PATCH 13/47] Missing test case for single non grouped message

---
 tests/test_cisco_ise.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/test_cisco_ise.py b/tests/test_cisco_ise.py
index db1e493..0093686 100644
--- a/tests/test_cisco_ise.py
+++ b/tests/test_cisco_ise.py
@@ -50,3 +50,23 @@ def test_cisco_ise(record_property, setup_wordlist, setup_splunk):
     record_property("message", message)
 
     assert resultCount == 1
+
+#<181>Oct 24 21:00:02 ciscohost CISE_RADIUS_Accounting 0006028545 1 0 2019-10-24 21:00:02.879 +00:00 0088472694 3002 NOTICE Radius-Accounting: RADIUS Accounting watchdog update, ConfigVersionId=336, Device IP Address=10.0.0.3, RequestLatency=3, NetworkDeviceName=nc-aaa-aaa1, User-Name=U100000.ent.corp, NAS-IP-Address=10.0.0.3, NAS-Port=50047, Service-Type=Framed, Framed-IP-Address=10.0.0.80, Class=CACS:0AEF12345677832097B3F362:ncsilsepsuie212/356139633/9969901, Called-Station-ID=00-08-00-00-1B-AF, Calling-Station-ID=00-00-00-00-A0-7E, Acct-Status-Type=Interim-Update, Acct-Delay-Time=0, Acct-Input-Octets=653293631, Acct-Output-Octets=1497972244, Acct-Session-Id=00000B68, Acct-Authentic=RADIUS, Acct-Session-Time=241598, Acct-Input-Packets=2656224, Acct-Output-Packets=7614179, Acct-Input-Gigawords=0, Acct-Output-Gigawords=1, NAS-Port-Type=Ethernet, NAS-Port-Id=FastEthernet0/47, undefined-151=31D7AADD, cisco-av-pair=audit-session-id=0AEF10030000032097B3F362, cisco-av-pair=connect-progress=Auth Open, AcsSessionID=ncsilsepsuie205/359238109/4017186, SelectedAccessService=Default Network Access, Step=11004, Step=11017, Step=15049, Step=15008, Step=15048, Step=15048, Step=15048, Step=15004, Step=22094, Step=11005, NetworkDeviceGroups=Location#All Locations#NC, NetworkDeviceGroups=Device Type#All Device Types#Switch#2960-Switches, NetworkDeviceGroups=All Network Device Groups#All Network Device Groups, CPMSessionID=0AEF10030000032097B3F362, AllowedProtocolMatchedRule=EAP-TLS, All Network Device Groups=All Network Device Groups#All Network Device Groups, Location=Location#All Locations#NC, Device Type=Device Type#All Device Types#Switch#2960-Switches, Network Device Profile=Cisco,
+def test_cisco_ise_single(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_RADIUS_Accounting 0006028545 1 0 2019-10-24 21:00:02.879 +00:00 0088472694 3002 NOTICE Radius-Accounting: RADIUS Accounting watchdog update, ConfigVersionId=336, Device IP Address=10.0.0.3, RequestLatency=3, NetworkDeviceName=nc-aaa-aaa1, User-Name=U100000.ent.corp, NAS-IP-Address=10.0.0.3, NAS-Port=50047, Service-Type=Framed, Framed-IP-Address=10.0.0.80, Class=CACS:0AEF12345677832097B3F362:ncsilsepsuie212/356139633/9969901, Called-Station-ID=00-08-00-00-1B-AF, Calling-Station-ID=00-00-00-00-A0-7E, Acct-Status-Type=Interim-Update, Acct-Delay-Time=0, Acct-Input-Octets=653293631, Acct-Output-Octets=1497972244, Acct-Session-Id=00000B68, Acct-Authentic=RADIUS, Acct-Session-Time=241598, Acct-Input-Packets=2656224, Acct-Output-Packets=7614179, Acct-Input-Gigawords=0, Acct-Output-Gigawords=1, NAS-Port-Type=Ethernet, NAS-Port-Id=FastEthernet0/47, undefined-151=31D7AADD, cisco-av-pair=audit-session-id=0AEF10030000032097B3F362, cisco-av-pair=connect-progress=Auth Open, AcsSessionID=ncsilsepsuie205/359238109/4017186, SelectedAccessService=Default Network Access, Step=11004, Step=11017, Step=15049, Step=15008, Step=15048, Step=15048, Step=15048, Step=15004, Step=22094, Step=11005, NetworkDeviceGroups=Location#All Locations#NC, NetworkDeviceGroups=Device Type#All Device Types#Switch#2960-Switches, NetworkDeviceGroups=All Network Device Groups#All Network Device Groups, CPMSessionID=0AEF10030000032097B3F362, AllowedProtocolMatchedRule=EAP-TLS, All Network Device Groups=All Network Device Groups#All Network Device Groups, Location=Location#All Locations#NC, Device Type=Device Type#All Device Types#Switch#2960-Switches, Network Device Profile=Cisco,\n")
+    message = mt.render(mark="<165>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:ise:syslog\" | head 11")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1

From 428386d979b058ba6936f85022f8e930d32ff92b Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 15:07:57 -0500
Subject: [PATCH 14/47] Update index.md

add missing default indexes
---
 docs/gettingstarted/index.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md
index c3822dc..f686dc0 100644
--- a/docs/gettingstarted/index.md
+++ b/docs/gettingstarted/index.md
@@ -42,6 +42,8 @@ using the SC4S defaults. SC4S can be easily customized to use different indexes
 * netops
 * netproxy
 * netipam
+* oswinsec
+* osnix
 * em_metrics (ensure this is created as a metrics index)
 
 #### Install Related Splunk Apps

From 1d533b7a8c6d498a181080286b7f3063eb3064a8 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 16:06:55 -0500
Subject: [PATCH 15/47] Fixes #152 add support for cisco ACS

---
 docs/sources/Cisco/index.md                   |  49 ++++++++
 package/etc/conf.d/filters/cisco/acs.conf     |   4 +
 .../log_paths/p_rfc3164-cisco_acs.conf.tmpl   | 109 ++++++++++++++++++
 .../etc/context_templates/splunk_index.csv    |   1 +
 tests/test_cisco_acs.py                       |  57 +++++++++
 5 files changed, 220 insertions(+)
 create mode 100644 package/etc/conf.d/filters/cisco/acs.conf
 create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl
 create mode 100644 tests/test_cisco_acs.py

diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md
index 9d6cad4..d2c67fd 100644
--- a/docs/sources/Cisco/index.md
+++ b/docs/sources/Cisco/index.md
@@ -1,5 +1,54 @@
 # Vendor - Cisco
 
+## Product - ACS
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/1811/                                                                 |
+| Product Manual | https://community.cisco.com/t5/security-documents/acs-5-x-configuring-the-external-syslog-server/ta-p/3143143 |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| cisco:acs     | Aggregation used                                                                                                    |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| cisco_acs    | cisco:acs    | netauth          | None     |
+
+
+### Filter type
+
+PATTERN MATCH
+
+### Setup and Configuration
+
+* No special steps required
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_CISCO_ACS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format |
+| SC4S_LISTEN_CISCO_ACS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined expecting RFC5424 format |
+| SC4S_ARCHIVE_CISCO_ACS | no | Enable archive to disk for this specific source |
+| SC4S_DEST_CISCO_ACS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+Use the following search to validate events are present
+
+```
+index=<asconfigured> sourcetype=cisco:acs
+```
+
+Verify timestamp, and host values match as expected    
+
+
 ## Product - ASA (Pre Firepower)
 
 | Ref            | Link                                                                                                    |
diff --git a/package/etc/conf.d/filters/cisco/acs.conf b/package/etc/conf.d/filters/cisco/acs.conf
new file mode 100644
index 0000000..42e9938
--- /dev/null
+++ b/package/etc/conf.d/filters/cisco/acs.conf
@@ -0,0 +1,4 @@
+
+filter f_cisco_acs {
+    program("CSCOacs.*");
+};
\ No newline at end of file
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl
new file mode 100644
index 0000000..5666854
--- /dev/null
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl
@@ -0,0 +1,109 @@
+# Cisco ACS
+{{ $context := dict "port_id" "CISCO_ACS" "parser" "common"}}
+{{ tmpl.Exec "t/source_network.t" $context }}
+
+#This filter uses a field we set to prevent the original messages before aggregation from being
+#sent to Splunk
+filter f_cisco_acs_complete{
+    match("yes", value("ACS.COMPLETE") type(glob));
+};
+
+#This parser adds messages from ACS to a context without sending them
+#forward to Splunk
+parser acs_grouping {
+    csv-parser(
+        columns(PID, ACS.num, ACS.seq, MESSAGE)
+        delimiters(chars(" "))
+        flags(greedy)
+    );
+    grouping-by(
+        scope(program)
+        key("$PID")
+        trigger("$(+ ${ACS.seq} 1)" == "${ACS.num}")
+        sort-key("${ACS.seq}")
+        aggregate(
+             value("MESSAGE" "$(implode '' $(context-values ${MESSAGE}))")
+             value("ACS.COMPLETE" "yes")
+        )
+        timeout(10)
+    );
+};
+
+#The syslog message includes a date with nano seconds and TZ which is not in the header
+#So must reparse the date
+parser acs_event_time {
+    csv-parser(
+        columns(ACS.DATE, ACS.TIME, ACS.TZ, MESSAGE)
+        delimiters(chars(" "))
+        flags(greedy)
+    );
+
+    date-parser(
+            #YYYY- MM-DD hh:mm:ss:xxx +/-zh:zm
+            format("%Y-%m-%d %H:%M:%S.%f %z" )
+            template("${ACS.DATE} ${ACS.TIME} ${ACS.TZ}")
+    );
+};
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes"}}
+    source(s_DEFAULT);
+    filter(f_is_rfc3164);
+    filter(f_cisco_acs);
+{{- end}}
+{{- if eq (.) "no"}}
+    source (s_CISCO_ACS);
+{{- end}}
+
+    parser(acs_grouping);
+
+    if {
+        filter(f_cisco_acs_complete);
+        parser(acs_event_time);
+        rewrite {
+            set("cisco_acs", value("fields.sc4s_vendor_product"));
+            r_set_splunk_dest_default(sourcetype("cisco:acs"), index("netauth"))
+        };
+
+        parser {p_add_context_splunk(key("cisco_acs")); };
+        parser (compliance_meta_by_source);
+
+        #We want to unset the fields we won't need, as this is copied into the
+        #disk queue for network destinations. This can be very disk expensive
+        #if we don't
+        rewrite {
+            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("PID"));
+            unset(value("LEGACY_MSGHDR"));
+            unset(value("EPOCH"));
+            unset(value("VERSION"));
+            unset(value("TIMESECFRAC"));
+            groupunset(values("ACS.*"));
+        };
+
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ACS_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ACS") }}
+        destination(d_archive);
+    {{- end}}
+
+        flags(flow-control,final);
+    };
+
+
+};
+{{- end}}
+
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ACS_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ACS_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ACS_TLS_PORT")) }}
+
+# Listen on the specified dedicated port(s) for CISCO_ACS traffic
+    {{tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for CISCO_ACS traffic
+{{tmpl.Exec "log_path" "yes" }}
diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv
index eaf846d..12927fb 100644
--- a/package/etc/context_templates/splunk_index.csv
+++ b/package/etc/context_templates/splunk_index.csv
@@ -11,6 +11,7 @@
 #checkpoint_splunk_web,index,netproxy
 #checkpoint_splunk,index,netops
 #checkpoint_splunk,index,netops
+#cisco_acs,index,netauth
 #cisco_asa,index,netfw
 #cisco_ios,index,netops
 #cisco_ise,index,netauth
diff --git a/tests/test_cisco_acs.py b/tests/test_cisco_acs.py
new file mode 100644
index 0000000..29bf81f
--- /dev/null
+++ b/tests/test_cisco_acs.py
@@ -0,0 +1,57 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import random
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+
+def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0765855540 1 0 2019-10-24 21:01:05.028 +00:00 0178632943 5202 NOTICE Device-Administration: Command Authorization succeeded, ACSVersion=acs-5.8.1.4-B.462.x86_64, ConfigVersionId=16489, Device IP Address=10.0.0.93, DestinationIPAddress=10.0.0.10, DestinationPort=49, UserName=nsdevman, CmdSet=[ CmdAV=show CmdArgAV=vpn-sessiondb CmdArgAV=full CmdArgAV=ra-ikev2-ipsec ], Protocol=Tacacs, MatchedCommandSet=fw3, RequestLatency=11, Type=Authorization, Privilege-Level=15, Authen-Type=ASCII, Service=None, User=nsdevman, Port=443, Remote-Address=10.0.0.15, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=mnsvdcfpiuac03/359448835/9871764, AuthenticationIdentityStore=AD1, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedCommandSet=fw3, IdentityGroup=IdentityGroup:All Groups:SystemID, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15004 , Step=15013 , Step=24210 , Step=24212 , Step=24432 , Step=24325 , Step=24313 , Step=24319 , Step=24323 , Step=24420 , Step=24355 , Step=24416 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15018 , Step=13024 , Step=13034 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceName=devicenamehere, NetworkDeviceGroups=Device Type:All Device Types:Firewall:Cisco Systems:Firewall:ASA5545, NetworkDeviceGroups=Location:All Locations:MN, ServiceSelectionMatchedRule=TACACS, IdentityPolicyMatchedRule=Firewall, AuthorizationPolicyMatchedRule=nsdevman, AD-User-Candidate-Identities=nsdevman@ent.example.corp, AD-User-DNS-Domain=ent.example.corp, AD-User-NetBios-Name=AD-ENT, AD-User-Resolved-Identities=nsdevman@ent.example.corp, AD-User-Join-Point=ENT.example.CORP, AD-User-Resolved-DNs=CN=nsdevman\,OU=Service Accounts\,OU=CAO\,OU=ENT\,DC=ent\,DC=wfb\,DC=example\,DC=corp, StepData=10=nsdevman, StepData=11=ent.example.corp, StepData=12=example.corp, StepData=15=ent.example.corp, AD-Domain=ent.example.corp, IdentityAccessRestricted=false, UserIdentityGroup=IdentityGroup:All Groups:SystemID, Cisco-Firewall=Superuser, Firewall=Superuser, NetSec-CSM=User, NetSec-Logging=Engineer, Response={Type=Authorization; Author-Reply-Status=PassAdd; ExternalIdentityStoreName=AD1; }\n")
+    message = mt.render(mark="<165>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" | head 11")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+def test_cisco_acs_multi(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 0 2011-08-01 22:32:53.032 +00:00 0000008450 5203 NOTICE Device-Administration: Session Authorization succeeded, ACSVersion=acs-5.2.0.26-B.3075, ConfigVersionId=117, Device IP Address=192.168.26.137, UserName=edward, CmdSet=[ CmdAV= ], Protocol=Tacacs, RequestLatency=10, NetworkDeviceName=switch, Type=Authorization, Privilege-Level=1, Authen-Type=ASCII, Service=Login, User=edward, Port=tty2, Remote-Address=10.78.167.190, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=ACS41/101085887/112, AuthenticationIdentityStore=Internal Users, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedShellProfile=Permit Access, IdentityGroup=IdentityGroup:All Groups, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15006 , Step=15013 , Step=24210 , Step=24212 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15017 , Step=13034 ,\n")
+    message = mt.render(mark="<165>", host=host)
+    sendsingle(message)
+
+    mt = env.from_string(
+        "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 1 Step=13015 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceGroups=s1Migrated_NDGs:All s1Migrated_NDGs, NetworkDeviceGroups=Device Type:All Device Types, NetworkDeviceGroups=Location:All Locations, ServiceSelectionMatchedRule=Rule-2, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Rule-0, Action=Login, Privilege-Level=1, Authen-Type=ASCII, Service=Login, Remote-Address=10.78.167.190, UserIdentityGroup=IdentityGroup:All\n")
+    message = mt.render(mark="<165>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" \"Step=13015\" | head 11")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
\ No newline at end of file

From a2d5a277d8e049853fef16508c04549e70df982b Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 17:14:39 -0500
Subject: [PATCH 16/47] Update Paloalto for IETF format

In most cases palo alto customers use BSD format over UDP for performance if a customer uses IETF. The firewall sends a malformed message which fails to parse the host. This change also add a feature flag
---
 docs/sources/PaloaltoNetworks/index.md        |  1 +
 .../conf.d/conflib/_common/syslog_format.conf | 13 ++++++
 .../etc/conf.d/filters/paloalto/panos.conf    |  6 +++
 .../p_rfc3164-paloalto_panos.conf.tmpl        |  7 ++-
 package/etc/go_templates/source_network.t     |  7 +++
 tests/test_palo_alto.py                       | 43 +++++++++++++++++++
 6 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/docs/sources/PaloaltoNetworks/index.md b/docs/sources/PaloaltoNetworks/index.md
index bc27602..814344b 100644
--- a/docs/sources/PaloaltoNetworks/index.md
+++ b/docs/sources/PaloaltoNetworks/index.md
@@ -53,6 +53,7 @@ MSG Parse: This filter parses message content
 | SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_PALOALTO_PANOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_PALOALTO_PANOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS | no | Use custom time stamp parsing with ms added |
 
 ### Verification
 
diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf
index 5b69e71..49e6529 100644
--- a/package/etc/conf.d/conflib/_common/syslog_format.conf
+++ b/package/etc/conf.d/conflib/_common/syslog_format.conf
@@ -7,6 +7,9 @@ filter f_rfc5424_noversion{
 filter f_rfc5424_epochtime{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>(?<EPOCH>\d{10})(?:.(?<TIMESECFRAC>\d{1,9})?)) (?<FROMHOST>[^ ]+) ))');
 };
+filter f_rfc5424_bsdtime{
+    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?<FROMHOST>[^ ]+) ))');
+};
 rewrite set_rfcnonconformant{
     set("rfc5424_nonconform" value("fields.sc4s_syslog_format"));
 };
@@ -28,6 +31,12 @@ rewrite set_rfc5424_epochtime{
 filter f_is_rfc5424_epochtime{
     match("rfc5424_epochtime" value("fields.sc4s_syslog_format"))
 };
+rewrite set_rfc5424_bsdtime{
+    set("rfc5424_bsdtime" value("fields.sc4s_syslog_format"));
+};
+filter f_is_rfc5424_bsdtime{
+    match("rfc5424_bsdtime" value("fields.sc4s_syslog_format"))
+};
 rewrite set_rfc3164{
     set("rfc3164" value("fields.sc4s_syslog_format"));
 };
@@ -45,4 +54,8 @@ rewrite set_no_parse{
 };
 filter f_is_no_parse{
     match("no_parse" value("fields.sc4s_syslog_format"))
+};
+
+rewrite set_rfc3894_bsdtime{
+    subst('^<\d+>(\d)', "", value("MESSAGE"));
 };
\ No newline at end of file
diff --git a/package/etc/conf.d/filters/paloalto/panos.conf b/package/etc/conf.d/filters/paloalto/panos.conf
index da0cb04..6b645b1 100644
--- a/package/etc/conf.d/filters/paloalto/panos.conf
+++ b/package/etc/conf.d/filters/paloalto/panos.conf
@@ -1,3 +1,9 @@
+filter f_is_palalto_format{
+    match("rfc5424_bsdtime" value("fields.sc4s_syslog_format"))
+    or
+    match("rfc3164" value("fields.sc4s_syslog_format"))
+};
+
 filter f_paloalto_panos {
     message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),');
 };
\ No newline at end of file
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
index 6ee0ad0..31b5013 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
@@ -6,7 +6,7 @@
 log {
 {{- if eq (.) "yes"}}
     source(s_DEFAULT);
-    filter(f_is_rfc3164);
+    filter(f_is_palalto_format);
     filter(f_paloalto_panos);
 {{- end}}
 {{- if eq (.) "no"}}
@@ -36,7 +36,12 @@ log {
         #2012/04/10 04:39:55
         #parse the date
         date-parser(
+            {{- if ((getenv "SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS") | conv.ToBool) }}
+                format("%Y/%m/%d %H:%M:%S.%f")
+            {{- else}}
                 format("%Y/%m/%d %H:%M:%S")
+            {{- end}}
+
                 template("${.pan.GeneratedTime}")
                 time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})
                 flags(guess-timezone)
diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t
index ff5be59..8a71d29 100644
--- a/package/etc/go_templates/source_network.t
+++ b/package/etc/go_templates/source_network.t
@@ -99,6 +99,13 @@ source s_{{ .port_id}} {
         } elif {
             parser {cisco-parser()};
             rewrite(set_cisco_ios);
+        } elif {
+            filter(f_rfc5424_bsdtime);
+            rewrite(set_rfc3894_bsdtime);
+            parser {
+                    syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone));
+                };
+            rewrite(set_rfc5424_bsdtime);
         } elif {
             parser (p_cisco_meraki);
             rewrite(set_rfc5424_epochtime);
diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py
index 2d40c8f..6211590 100644
--- a/tests/test_palo_alto.py
+++ b/tests/test_palo_alto.py
@@ -6,6 +6,7 @@
 import random
 
 from jinja2 import Environment
+from pytest import mark
 
 from .sendmessage import *
 from .splunkutils import *
@@ -54,3 +55,45 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk):
     record_property("message", message)
 
     assert resultCount == 1
+
+def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }}1 {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n")
+    message = mt.render(mark="<111>", host=host)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+
+@mark.skip()
+def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n")
+    message = mt.render(mark="<111>", host=host)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1

From 46ff4a92ab09fa410edaea6b5cdbf9b7855d24a3 Mon Sep 17 00:00:00 2001
From: mbonsack <mbonsack@splunk.com>
Date: Thu, 12 Dec 2019 15:00:30 -0800
Subject: [PATCH 17/47] Update index.md

---
 docs/sources/Cisco/index.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md
index d2c67fd..d6f583a 100644
--- a/docs/sources/Cisco/index.md
+++ b/docs/sources/Cisco/index.md
@@ -91,8 +91,8 @@ MSG Parse: This filter parses message content
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_CISCO_ASA_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format |
-| SC4S_LISTEN_CISCO_ASA_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined expecting RFC5424 format |
+| SC4S_LISTEN_CISCO_ASA_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CISCO_ASA_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_CISCO_ASA | no | Enable archive to disk for this specific source |
 | SC4S_DEST_CISCO_ASA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 | SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format |

From 55e17a2d7cb5d8c1b17f991bd67d7c7db058387e Mon Sep 17 00:00:00 2001
From: mbonsack <mbonsack@splunk.com>
Date: Thu, 12 Dec 2019 15:01:33 -0800
Subject: [PATCH 18/47] Update index.md

---
 docs/sources/Cisco/index.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md
index d6f583a..b0234dd 100644
--- a/docs/sources/Cisco/index.md
+++ b/docs/sources/Cisco/index.md
@@ -33,8 +33,8 @@ PATTERN MATCH
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_CISCO_ACS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format |
-| SC4S_LISTEN_CISCO_ACS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined expecting RFC5424 format |
+| SC4S_LISTEN_CISCO_ACS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CISCO_ACS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_CISCO_ACS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_CISCO_ACS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 

From 38e86fb075560a349034eb13884d89753372e772 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 18:55:08 -0500
Subject: [PATCH 19/47] Provide support for SMG

Initial support for SMG
---
 docs/sources/Symantec/index.md                | 50 ++++++++++++++++
 .../conf.d/filters/symantec/brightmail.conf   |  3 +
 .../p_rfc3165-symantec_brightmail.conf.tmpl   | 59 +++++++++++++++++++
 .../apps/SA-syslog-ng/default/indexes.conf    |  5 ++
 tests/test_symantec_brightmail.py             | 34 +++++++++++
 5 files changed, 151 insertions(+)
 create mode 100644 package/etc/conf.d/filters/symantec/brightmail.conf
 create mode 100644 package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
 create mode 100644 tests/test_symantec_brightmail.py

diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md
index e169c78..c92bc05 100644
--- a/docs/sources/Symantec/index.md
+++ b/docs/sources/Symantec/index.md
@@ -49,3 +49,53 @@ An active proxy will generate frequent events. Use the following search to valid
 ```
 index=<asconfigured> sourcetype=bluecoat:proxysg:access:kv | stats count by host
 ```
+
+## Product - Mail Gateway (Brightmail)
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | TBD                                                            |
+| Product Manual | https://support.symantec.com/us/en/article.howto38250.html                                                       |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| symantec:smg        | Requires version TA 3.6                                                                                                    |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| symantec_brightmail      | symantec:smg     | email          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* No TA available
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_SYMANTEC_PROXY | no | Enable archive to disk for this specific source |
+| SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active mail server will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=symantec:smg | stats count by host
+```
diff --git a/package/etc/conf.d/filters/symantec/brightmail.conf b/package/etc/conf.d/filters/symantec/brightmail.conf
new file mode 100644
index 0000000..af6f53e
--- /dev/null
+++ b/package/etc/conf.d/filters/symantec/brightmail.conf
@@ -0,0 +1,3 @@
+filter f_symantec_brightmail {
+    message("[Brightmail]")
+};
\ No newline at end of file
diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
new file mode 100644
index 0000000..41cc15d
--- /dev/null
+++ b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
@@ -0,0 +1,59 @@
+# Proofpoint
+{{ $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_DEFAULT);
+    filter(f_symantec_brightmail);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_SYMANTEC_BRIGHTMAIL);
+{{- end }}
+
+    rewrite {
+        set("symantec_brightmail", value("fields.sc4s_vendor_product"));
+        subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+        set("${PROGRAM}", value(".PROGRAM"));
+        subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+    };
+
+    rewrite {
+        r_set_splunk_dest_default(sourcetype("symantec:smg"), index("email"), source("program:${.PROGRAM}") )
+
+    };
+
+    parser { p_add_context_splunk(key("symantec_brightmail")); };
+
+    parser (compliance_meta_by_source);
+
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+        set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+    };
+
+{{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};
+{{- end}}
+
+{{- if or (or (getenv  (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT")) (getenv  (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT"))) (getenv  (print "SC4S_SYMANTEC_BRIGHTMAIL_NSS_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for SYMANTEC_BRIGHTMAIL traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for SYMANTEC_BRIGHTMAIL traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
index 9c64a84..b2ecfa9 100644
--- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
+++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
@@ -4,6 +4,11 @@ homePath   = $SPLUNK_DB/syslogng_metrics/db
 coldPath   = $SPLUNK_DB/syslogng_metrics/colddb
 thawedPath = $SPLUNK_DB/syslogng_metrics/thaweddb
 
+[email]
+homePath   = $SPLUNK_DB/email/db
+coldPath   = $SPLUNK_DB/email/colddb
+thawedPath = $SPLUNK_DB/email/thaweddb
+
 [em_metrics]
 datatype=metric
 homePath   = $SPLUNK_DB/em_metrics/db
diff --git a/tests/test_symantec_brightmail.py b/tests/test_symantec_brightmail.py
new file mode 100644
index 0000000..5fa0498
--- /dev/null
+++ b/tests/test_symantec_brightmail.py
@@ -0,0 +1,34 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import random
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+# <141>Oct 24 21:05:43 smg-1 conduit: [Brightmail] (NOTICE:7500.3119331456): [12066] 'BrightSig3 Newsletter Rules' were updated successfully.
+def test_symantec_brightmail(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} conduit: [Brightmail] (NOTICE:7500.3119331456): [12066] 'BrightSig3 Newsletter Rules' were updated successfully.")
+    message = mt.render(mark="<134>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"symantec:smg\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#

From bd0a9d6a77e15da04ae71b8f2576c913cb6ca6ed Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 22:53:01 -0500
Subject: [PATCH 20/47] Final updates

---
 docs/sources/Symantec/index.md                |   2 +-
 .../conf.d/filters/symantec/brightmail.conf   |   5 +-
 .../p_rfc3165-symantec_brightmail.conf.tmpl   | 119 ++++++++++++++----
 tests/test_symantec_brightmail.py             |  37 ++++++
 4 files changed, 136 insertions(+), 27 deletions(-)

diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md
index c92bc05..dc6dcf2 100644
--- a/docs/sources/Symantec/index.md
+++ b/docs/sources/Symantec/index.md
@@ -91,7 +91,7 @@ MSG Parse: This filter parses message content
 | SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_SYMANTEC_PROXY | no | Enable archive to disk for this specific source |
 | SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
-
+| SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG | yes | Email processing events generated by the bmserver process will be grouped by host+program+pid+msg ID into a single event |
 ### Verification
 
 An active mail server will generate frequent events. Use the following search to validate events are present per source device
diff --git a/package/etc/conf.d/filters/symantec/brightmail.conf b/package/etc/conf.d/filters/symantec/brightmail.conf
index af6f53e..7d78535 100644
--- a/package/etc/conf.d/filters/symantec/brightmail.conf
+++ b/package/etc/conf.d/filters/symantec/brightmail.conf
@@ -1,3 +1,6 @@
 filter f_symantec_brightmail {
-    message("[Brightmail]")
+    message('\[Brightmail\]') or program("bmserver")
+};
+filter f_symantec_brightmail_details {
+    program("bmserver") and not message('\[Brightmail\]')
 };
\ No newline at end of file
diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
index 41cc15d..ae26f0f 100644
--- a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
@@ -1,4 +1,25 @@
-# Proofpoint
+filter f_symantec_brightmail_complete{
+    match("yes", value("SMG.COMPLETE") type(glob));
+};
+
+parser symantec_brightmail_grouping {
+    csv-parser(
+        columns(PID, SMG.seq, MESSAGE)
+        delimiters(chars("|"))
+        flags(greedy)
+    );
+    grouping-by(
+        scope(program)
+        key("${SMG.seq}")
+        timeout(2)
+        aggregate(
+             value("MESSAGE" "${PID}|${SMG.seq}|$(implode ';' $(context-values ${MESSAGE}))")
+             value("SMG.COMPLETE" "yes")
+        )
+    );
+};
+
+
 {{ $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }}
 {{ tmpl.Exec "t/source_network.t" $context }}
 # The following is an inline template; we will use this to generate the actual log path
@@ -6,46 +27,94 @@
 log {
 {{- if eq (.) "yes" }}
     source(s_DEFAULT);
+    filter(f_is_rfc3164);
     filter(f_symantec_brightmail);
 {{- end }}
 {{- if eq (.) "no" }}
     source (s_SYMANTEC_BRIGHTMAIL);
 {{- end }}
 
-    rewrite {
-        set("symantec_brightmail", value("fields.sc4s_vendor_product"));
-        subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
-        set("${PROGRAM}", value(".PROGRAM"));
-        subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
-    };
+{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)
 
-    rewrite {
-        r_set_splunk_dest_default(sourcetype("symantec:smg"), index("email"), source("program:${.PROGRAM}") )
+    if {
+        filter(f_symantec_brightmail_details);
 
-    };
+        parser(symantec_brightmail_grouping);
+    {{- end}}
+        if {
+            filter(f_symantec_brightmail_complete);
 
-    parser { p_add_context_splunk(key("symantec_brightmail")); };
+            rewrite {
+                set("symantec_brightmail", value("fields.sc4s_vendor_product"));
+            };
 
-    parser (compliance_meta_by_source);
+            rewrite {
+                r_set_splunk_dest_default(sourcetype("symantec:smg:mail"), index("email"), source("program:${.PROGRAM}") )
 
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-    };
+            };
+
+            parser { p_add_context_splunk(key("symantec_brightmail")); };
+
+            parser (compliance_meta_by_source);
+
+            #We want to unset the fields we won't need, as this is copied into the
+            #disk queue for network destinations. This can be very disk expensive
+            #if we don't
+            rewrite {
+                set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+                unset(value("RAWMSG"));
+                unset(value("PROGRAM"));
+                unset(value("LEGACY_MSGHDR"));
+            };
+
+            {{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }}
+                destination(d_hec);
+            {{- end}}
 
-{{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }}
-    destination(d_hec);
+            {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }}
+                destination(d_archive);
+            {{- end}}
+
+        };
+    } else {
 {{- end}}
+        rewrite {
+            set("symantec_brightmail", value("fields.sc4s_vendor_product"));
+        };
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("symantec:smg"), index("email"), source("program:${.PROGRAM}") )
+
+        };
+
+        parser { p_add_context_splunk(key("symantec_brightmail")); };
+
+        parser (compliance_meta_by_source);
+
+        #We want to unset the fields we won't need, as this is copied into the
+        #disk queue for network destinations. This can be very disk expensive
+        #if we don't
+        rewrite {
+            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("LEGACY_MSGHDR"));
+        };
 
-{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }}
-    destination(d_archive);
+    {{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }}
+        destination(d_hec);
+    {{- end}}
+
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }}
+        destination(d_archive);
+    {{- end}}
+
+{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)
+
+    };
 {{- end}}
 
+
     flags(flow-control,final);
 };
 {{- end}}
diff --git a/tests/test_symantec_brightmail.py b/tests/test_symantec_brightmail.py
index 5fa0498..6e877f9 100644
--- a/tests/test_symantec_brightmail.py
+++ b/tests/test_symantec_brightmail.py
@@ -4,6 +4,7 @@
 # license that can be found in the LICENSE-BSD2 file or at
 # https://opensource.org/licenses/BSD-2-Clause
 import random
+import uuid
 
 from jinja2 import Environment
 
@@ -31,4 +32,40 @@ def test_symantec_brightmail(record_property, setup_wordlist, setup_splunk):
 
     assert resultCount == 1
 
+def test_symantec_brightmail_msg(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    msgid = uuid.uuid4()
+
+    mt = env.from_string("""{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|VERDICT|someone@example.com|none|default|default\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|FIRED|someone@example.com|none\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|UNTESTED|someone@example.com|safe|opl|content_1574820902092|content_1574820956288|content_1574821059194|content_1574821017042|sys_deny_ip|sys_allow_ip|sys_deny_email|dns_allow|dns_deny|user_allow|user_deny|freq_va|freq_dha|freq_sa|connection_class_0|connection_class_1|connection_class_2|connection_class_3|connection_class_4|connection_class_5|connection_class_6|connection_class_7|connection_class_8|connection_class_9|blockedlang|knownlang\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|LOGICAL_IP|200.200.200.154\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|google-play_111-33.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|mac_appstore_136_33.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|6f0c8ad8-e0da-4bcc-aac9-8fa71ba43bb6.jpg\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|b340ec99-9c66-4a19-a2f4-ee467e0d63e1.jpg\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|product-logo.update.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|header-research.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|ms-logo-138.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|ATTACH|ms-logo-138.png|header-research.png|product-logo.update.png|b340ec99-9c66-4a19-a2f4-ee467e0d63e1.jpg|6f0c8ad8-e0da-4bcc-aac9-8fa71ba43bb6.jpg|mac_appstore_136_33.png|google-play_111-33.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|EHLO|mail6.bemta23.messagelabs.com\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|MSG_SIZE|94239\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|MSGID| <7jszytr60wmja@example.com>\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|SUBJECT|pulse: this is a subject\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|SOURCE|external\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195987|{{ MSGID }}|VERDICT|<none>|connection_class_1|default|static connection class 1\n""")
+    message = mt.render(mark="<1>", host=host, MSGID=msgid)
+    sendsingle(message)
+
+    st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"symantec:smg:mail\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
 #

From 1f02296db1c35107f6b610b976c48a085dbf859d Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 23:14:16 -0500
Subject: [PATCH 21/47] Update p_rfc3165-symantec_brightmail.conf.tmpl

---
 .../conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
index ae26f0f..21ca1eb 100644
--- a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
@@ -13,7 +13,7 @@ parser symantec_brightmail_grouping {
         key("${SMG.seq}")
         timeout(2)
         aggregate(
-             value("MESSAGE" "${PID}|${SMG.seq}|$(implode ';' $(context-values ${MESSAGE}))")
+             value("MESSAGE" ";CONTEXT|${PID};${SMG.seq}|$(implode ';' $(context-values ${MESSAGE}))")
              value("SMG.COMPLETE" "yes")
         )
     );

From 3df2bce210bee49598b7043934cd8a02a0a0342e Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 23:55:53 -0500
Subject: [PATCH 22/47] Fix feature flag

---
 docs/sources/Symantec/index.md                    |  2 +-
 .../p_rfc3165-symantec_brightmail.conf.tmpl       | 15 +++++++--------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md
index dc6dcf2..58e79a5 100644
--- a/docs/sources/Symantec/index.md
+++ b/docs/sources/Symantec/index.md
@@ -89,7 +89,7 @@ MSG Parse: This filter parses message content
 |----------------|----------------|----------------|
 | SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
 | SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
-| SC4S_ARCHIVE_SYMANTEC_PROXY | no | Enable archive to disk for this specific source |
+| SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL | no | Enable archive to disk for this specific source |
 | SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 | SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG | yes | Email processing events generated by the bmserver process will be grouped by host+program+pid+msg ID into a single event |
 ### Verification
diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
index 21ca1eb..6e71835 100644
--- a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
@@ -1,3 +1,4 @@
+{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)
 filter f_symantec_brightmail_complete{
     match("yes", value("SMG.COMPLETE") type(glob));
 };
@@ -13,12 +14,12 @@ parser symantec_brightmail_grouping {
         key("${SMG.seq}")
         timeout(2)
         aggregate(
-             value("MESSAGE" ";CONTEXT|${PID};${SMG.seq}|$(implode ';' $(context-values ${MESSAGE}))")
+             value("MESSAGE" "${PID}|${SMG.seq}|$(implode ';' $(context-values ${MESSAGE}))")
              value("SMG.COMPLETE" "yes")
         )
     );
 };
-
+{{- end }}
 
 {{ $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }}
 {{ tmpl.Exec "t/source_network.t" $context }}
@@ -35,12 +36,11 @@ log {
 {{- end }}
 
 {{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)
-
     if {
-        filter(f_symantec_brightmail_details);
 
+        filter(f_symantec_brightmail_details);
         parser(symantec_brightmail_grouping);
-    {{- end}}
+
         if {
             filter(f_symantec_brightmail_complete);
 
@@ -77,7 +77,8 @@ log {
 
         };
     } else {
-{{- end}}
+{{- end }}
+
         rewrite {
             set("symantec_brightmail", value("fields.sc4s_vendor_product"));
         };
@@ -108,9 +109,7 @@ log {
     {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }}
         destination(d_archive);
     {{- end}}
-
 {{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)
-
     };
 {{- end}}
 

From 578829890d1cf642c87c5f5e1a05d6fbf3717108 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Thu, 12 Dec 2019 23:57:15 -0500
Subject: [PATCH 23/47] Update p_rfc3165-symantec_brightmail.conf.tmpl

---
 .../log_paths/p_rfc3165-symantec_brightmail.conf.tmpl       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
index 6e71835..a81ca33 100644
--- a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
@@ -1,4 +1,4 @@
-{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)
+{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }}
 filter f_symantec_brightmail_complete{
     match("yes", value("SMG.COMPLETE") type(glob));
 };
@@ -35,7 +35,7 @@ log {
     source (s_SYMANTEC_BRIGHTMAIL);
 {{- end }}
 
-{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)
+{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)  }}
     if {
 
         filter(f_symantec_brightmail_details);
@@ -109,7 +109,7 @@ log {
     {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }}
         destination(d_archive);
     {{- end}}
-{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)
+{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }}
     };
 {{- end}}
 

From b6d6bb82e1b61673a2bbc7ccacb1435ca9210d7c Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Fri, 13 Dec 2019 00:24:06 -0500
Subject: [PATCH 24/47] Update syslog-ng to 3.25.1

---
 package/syslog-ng | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package/syslog-ng b/package/syslog-ng
index f219fbb..b880087 160000
--- a/package/syslog-ng
+++ b/package/syslog-ng
@@ -1 +1 @@
-Subproject commit f219fbbb12dad0b0b6cb458a0bb4415b2cd94d8a
+Subproject commit b880087c95ce44e3d33722a825e895b9351aca58

From 66c1387a584518681d7bf298016550bed8a3539b Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Fri, 13 Dec 2019 11:13:00 -0500
Subject: [PATCH 25/47] Switch to UBI 8

This switch will also remove kafka support from the image. At this time confluent nor EPEL contain a current conpatible version of the librarys
---
 package/Dockerfile     | 64 ++++++++++++++++--------------------------
 package/confluent.repo |  8 +++---
 2 files changed, 28 insertions(+), 44 deletions(-)

diff --git a/package/Dockerfile b/package/Dockerfile
index a5cbb4b..5a0711d 100644
--- a/package/Dockerfile
+++ b/package/Dockerfile
@@ -6,35 +6,27 @@
 #
 #You should have received a copy of the CC0 legalcode along with this
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
-FROM registry.access.redhat.com/rhel7/rhel
+FROM centos:centos8
 
-ARG RH_ORG
-ARG RH_ACTIVATION
 
-ENV CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --enable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --enable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking"
+ENV CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --disable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --disable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking"
 
-ENV DISTCHECK_CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --enable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --enable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking"
-COPY confluent.repo /etc/yum.repos.d/confluent.repo
-RUN subscription-manager register --org=$RH_ORG --activationkey=$RH_ACTIVATION --force
-RUN subscription-manager repos --enable=rhel-7-server-extras-rpms --enable=rhel-7-server-optional-rpms --enable=rhel-server-rhscl-7-rpms
-RUN yum update -y; yum install wget -y
-RUN rpm --import https://packages.confluent.io/rpm/5.2/archive.key
-RUN cd /tmp ;\
-    wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm ;\
-    yum install epel-release-latest-7.noarch.rpm -y ;\
-    rm epel-release-latest-7.noarch.rpm
-RUN yum install findutils autoconf \
-    autoconf-archive automake ca-certificates git libtool pkgconfig bison byacc file \
-    flex pcre-devel glib2-devel openssl-devel librdkafka-devel libcurl-devel \
-    rh-python36 rh-python36-python-tools rh-python36-scldevel\
-    net-snmp-devel \
-    libuuid-devel cmake make libxslt docbook-style-xsl gcc-c++ tzdata libxml2 sqlite \
-    json-c-devel gnupg wget curl which bzip2 doxygen libsecret ivykis-devel -y
+ENV DISTCHECK_CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --disable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --disable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking"
 
-RUN subscription-manager unregister
+RUN dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y ;\
+    dnf install 'dnf-command(config-manager)' -y ;\
+    dnf config-manager --set-enabled PowerTools -y; \
+    dnf update -y
+
+RUN dnf group install "Development Tools" ;\
+    dnf install findutils autoconf \
+    autoconf automake ca-certificates git libtool pkgconfig bison byacc file \
+    flex pcre-devel glib2-devel openssl-devel libcurl-devel \
+    python3 python3-devel \
+    net-snmp-devel \
+    libuuid-devel cmake make libxslt gcc-c++ tzdata libxml2 sqlite \
+    gnupg wget curl which bzip2 libsecret ivykis-devel autoconf-archive json-c-devel -y
 
-RUN echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh ;\
-    source scl_source enable rh-python36
 
 RUN CRITERION_VERSION=2.3.3 ;\
     cd /tmp/;\
@@ -48,32 +40,24 @@ RUN CRITERION_VERSION=2.3.3 ;\
 
 COPY syslog-ng /work
 
-RUN source scl_source enable rh-python36 ;\
-    cd /work;\
-    pip install -r requirements.txt ;\
+RUN cd /work;\
+    pip3 install -r requirements.txt ;\
     ./autogen.sh ;\
     ./configure $CONFIGURE_FLAGS ;\
     make -j -l 2.5 install
 
 
-FROM registry.access.redhat.com/ubi7/ubi
-
-COPY confluent.repo /etc/yum.repos.d/confluent.repo
+FROM registry.redhat.io/ubi8/ubi
 
 RUN cd /tmp ;\
-    yum install -y wget ;\
-    wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm ;\
-    yum install epel-release-latest-7.noarch.rpm -y ;\
-    rm epel-release-latest-7.noarch.rpm ;\
-    rpm --import https://packages.confluent.io/rpm/5.2/archive.key ;\
-    yum install gcc tzdata libdbi libsecret libxml2 sqlite tcp_wrappers librdkafka \
-    rh-python36 rh-python36-python-tools libcurl ivykis scl-utils tcp_wrappers-libs curl wget openssl -y;\
-    echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh ;\
-    source scl_source enable rh-python36
+    dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y; \
+    dnf update -y ;\
+    dnf install wget gcc tzdata libdbi libsecret libxml2 sqlite \
+    python3 libcurl ivykis scl-utils curl wget openssl -y
 
 ENV DEBCONF_NONINTERACTIVE_SEEN=true
 
-RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh
+RUN curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh
 COPY goss.yaml /etc/goss.yaml
 
 COPY --from=0 /opt/syslog-ng /opt/syslog-ng
diff --git a/package/confluent.repo b/package/confluent.repo
index 26ecf54..ff4016a 100644
--- a/package/confluent.repo
+++ b/package/confluent.repo
@@ -8,14 +8,14 @@
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
 [Confluent.dist]
 name=Confluent repository (dist)
-baseurl=https://packages.confluent.io/rpm/5.2/7
+baseurl=https://packages.confluent.io/rpm/5.3/7
 gpgcheck=1
-gpgkey=https://packages.confluent.io/rpm/5.2/archive.key
+gpgkey=https://packages.confluent.io/rpm/5.3/archive.key
 enabled=1
 
 [Confluent]
 name=Confluent repository
-baseurl=https://packages.confluent.io/rpm/5.2
+baseurl=https://packages.confluent.io/rpm/5.3
 gpgcheck=1
-gpgkey=https://packages.confluent.io/rpm/5.2/archive.key
+gpgkey=https://packages.confluent.io/rpm/5.3/archive.key
 enabled=1
\ No newline at end of file

From 99e339eb68d8b3cebacc64a4838be90d36e49365 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Fri, 13 Dec 2019 11:37:15 -0500
Subject: [PATCH 26/47] Update Dockerfile

---
 package/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package/Dockerfile b/package/Dockerfile
index 5a0711d..7ea7def 100644
--- a/package/Dockerfile
+++ b/package/Dockerfile
@@ -47,7 +47,7 @@ RUN cd /work;\
     make -j -l 2.5 install
 
 
-FROM registry.redhat.io/ubi8/ubi
+FROM registry.access.redhat.com/ubi8/ubi
 
 RUN cd /tmp ;\
     dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y; \

From 49f469239ca8037c4964c765edc82efb400e7890 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Fri, 13 Dec 2019 11:38:58 -0500
Subject: [PATCH 27/47] remove rhel dev network use

---
 .circleci/config.yml     | 1 -
 .env.template            | 2 --
 docker-compose-debug.yml | 3 ---
 docker-compose-perf.yml  | 3 ---
 docker-compose.yml       | 3 ---
 5 files changed, 12 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8840fcf..a455417 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -39,7 +39,6 @@ jobs:
           image: $CI_IMAGE
           registry: $REGISTRY
           path: package
-          extra_build_args: --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION
 
       - docker/install-goss:
           version: v0.3.7
diff --git a/.env.template b/.env.template
index f360cd6..b795fde 100644
--- a/.env.template
+++ b/.env.template
@@ -6,8 +6,6 @@
 #
 #You should have received a copy of the CC0 legalcode along with this
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
-RH_ORG=xxxx
-RH_ACTIVATION=xxxxx
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SPLUNK_PASSWORD=Changed@11
 SPLUNK_START_ARGS=--accept-license
diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml
index bb22763..ee8d0e0 100644
--- a/docker-compose-debug.yml
+++ b/docker-compose-debug.yml
@@ -13,9 +13,6 @@ services:
     image: splunk/scs:latest
     build:
       context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
     entrypoint:
       - "tail"
       - "-f"
diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml
index e1bd795..b552c66 100644
--- a/docker-compose-perf.yml
+++ b/docker-compose-perf.yml
@@ -12,9 +12,6 @@ services:
     image: rfaircloth/scs:edge
     build:
       context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
     ports:
       - "514"
diff --git a/docker-compose.yml b/docker-compose.yml
index d24be63..97a759b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -25,9 +25,6 @@ services:
     image: splunk/scs:latest
     build:
       context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
 #When this is enabled test_common will fail
     command: -det

From 77c4cc6b1f832f0255a3e12ffe4cbfcafc3e42db Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Fri, 13 Dec 2019 11:43:29 -0500
Subject: [PATCH 28/47] Update docker-compose.yml

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 97a759b..dff7a5c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -27,7 +27,7 @@ services:
       context: ./package
     hostname: sc4s
 #When this is enabled test_common will fail
-    command: -det
+#    command: -det
     ports:
       - "514:514"
       - "601:601"

From d818b87c9d6feaaa7c7c550a8cdd024dd1d93963 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Fri, 13 Dec 2019 16:38:21 -0500
Subject: [PATCH 29/47] Support nix on dedicated port

When checkpoint devices are configured for syslog they also output os syslog.
---
 .../p_rfc3164-checkpoint_splunk.conf.tmpl     | 92 +++++++++++--------
 tests/test_checkpoint.py                      | 24 +++++
 2 files changed, 78 insertions(+), 38 deletions(-)

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
index 316bd6d..e0d5c88 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
@@ -14,47 +14,63 @@ log {
     source (s_CHECKPOINT_SPLUNK);
 {{- end}}
 
-    parser {
-        kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}"));
-        date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
-    };
+    if {
+        filter(f_nix_syslog);
+        rewrite {
+          set("checkpoint_splunk", value("fields.sc4s_vendor_product"));
+            subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+        };
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}") )
+        };
+        parser { p_add_context_splunk(key("checkpoint_os")); };
 
-    rewrite {
-        set("${.kv.hostname}", value("HOST"));
-        set("checkpoint_splunk", value("fields.sc4s_vendor_product"));
-        r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"))
-    };
 
-    parser {p_add_context_splunk(key("checkpoint_splunk")); };
+    } else {
+        parser {
+            kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}"));
+            date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
+        };
+
+        rewrite {
+            set("${.kv.hostname}", value("HOST"));
+            set("checkpoint_splunk", value("fields.sc4s_vendor_product"));
+            r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"))
+        };
+
+        parser {p_add_context_splunk(key("checkpoint_splunk")); };
 
-    if  {
-       filter(f_checkpoint_splunk_NetworkTraffic);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); };
-    } elif  {
-       filter(f_checkpoint_splunk_Web);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_web")); };
-    } elif  {
-       filter(f_checkpoint_splunk_NetworkSessions);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); };
-    } elif  {
-       filter(f_checkpoint_splunk_IDS_Malware);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
-    } elif  {
-       filter(f_checkpoint_splunk_IDS);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
-    } elif  {
-       filter(f_checkpoint_splunk_email);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_email")); };
-    } elif  {
-       filter(f_checkpoint_splunk_DLP);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); };
+        if  {
+           filter(f_checkpoint_splunk_NetworkTraffic);
+           rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))};
+           parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); };
+        } elif  {
+           filter(f_checkpoint_splunk_Web);
+           rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))};
+           parser {p_add_context_splunk(key("checkpoint_splunk_web")); };
+        } elif  {
+           filter(f_checkpoint_splunk_NetworkSessions);
+           rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))};
+           parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); };
+        } elif  {
+           filter(f_checkpoint_splunk_IDS_Malware);
+           rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))};
+           parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
+        } elif  {
+           filter(f_checkpoint_splunk_IDS);
+           rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))};
+           parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
+        } elif  {
+           filter(f_checkpoint_splunk_email);
+           rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))};
+           parser {p_add_context_splunk(key("checkpoint_splunk_email")); };
+        } elif  {
+           filter(f_checkpoint_splunk_DLP);
+           rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))};
+           parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); };
+        };
     };
 
     parser (compliance_meta_by_source);
diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py
index 92d1717..ab0a630 100644
--- a/tests/test_checkpoint.py
+++ b/tests/test_checkpoint.py
@@ -138,3 +138,27 @@ def test_checkpoint_splunk_smartconsole(record_property, setup_wordlist, setup_s
     record_property("message", message)
 
     assert resultCount == 1
+
+
+#<6>kernel: sd 2:0:0:0: SCSI error: return code = 0x00040000
+def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+
+    mt = env.from_string(
+        "{{ mark }}kernel: sd 2:0:0:0: SCSI error: return code = 0x{{pid}}\n")
+    message = mt.render(mark="<6>", pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2")
+    search = st.render(pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
\ No newline at end of file

From 17e972bdb344b237f705393e540cf9b3949d64fa Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Fri, 13 Dec 2019 20:02:34 -0500
Subject: [PATCH 30/47] Support InfoBlox DNS

---
 docs/sources/InfoBlox/index.md                |  54 +++++++++
 .../etc/conf.d/filters/infoblox/syslog.conf   |   4 +
 .../etc/conf.d/filters/juniper/legacy.conf    |   2 +-
 .../log_paths/p_rfc3164-infoblox.conf.tmpl    | 104 ++++++++++++++++++
 .../log_paths/p_za_nix_syslog.conf.tmpl       |   2 +-
 .../etc/context_templates/splunk_index.csv    |   3 +
 .../vendor_product_by_source.conf             |   4 +
 .../vendor_product_by_source.csv              |   1 +
 .../apps/SA-syslog-ng/default/indexes.conf    |  10 ++
 tests/test_infoblox.py                        |  58 ++++++++++
 10 files changed, 240 insertions(+), 2 deletions(-)
 create mode 100644 docs/sources/InfoBlox/index.md
 create mode 100644 package/etc/conf.d/filters/infoblox/syslog.conf
 create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl
 create mode 100644 tests/test_infoblox.py

diff --git a/docs/sources/InfoBlox/index.md b/docs/sources/InfoBlox/index.md
new file mode 100644
index 0000000..5e24f28
--- /dev/null
+++ b/docs/sources/InfoBlox/index.md
@@ -0,0 +1,54 @@
+# Vendor - Infoblox
+
+## Product - NIOS
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/2934/                                                                 |
+| Product Manual | http://dloads.infoblox.com/direct/appliance//NIOS/NIOS_AdminGuide_6.10.pdf    |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| infoblox:dns        | None                                                                                                    |
+| infoblox:dhcp    | None                                                                                         |
+| infoblox:threat     | None                                                                                          |
+| nix:syslog     | None                                                                                          |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| infoblox_dns      | infoblox:dns       | netdns          | none          |
+| infoblox_dhcp    | infoblox:dhcp      | netipam          | none          |
+| infoblox_threat    | infoblox:threat      | netids          | none          |
+| nix_syslog    | nix:syslog      | osnix          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the admin manual for specific details of configuration
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_INFOBLOX_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_INFOBLOX_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_INFOBLOX | no | Enable archive to disk for this specific source |
+| SC4S_DEST_INFOBLOX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active device will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=infoblox:*| stats count by host
+```
diff --git a/package/etc/conf.d/filters/infoblox/syslog.conf b/package/etc/conf.d/filters/infoblox/syslog.conf
new file mode 100644
index 0000000..f131781
--- /dev/null
+++ b/package/etc/conf.d/filters/infoblox/syslog.conf
@@ -0,0 +1,4 @@
+filter f_infoblox {
+    match("^infoblox", value("fields.sc4s_vendor_product"));
+
+};
\ No newline at end of file
diff --git a/package/etc/conf.d/filters/juniper/legacy.conf b/package/etc/conf.d/filters/juniper/legacy.conf
index 27b3f52..50383ab 100644
--- a/package/etc/conf.d/filters/juniper/legacy.conf
+++ b/package/etc/conf.d/filters/juniper/legacy.conf
@@ -14,7 +14,7 @@ filter f_juniper_netscreen {
 
 };
 
- filter f_juniper_idp {
+filter f_juniper_idp {
     match("juniper_idp", value("fields.sc4s_vendor_product") type(glob))
       or match('^\[syslog@juniper' value("SDATA"))
 };
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl
new file mode 100644
index 0000000..14bb05a
--- /dev/null
+++ b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl
@@ -0,0 +1,104 @@
+# Juniper IDP
+{{ $context := dict "port_id" "INFOBLOX" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes"}}
+    source(s_DEFAULT);
+    filter(f_is_rfc3164);
+    filter(f_infoblox);
+{{- end}}
+{{- if eq (.) "no"}}
+    source (s_INFOBLOX);
+{{- end}}
+
+
+    if {
+        filter{program("named")};
+        rewrite {
+            set("infoblox_dns", value("fields.sc4s_vendor_product"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("infoblox:dns"), index("netdns"), source("program:${.PROGRAM}"))
+            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("infoblox_dns"));
+        };
+    } elif {
+        filter{program("dhcpd")};
+        rewrite {
+            set("infoblox_dhcp", value("fields.sc4s_vendor_product"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("infoblox:dhcp"), index("netipam"), source("program:${.PROGRAM}"))
+            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("infoblox_dhcp"));
+        };
+    } elif {
+        filter{program("threat-protect-log")};
+        rewrite {
+            set("infoblox_dns", value("fields.sc4s_vendor_product"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("infoblox:threat"), index("netids"), source("program:${.PROGRAM}"))
+            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("infoblox_threat"));
+        };
+    } else {
+        rewrite {
+            set("nix_syslog", value("fields.sc4s_vendor_product"));
+            subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+        };
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") )
+            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+
+        };
+
+        parser { p_add_context_splunk(key("nix_syslog")); };
+    };
+
+
+    parser (compliance_meta_by_source);
+
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+        unset(value("PID"));
+        groupunset(values(".kv.*"));
+    };
+
+
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INFOBLOX_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_INFOBLOX") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};
+{{- end}}
+
+{{- if or (or (getenv  (print "SC4S_LISTEN_INFOBLOX_TCP_PORT")) (getenv  (print "SC4S_LISTEN_INFOBLOX_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_INFOBLOX_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for INFOBLOX traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for INFOBLOX traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
index 9e2547e..805ceff 100644
--- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
@@ -20,7 +20,7 @@ log {
     };
 
     rewrite {
-        r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"), source("program:${.PROGRAM}") )
+        r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") )
 
     };
 
diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv
index 12927fb..6c19bc3 100644
--- a/package/etc/context_templates/splunk_index.csv
+++ b/package/etc/context_templates/splunk_index.csv
@@ -22,6 +22,9 @@
 #fortinet_fortios_log,index,netops
 #fortinet_fortios_traffic,index,netfw
 #fortinet_fortios_utm,index,netids
+#infoblox_dns,index,netdns
+#infoblox_dhcp,index,netipam
+#infoblox_threat,index,netids
 #juniper_idp,index,netids
 #juniper_structured,index,netops
 #juniper_idp_structured,index,netids
diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf
index 4c8127f..911117d 100644
--- a/package/etc/context_templates/vendor_product_by_source.conf
+++ b/package/etc/context_templates/vendor_product_by_source.conf
@@ -28,6 +28,10 @@ filter f_cisco_nx_os {
     host("csconx-*" type(glob)) or
     netmask(169.254.7.0/24)
 };
+filter f_infoblox {
+    host("vib-*" type(glob)) or
+    netmask(169.254.7.0/24)
+};
 filter f_proofpoint_pps_filter {
     host("pps-*" type(glob)) or
     netmask(169.254.8.0/24)
diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv
index 975d558..510cc19 100644
--- a/package/etc/context_templates/vendor_product_by_source.csv
+++ b/package/etc/context_templates/vendor_product_by_source.csv
@@ -1,5 +1,6 @@
 f_test_test,sc4s_vendor_product,"test_test"
 f_cisco_meraki,sc4s_vendor_product,"cisco_meraki"
+f_infoblox,sc4s_vendor_product,"infoblox"
 f_juniper_nsm,sc4s_vendor_product,"juniper_nsm"
 f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp"
 f_juniper_idp,sc4s_vendor_product,"juniper_idp"
diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
index b2ecfa9..1a37448 100644
--- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
+++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
@@ -56,6 +56,11 @@ homePath   = $SPLUNK_DB/netdlp/db
 coldPath   = $SPLUNK_DB/netdlp/colddb
 thawedPath = $SPLUNK_DB/netdlp/thaweddb
 
+[netdns]
+homePath   = $SPLUNK_DB/netdns/db
+coldPath   = $SPLUNK_DB/netdns/colddb
+thawedPath = $SPLUNK_DB/netdns/thaweddb
+
 [netfw]
 homePath   = $SPLUNK_DB/netfw/db
 coldPath   = $SPLUNK_DB/netfw/colddb
@@ -66,6 +71,11 @@ homePath   = $SPLUNK_DB/netids/db
 coldPath   = $SPLUNK_DB/netids/colddb
 thawedPath = $SPLUNK_DB/netids/thaweddb
 
+[netipam]
+homePath   = $SPLUNK_DB/netipam/db
+coldPath   = $SPLUNK_DB/netipam/colddb
+thawedPath = $SPLUNK_DB/netipam/thaweddb
+
 [netops]
 homePath   = $SPLUNK_DB/netops/db
 coldPath   = $SPLUNK_DB/netops/colddb
diff --git a/tests/test_infoblox.py b/tests/test_infoblox.py
new file mode 100644
index 0000000..f7da9c7
--- /dev/null
+++ b/tests/test_infoblox.py
@@ -0,0 +1,58 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import datetime
+import random
+import pytz
+
+from jinja2 import Environment, environment
+
+from .sendmessage import *
+from .splunkutils import *
+import random
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+#<142>Oct 25 13:08:00 161.231.218.156 named[6597]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53
+def test_infoblox_dns(record_property, setup_wordlist, setup_splunk):
+    host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} named[{{ pid }}]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53\n")
+    message = mt.render(mark="<111>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netdns host={{ host }} sourcetype=\"infoblox:dns\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#<150>Oct 24 21:09:00 162.101.157.246 dhcpd[28922]: DHCPREQUEST for 10.130.151.62 from 80:ce:62:9c:0e:70 (DTCCE0826E00C97) via eth2 TransID 802c562c uid 01:80:ce:62:9c:0e:70 (RENEW)
+def test_infoblox_dhcp(record_property, setup_wordlist, setup_splunk):
+    host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dhcpd[{{ pri }}]: DHCPREQUEST for 10.00.00.62 from 80:00:00:00:0e:70 (EXAMPLE) via eth2 TransID 802c562c uid 01:80:00:00:00:00:70 (RENEW)\n")
+    message = mt.render(mark="<150>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netipam host={{ host }} sourcetype=\"infoblox:dhcp\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
\ No newline at end of file

From fb9c900950d408da42686af8c58bc8df41605bc6 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Fri, 13 Dec 2019 20:33:01 -0500
Subject: [PATCH 31/47] Update index.md

---
 docs/sources/InfoBlox/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sources/InfoBlox/index.md b/docs/sources/InfoBlox/index.md
index 5e24f28..92cea8f 100644
--- a/docs/sources/InfoBlox/index.md
+++ b/docs/sources/InfoBlox/index.md
@@ -28,7 +28,7 @@
 
 ### Filter type
 
-MSG Parse: This filter parses message content
+Must be identified by host or ip assignment. Update the filter `f_infoblox` or configure a dedicated port as required
 
 ### Setup and Configuration
 

From 007965854464f9449722f9de5fbd655154289daf Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Sat, 14 Dec 2019 07:19:28 -0500
Subject: [PATCH 32/47] change all netmasks to place holders

---
 .../vendor_product_by_source.conf             | 52 +++++++++----------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf
index 911117d..ac87c2e 100644
--- a/package/etc/context_templates/vendor_product_by_source.conf
+++ b/package/etc/context_templates/vendor_product_by_source.conf
@@ -1,54 +1,54 @@
 @version: 3.24
 
 filter f_test_test {
-    host("testvp-*" type(glob)) or
-    netmask(169.254.1.0/24)
+    host("testvp-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_juniper_idp {
-    host("jnpidp-*" type(glob)) or
-    netmask(169.254.2.0/24)
+    host("jnpidp-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_juniper_netscreen {
-    host("jnpns-*" type(glob)) or
-    netmask(169.254.3.0/24)
+    host("jnpns-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_juniper_nsm {
-    host("jnpnsm-*" type(glob)) or
-    netmask(169.254.4.0/24)
+    host("jnpnsm-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_juniper_nsm_idp {
-    host("jnpnsmidp-*" type(glob)) or
-    netmask(169.254.5.0/24)
+    host("jnpnsmidp-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_cisco_meraki {
-    host("testcm-*" type(glob)) or
-    netmask(169.254.6.0/24)
+    host("testcm-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_cisco_nx_os {
-    host("csconx-*" type(glob)) or
-    netmask(169.254.7.0/24)
+    host("csconx-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_infoblox {
-    host("vib-*" type(glob)) or
-    netmask(169.254.7.0/24)
+    host("vib-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_proofpoint_pps_filter {
-    host("pps-*" type(glob)) or
-    netmask(169.254.8.0/24)
+    host("pps-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_proofpoint_pps_sendmail {
-    host("pps-*" type(glob)) or
-    netmask(169.254.9.0/24)
+    host("pps-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_ubiquiti_unifi_fw {
-    host("usg-*" type(glob)) or
-    netmask(169.254.10.0/24)
+    host("usg-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_tzfixhst {
-    host("tzfhst-*" type(glob)) or
-    netmask(169.254.11.0/24)
+    host("tzfhst-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_tzfixny {
-    host("tzfny-*" type(glob)) or
-    netmask(169.254.12.0/24)
+    host("tzfny-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };

From 891ae1df46137aad98469f4e0f3f5905e8f783b4 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Mon, 16 Dec 2019 07:55:43 -0500
Subject: [PATCH 33/47] Add support vmware nsx

---
 docs/sources/VMWare/index.md                  | 52 +++++++++++
 .../etc/conf.d/conflib/_common/templates.conf |  2 +
 .../conf.d/destinations/splunk_hec.conf.tmpl  |  2 +-
 package/etc/conf.d/filters/VMware/nsx.conf    |  8 ++
 .../log_paths/p_multi-vmware_nsx.conf.tmpl    | 92 +++++++++++++++++++
 .../conf.d/log_paths/p_zz_fallback.conf.tmpl  | 70 +++++++++-----
 .../etc/context_templates/splunk_index.csv    |  1 +
 tests/test_vmware.py                          | 59 ++++++++++++
 8 files changed, 264 insertions(+), 22 deletions(-)
 create mode 100644 docs/sources/VMWare/index.md
 create mode 100644 package/etc/conf.d/filters/VMware/nsx.conf
 create mode 100644 package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
 create mode 100644 tests/test_vmware.py

diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md
new file mode 100644
index 0000000..665b105
--- /dev/null
+++ b/docs/sources/VMWare/index.md
@@ -0,0 +1,52 @@
+# Vendor - Dell - VMWare
+
+## Product - NSX Controller, Manager, Edge
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | None                                                                |
+| Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| vmware:nsx:vsphere:syslog | None |
+| nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx  |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| vmware_nsx      | vmware:nsx:vsphere:syslog | main          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content when using the default configuration
+
+### Setup and Configuration
+
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_VMWARE_NSX_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_NSX_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_NSX_TLS_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_VMWARE_NSX | no | Enable archive to disk for this specific source |
+| SC4S_DEST_VMWARE_NSX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=vmware:nsx:vsphere:syslog | stats count by host
+```
diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf
index 729e5e2..97db7ef 100644
--- a/package/etc/conf.d/conflib/_common/templates.conf
+++ b/package/etc/conf.d/conflib/_common/templates.conf
@@ -63,6 +63,7 @@ template t_JSON {
 	            --exclude .splunk.*
 	            --exclude HOST
 	            --exclude HOST_FROM
+	            --exclude SOURCE
 	            )");
 	};
 
@@ -77,5 +78,6 @@ template t_JSON_5424 {
 	            --exclude HOST
 	            --exclude HOST_FROM
 	            --exclude RAWMSG
+	            --exclude SOURCE
 	            )");
 	};
diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl
index bc92162..1b2d8fe 100644
--- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl
+++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl
@@ -42,7 +42,7 @@ destination d_hec {
                  source=${.splunk.source}
                  sourcetype=${.splunk.sourcetype}
                  index=${.splunk.index}
-                 event=$MSG
+                 event="$MSG"
                  fields.*)')
         );
 };
\ No newline at end of file
diff --git a/package/etc/conf.d/filters/VMware/nsx.conf b/package/etc/conf.d/filters/VMware/nsx.conf
new file mode 100644
index 0000000..88744dd
--- /dev/null
+++ b/package/etc/conf.d/filters/VMware/nsx.conf
@@ -0,0 +1,8 @@
+
+filter f_vmware_nsx {
+    program("NSX")
+    or
+    program("NSXV")
+    or
+    program("dfwpktlogs")
+};
\ No newline at end of file
diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
new file mode 100644
index 0000000..6e623a3
--- /dev/null
+++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
@@ -0,0 +1,92 @@
+# Generate the custom port if defined
+{{ $context := dict "port_id" "VMWARE_NSX" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes"}}
+    source(s_DEFAULT);
+
+    filter(f_vmware_nsx);
+{{- end}}
+{{- if eq (.) "no"}}
+    source (s_VMWARE_NSX);
+{{- end}}
+
+
+    if {
+        filter(f_is_rfc5424_strict);
+        filter(f_vmware_nsx);
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}"));
+            set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_nsx"));
+        };
+    } elif {
+
+        filter(f_vmware_nsx);
+
+        rewrite {
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}"));
+            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_nsx"));
+        };
+
+    } else {
+
+        rewrite {
+            set("nix_syslog", value("fields.sc4s_vendor_product"));
+            subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+        };
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") )
+
+        };
+
+        parser { p_add_context_splunk(key("nix_syslog")); };
+
+        parser (compliance_meta_by_source);
+
+        #We want to unset the fields we won't need, as this is copied into the
+        #disk queue for network destinations. This can be very disk expensive
+        #if we don't
+        rewrite {
+            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("LEGACY_MSGHDR"));
+        };
+    };
+
+    parser (compliance_meta_by_source);
+
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_NSX_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE_NSX") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};
+{{- end}}
+{{- if or (or (getenv  (print "SC4S_LISTEN_VMWARE_NSX_TCP_PORT")) (getenv  (print "SC4S_LISTEN_VMWARE_NSX_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_VMWARE_NSX_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for VMWARE_NSX traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for VMWARE_NSX traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
index 15c3931..b6e1f40 100644
--- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
@@ -1,31 +1,59 @@
 log {
     source(s_DEFAULT);
 
-    rewrite {
-        r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
-        set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG"));
-    };
-    parser {
-        p_add_context_splunk(key("sc4s_fallback"));
-    };
+    if {
+        filter(f_is_rfc5424_strict);
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
+            set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("sc4s_fallback"));
+        };
+    {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
+        destination(d_hec);
+    {{- end}}
+
+
+        #in fallback archive only write rawmsg as msg
+        rewrite {
+            unset(value("RAWMSG"));
+            groupunset(values(".kv.*"));
+        };
 
-{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
-    destination(d_hec);
-{{- end}}
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
+        destination(d_archive);
+    {{- end}}
 
+    } else {
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
+            set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("sc4s_fallback"));
+        };
 
-    #in fallback archive only write rawmsg as msg
-    rewrite {
-        set("$RAWMSG" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-        groupunset(values(".kv.*"));
+    {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
+        destination(d_hec);
+    {{- end}}
+
+
+        #in fallback archive only write rawmsg as msg
+        rewrite {
+            set("$RAWMSG" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("LEGACY_MSGHDR"));
+            groupunset(values(".kv.*"));
+        };
+
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
+        destination(d_archive);
+    {{- end}}
     };
 
-{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
-    destination(d_archive);
-{{- end}}
 
-    flags(flow-control,fallback,final);
+
+    flags(flow-control,fallback);
 };
diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv
index 6c19bc3..51b71c0 100644
--- a/package/etc/context_templates/splunk_index.csv
+++ b/package/etc/context_templates/splunk_index.csv
@@ -53,3 +53,4 @@
 #sc4s_events,index,main
 #sc4s_fallback,index,main
 #sc4s_metrics,index,em_metrics
+#vmware_nsx,index,main
\ No newline at end of file
diff --git a/tests/test_vmware.py b/tests/test_vmware.py
new file mode 100644
index 0000000..e64b3ca
--- /dev/null
+++ b/tests/test_vmware.py
@@ -0,0 +1,59 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import datetime
+import random
+import pytz
+
+from jinja2 import Environment, environment
+
+from .sendmessage import *
+from .splunkutils import *
+import random
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+
+#<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704
+def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk):
+    host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV {{ pid }} - [nsxv@6876 comp=\"nsx-manager\" subcomp=\"manager\"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704\n")
+    message = mt.render(mark="<144>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#
+def test_linux_vmware_nsx_fw(record_property, setup_wordlist, setup_splunk):
+    host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dfwpktlogs: {{ pid }} INET match PASS domain-c7/1001 IN 60 TCP 10.33.24.50/45926->10.33.24.9/8140 S\n")
+    message = mt.render(mark="<144>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main host={{ host }} {{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1

From 3b319995e6e0fec8c4ca54d04850a48cae5babd4 Mon Sep 17 00:00:00 2001
From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com>
Date: Mon, 16 Dec 2019 11:10:57 -0500
Subject: [PATCH 34/47] Update .gitsubmodules to match link file

---
 .gitmodules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitmodules b/.gitmodules
index b460d99..fd4c8a0 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,6 @@
 [submodule "package/syslog-ng"]
 	path = package/syslog-ng
 	url = https://github.com/balabit/syslog-ng.git
-	branch = syslog-ng-3.24.1
+	branch = syslog-ng-3.25.1
 #
 

From f3d76e0a0bd7d86901b29bce0c1dad837a0c7dec Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Mon, 16 Dec 2019 12:11:10 -0500
Subject: [PATCH 35/47] fix osnix index in tests

---
 tests/test_checkpoint.py   | 2 +-
 tests/test_linux_syslog.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py
index ab0a630..c9adef0 100644
--- a/tests/test_checkpoint.py
+++ b/tests/test_checkpoint.py
@@ -152,7 +152,7 @@ def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk):
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2")
     search = st.render(pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
diff --git a/tests/test_linux_syslog.py b/tests/test_linux_syslog.py
index 914dea6..9297929 100644
--- a/tests/test_linux_syslog.py
+++ b/tests/test_linux_syslog.py
@@ -25,7 +25,7 @@ def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_sp
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -45,7 +45,7 @@ def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splu
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -65,7 +65,7 @@ def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_s
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -85,7 +85,7 @@ def test_linux__host_program_conforms(record_property, setup_wordlist, setup_spl
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)

From 5d4e597abd3ca42bf6ef07abb57872bacb3a3491 Mon Sep 17 00:00:00 2001
From: mbonsack <mbonsack@splunk.com>
Date: Mon, 16 Dec 2019 09:23:30 -0800
Subject: [PATCH 36/47] Update index.md

---
 docs/sources/VMWare/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md
index 665b105..d8954da 100644
--- a/docs/sources/VMWare/index.md
+++ b/docs/sources/VMWare/index.md
@@ -39,7 +39,7 @@ MSG Parse: This filter parses message content when using the default configurati
 |----------------|----------------|----------------|
 | SC4S_LISTEN_VMWARE_NSX_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
 | SC4S_LISTEN_VMWARE_NSX_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
-| SC4S_LISTEN_VMWARE_NSX_TLS_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_NSX_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_VMWARE_NSX | no | Enable archive to disk for this specific source |
 | SC4S_DEST_VMWARE_NSX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 

From ea3b9a2a7ac12d7c512c359b6e69a016669b20ef Mon Sep 17 00:00:00 2001
From: Mark Bonsack <mbonsack@splunk.com>
Date: Mon, 16 Dec 2019 11:24:01 -0800
Subject: [PATCH 37/47] fix template override variable

fix template override variable in log paths
---
 .../log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl       | 2 +-
 .../etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl    | 2 +-
 .../etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl    | 2 +-
 .../etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl    | 2 +-
 .../etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl    | 2 +-
 .../etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl   | 2 +-
 .../log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl   | 2 +-
 .../conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl | 2 +-
 package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl | 8 ++++----
 .../etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl  | 2 +-
 .../conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl    | 2 +-
 .../log_paths/p_rfc3164-juniper_netscreen.conf.tmpl       | 2 +-
 .../etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl  | 2 +-
 .../conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl  | 2 +-
 .../log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl     | 2 +-
 .../conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl   | 2 +-
 .../conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl   | 2 +-
 .../etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl  | 2 +-
 .../log_paths/p_rfc3165-symantec_brightmail.conf.tmpl     | 4 ++--
 .../log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl     | 2 +-
 .../p_rfc5424-noversion_symantec_proxy.conf.tmpl          | 2 +-
 .../log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl    | 2 +-
 .../log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl      | 2 +-
 package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl    | 2 +-
 package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl      | 2 +-
 25 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
index e0d5c88..da7ffeb 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
@@ -79,7 +79,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl
index 5666854..0b9ccfd 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl
@@ -73,7 +73,7 @@ log {
         #disk queue for network destinations. This can be very disk expensive
         #if we don't
         rewrite {
-            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
             unset(value("RAWMSG"));
             unset(value("PROGRAM"));
             unset(value("PID"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl
index 26dfaea..172445d 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl
@@ -26,7 +26,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
index d03081a..cb90f70 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
@@ -28,7 +28,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl
index bdd92d4..544c0a3 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl
@@ -73,7 +73,7 @@ log {
         #disk queue for network destinations. This can be very disk expensive
         #if we don't
         rewrite {
-            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
             unset(value("RAWMSG"));
             unset(value("PROGRAM"));
             unset(value("PID"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl
index 0d28a3a..59db061 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl
@@ -28,7 +28,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl
index d86957d..72d4de7 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl
@@ -27,7 +27,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl
index 02be55a..29f17fc 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl
@@ -44,7 +44,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl
index 14bb05a..ecfad6e 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl
@@ -22,7 +22,7 @@ log {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
             r_set_splunk_dest_default(sourcetype("infoblox:dns"), index("netdns"), source("program:${.PROGRAM}"))
-            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
         };
         parser {
             p_add_context_splunk(key("infoblox_dns"));
@@ -34,7 +34,7 @@ log {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
             r_set_splunk_dest_default(sourcetype("infoblox:dhcp"), index("netipam"), source("program:${.PROGRAM}"))
-            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
         };
         parser {
             p_add_context_splunk(key("infoblox_dhcp"));
@@ -46,7 +46,7 @@ log {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
             r_set_splunk_dest_default(sourcetype("infoblox:threat"), index("netids"), source("program:${.PROGRAM}"))
-            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
         };
         parser {
             p_add_context_splunk(key("infoblox_threat"));
@@ -61,7 +61,7 @@ log {
 
         rewrite {
             r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") )
-            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
 
         };
 
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl
index b9d1ca1..16f2c0c 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl
@@ -28,7 +28,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl
index 0ecaee2..f0c8a1d 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl
@@ -44,7 +44,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl
index ca717e8..dde7d54 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl
@@ -28,7 +28,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl
index 94bbba5..8d8d999 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl
@@ -29,7 +29,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl
index e571083..c1196c5 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl
@@ -26,7 +26,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_standard))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_standard))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl
index 0114932..8d26d45 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl
@@ -73,7 +73,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
index 31b5013..ee4caaf 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
@@ -82,7 +82,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl
index 4b383e8..0c805e7 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl
@@ -35,7 +35,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl
index 21ffa8b..0bcdf3d 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl
@@ -66,7 +66,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
index a81ca33..70c1298 100644
--- a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
@@ -61,7 +61,7 @@ log {
             #disk queue for network destinations. This can be very disk expensive
             #if we don't
             rewrite {
-                set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+                set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
                 unset(value("RAWMSG"));
                 unset(value("PROGRAM"));
                 unset(value("LEGACY_MSGHDR"));
@@ -96,7 +96,7 @@ log {
         #disk queue for network destinations. This can be very disk expensive
         #if we don't
         rewrite {
-            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
             unset(value("RAWMSG"));
             unset(value("PROGRAM"));
             unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl
index 0e803a7..065c824 100644
--- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl
@@ -24,7 +24,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl
index 2c3c547..c455a9d 100644
--- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl
@@ -27,7 +27,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl
index 766bbff..0d103a8 100644
--- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl
@@ -47,7 +47,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG"));
         unset(value("RAWMSG"));
         groupunset(values(".kv.*"));
     };
diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl
index 949fcce..793b356 100644
--- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl
@@ -24,7 +24,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
index 805ceff..c8f4e2a 100644
--- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl
@@ -32,7 +32,7 @@ log {
     #disk queue for network destinations. This can be very disk expensive
     #if we don't
     rewrite {
-        set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
index 15c3931..f0abfaa 100644
--- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
@@ -3,7 +3,7 @@ log {
 
     rewrite {
         r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
-        set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG"));
+        set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG"));
     };
     parser {
         p_add_context_splunk(key("sc4s_fallback"));

From 9e1992c8d08f69a0f2d1aa66c954aba7b0348b1c Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Mon, 16 Dec 2019 15:13:17 -0500
Subject: [PATCH 38/47] vmware support

---
 docs/sources/VMWare/index.md                  | 17 +++---
 package/etc/conf.d/filters/VMware/nsx.conf    |  8 ---
 .../etc/conf.d/filters/VMware/vsphere.conf    | 58 +++++++++++++++++++
 .../log_paths/p_multi-vmware_nsx.conf.tmpl    | 41 ++++++++++---
 tests/test_vmware.py                          | 20 +++++++
 5 files changed, 120 insertions(+), 24 deletions(-)
 delete mode 100644 package/etc/conf.d/filters/VMware/nsx.conf
 create mode 100644 package/etc/conf.d/filters/VMware/vsphere.conf

diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md
index d8954da..3b9205e 100644
--- a/docs/sources/VMWare/index.md
+++ b/docs/sources/VMWare/index.md
@@ -1,6 +1,6 @@
 # Vendor - Dell - VMWare
 
-## Product - NSX Controller, Manager, Edge
+## Product - vSphwere - ESX NSX (Controller, Manager, Edge)
 
 
 | Ref            | Link                                                                                                    |
@@ -8,12 +8,12 @@
 | Splunk Add-on  | None                                                                |
 | Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html |
 
-
 ### Sourcetypes
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
 | vmware:nsx:vsphere:syslog | None |
+| vmware:esx:vsphere:syslog | None |
 | nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx  |
 
 ### Sourcetype and Index Configuration
@@ -21,6 +21,7 @@
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
 | vmware_nsx      | vmware:nsx:vsphere:syslog | main          | none          |
+| vmware_esx      | vmware:esx:vsphere:syslog | main          | none          |
 
 ### Filter type
 
@@ -37,16 +38,16 @@ MSG Parse: This filter parses message content when using the default configurati
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_VMWARE_NSX_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
-| SC4S_LISTEN_VMWARE_NSX_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
-| SC4S_LISTEN_VMWARE_NSX_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
-| SC4S_ARCHIVE_VMWARE_NSX | no | Enable archive to disk for this specific source |
-| SC4S_DEST_VMWARE_NSX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_VMWARE_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source |
+| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Verification
 
 An active proxy will generate frequent events. Use the following search to validate events are present per source device
 
 ```
-index=<asconfigured> sourcetype=vmware:nsx:vsphere:syslog | stats count by host
+index=<asconfigured> sourcetype="vmware:*:vsphere:*" | stats count by host
 ```
diff --git a/package/etc/conf.d/filters/VMware/nsx.conf b/package/etc/conf.d/filters/VMware/nsx.conf
deleted file mode 100644
index 88744dd..0000000
--- a/package/etc/conf.d/filters/VMware/nsx.conf
+++ /dev/null
@@ -1,8 +0,0 @@
-
-filter f_vmware_nsx {
-    program("NSX")
-    or
-    program("NSXV")
-    or
-    program("dfwpktlogs")
-};
\ No newline at end of file
diff --git a/package/etc/conf.d/filters/VMware/vsphere.conf b/package/etc/conf.d/filters/VMware/vsphere.conf
new file mode 100644
index 0000000..624c6e0
--- /dev/null
+++ b/package/etc/conf.d/filters/VMware/vsphere.conf
@@ -0,0 +1,58 @@
+filter f_vmware_all {
+    #begin base vmware
+    program("cimslp", flags(ignore-case))
+    or program("Fdm", flags(ignore-case))
+    or program("Hostd", flags(ignore-case))
+    or program("hostd-probe", flags(ignore-case))
+    or program("indcfg", flags(ignore-case))
+    or program("lwsmd", flags(ignore-case))
+    or program("netcpa", flags(ignore-case))
+    or program("pktcap-agent", flags(ignore-case))
+    or program("Rhttpproxy", flags(ignore-case))
+    or program("sdrsInjector", flags(ignore-case))
+    or program("sfcb-.*", flags(ignore-case))
+    or program("storageRM", flags(ignore-case))
+    or program("vmkernel", flags(ignore-case))
+    or program("vmkwarning", flags(ignore-case))
+    or program("vobd", flags(ignore-case))
+    or program("Vpxa", flags(ignore-case))
+    or program("Vpxd", flags(ignore-case))
+    or program("VSANMGMTSVC", flags(ignore-case))
+    or program("vsfwd", flags(ignore-case))
+    #begin nsx
+    or program("NSX", flags(ignore-case))
+    or program("NSXV", flags(ignore-case))
+    or program("dfwpktlogs", flags(ignore-case))
+    or program("nsx-.*", flags(ignore-case))};
+
+filter f_vmware_vsphere {
+    program("cimslp", flags(ignore-case))
+    or program("Fdm", flags(ignore-case))
+    or program("Hostd", flags(ignore-case))
+    or program("hostd-probe", flags(ignore-case))
+    or program("indcfg", flags(ignore-case))
+    or program("lwsmd", flags(ignore-case))
+    or program("netcpa", flags(ignore-case))
+    or program("pktcap-agent", flags(ignore-case))
+    or program("Rhttpproxy", flags(ignore-case))
+    or program("sdrsInjector", flags(ignore-case))
+    or program("sfcb-.*", flags(ignore-case))
+    or program("storageRM", flags(ignore-case))
+    or program("vmkernel", flags(ignore-case))
+    or program("vmkwarning", flags(ignore-case))
+    or program("vobd", flags(ignore-case))
+    or program("Vpxa", flags(ignore-case))
+    or program("Vpxd", flags(ignore-case))
+    or program("VSANMGMTSVC", flags(ignore-case))
+    or program("vsfwd", flags(ignore-case))
+};
+
+filter f_vmware_nsx {
+    program("NSX", flags(ignore-case))
+    or
+    program("NSXV", flags(ignore-case))
+    or
+    program("dfwpktlogs", flags(ignore-case))
+    or
+    program("nsx-.*", flags(ignore-case))
+};
\ No newline at end of file
diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
index 6e623a3..23a480f 100644
--- a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
@@ -1,5 +1,5 @@
 # Generate the custom port if defined
-{{ $context := dict "port_id" "VMWARE_NSX" "parser" "common" }}
+{{ $context := dict "port_id" "VMWARE" "parser" "common" }}
 {{ tmpl.Exec "t/source_network.t" $context }}
 
 # The following is an inline template; we will use this to generate the actual log path
@@ -8,13 +8,14 @@ log {
 {{- if eq (.) "yes"}}
     source(s_DEFAULT);
 
-    filter(f_vmware_nsx);
+    filter(f_vmware_all);
 {{- end}}
 {{- if eq (.) "no"}}
-    source (s_VMWARE_NSX);
+    source (s_VMWARE);
 {{- end}}
 
 
+    #NSX first because its the cheapest check
     if {
         filter(f_is_rfc5424_strict);
         filter(f_vmware_nsx);
@@ -39,7 +40,31 @@ log {
         parser {
             p_add_context_splunk(key("vmware_nsx"));
         };
+    #esx things
+    } elif {
+        filter(f_is_rfc5424_strict);
+        filter(f_vmware_vsphere);
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}"));
+            set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_esx"));
+        };
+    } elif {
 
+        filter(f_vmware_vsphere);
+
+        rewrite {
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}"));
+            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_esx"));
+        };
     } else {
 
         rewrite {
@@ -71,22 +96,22 @@ log {
 
     parser (compliance_meta_by_source);
 
-{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_NSX_HEC" "no") | conv.ToBool) }}
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
 {{- end}}
 
 
-{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE_NSX") }}
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE") }}
     destination(d_archive);
 {{- end}}
 
     flags(flow-control,final);
 };
 {{- end}}
-{{- if or (or (getenv  (print "SC4S_LISTEN_VMWARE_NSX_TCP_PORT")) (getenv  (print "SC4S_LISTEN_VMWARE_NSX_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_VMWARE_NSX_TLS_PORT")) }}
-# Listen on the specified dedicated port(s) for VMWARE_NSX traffic
+{{- if or (or (getenv  (print "SC4S_LISTEN_VMWARE_TCP_PORT")) (getenv  (print "SC4S_LISTEN_VMWARE_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_VMWARE_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for VMWARE traffic
     {{ tmpl.Exec "log_path" "no" }}
 {{- end}}
 
-# Listen on the default port (typically 514) for VMWARE_NSX traffic
+# Listen on the default port (typically 514) for VMWARE traffic
 {{ tmpl.Exec "log_path" "yes" }}
diff --git a/tests/test_vmware.py b/tests/test_vmware.py
index e64b3ca..e406ee8 100644
--- a/tests/test_vmware.py
+++ b/tests/test_vmware.py
@@ -15,6 +15,26 @@
 
 env = Environment(extensions=['jinja2_time.TimeExtension'])
 
+#vpxd 123 - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)]
+def test_linux_vmware(record_property, setup_wordlist, setup_splunk):
+    host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} vpxd {{ pid }} - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)]\n")
+    message = mt.render(mark="<144>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main {{ pid }} sourcetype=\"vmware:esx:vsphere:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
 
 #<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704
 def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk):

From 8ddb10e2204a0c69bb707fdcf5cb942218bcf371 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Tue, 17 Dec 2019 09:42:19 -0500
Subject: [PATCH 39/47] Update p_multi-vmware_nsx.conf.tmpl

---
 .../etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl  | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
index 23a480f..797b513 100644
--- a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
@@ -22,7 +22,7 @@ log {
 
         rewrite {
             r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}"));
-            set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG"));
         };
         parser {
             p_add_context_splunk(key("vmware_nsx"));
@@ -35,7 +35,7 @@ log {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
             r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}"));
-            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
         };
         parser {
             p_add_context_splunk(key("vmware_nsx"));
@@ -47,7 +47,7 @@ log {
 
         rewrite {
             r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}"));
-            set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG"));
         };
         parser {
             p_add_context_splunk(key("vmware_esx"));
@@ -60,7 +60,7 @@ log {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
             r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}"));
-            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
         };
         parser {
             p_add_context_splunk(key("vmware_esx"));
@@ -87,7 +87,7 @@ log {
         #disk queue for network destinations. This can be very disk expensive
         #if we don't
         rewrite {
-            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
             unset(value("RAWMSG"));
             unset(value("PROGRAM"));
             unset(value("LEGACY_MSGHDR"));

From 4b87e50304ef54445afb41633eefad0bc21f99e5 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Tue, 17 Dec 2019 11:04:42 -0500
Subject: [PATCH 40/47] Update p_multi-vmware_nsx.conf.tmpl

---
 package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
index 797b513..f5c1fa0 100644
--- a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
@@ -21,7 +21,7 @@ log {
         filter(f_vmware_nsx);
 
         rewrite {
-            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}"));
+            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${PROGRAM}"));
             set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG"));
         };
         parser {
@@ -34,7 +34,7 @@ log {
         rewrite {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
-            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}"));
+            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}"));
             set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
         };
         parser {
@@ -46,7 +46,7 @@ log {
         filter(f_vmware_vsphere);
 
         rewrite {
-            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}"));
+            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${PROGRAM}"));
             set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG"));
         };
         parser {
@@ -59,7 +59,7 @@ log {
         rewrite {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
-            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}"));
+            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}"));
             set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
         };
         parser {

From b6397f53ea4a1aa9ed5b6279733c3756b2b59741 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Tue, 17 Dec 2019 11:21:22 -0500
Subject: [PATCH 41/47] Update p_zz_fallback.conf.tmpl

---
 package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
index 11f90fa..011f670 100644
--- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
@@ -4,7 +4,7 @@ log {
     if {
         filter(f_is_rfc5424_strict);
         rewrite {
-            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
+            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"));
             set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG"));
         };
         parser {
@@ -27,7 +27,7 @@ log {
 
     } else {
         rewrite {
-            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
+            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main") );
             set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG"));
         };
         parser {

From 857b6d0081ac2881ebb2331844e530d5c4e6bfd2 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Tue, 17 Dec 2019 11:50:22 -0500
Subject: [PATCH 42/47] Update p_zz_fallback.conf.tmpl

---
 package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
index 011f670..cb9ed87 100644
--- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
@@ -28,7 +28,7 @@ log {
     } else {
         rewrite {
             r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main") );
-            set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG"));
         };
         parser {
             p_add_context_splunk(key("sc4s_fallback"));

From acde21e27a1fbee4fd2c1a01342bc533aff4d25f Mon Sep 17 00:00:00 2001
From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com>
Date: Tue, 17 Dec 2019 17:46:26 -0500
Subject: [PATCH 43/47] Update Infoblox manual link

Fixes #246
---
 docs/sources/InfoBlox/index.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/sources/InfoBlox/index.md b/docs/sources/InfoBlox/index.md
index 92cea8f..627d8fc 100644
--- a/docs/sources/InfoBlox/index.md
+++ b/docs/sources/InfoBlox/index.md
@@ -1,11 +1,13 @@
 # Vendor - Infoblox
 
+Warning: Despite the TA indication this data source is CIM compliant the all versions of NIOS including the most recent available as of 2019-12-17 do not support the DNS data model correctly. For DNS security use cases use Splunk Stream instead.
+
 ## Product - NIOS
 
 | Ref            | Link                                                                                                    |
 |----------------|---------------------------------------------------------------------------------------------------------|
 | Splunk Add-on  | https://splunkbase.splunk.com/app/2934/                                                                 |
-| Product Manual | http://dloads.infoblox.com/direct/appliance//NIOS/NIOS_AdminGuide_6.10.pdf    |
+| Product Manual | https://docs.infoblox.com/display/ILP/NIOS?preview=/8945695/43728387/NIOS_8.4_Admin_Guide.pdf   |
 
 
 ### Sourcetypes

From 40d80905db71c81bb4e7f12b7243ac8a6e26faed Mon Sep 17 00:00:00 2001
From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com>
Date: Tue, 17 Dec 2019 21:06:56 -0500
Subject: [PATCH 44/47] Resolve errors in documentation of ports

---
 docs/sources/Juniper/index.md | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/docs/sources/Juniper/index.md b/docs/sources/Juniper/index.md
index 98cd701..e497d0c 100644
--- a/docs/sources/Juniper/index.md
+++ b/docs/sources/Juniper/index.md
@@ -37,9 +37,11 @@
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_JUNIPER_JUNOS_LEGACY_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format|
-| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using 5424 format |
-| SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source |
+| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format|
+| SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined using legacy 3164 format|
+| SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined using legacy 3164 format|
+| SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using 5424 format |
+| SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined using 5424 format || SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_JUNOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Verification
@@ -90,6 +92,7 @@ Verify timestamp, and host values match as expected
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
 | SC4S_LISTEN_JUNIPER_NSM_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_JUNIPER_NSM_TLS_PORT      | empty string      | Enable at TLS port for this specific vendor product using the number defined |
 | SC4S_LISTEN_JUNIPER_NSM_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_JUNIPER_NSM | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_NSM_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
@@ -142,6 +145,7 @@ Verify timestamp, and host values match as expected
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
 | SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
 | SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_JUNIPER_NETSCREEN | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_NETSCREEN_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
@@ -192,6 +196,7 @@ Verify timestamp, and host values match as expected
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
 | SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
 | SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_JUNOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
@@ -204,4 +209,4 @@ Use the following search to validate events are present; for Juniper SSL VPN ens
 index=<asconfigured> sourcetype=juniper:sslvpn | stats count by host
 ```
 
-Verify timestamp, and host values match as expected
\ No newline at end of file
+Verify timestamp, and host values match as expected

From 744021d93054e409df2a2b9824e6e320aa0b0da2 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Wed, 18 Dec 2019 11:48:17 -0500
Subject: [PATCH 45/47] Update p_multi-vmware_nsx.conf.tmpl

---
 package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
index f5c1fa0..5396355 100644
--- a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
+++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
@@ -21,7 +21,7 @@ log {
         filter(f_vmware_nsx);
 
         rewrite {
-            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${PROGRAM}"));
+            r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${PROGRAM}"));
             set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG"));
         };
         parser {
@@ -34,7 +34,7 @@ log {
         rewrite {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
-            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}"));
+            r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${.PROGRAM}"));
             set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
         };
         parser {
@@ -46,7 +46,7 @@ log {
         filter(f_vmware_vsphere);
 
         rewrite {
-            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${PROGRAM}"));
+            r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${PROGRAM}"));
             set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG"));
         };
         parser {
@@ -59,7 +59,7 @@ log {
         rewrite {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
-            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}"));
+            r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${.PROGRAM}"));
             set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
         };
         parser {

From eb73ca161c9a196cdb6a7dc76f8ceb88526d1bd5 Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Wed, 18 Dec 2019 11:48:55 -0500
Subject: [PATCH 46/47] Update index.md

---
 docs/sources/VMWare/index.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md
index 3b9205e..7a9e54f 100644
--- a/docs/sources/VMWare/index.md
+++ b/docs/sources/VMWare/index.md
@@ -12,16 +12,16 @@
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| vmware:nsx:vsphere:syslog | None |
-| vmware:esx:vsphere:syslog | None |
+| vmware:vsphere:nsx | None |
+| vmware:vsphere:esx | None |
 | nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx  |
 
 ### Sourcetype and Index Configuration
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| vmware_nsx      | vmware:nsx:vsphere:syslog | main          | none          |
-| vmware_esx      | vmware:esx:vsphere:syslog | main          | none          |
+| vmware_nsx      | vmware:vsphere:nsx | main          | none          |
+| vmware_esx      | vmware:vsphere:esx | main          | none          |
 
 ### Filter type
 

From 704a40cc150fd278bac80f75dcbd9566f2edfcef Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Wed, 18 Dec 2019 15:08:44 -0500
Subject: [PATCH 47/47] fix test case

---
 ...ulti-vmware_nsx.conf.tmpl => p_vmware_vsphere.conf.tmpl} | 0
 tests/test_vmware.py                                        | 6 +++---
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename package/etc/conf.d/log_paths/{p_multi-vmware_nsx.conf.tmpl => p_vmware_vsphere.conf.tmpl} (100%)

diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl
similarity index 100%
rename from package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl
rename to package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl
diff --git a/tests/test_vmware.py b/tests/test_vmware.py
index e406ee8..4ed57ca 100644
--- a/tests/test_vmware.py
+++ b/tests/test_vmware.py
@@ -25,7 +25,7 @@ def test_linux_vmware(record_property, setup_wordlist, setup_splunk):
 
     sendsingle(message)
 
-    st = env.from_string("search index=main {{ pid }} sourcetype=\"vmware:esx:vsphere:syslog\" | head 2")
+    st = env.from_string("search index=main {{ pid }} sourcetype=\"vmware:vsphere:esx\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -46,7 +46,7 @@ def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk):
 
     sendsingle(message)
 
-    st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2")
+    st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:vsphere:nsx\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -67,7 +67,7 @@ def test_linux_vmware_nsx_fw(record_property, setup_wordlist, setup_splunk):
 
     sendsingle(message)
 
-    st = env.from_string("search index=main host={{ host }} {{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2")
+    st = env.from_string("search index=main host={{ host }} {{ pid }} sourcetype=\"vmware:vsphere:nsx\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)