Improve archive only use case allow disable of HEC out and archive of…

… fallback (#217) Allow use of archive on fallback path, allow the user to disable hec out for archive only use cases
CSVD · Dec 6, 2019 · 5a8e39b · 5a8e39b
1 parent 86eeea6
commit 5a8e39b
Show file tree

Hide file tree

Showing 28 changed files with 96 additions and 23 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -6,7 +6,7 @@
 #
 #You should have received a copy of the CC0 legalcode along with this
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
-version: "3.2"
+version: "3.7"
 services:
   test:
     build: ./tests

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -15,11 +15,12 @@ and variables needed to properly configure SC4S for your environment.
 
 | Variable | Values        | Description |
 |----------|---------------|-------------|
-| SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (threads).  Set this to the number of HEC endpoints up to a max of 32. |
-| SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate |
+| SC4S_DEST_SPLUNK_HEC_GLOBAL | yes | Send events to Splunk using HEC |
 | SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE | comma separated list | Open SSL cipher suite list |
 | SC4S_DEST_SPLUNK_HEC_SSL_VERSION |  comma separated list | Open SSL version list |
 | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file |
+| SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate |
+| SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (threads).  Set this to the number of HEC endpoints up to a max of 32. |
 
 ## SC4S Disk Buffer Configuration
 

diff --git a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl
@@ -55,7 +55,7 @@ source (s_LOCAL_EXAMPLE);
 
 # Send it to Splunk
 
-    destination(d_hec);  #--HEC--
+    destination(d_hec);
 
 # Note: We normally do not use the "final" flag; this will allow another plugin to be created that will
 # forward events to another system

diff --git a/package/etc/conf.d/log_paths/internal.conf.tmpl b/package/etc/conf.d/log_paths/internal.conf.tmpl
@@ -16,15 +16,23 @@ log {
                      value("MESSAGE") flags("utf8" "global")
             );
         };
-        destination(d_hecmetrics);  #--HEC--
+
+        {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INTERNAL_METRICS_HEC" "no") | conv.ToBool) }}
+        destination(d_hecmetrics);
+        {{- end}}
+
     } else {
 
         {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}}
         destination(d_stdout);
         {{- end}}
         rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))};
         parser {p_add_context_splunk(key("sc4s_events")); };
+
+        {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no") | conv.ToBool) }}
         destination(d_hec_internal);
+        {{- end}}
+
     };
  };
 {{- end}}

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
@@ -69,8 +69,10 @@ log {
         unset(value("LEGACY_MSGHDR"));
         groupunset(values(".kv.*"));
     };
-
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
+
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl
@@ -32,7 +32,9 @@ log {
         unset(value("LEGACY_MSGHDR"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ASA_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ASA_LEGACY") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
@@ -35,7 +35,9 @@ log {
         groupunset(values(".cisco.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_IOS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_IOS") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl
@@ -84,7 +84,9 @@ log {
             groupunset(values("ISE.*"));
         };
 
-        destination(d_hec);
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ISE_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
 
     {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ISE") }}
         destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl
@@ -34,7 +34,10 @@ log {
         unset(value("LEGACY_MSGHDR"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_NXOS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
+
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_NXOS") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl
@@ -33,7 +33,9 @@ log {
         unset(value("LEGACY_MSGHDR"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORCEPOINT_WEBPROTECT_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORCEPOINT_WEBPROTECT") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl
@@ -51,7 +51,10 @@ log {
         groupunset(values(".kv.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORTINET_FORTIOS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
+
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORTINET_FORTIOS") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl
@@ -36,7 +36,10 @@ log {
         groupunset(values(".kv.*"));
     };
 
+
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_IDP_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_IDP") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl
@@ -50,7 +50,9 @@ log {
         unset(value("LEGACY_MSGHDR"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_JUNOS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_JUNOS") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl
@@ -34,7 +34,9 @@ log {
         unset(value("LEGACY_MSGHDR"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_NETSCREEN_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NETSCREEN") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl
@@ -35,7 +35,9 @@ log {
         unset(value("LEGACY_MSGHDR"));
     };
 
+ {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_NSM_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NSM") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl
@@ -32,7 +32,9 @@ log {
         unset(value("LEGACY_MSGHDR"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_NSM_IDP_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NSM_IDP") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl
@@ -80,7 +80,9 @@ log {
         groupunset(values(".cef.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl
@@ -84,7 +84,9 @@ log {
         groupunset(values(".pan.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_PALOALTO_PANOS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PALOALTO_PANOS") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl
@@ -42,7 +42,9 @@ log {
         groupunset(values(".kv.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_PROOFPOINT_PPS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl
@@ -117,7 +117,10 @@ log {
         unset(value("LEGACY_MSGHDR"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_PROOFPOINT_UBIQUITI_UNIFI_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
+
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_UBIQUITI_UNIFI") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl
@@ -73,7 +73,9 @@ log {
         groupunset(values(".kv.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ZSCALER_NSS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_ZSCALER_NSS") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl
@@ -31,7 +31,9 @@ log {
         groupunset(values(".kv.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ASA_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ASA") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl
@@ -34,7 +34,9 @@ log {
         groupunset(values(".kv.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_PROXY_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_PROXY") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl
@@ -52,9 +52,12 @@ log {
         groupunset(values(".kv.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNOS_STRUCTURED_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
+
 
-{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT") }}
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNOS_STRUCTURED") }}
     destination(d_archive);
 {{- end}}
 

diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl
@@ -31,7 +31,9 @@ log {
         groupunset(values(".kv.*"));
     };
 
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_MERAKI_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
+{{- end}}
 
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_MERAKI") }}
     destination(d_archive);

diff --git a/package/etc/conf.d/log_paths/zfallback.conf b/package/etc/conf.d/log_paths/zfallback.conf
diff --git a/package/etc/conf.d/log_paths/zfallback.conf.tmpl b/package/etc/conf.d/log_paths/zfallback.conf.tmpl
@@ -0,0 +1,31 @@
+log {
+    source(s_DEFAULT);
+
+    rewrite {
+        r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
+        set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG"));
+    };
+    parser {
+        p_add_context_splunk(key("sc4s_fallback"));
+    };
+
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+
+    #in fallback archive only write rawmsg as msg
+    rewrite {
+        set("value(RAWMSG)" value("MSG"));
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+        groupunset(values(".kv.*"));
+    };
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,fallback);
+};
diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl
@@ -53,7 +53,7 @@ log {
 
 # Send it to Splunk
 
-    destination(d_hec);  #--HEC--
+    destination(d_hec);
 
 # Note: We normally do not use the "final" flag; this will allow another plugin to be created that will
 # forward events to another system