Remove RAWMSG from all configs

* Remove storage of RAWMSG to save memory/disk * Update Ubiquiti and Checkpoint log paths to remove dependency on RAWMSG * Remove ineffective "unset" directives from all log paths * Fix Ubiqutit log path for regex parsing * Simplify/remove extraneous processing in log paths * Add `sc4s_vendor_product` to vmware log path * Fix product comment at top of all log paths * Destination env var gomplate logic fixed for fallback; subsequent PR will address all other log paths
CSVD · Jan 2, 2020 · 2f64327 · 2f64327
1 parent db6310a
commit 2f64327
Show file tree

Hide file tree

Showing 29 changed files with 122 additions and 485 deletions.
diff --git a/package/etc/conf.d/filters/checkpoint/splunk.conf b/package/etc/conf.d/filters/checkpoint/splunk.conf
@@ -1,5 +1,6 @@
 filter f_checkpoint_splunk {
-    match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("RAWMSG") type("pcre"));
+    match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("MSG") type("pcre")) or
+    match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("LEGACY_MSGHDR") type("pcre"));
 };
 
 filter f_checkpoint_splunk_alerts {

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl
@@ -1,3 +1,4 @@
+# Checkpoint
 # Generate the custom port if defined
 {{ $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }}
 {{ tmpl.Exec "t/source_network.t" $context }}
@@ -22,12 +23,9 @@ log {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
         };
-        rewrite {
-            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}") )
-        };
+        rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) };
         parser { p_add_context_splunk(key("checkpoint_os")); };
 
-
     } else {
         parser {
             kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}"));
@@ -74,22 +72,12 @@ log {
     };
 
     parser (compliance_meta_by_source);
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
 
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-        groupunset(values(".kv.*"));
-    };
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no") | conv.ToBool) }}
     destination(d_hec);
 {{- end}}
 
-
 {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK") }}
     destination(d_archive);
 {{- end}}

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl
@@ -68,21 +68,7 @@ log {
 
         parser {p_add_context_splunk(key("cisco_acs")); };
         parser (compliance_meta_by_source);
-
-        #We want to unset the fields we won't need, as this is copied into the
-        #disk queue for network destinations. This can be very disk expensive
-        #if we don't
-        rewrite {
-            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
-            unset(value("RAWMSG"));
-            unset(value("PROGRAM"));
-            unset(value("PID"));
-            unset(value("LEGACY_MSGHDR"));
-            unset(value("EPOCH"));
-            unset(value("VERSION"));
-            unset(value("TIMESECFRAC"));
-            groupunset(values("ACS.*"));
-        };
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ACS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl
@@ -19,18 +19,8 @@ log {
         r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))
     };
     parser {p_add_context_splunk(key("cisco_asa")); };
-
     parser (compliance_meta_by_source);
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-    };
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ASA_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
@@ -18,22 +18,9 @@ log {
         guess-time-zone();
         r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"))
     };
-    parser {
-        p_add_context_splunk(key("cisco_ios"));
-    };
-
+    parser { p_add_context_splunk(key("cisco_ios")); };
     parser (compliance_meta_by_source);
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-        groupunset(values(".cisco.*"));
-    };
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_IOS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl
@@ -68,21 +68,7 @@ log {
 
         parser {p_add_context_splunk(key("cisco_ise")); };
         parser (compliance_meta_by_source);
-
-        #We want to unset the fields we won't need, as this is copied into the
-        #disk queue for network destinations. This can be very disk expensive
-        #if we don't
-        rewrite {
-            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
-            unset(value("RAWMSG"));
-            unset(value("PROGRAM"));
-            unset(value("PID"));
-            unset(value("LEGACY_MSGHDR"));
-            unset(value("EPOCH"));
-            unset(value("VERSION"));
-            unset(value("TIMESECFRAC"));
-            groupunset(values("ISE.*"));
-        };
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ISE_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl
@@ -19,20 +19,9 @@ log {
         r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))
     };
 
-    parser {
-        p_add_context_splunk(key("cisco_nx_os"));
-    };
+    parser { p_add_context_splunk(key("cisco_nx_os")); };
     parser (compliance_meta_by_source);
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-    };
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_NXOS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl
@@ -20,18 +20,8 @@ log {
         r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg"))
     };
     parser {p_add_context_splunk(key("forcepoint_webprotect")); };
-
     parser (compliance_meta_by_source);
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-    };
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORCEPOINT_WEBPROTECT_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl
@@ -39,17 +39,7 @@ log {
     };
 
     parser (compliance_meta_by_source);
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-        groupunset(values(".kv.*"));
-    };
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORTINET_FORTIOS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl
@@ -1,4 +1,4 @@
-# Juniper IDP
+# Infoblox
 {{ $context := dict "port_id" "INFOBLOX" "parser" "common" }}
 {{ tmpl.Exec "t/source_network.t" $context }}
 
@@ -22,66 +22,39 @@ log {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
             r_set_splunk_dest_default(sourcetype("infoblox:dns"), index("netdns"), source("program:${.PROGRAM}"))
-            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
-        };
-        parser {
-            p_add_context_splunk(key("infoblox_dns"));
         };
+        parser { p_add_context_splunk(key("infoblox_dns")); };
     } elif {
         filter{program("dhcpd")};
         rewrite {
             set("infoblox_dhcp", value("fields.sc4s_vendor_product"));
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
             r_set_splunk_dest_default(sourcetype("infoblox:dhcp"), index("netipam"), source("program:${.PROGRAM}"))
-            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
-        };
-        parser {
-            p_add_context_splunk(key("infoblox_dhcp"));
         };
+        parser {  p_add_context_splunk(key("infoblox_dhcp")); };
     } elif {
         filter{program("threat-protect-log")};
         rewrite {
             set("infoblox_dns", value("fields.sc4s_vendor_product"));
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
             r_set_splunk_dest_default(sourcetype("infoblox:threat"), index("netids"), source("program:${.PROGRAM}"))
-            set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG"));
-        };
-        parser {
-            p_add_context_splunk(key("infoblox_threat"));
         };
+        parser { p_add_context_splunk(key("infoblox_threat")); };
     } else {
         rewrite {
             set("nix_syslog", value("fields.sc4s_vendor_product"));
             subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}"))
         };
-
-        rewrite {
-            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") )
-            set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
-
-        };
-
         parser { p_add_context_splunk(key("nix_syslog")); };
     };
 
-
     parser (compliance_meta_by_source);
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-        unset(value("PID"));
-        groupunset(values(".kv.*"));
-    };
-
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INFOBLOX_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl
@@ -18,24 +18,9 @@ log {
         set("juniper_idp", value("fields.sc4s_vendor_product"));
         r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"))
     };
-    parser {
-        p_add_context_splunk(key("juniper_idp"));
-    };
-
+    parser {  p_add_context_splunk(key("juniper_idp")); };
     parser (compliance_meta_by_source);
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-        unset(value("PID"));
-        groupunset(values(".kv.*"));
-    };
-
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_IDP_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl
@@ -39,16 +39,7 @@ log {
     };
 
     parser (compliance_meta_by_source);
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-    };
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_JUNOS_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl
@@ -17,22 +17,9 @@ log {
         set("juniper_netscreen", value("fields.sc4s_vendor_product"));
         r_set_splunk_dest_default(sourcetype("netscreen:firewall"), index("netfw"))
     };
-
-    parser {
-            p_add_context_splunk(key("juniper_netscreen"));
-    };
-
+    parser { p_add_context_splunk(key("juniper_netscreen")); };
     parser (compliance_meta_by_source);
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-    };
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
 
 {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_NETSCREEN_HEC" "no") | conv.ToBool) }}
     destination(d_hec);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl
@@ -16,24 +16,11 @@ log {
 
     rewrite {
         set("juniper_nsm", value("fields.sc4s_vendor_product"));
-        r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"))};
-
-    parser {
-            p_add_context_splunk(key("juniper_nsm"));
+        r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"))
     };
-
+    parser { p_add_context_splunk(key("juniper_nsm")); };
     parser (compliance_meta_by_source);
-
-
-    #We want to unset the fields we won't need, as this is copied into the
-    #disk queue for network destinations. This can be very disk expensive
-    #if we don't
-    rewrite {
-        set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-    };
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
 
  {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_NSM_HEC" "no") | conv.ToBool) }}
     destination(d_hec);