Merge pull request #522 from splunk/featture/handle-bad-user-indexes

Improve index definition handling
CSVD · Jun 14, 2020 · dea2776 · dea2776
2 parents e823604 + d92a6d4
commit dea2776
Show file tree

Hide file tree

Showing 49 changed files with 258 additions and 234 deletions.
diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md
@@ -405,9 +405,9 @@ Verify timestamp, and host values match as expected
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| cisco_wsa_l4tm    | cisco:wsa:l4tm    | netops          | None     |
-| cisco_wsa_squid    | cisco:wsa:squid    | netops          | None     |
-| cisco_wsa_squid_new    | cisco:wsa:squid:new    | netops          | None     |
+| cisco_wsa    | cisco:wsa:l4tm    | netproxy          | None     |
+| cisco_wsa    | cisco:wsa:squid    | netproxy          | None     |
+| cisco_wsa    | cisco:wsa:squid:new    | netproxy          | None     |
 
 ### Filter type
 

diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl
@@ -33,15 +33,13 @@ rewrite r_set_splunk_default {
     };
 {{- end}}    
 };
-#used by each log-path to set index and sourcetype which may be
+#used by each log-path to set source and sourcetype which may be
 #overridden by user defined values
 block rewrite r_set_splunk_dest_default(
-                              index()
                               source("${.splunk.source}")
                               sourcetype()
                               template(`splunk-template`)
                            ) {
-      set("`index`", value(".splunk.index"));
       set("`source`", value(".splunk.source"));
       set("`sourcetype`", value(".splunk.sourcetype"));
 };

diff --git a/package/etc/conf.d/context/common_event_format_source.csv b/package/etc/conf.d/context/common_event_format_source.csv
@@ -1,4 +1,5 @@
 ArcSight_ArcSight,source,ArcSight:ArcSight
+ArcSight_ArcSight,index,main
 Carbon Black_Protection,sourcetype,carbonblack:protection:cef
 Carbon Black_Protection,index,cb:cef
 Cyber-Ark_Vault,sourcetype,cyberark:epv:cef

diff --git a/package/etc/conf.d/local/config/log_paths/lp-example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/lp-example.conf.tmpl
@@ -53,7 +53,7 @@ log {
 
     rewrite {
         set("local_example", value("fields.sc4s_vendor_product"));
-        r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"));
+        r_set_splunk_dest_default(sourcetype("sc4s:local_example"));
     };
 
 # using the key "local_example" find any customized index,source or sourcetype meta values

diff --git a/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl b/package/etc/conf.d/log_paths/lp-bbb-ietf_syslog.conf.tmpl
@@ -13,7 +13,7 @@ log {
         set("IETF_SYSLOG", value("fields.sc4s_vendor_product"));
     };
 
-    rewrite { r_set_splunk_dest_default(sourcetype("ietf:syslog"), index("main"), source("${APP}:${PROGRAM}")) };
+    rewrite { r_set_splunk_dest_default(sourcetype("ietf:syslog"),  source("${APP}:${PROGRAM}")) };
     parser { p_add_context_splunk(key("IETF_SYSLOG")); };
     parser (compliance_meta_by_source);
     rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); };

diff --git a/package/etc/conf.d/log_paths/lp-brocade.conf.tmpl b/package/etc/conf.d/log_paths/lp-brocade.conf.tmpl
@@ -27,7 +27,7 @@ log {
         subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
         set("${PROGRAM}", value(".PROGRAM"));
         subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
-        r_set_splunk_dest_default(sourcetype("brocade:syslog"), index("netops"), source("program:${.PROGRAM}"))
+        r_set_splunk_dest_default(sourcetype("brocade:syslog"),  source("program:${.PROGRAM}"))
     };
     parser { p_add_context_splunk(key("brocade_syslog")); };
 

diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl
@@ -45,7 +45,7 @@ log {
            set("${.kv.hostname}", value("HOST"));
            set("${.kv.hostname}", value("fields.cp_lm"));
            set("checkpoint_splunk", value("fields.sc4s_vendor_product"));
-           r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"))
+           r_set_splunk_dest_default(sourcetype("cp_log"))
         };
 
         if {
@@ -89,31 +89,31 @@ log {
 
             if  {
                 filter(f_checkpoint_splunk_NetworkTraffic);
-                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))};
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"))};
                 parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); };
             } elif  {
                 filter(f_checkpoint_splunk_Web);
-                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))};
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"))};
                 parser {p_add_context_splunk(key("checkpoint_splunk_web")); };
             } elif  {
                 filter(f_checkpoint_splunk_NetworkSessions);
-                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))};
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"))};
                 parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); };
             } elif  {
                 filter(f_checkpoint_splunk_IDS_Malware);
-                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))};
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"))};
                 parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
             } elif  {
                 filter(f_checkpoint_splunk_IDS);
-                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))};
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"))};
                 parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
             } elif  {
                 filter(f_checkpoint_splunk_email);
-                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))};
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"))};
                 parser {p_add_context_splunk(key("checkpoint_splunk_email")); };
             } elif  {
                 filter(f_checkpoint_splunk_DLP);
-                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))};
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"))};
                 parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); };
             } elif {
                 filter(f_checkpoint_splunk_syslog);
@@ -130,7 +130,7 @@ log {
                         set("${PROGRAM}", value(".PROGRAM"));
                         subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
                     };
-                rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) };
+                rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"),  source("program:${.PROGRAM}")) };
                 parser { p_add_context_splunk(key("checkpoint_os")); };                
 
             };
@@ -163,7 +163,7 @@ log {
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
         };
-        rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) };
+        rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"),  source("program:${.PROGRAM}")) };
         parser { p_add_context_splunk(key("checkpoint_os")); };
 
         parser (compliance_meta_by_source);

diff --git a/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl
@@ -86,7 +86,7 @@ log {
         parser(acs_event_time);
         rewrite {
             set("cisco_acs", value("fields.sc4s_vendor_product"));
-            r_set_splunk_dest_default(sourcetype("cisco:acs"), index("netauth"))
+            r_set_splunk_dest_default(sourcetype("cisco:acs"))
         };
 
         parser {p_add_context_splunk(key("cisco_acs")); };

diff --git a/package/etc/conf.d/log_paths/lp-cisco_apic.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_apic.conf.tmpl
@@ -29,14 +29,14 @@ log {
         };
         rewrite {
             set("cisco_APIC_acl", value("fields.sc4s_vendor_product"));
-            r_set_splunk_dest_default(sourcetype("cisco:apic:acl"), index("netfw"), template("t_hdr_msg"))
+            r_set_splunk_dest_default(sourcetype("cisco:apic:acl"),  template("t_hdr_msg"))
         };
         parser { p_add_context_splunk(key("cisco_apic_acl")); };
 
     } elif {
         rewrite {
             set("cisco_APIC_events", value("fields.sc4s_vendor_product"));
-            r_set_splunk_dest_default(sourcetype("cisco:apic:events"), index("netops"), template("t_hdr_msg"))
+            r_set_splunk_dest_default(sourcetype("cisco:apic:events"),  template("t_hdr_msg"))
         };
         parser { p_add_context_splunk(key("cisco_apic_events")); };
     };

diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl
@@ -28,7 +28,7 @@ log {
         };
         rewrite {
             set("cisco_ftd", value("fields.sc4s_vendor_product"));
-            r_set_splunk_dest_default(sourcetype("cisco:firepower:syslog"), index("netfw"))
+            r_set_splunk_dest_default(sourcetype("cisco:firepower:syslog"))
         };
         parser {p_add_context_splunk(key("cisco_ftd")); };
         parser (compliance_meta_by_source);
@@ -37,7 +37,7 @@ log {
     } else {
         rewrite {
             set("cisco_asa", value("fields.sc4s_vendor_product"));
-            r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))
+            r_set_splunk_dest_default(sourcetype("cisco:asa"))
         };
         parser {p_add_context_splunk(key("cisco_asa")); };
         parser (compliance_meta_by_source);

diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl
@@ -23,7 +23,7 @@ log {
 
     rewrite {
         set("cisco_asa", value("fields.sc4s_vendor_product"));
-        r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))
+        r_set_splunk_dest_default(sourcetype("cisco:asa"))
     };
     parser {p_add_context_splunk(key("cisco_asa")); };
     parser (compliance_meta_by_source);

diff --git a/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl
@@ -86,7 +86,7 @@ log {
         parser(ise_event_time);
         rewrite {
             set("cisco_ise", value("fields.sc4s_vendor_product"));
-            r_set_splunk_dest_default(sourcetype("cisco:ise:syslog"), index("netauth"))
+            r_set_splunk_dest_default(sourcetype("cisco:ise:syslog"))
         };
 
         parser {p_add_context_splunk(key("cisco_ise")); };

diff --git a/package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl
@@ -22,7 +22,7 @@ log {
 
     rewrite {
         set("cisco_meraki", value("fields.sc4s_vendor_product"));
-        r_set_splunk_dest_default(sourcetype("meraki"), index("netfw"))
+        r_set_splunk_dest_default(sourcetype("meraki"))
     };
     parser {p_add_context_splunk(key("cisco_meraki")); };
     parser (compliance_meta_by_source);

diff --git a/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl
@@ -23,7 +23,7 @@ log {
     rewrite {
         set("cisco_nxos", value("fields.sc4s_vendor_product"));
         guess-time-zone();
-        r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))
+        r_set_splunk_dest_default(sourcetype("cisco:ios"),  template("t_hdr_msg"))
     };
 
     parser { p_add_context_splunk(key("cisco_nx_os")); };

diff --git a/package/etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl
@@ -44,7 +44,7 @@ log {
 
     rewrite {
         set("cisco_ucm", value("fields.sc4s_vendor_product"));
-        r_set_splunk_dest_default(sourcetype("cisco:ucm"), index("main"))
+        r_set_splunk_dest_default(sourcetype("cisco:ucm"))
     };
     parser {p_add_context_splunk(key("cisco_ucm")); };
     parser (compliance_meta_by_source);

diff --git a/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl
@@ -26,7 +26,7 @@ log{
         };
         rewrite {
                 set("cisco_wsa", value("fields.sc4s_vendor_product"));
-                r_set_splunk_dest_default(sourcetype("cisco:wsa:l4tm"), index("netops"))
+                r_set_splunk_dest_default(sourcetype("cisco:wsa:l4tm"))
         };
         parser { p_add_context_splunk(key("cisco_wsa")); };
         parser (compliance_meta_by_source);
@@ -51,7 +51,7 @@ log{
         };
         rewrite {
                 set("cisco_wsa11_7", value("fields.sc4s_vendor_product"));
-                r_set_splunk_dest_default(sourcetype("cisco:wsa:squid:new"), index("netops"),source("wsa_11.7"))
+                r_set_splunk_dest_default(sourcetype("cisco:wsa:squid:new"), source("wsa_11.7"))
         };
         parser { p_add_context_splunk(key("cisco_wsa")); };
         parser (compliance_meta_by_source);
@@ -75,7 +75,7 @@ log{
         };
         rewrite {
                 set("cisco_wsa", value("fields.sc4s_vendor_product"));
-                r_set_splunk_dest_default(sourcetype("cisco:wsa:squid"), index("netops"))
+                r_set_splunk_dest_default(sourcetype("cisco:wsa:squid"))
         };
         parser { p_add_context_splunk(key("cisco_wsa")); };
         parser (compliance_meta_by_source);

diff --git a/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl
@@ -23,7 +23,7 @@ log {
     rewrite {
         set("cisco_ios", value("fields.sc4s_vendor_product"));
         guess-time-zone();
-        r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"))
+        r_set_splunk_dest_default(sourcetype("cisco:ios"))
     };
     parser { p_add_context_splunk(key("cisco_ios")); };
     parser (compliance_meta_by_source);

diff --git a/package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl b/package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl
@@ -22,7 +22,7 @@ log {
 
     rewrite {
         set("citrix_netscaler", value("fields.sc4s_vendor_product"));
-        r_set_splunk_dest_default(sourcetype("citrix:netscaler:syslog"), index("netfw"))
+        r_set_splunk_dest_default(sourcetype("citrix:netscaler:syslog"))
     };
 
     parser {p_add_context_splunk(key("citrix_netscaler")); };

diff --git a/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl b/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl
@@ -63,7 +63,7 @@ log {
     };
 
     rewrite {
-        r_set_splunk_dest_default(sourcetype("cef"), index("main"))
+        r_set_splunk_dest_default(sourcetype("cef"))
     };
 
     parser (p_cef_header);

diff --git a/package/etc/conf.d/log_paths/lp-dell_rsa_secureid.conf.tmpl b/package/etc/conf.d/log_paths/lp-dell_rsa_secureid.conf.tmpl
@@ -44,27 +44,27 @@ log {
             filter{match('audit\.admin' value('.rsa.type'))};
             rewrite {
                 set("dell_rsa_secureid", value("fields.sc4s_vendor_product"));
-                r_set_splunk_dest_default(sourcetype("rsa:securid:admin:syslog"), index("netauth"))
+                r_set_splunk_dest_default(sourcetype("rsa:securid:admin:syslog"))
             };
             parser { p_add_context_splunk(key("dell_rsa_secureid")); };
         } elif {
             filter{match('system\.com\.rsa|,\s+system\.erationsconsole' value('.rsa.type'))};
             rewrite {
                 set("dell_rsa_secureid", value("fields.sc4s_vendor_product"));
-                r_set_splunk_dest_default(sourcetype("rsa:securid:system:syslog"), index("netauth"))
+                r_set_splunk_dest_default(sourcetype("rsa:securid:system:syslog"))
             };
             parser { p_add_context_splunk(key("dell_rsa_secureid")); };
         } elif {
             filter{match('audit\.runtime\.com\.rsa' value('.rsa.type'))};
             rewrite {
                 set("dell_rsa_secureid", value("fields.sc4s_vendor_product"));
-                r_set_splunk_dest_default(sourcetype("rsa:securid:runtime:syslog"), index("netauth"))
+                r_set_splunk_dest_default(sourcetype("rsa:securid:runtime:syslog"))
             };
             parser { p_add_context_splunk(key("dell_rsa_secureid")); };
         } else {
             rewrite {
                 set("dell_rsa_secureid", value("fields.sc4s_vendor_product"));
-                r_set_splunk_dest_default(sourcetype("rsa:securid:syslog"), index("netauth"))
+                r_set_splunk_dest_default(sourcetype("rsa:securid:syslog"))
             };
             parser { p_add_context_splunk(key("dell_rsa_secureid")); };
         };
@@ -81,9 +81,9 @@ log {
             subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
-            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}"))
+            r_set_splunk_dest_default(sourcetype("nix:syslog"),  source("program:${.PROGRAM}"))
         };
-        parser { p_add_context_splunk(key("nix_syslog")); };
+        parser { p_add_context_splunk(key("dell_rsa_secureid")); };
         parser (compliance_meta_by_source);
         rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };
     } else {
@@ -99,9 +99,9 @@ log {
         };
         rewrite {
             set("dell_rsa_secureid", value("fields.sc4s_vendor_product"));
-            r_set_splunk_dest_default(sourcetype("rsa:securid:trace"), index("netauth"));
+            r_set_splunk_dest_default(sourcetype("rsa:securid:trace"));
         };
-        parser { p_add_context_splunk(key("nix_syslog")); };
+        parser { p_add_context_splunk(key("dell_rsa_secureid")); };
         parser (compliance_meta_by_source);
         rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };