Fix/dynamic template (#82)

* Template fixes

Add dynamic templates
Introduce new t_JSON_5424 template
Add default source

* change default source preamble

change source preamble to sc4s per official branding

package/etc/conf.d/conflib/_common/templates.conf

@@ -44,13 +44,28 @@ template t_hdr_sdata_msg {
         };
 
 # ===============================================================================================
-# JSON; for JSON pretty-printing
+# JSON; for JSON pretty-printing (for debugging)
 # ===============================================================================================
- 
+
 template t_JSON {
 	template("$(format-json --scope all-nv-pairs
 	            --exclude fields.*
 	            --exclude .splunk.*
-	            --exclude HOST --exclude HOSTFROMgit m
+	            --exclude HOST
+	            --exclude HOST_FROM
+	            )");
+	};
+
+# ===============================================================================================
+# JSON; for JSON pretty-printing (for RFC5424)
+# ===============================================================================================
+
+template t_JSON_5424 {
+	template("$(format-json --scope all-nv-pairs
+	            --exclude fields.*
+	            --exclude .splunk.*
+	            --exclude HOST
+	            --exclude HOST_FROM
+	            --exclude RAWMSG
 	            )");
 	};

package/etc/conf.d/conflib/_splunk/splunkfields.conf

@@ -2,8 +2,9 @@
 rewrite r_set_splunk_default {
       set("`index`", value(".splunk.index"));
 
-      set("`sourcetype`", value(".splunk.sourcetype"));
-      set("`source`", value(".splunk.source"));
+      set("`splunk-sourcetype`", value(".splunk.sourcetype"));
+      set("`splunk-source`:$LOGHOST", value(".splunk.source"));
+      set("`splunk-template`", value(".splunk.template"));
       set($FACILITY, value("fields.sc4s_syslog_facility"));
       set($LEVEL, value("fields.sc4s_syslog_severity"));
       set($LOGHOST, value("fields.sc4s_syslog_server"));
@@ -14,16 +15,12 @@ rewrite r_set_splunk_default {
 block rewrite r_set_splunk_dest_default(
                               index()
                               sourcetype()
+                              template(`splunk-template`)
                            ) {
       set("`index`", value(".splunk.index"));
       set("`sourcetype`", value(".splunk.sourcetype"));
+      set("`template`", value(".splunk.template"));
 };
 
-block rewrite r_set_splunk(
-                              template(`splunk-default-template`)
-                           ) {
-      set("`template`", value("fields.sc4s_template"));
-  };
-
 
 

package/etc/conf.d/destinations/splunk_hec.conf

@@ -45,7 +45,7 @@ destination d_hec {
                  source=${.splunk.source}
                  sourcetype=${.splunk.sourcetype}
                  index=${.splunk.index}
-                 event=$(template ${fields.sc4s_template} $(template `splunk-default-template`))
+                 event=$(template ${.splunk.template} $(template `splunk-template`))
                  fields.*)')
         );
     };

package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf

@@ -4,13 +4,11 @@ log {
     filter(f_is_rfc3164);
     filter(f_juniper_nsm_idp);
 
-    rewrite {r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"))};
+    rewrite {r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"), template("t_standard"))};
     parser {
                 p_add_context_splunk(key("juniper_idp"));
     };
-    rewrite {
-        r_set_splunk(template("t_standard"))
-    };
+
     destination(d_hec);  #--HEC--
     flags(flow-control);
 };

package/etc/conf.d/log_paths/internal.conf

@@ -30,10 +30,10 @@ log {
         };
         destination(d_hecmetrics);  #--HEC--
     } else {
-        rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:events"), index("_internal"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:events"), index("_internal"), template("t_msg_only"))};
 
         parser {p_add_context_splunk(key("syslog-ng_events")); };
-        rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
+
         destination(d_hec);  #--HEC--
     };
 };

package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf

@@ -3,13 +3,12 @@ log {
     source(s_default-ports);
     filter(f_cisco_nx_os);
 
-    rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))};
     parser {
             p_add_context_splunk(key("cisco_nx_os"));
     };
-    rewrite {
-        r_set_splunk(template("t_hdr_msg"))
-    };
+
     destination(d_hec);  #--HEC--
+
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf

@@ -9,13 +9,11 @@ log {
     filter(f_is_rfc3164);
     filter(f_cisco_asa);
     #set the source type based on program field and lookup index from the splunk context csv
+    #Using the 5424 parser the message content is all we need
 
-    rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))};
     parser {p_add_context_splunk(key("cisco_asa")); };
 
-    #Using the 5424 parser the message content is all we need
-    rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
-
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf

@@ -9,13 +9,11 @@ log {
     filter(f_cisco_ios);
 
 
-    rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_msg_only"))};
     parser {
         p_add_context_splunk(key("cisco_ios"));
     };
-    rewrite {
-        r_set_splunk(template("t_msg_only")) #--HEC--
-    };
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf

@@ -14,22 +14,19 @@ log {
 
 
     if (match("traffic" value(".kv.type"))) {
-       rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"), template("t_standard"))};
        parser {p_add_context_splunk(key("fortinet_fortios_traffic")); };
     } elif (match("utm" value(".kv.type"))) {
-       rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"), template("t_standard"))};
        parser {p_add_context_splunk(key("fortinet_fortios_utm")); };
     } elif (match("event" value(".kv.type"))) {
-       rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"), template("t_standard"))};
        parser {p_add_context_splunk(key("fortinet_fortios_event")); };
     } else {
-       rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"), template("t_standard"))};
        parser {p_add_context_splunk(key("fortinet_fortios_log")); };
     };
 
-    #rewrite the final message for splunk json
-    #sourcetype and index are set in the filter defaults won't be used
-    rewrite {r_set_splunk(template("t_standard")) }; #--HEC--
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf

@@ -4,13 +4,12 @@ log {
     filter(f_is_rfc5424_strict);
     filter(f_juniper_idp);
 
-    rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"), template("t_hdr_sdata_msg"))};
     parser {
                 p_add_context_splunk(key("juniper_idp"));
     };
-    rewrite {
-        r_set_splunk(template("t_hdr_sdata_msg"))
-    };
+
     destination(d_hec);  #--HEC--
+
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf

@@ -9,26 +9,26 @@ log {
     filter(f_juniper_junos_standard);
 
     if (program('RT_IDP')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))};
         parser {p_add_context_splunk(key("juniper_idp")); };
     } elif (program('RT_FLOW')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"), template("t_standard"))};
         parser {p_add_context_splunk(key("juniper_junos_flow")); };
     } elif (program('RT_IDS')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))};
         parser {p_add_context_splunk(key("juniper_junos_ids")); };
     } elif (program('RT_UTM')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"), template("t_standard"))};
         parser {p_add_context_splunk(key("juniper_junos_utm")); };
     } elif (program('Juniper')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"), template("t_standard"))};
         parser {p_add_context_splunk(key("juniper_sslvpn")); };
     } else {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"), template("t_standard"))};
         parser {p_add_context_splunk(key("juniper_legacy")); };
     };
 
-    rewrite { r_set_splunk(template("t_standard")) };
     destination(d_hec);  #--HEC--
+
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf

@@ -3,14 +3,12 @@ log {
     source(s_default-ports);
     filter(f_juniper_netscreen);
 
-    rewrite { r_set_splunk_dest_default(sourcetype("netscreen:firewall"), index("netfw"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("netscreen:firewall"), index("netfw"), template("t_standard"))};
 
     parser {
             p_add_context_splunk(key("juniper_netscreen"));
     };
-    rewrite {
-        r_set_splunk(template("t_standard"))
-    };
+
     destination(d_hec);  #--HEC--
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf

@@ -8,14 +8,12 @@ log {
     filter(f_is_rfc3164);
     filter(f_juniper_nsm);
 
-    rewrite { r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"), template("t_standard"))};
 
     parser {
                 p_add_context_splunk(key("juniper_nsm"));
     };
-    rewrite {
-        r_set_splunk(template("t_standard"))
-    };
+
     destination(d_hec);  #--HEC--
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf

@@ -33,35 +33,31 @@ log {
 
     #set the source type based on program field and lookup index from the splunk context csv
     if (message(',\d+,THREAT')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"), template("t_msg_only"))};
         parser {p_add_context_splunk(key("pan_threat")); };
     } elif (message(',\d+,TRAFFIC')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"), template("t_msg_only"))};
         parser {p_add_context_splunk(key("pan_traffic")); };
     } elif (message(',\d+,SYSTEM')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"), template("t_msg_only"))};
         parser {p_add_context_splunk(key("pan_system")); };
     } elif (message(',\d+,CONFIG')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"), template("t_msg_only"))};
         parser {p_add_context_splunk(key("pan_config")); };
     } elif (message(',\d+,HIPWATCH')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"), template("t_msg_only"))};
         parser {p_add_context_splunk(key("pan_hipwatch")); };
     } elif (message(',\d+,CORRELATION')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"), template("t_msg_only"))};
         parser {p_add_context_splunk(key("pan_correlation")); };
     } elif (message(',\d+,USERID')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"), template("t_msg_only"))};
         parser {p_add_context_splunk(key("pan_userid")); };
     } else {
-        rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"), template("t_msg_only"))};
         parser {p_add_context_splunk(key("pan_log")); };
     };
 
-    #rewrite the final message for splunk json
-    #sourcetype and index are set in the filter defaults won't be used
-    rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
-
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf

@@ -4,13 +4,11 @@ log {
     filter(f_cisco_asa);
 
     #set the source type based on program field and lookup index from the splunk context csv
+    #Using the 5424 parser the message content is all we need
 
-    rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))};
     parser {p_add_context_splunk(key("cisco_asa")); };
 
-    #Using the 5424 parser the message content is all we need
-    rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
-
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf

@@ -3,15 +3,13 @@ log {
     filter(f_is_rfc5424_noversion);
     filter(f_symantec_bluecoat_proxy);
 
-    #set the source type based on program field and lookup index from the splunk context csv
+    #set the source type based on program field and lookup index from the splunk context csv.
+    #With the 5424 parser the message is the only portion needed in the template.
 
-    rewrite { r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy"), template("t_msg_only")) };
 
     parser {p_add_context_splunk(key("bluecoat_proxy")); };
 
-    #Using the 5424 parser the message content is all we need
-    rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
-
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf

@@ -9,17 +9,17 @@ log {
     filter(f_juniper_junos_structured);
 
     if (program('RT_IDP')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids")) };
-        parser {p_add_context_splunk(key("juniper_idp")); };
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) };
+        parser {p_add_context_splunk(key("juniper_idp_structured")); };
     } elif (program('RT_FLOW')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw")) };
-        parser {p_add_context_splunk(key("juniper_junos_flow")); };
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) };
+        parser {p_add_context_splunk(key("juniper_junos_flow_structured")); };
     } elif (program('RT_IDS')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netids")) };
-        parser {p_add_context_splunk(key("juniper_junos_ids")); };
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) };
+        parser {p_add_context_splunk(key("juniper_junos_ids_structured")); };
     } elif (program('RT_UTM')) {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netids")) };
-        parser {p_add_context_splunk(key("juniper_junos_utm")); };
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) };
+        parser {p_add_context_splunk(key("juniper_junos_utm_structured")); };
     }
 # Legacy Netscreen IDP is handled in the "p_rfc3164-juniper-idp.conf" log path
 #
@@ -28,11 +28,10 @@ log {
 #        parser {p_add_context_splunk(key("juniper_junos_idp")); };
 #        }
     else {
-        rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops")) };
+        rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops"), template("t_JSON_5424")) };
         parser {p_add_context_splunk(key("juniper_structured")); };
     };
 
-    rewrite { r_set_splunk(template("t_hdr_sdata_msg")) };
     destination(d_hec);  #--HEC--
 
 

package/etc/conf.d/log_paths/zfallback.conf

@@ -1,12 +1,11 @@
 log {
     source(s_default-ports);
 
-    rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:fallback"), index("main"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:fallback"), index("main"), template("t_JSON")) };
     parser {
         p_add_context_splunk(key("syslog-ng_fallback"));
     };
 
-    rewrite { r_set_splunk(template("t_JSON"))}; #--HEC--
     destination(d_hec);  #--HEC--
     flags(flow-control,fallback);
 };