diff --git a/package/etc/conf.d/filters/checkpoint/splunk.conf b/package/etc/conf.d/filters/checkpoint/splunk.conf index 701ebc2..87643d1 100644 --- a/package/etc/conf.d/filters/checkpoint/splunk.conf +++ b/package/etc/conf.d/filters/checkpoint/splunk.conf @@ -1,5 +1,6 @@ filter f_checkpoint_splunk { - match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("RAWMSG") type("pcre")); + match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("MSG") type("pcre")) or + match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("LEGACY_MSGHDR") type("pcre")); }; filter f_checkpoint_splunk_alerts { diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index da7ffeb..6a466cb 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -1,3 +1,4 @@ +# Checkpoint # Generate the custom port if defined {{ $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} @@ -22,12 +23,9 @@ log { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); }; - rewrite { - r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}") ) - }; + rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) }; parser { p_add_context_splunk(key("checkpoint_os")); }; - } else { parser { kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); @@ -74,22 +72,12 @@ log { }; parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no") | conv.ToBool) }} destination(d_hec); {{- end}} - {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK") }} destination(d_archive); {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl index 0b9ccfd..492591e 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl @@ -68,21 +68,7 @@ log { parser {p_add_context_splunk(key("cisco_acs")); }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("PID")); - unset(value("LEGACY_MSGHDR")); - unset(value("EPOCH")); - unset(value("VERSION")); - unset(value("TIMESECFRAC")); - groupunset(values("ACS.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ACS_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index 172445d..932085a 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -19,18 +19,8 @@ log { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw")) }; parser {p_add_context_splunk(key("cisco_asa")); }; - parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ASA_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index cb90f70..aa71cf5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -18,22 +18,9 @@ log { guess-time-zone(); r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops")) }; - parser { - p_add_context_splunk(key("cisco_ios")); - }; - + parser { p_add_context_splunk(key("cisco_ios")); }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".cisco.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_IOS_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl index 544c0a3..15e50f8 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -68,21 +68,7 @@ log { parser {p_add_context_splunk(key("cisco_ise")); }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("PID")); - unset(value("LEGACY_MSGHDR")); - unset(value("EPOCH")); - unset(value("VERSION")); - unset(value("TIMESECFRAC")); - groupunset(values("ISE.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ISE_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index 59db061..f3b3987 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -19,20 +19,9 @@ log { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg")) }; - parser { - p_add_context_splunk(key("cisco_nx_os")); - }; + parser { p_add_context_splunk(key("cisco_nx_os")); }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_NXOS_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index 72d4de7..a3977ca 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -20,18 +20,8 @@ log { r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg")) }; parser {p_add_context_splunk(key("forcepoint_webprotect")); }; - parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORCEPOINT_WEBPROTECT_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 29f17fc..d313ef7 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -39,17 +39,7 @@ log { }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORTINET_FORTIOS_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl index ecfad6e..bbf33ee 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl @@ -1,4 +1,4 @@ -# Juniper IDP +# Infoblox {{ $context := dict "port_id" "INFOBLOX" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} @@ -22,11 +22,8 @@ log { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_default(sourcetype("infoblox:dns"), index("netdns"), source("program:${.PROGRAM}")) - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - }; - parser { - p_add_context_splunk(key("infoblox_dns")); }; + parser { p_add_context_splunk(key("infoblox_dns")); }; } elif { filter{program("dhcpd")}; rewrite { @@ -34,11 +31,8 @@ log { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_default(sourcetype("infoblox:dhcp"), index("netipam"), source("program:${.PROGRAM}")) - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - }; - parser { - p_add_context_splunk(key("infoblox_dhcp")); }; + parser { p_add_context_splunk(key("infoblox_dhcp")); }; } elif { filter{program("threat-protect-log")}; rewrite { @@ -46,42 +40,21 @@ log { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_default(sourcetype("infoblox:threat"), index("netids"), source("program:${.PROGRAM}")) - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - }; - parser { - p_add_context_splunk(key("infoblox_threat")); }; + parser { p_add_context_splunk(key("infoblox_threat")); }; } else { rewrite { set("nix_syslog", value("fields.sc4s_vendor_product")); subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}")) }; - - rewrite { - r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) - set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); - - }; - parser { p_add_context_splunk(key("nix_syslog")); }; }; - parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - unset(value("PID")); - groupunset(values(".kv.*")); - }; - + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INFOBLOX_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 16f2c0c..de79a15 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -18,24 +18,9 @@ log { set("juniper_idp", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids")) }; - parser { - p_add_context_splunk(key("juniper_idp")); - }; - + parser { p_add_context_splunk(key("juniper_idp")); }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - unset(value("PID")); - groupunset(values(".kv.*")); - }; - + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_IDP_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index f0c8a1d..dd912d0 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -39,16 +39,7 @@ log { }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_JUNOS_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index dde7d54..478272a 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -17,22 +17,9 @@ log { set("juniper_netscreen", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("netscreen:firewall"), index("netfw")) }; - - parser { - p_add_context_splunk(key("juniper_netscreen")); - }; - + parser { p_add_context_splunk(key("juniper_netscreen")); }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_NETSCREEN_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index 8d8d999..a3dc302 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -16,24 +16,11 @@ log { rewrite { set("juniper_nsm", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"))}; - - parser { - p_add_context_splunk(key("juniper_nsm")); + r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw")) }; - + parser { p_add_context_splunk(key("juniper_nsm")); }; parser (compliance_meta_by_source); - - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_NSM_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index c1196c5..fce90bb 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -16,21 +16,10 @@ log { rewrite { set("juniper_nsm_idp", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"))}; - parser { - p_add_context_splunk(key("juniper_nsm_idp")); - }; - - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_standard))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); + r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids")) }; + parser { p_add_context_splunk(key("juniper_nsm_idp")); }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_standard))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNIPER_NSM_IDP_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index 8d26d45..5e1fd20 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -49,9 +49,7 @@ log { r_set_splunk_dest_default(sourcetype("cef"), index("main")) }; - parser { - p_add_context_splunk(key("cef_{fields.cef_device_vendor}_${fields.cef_device_product}")); - }; + parser { p_add_context_splunk(key("cef_{fields.cef_device_vendor}_${fields.cef_device_product}")); }; parser (p_microfocus_arcsight_header); @@ -72,13 +70,7 @@ log { #We want to unset the fields we won't need, as this is copied into the #disk queue for network destinations. This can be very disk expensive #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".cef.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index ee4caaf..8c0abe7 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -16,13 +16,11 @@ log { # The palo message does not include a program value in the header, unfortunately. # The use of colon in the message tricks the RFC3164 parser; We will re-write the message # so the parser will not incorrectly break it apart. - # While we are at it we will save the mesage type into the program field so parser can find it. + # While we are at it we will save the message type into the program field so parser can find it. rewrite { set("paloalto_panos", value("fields.sc4s_vendor_product")); set("${LEGACY_MSGHDR}${MESSAGE}" value("MESSAGE")); - unset(value("LEGACY_MSGHDR")); - unset(value("PROGRAM")); }; parser { #basic parsing @@ -77,17 +75,7 @@ log { }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".pan.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_PALOALTO_PANOS_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl index 0c805e7..f981341 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl @@ -16,31 +16,17 @@ log { rewrite { set("proofpoint_pps_filter", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"))}; - parser { - p_add_context_splunk(key("proofpoint_pps_filter")); - }; + parser { p_add_context_splunk(key("proofpoint_pps_filter")); }; } else { rewrite { set("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"))}; - parser { - p_add_context_splunk(key("proofpoint_pps_sendmail")); - }; + parser { p_add_context_splunk(key("proofpoint_pps_sendmail")); }; }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_PROOFPOINT_PPS_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl index 7ca3176..d203626 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -17,28 +17,26 @@ log { parser {p_add_context_splunk(key("ubiquiti_unifi")); }; - #Fiewall + #Firewall if { filter { match("ubiquiti_unifi_fw", value("fields.sc4s_vendor_product") type(glob) ); }; - if (match("[^)]\s\S+\skernel:\s[^ll\sheader][^\[\d+.\d+\]]\S+\s\w+:" value("RAWMSG"))) { - rewrite { r_set_splunk_dest_default(sourcetype("ubnt:threat"), index("netids")) - set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + rewrite { set("${LEGACY_MSGHDR}${MSG}" value("MSG")); }; + + if (match("[^)]\s\S+\skernel:\s[^ll\sheader][^\[\d+.\d+\]]\S+\s\w+:" value("MSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:threat"), index("netids")) }; parser {p_add_context_splunk(key("ubiquiti_unifi_threat")); }; - } elif (match("\S+\slinkcheck:" value("RAWMSG"))) { - rewrite { r_set_splunk_dest_default(sourcetype("ubnt:link"), index("netops")) - set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + } elif (match("\S+\slinkcheck:" value("MSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:link"), index("netops")) }; parser {p_add_context_splunk(key("ubiquiti_unifi_link")); }; - } elif (match("\d+:\d+:\d+\s\S+\ssudo:" value("RAWMSG"))) { - rewrite { r_set_splunk_dest_default(sourcetype("ubnt:sudo"), index("netops")) - set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + } elif (match("\d+:\d+:\d+\s\S+\ssudo:" value("MSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:sudo"), index("netops")) }; parser {p_add_context_splunk(key("ubiquiti_unifi_sudo")); }; } else { rewrite { r_set_splunk_dest_default(sourcetype("ubnt:fw"), index("netfw")); - set("${LEGACY_MSGHDR}${MSG}" value("MSG")); }; parser {p_add_context_splunk(key("ubiquiti_unifi_fw")); }; }; @@ -47,25 +45,25 @@ log { filter { host('^(?US[^,]{1,10}),(?[a-z0-9]{9,16}),(?v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches")); }; - if (match("hostapd:\s+ath" value("RAWMSG"))) { + + rewrite { set("${LEGACY_MSGHDR}${MSG}" value("MSG")); }; + + if (match('hostapd:\s+ath' value("MSG"))) { rewrite { r_set_splunk_dest_default(sourcetype("ubnt:hostapd"), index("netops")); - set("${LEGACY_MSGHDR}${MSG}" value("MSG")); - set("${HOST_FROM}", value("HOST")); + set("${FULLHOST_FROM}", value("HOST")); }; parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; - } elif (match("\d+:\d+:\d+\s\S+\smcad:" value("RAWMSG"))) { + } elif (match('\d+:\d+:\d+\s\S+\smcad:' value("MSG"))) { rewrite { r_set_splunk_dest_default(sourcetype("ubnt:mcad"), index("netops")); - set("${LEGACY_MSGHDR}${MSG}" value("MSG")); - set("${HOST_FROM}", value("HOST")); + set("${FULLHOST_FROM}", value("HOST")); }; parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; } else { rewrite { r_set_splunk_dest_default(sourcetype("ubnt:switch"), index("netops")); - set("${LEGACY_MSGHDR}${MSG}" value("MSG")); - set("${FROM_HOST}",value("HOST")); + set("${FULLHOST_FROM}",value("HOST")); set("${model}", value("fields.model")); set("${serial}", value("fields.serial")); set("${firmware}", value("fields.firmware")); @@ -80,48 +78,36 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("ubnt:wireless"), index("netops")); - set("${FROM_HOST}",value("HOST")); + set("${FULLHOST_FROM}",value("HOST")); set("${model}", value("fields.model")); set("${serial}", value("fields.serial")); set("${firmware}", value("fields.firmware")); }; parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; - } elif (match("traputil.c\(696\) " value("RAWMSG"))) { + } elif (match("traputil.c\(696\) " value("MSG"))) { rewrite { r_set_splunk_dest_default(sourcetype("ubnt:edgeswitch"), index("netops")); - set("${HOST_FROM}", value("HOST")); - set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${FULLHOST_FROM}", value("HOST")); }; parser {p_add_context_splunk(key("ubiquiti_unifi_edge_switch")); }; } else { rewrite { r_set_splunk_dest_default(sourcetype("ubnt"), index("netops")); - set("${HOST_FROM}", value("HOST")); - set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${FULLHOST_FROM}", value("HOST")); }; parser {p_add_context_splunk(key("ubiquiti_unifi")); }; }; - parser (compliance_meta_by_source); - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_PROOFPOINT_UBIQUITI_UNIFI_HEC" "no") | conv.ToBool) }} destination(d_hec); {{- end}} - {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_UBIQUITI_UNIFI") }} destination(d_archive); {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index 0bcdf3d..096119b 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -1,4 +1,4 @@ -# Proofpoint +# Zscaler {{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} # The following is an inline template; we will use this to generate the actual log path @@ -58,20 +58,8 @@ log { }; }; - parser (compliance_meta_by_source); - - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ZSCALER_NSS_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl index 70c1298..5f8ea54 100644 --- a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl @@ -1,3 +1,4 @@ +#Symantec Brightmail {{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} filter f_symantec_brightmail_complete{ match("yes", value("SMG.COMPLETE") type(glob)); @@ -46,26 +47,11 @@ log { rewrite { set("symantec_brightmail", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("symantec:smg:mail"), index("email"), source("program:${.PROGRAM}")) }; - - rewrite { - r_set_splunk_dest_default(sourcetype("symantec:smg:mail"), index("email"), source("program:${.PROGRAM}") ) - - }; - parser { p_add_context_splunk(key("symantec_brightmail")); }; - parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }} destination(d_hec); @@ -81,26 +67,11 @@ log { rewrite { set("symantec_brightmail", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("symantec:smg"), index("email"), source("program:${.PROGRAM}")) }; - - rewrite { - r_set_splunk_dest_default(sourcetype("symantec:smg"), index("email"), source("program:${.PROGRAM}") ) - - }; - parser { p_add_context_splunk(key("symantec_brightmail")); }; - parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index 065c824..d07e8bb 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -15,21 +15,11 @@ log { rewrite { set("cisco_asa", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))}; + r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw")) + }; parser {p_add_context_splunk(key("cisco_asa")); }; - parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ASA_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index c455a9d..86ac72a 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -19,20 +19,8 @@ log { }; parser {p_add_context_splunk(key("bluecoat_proxy")); }; - parser (compliance_meta_by_source); - - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_PROXY_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index 0d103a8..580770a 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -46,22 +46,16 @@ log { #We want to unset the fields we won't need, as this is copied into the #disk queue for network destinations. This can be very disk expensive #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); - unset(value("RAWMSG")); - groupunset(values(".kv.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_JUNOS_STRUCTURED_HEC" "no") | conv.ToBool) }} destination(d_hec); {{- end}} - {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNOS_STRUCTURED") }} destination(d_archive); {{- end}} - flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl index 793b356..c693dae 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -1,4 +1,4 @@ -# Checkpoint Splunk format +# Cisco Meraki {{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} # The following is an inline template; we will use this to generate the actual log path @@ -17,19 +17,9 @@ log { r_set_splunk_dest_default(sourcetype("meraki"), index("netfw")) }; parser {p_add_context_splunk(key("cisco_meraki")); }; - parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_MERAKI_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl index 5396355..b48f032 100644 --- a/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl @@ -1,3 +1,4 @@ +#VMware ESXi and NSX # Generate the custom port if defined {{ $context := dict "port_id" "VMWARE" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} @@ -7,26 +8,24 @@ log { {{- if eq (.) "yes"}} source(s_DEFAULT); - filter(f_vmware_all); {{- end}} {{- if eq (.) "no"}} source (s_VMWARE); {{- end}} + rewrite { set("vmware", value("fields.sc4s_vendor_product")); }; #NSX first because its the cheapest check if { filter(f_is_rfc5424_strict); filter(f_vmware_nsx); - rewrite { - r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${PROGRAM}")); - set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); - }; - parser { - p_add_context_splunk(key("vmware_nsx")); - }; + rewrite { r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${PROGRAM}")); }; + parser { p_add_context_splunk(key("vmware_nsx")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; + } elif { filter(f_vmware_nsx); @@ -35,23 +34,21 @@ log { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${.PROGRAM}")); - set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); - }; - parser { - p_add_context_splunk(key("vmware_nsx")); }; + parser { p_add_context_splunk(key("vmware_nsx")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + #esx things } elif { filter(f_is_rfc5424_strict); filter(f_vmware_vsphere); - rewrite { - r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${PROGRAM}")); - set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); - }; - parser { - p_add_context_splunk(key("vmware_esx")); - }; + rewrite { r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${PROGRAM}")); }; + parser { p_add_context_splunk(key("vmware_esx")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; + } elif { filter(f_vmware_vsphere); @@ -60,11 +57,11 @@ log { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${.PROGRAM}")); - set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); - }; - parser { - p_add_context_splunk(key("vmware_esx")); }; + parser { p_add_context_splunk(key("vmware_esx")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + } else { rewrite { @@ -74,28 +71,12 @@ log { subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); }; - rewrite { - r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) - - }; - + rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}")) }; parser { p_add_context_splunk(key("nix_syslog")); }; - parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; }; - parser (compliance_meta_by_source); - {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_HEC" "no") | conv.ToBool) }} destination(d_hec); {{- end}} diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl index c8f4e2a..aa6750d 100644 --- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl @@ -1,4 +1,4 @@ -# Proofpoint +# Linux/Unix OS system logs {{ $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} # The following is an inline template; we will use this to generate the actual log path @@ -19,24 +19,10 @@ log { subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); }; - rewrite { - r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) - - }; - + rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}")) }; parser { p_add_context_splunk(key("nix_syslog")); }; - parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; {{- if ((getenv "SC4S_NIX_SYSLOG_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_NIX_SYSLOG_HEC" "no") | conv.ToBool) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index 70da153..7427ed3 100644 --- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -1,52 +1,30 @@ +# Fallback for un-parsed sources log { source(s_DEFAULT); if { filter(f_is_rfc5424_strict); - rewrite { - r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main")); - set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); - unset(value("RAWMSG")); - }; - parser { - p_add_context_splunk(key("sc4s_fallback")); - }; - {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} - destination(d_hec); - {{- end}} - - - {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} - destination(d_archive); - {{- end}} + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main")); }; + parser { p_add_context_splunk(key("sc4s_fallback")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; } else { - rewrite { - r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main") ); - set("$(template ${.splunk.sc4s_template} $(template t_JSON_3164))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; - parser { - p_add_context_splunk(key("sc4s_fallback")); - }; - - {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} - destination(d_hec); - {{- end}} - - - #in fallback archive only write rawmsg as msg - - {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} - destination(d_archive); - {{- end}} + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main")); }; + parser { p_add_context_splunk(key("sc4s_fallback")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_3164))" value("MSG")); }; }; + {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_FALLBACK_HEC" "no")) }} + destination(d_hec); + {{- end}} + {{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_FALLBACK" "no")) }} + destination(d_archive); + {{- end}} flags(flow-control,fallback); + }; \ No newline at end of file diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 69021a5..8b43665 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -62,13 +62,13 @@ source s_{{ .port_id}} { {{ if eq .parser "rfc5424_strict" }} filter(f_rfc5424_strict); parser { - syslog-parser(flags(syslog-protocol store-raw-message)); + syslog-parser(flags(syslog-protocol)); }; rewrite(set_rfc5424_strict); {{- else if eq .parser "rfc5424_noversion" }} filter(f_rfc5424_noversion); parser { - syslog-parser(flags(syslog-protocol store-raw-message)); + syslog-parser(flags(syslog-protocol)); }; rewrite(set_rfc5424_noversion); {{- else if eq .parser "cisco_parser" }} @@ -79,7 +79,7 @@ source s_{{ .port_id}} { rewrite(set_rfc5424_epochtime); {{- else if eq .parser "rfc3164" }} parser { - syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); + syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) ); }; rewrite(set_rfc3164); {{- else if eq .parser "no_parse" }} @@ -87,13 +87,13 @@ source s_{{ .port_id}} { {{- else }} if {filter(f_rfc5424_strict); parser { - syslog-parser(flags(syslog-protocol store-raw-message)); + syslog-parser(flags(syslog-protocol)); }; rewrite(set_rfc5424_strict); } elif { filter(f_rfc5424_noversion); parser { - syslog-parser(flags(syslog-protocol store-raw-message)); + syslog-parser(flags(syslog-protocol)); }; rewrite(set_rfc5424_noversion); } elif { @@ -103,7 +103,7 @@ source s_{{ .port_id}} { filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite(set_rfc3164_version); } elif { @@ -111,7 +111,7 @@ source s_{{ .port_id}} { rewrite(set_rfc5424_epochtime); } else { parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite(set_rfc3164); };