From 1ef1267af8a42d96be8feddc87d0191dd402d90d Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 2 Jan 2020 22:41:59 -0800 Subject: [PATCH 1/2] Update sc4s_vendor_product values * Provide more granular `sc4s_vendor_product` values to help distinguish the actual log path used. * Results in far cleaner archive files on disk --- .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 2 +- ...mpl => p_rfc3164-symantec_brightmail.conf.tmpl} | 2 +- .../log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl | 6 ++++++ .../conf.d/log_paths/p_vmware_vsphere.conf.tmpl | 14 ++++++++++---- .../etc/conf.d/log_paths/p_zz_fallback.conf.tmpl | 2 ++ 5 files changed, 20 insertions(+), 6 deletions(-) rename package/etc/conf.d/log_paths/{p_rfc3165-symantec_brightmail.conf.tmpl => p_rfc3164-symantec_brightmail.conf.tmpl} (98%) diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index 5697363..dc65e02 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -17,7 +17,7 @@ log { rewrite { subst(" [^ =]+\=\-", "", value("MESSAGE"), flags("global")); set("forcepoint_webprotect", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg")) + r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy")) }; parser {p_add_context_splunk(key("forcepoint_webprotect")); }; parser (compliance_meta_by_source); diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl similarity index 98% rename from package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl index fee6fa3..a2b4c57 100644 --- a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl @@ -77,7 +77,7 @@ log { destination(d_hec); {{- end}} -{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL")) }} +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL" "no")) }} destination(d_archive); {{- end}} {{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl index 02c3326..a6ab503 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -50,18 +50,21 @@ log { if (match('hostapd:\s+ath' value("MSG"))) { rewrite { + set("ubiquiti_unifi_wireless", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("ubnt:hostapd"), index("netops")); set("${FULLHOST_FROM}", value("HOST")); }; parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; } elif (match('\d+:\d+:\d+\s\S+\smcad:' value("MSG"))) { rewrite { + set("ubiquiti_unifi_wireless", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("ubnt:mcad"), index("netops")); set("${FULLHOST_FROM}", value("HOST")); }; parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; } else { rewrite { + set("ubiquiti_unifi_switch", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("ubnt:switch"), index("netops")); set("${FULLHOST_FROM}",value("HOST")); set("${model}", value("fields.model")); @@ -77,6 +80,7 @@ log { program('^(?U\d[^,]{1,10}),(?[a-z0-9]{9,16}),(?v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches")); }; rewrite { + set("ubiquiti_unifi_wireless", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("ubnt:wireless"), index("netops")); set("${FULLHOST_FROM}",value("HOST")); set("${model}", value("fields.model")); @@ -87,6 +91,7 @@ log { } elif (match("traputil.c\(696\) " value("MSG"))) { rewrite { + set("ubiquiti_unifi_edge_switch", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("ubnt:edgeswitch"), index("netops")); set("${FULLHOST_FROM}", value("HOST")); }; @@ -94,6 +99,7 @@ log { } else { rewrite { + set("ubiquiti_unifi", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("ubnt"), index("netops")); set("${FULLHOST_FROM}", value("HOST")); }; diff --git a/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl index 311fcc8..5d31966 100644 --- a/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl @@ -14,14 +14,15 @@ log { source (s_VMWARE); {{- end}} - rewrite { set("vmware", value("fields.sc4s_vendor_product")); }; - #NSX first because its the cheapest check if { filter(f_is_rfc5424_strict); filter(f_vmware_nsx); - rewrite { r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${PROGRAM}")); }; + rewrite { + set("vmware_nsx", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${PROGRAM}")); + }; parser { p_add_context_splunk(key("vmware_nsx")); }; parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; @@ -31,6 +32,7 @@ log { filter(f_vmware_nsx); rewrite { + set("vmware_nsx", value("fields.sc4s_vendor_product")); set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${.PROGRAM}")); @@ -44,7 +46,10 @@ log { filter(f_is_rfc5424_strict); filter(f_vmware_vsphere); - rewrite { r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${PROGRAM}")); }; + rewrite { + set("vmware_vsphere_esx", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${PROGRAM}")); + }; parser { p_add_context_splunk(key("vmware_esx")); }; parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; @@ -54,6 +59,7 @@ log { filter(f_vmware_vsphere); rewrite { + set("vmware_vsphere_esx", value("fields.sc4s_vendor_product")); set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${.PROGRAM}")); diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index d76a959..44be960 100644 --- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -2,6 +2,8 @@ log { source(s_DEFAULT); + rewrite { set("SC4S_fallback", value("fields.sc4s_vendor_product")); }; + if { filter(f_is_rfc5424_strict); rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main")); }; From dd2969a6501d8c95169ab859c5cca42eb8a58641 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 3 Jan 2020 08:08:42 -0800 Subject: [PATCH 2/2] Update test_common.py --- tests/test_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_common.py b/tests/test_common.py index 5b57207..eecec2d 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -52,7 +52,7 @@ def test_internal(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_tag(record_property, setup_wordlist, setup_splunk): +def test_fallback(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} test\n") @@ -60,7 +60,7 @@ def test_tag(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"testvp-{{ host }}\" sourcetype=\"sc4s:fallback\" sc4s_vendor_product=test_test | head 2") + st = env.from_string("search index=main host=\"testvp-{{ host }}\" sourcetype=\"sc4s:fallback\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search)