From 17f94a13c71f4f41392464c2ba08cd252e833db8 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 22:25:03 -0400 Subject: [PATCH 1/3] Reduce noise from checkpoint by loguid --- docs/sources/Checkpoint/index.md | 1 + .../log_paths/lp-checkpoint_splunk.conf.tmpl | 169 +++++++++++------- tests/docker-compose.yml | 1 + tests/test_checkpoint.py | 31 ++++ 4 files changed, 140 insertions(+), 62 deletions(-) diff --git a/docs/sources/Checkpoint/index.md b/docs/sources/Checkpoint/index.md index ee2c590..5c1186b 100644 --- a/docs/sources/Checkpoint/index.md +++ b/docs/sources/Checkpoint/index.md @@ -51,6 +51,7 @@ MSG Parse: This filter parses message content | SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the port number defined | | SC4S_ARCHIVE_CHECKPOINT_SPLUNK | no | Enable archive to disk for this specific source | | SC4S_DEST_CHECKPOINT_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL | no | Suppress any duplicate product+loguid pairs processed with 2 seconds | ### Verification diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index 673b11c..cd227fc 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -1,3 +1,17 @@ + +parser checkpoint_grouping { + grouping-by( + scope(host) + key("${.kv.product}${.kv.loguid}") + aggregate( + value("MESSAGE" "${MESSAGE}") + value(".cpfinal" "1" ) + inherit-mode(context) + ) + timeout(1) + ); +}; + # Checkpoint {{- /* The following provides a unique port source configuration if env var(s) are set */}} {{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "rfc3164" }} @@ -32,55 +46,85 @@ log { r_set_splunk_dest_default(sourcetype("cp_log"), index("netops")) }; - parser {p_add_context_splunk(key("checkpoint_splunk")); }; - - if { - filter(f_checkpoint_splunk_NetworkTraffic); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; - } elif { - filter(f_checkpoint_splunk_Web); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; - } elif { - filter(f_checkpoint_splunk_NetworkSessions); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; - } elif { - filter(f_checkpoint_splunk_IDS_Malware); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; - } elif { - filter(f_checkpoint_splunk_IDS); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; - } elif { - filter(f_checkpoint_splunk_email); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; - } elif { - filter(f_checkpoint_splunk_DLP); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; - } elif { - filter(f_checkpoint_splunk_syslog); - if { - parser { - syslog-parser(template("${.kv.default_device_message}") flags(guess-timezone, no-hostname)); - date-parser-nofilter(format("%s") template("${.kv.time}")); - }; + + if { + parser {p_add_context_splunk(key("checkpoint_splunk")); }; + + {{- if (conv.ToBool (getenv "SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL" "no")) }} + parser(checkpoint_grouping); + filter { + match("1" value(".cpfinal")) }; - - rewrite { - set("${.kv.hostname}", value("HOST")); - set("checkpoint_splunk", value("fields.sc4s_vendor_product")); - subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); - set("${PROGRAM}", value(".PROGRAM")); - subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + {{- end }} + + if { + filter(f_checkpoint_splunk_NetworkTraffic); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; + } elif { + filter(f_checkpoint_splunk_Web); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; + } elif { + filter(f_checkpoint_splunk_NetworkSessions); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; + } elif { + filter(f_checkpoint_splunk_IDS_Malware); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; + } elif { + filter(f_checkpoint_splunk_IDS); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; + } elif { + filter(f_checkpoint_splunk_email); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; + } elif { + filter(f_checkpoint_splunk_DLP); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; + } elif { + filter(f_checkpoint_splunk_syslog); + if { + parser { + syslog-parser(template("${.kv.default_device_message}") flags(guess-timezone, no-hostname)); + date-parser-nofilter(format("%s") template("${.kv.time}")); + }; }; - rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) }; - parser { p_add_context_splunk(key("checkpoint_os")); }; - + + rewrite { + set("${.kv.hostname}", value("HOST")); + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) }; + parser { p_add_context_splunk(key("checkpoint_os")); }; + + }; + + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + + {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no")) }} + destination(d_hec); + {{- end}} + + {{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK" "no")) }} + destination(d_archive); + {{- end}} + + {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} + + {{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} + }; } else { filter(f_nix_syslog); @@ -93,25 +137,26 @@ log { rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) }; parser { p_add_context_splunk(key("checkpoint_os")); }; - }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; -{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no")) }} - destination(d_hec); -{{- end}} + {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no")) }} + destination(d_hec); + {{- end}} -{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK" "no")) }} - destination(d_archive); -{{- end}} + {{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK" "no")) }} + destination(d_archive); + {{- end}} -{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} - {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); -{{- end }} + {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} -{{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }} - {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); -{{- end }} + {{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} + + }; flags(flow-control,final); }; diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 0f070a0..04944c4 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -47,6 +47,7 @@ services: - SC4S_LISTEN_F5_BIGIP_UDP_PORT=5007 - SC4S_ARCHIVE_GLOBAL=no - SC4S_SOURCE_STORE_RAWMSG=yes + - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes splunk: build: context: ../splunk diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index d6ac8aa..06714f6 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -68,6 +68,37 @@ def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splun assert resultCount == 1 + +def test_checkpoint_splunk_firewall_noise(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions for Checkpoint + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") + message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"") + search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + def test_checkpoint_splunk_firewall2(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) From d1ccba9f584e8fbaa633fcfe58f1a68671bd8607 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 07:19:20 -0400 Subject: [PATCH 2/3] new test and docs fix --- docs/sources/Checkpoint/index.md | 2 +- .../log_paths/lp-checkpoint_splunk.conf.tmpl | 2 +- tests/test_checkpoint.py | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/sources/Checkpoint/index.md b/docs/sources/Checkpoint/index.md index 5c1186b..7bcde6e 100644 --- a/docs/sources/Checkpoint/index.md +++ b/docs/sources/Checkpoint/index.md @@ -51,7 +51,7 @@ MSG Parse: This filter parses message content | SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the port number defined | | SC4S_ARCHIVE_CHECKPOINT_SPLUNK | no | Enable archive to disk for this specific source | | SC4S_DEST_CHECKPOINT_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | -| SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL | no | Suppress any duplicate product+loguid pairs processed with 2 seconds | +| SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL | no | Suppress any duplicate product+loguid pairs processed within 2 seconds of the last matching event | ### Verification diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index cd227fc..c5a884a 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -8,7 +8,7 @@ parser checkpoint_grouping { value(".cpfinal" "1" ) inherit-mode(context) ) - timeout(1) + timeout(2) ); }; diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 06714f6..7baf2e8 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -98,6 +98,35 @@ def test_checkpoint_splunk_firewall_noise(record_property, setup_wordlist, setup assert resultCount == 1 +def test_checkpoint_splunk_firewall_noise2(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions for Checkpoint + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") + message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sleep(5) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"") + search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 2 def test_checkpoint_splunk_firewall2(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) From caaf5dc2d37d9cb362e1c727ad981e37f17cf08b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 14:33:42 -0400 Subject: [PATCH 3/3] Update docker-compose-ci.yml --- tests/docker-compose-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/docker-compose-ci.yml b/tests/docker-compose-ci.yml index e45bda0..6221947 100644 --- a/tests/docker-compose-ci.yml +++ b/tests/docker-compose-ci.yml @@ -55,6 +55,8 @@ services: - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005 - SC4S_LISTEN_PFSENSE_UDP_PORT=5006 - SC4S_ARCHIVE_GLOBAL=no + - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + splunk: build: context: ../splunk