Merge pull request #445 from splunk/feature/checkpoint-noise

Reduce noise from checkpoint by loguid
CSVD · May 8, 2020 · 638d5c1 · 638d5c1
2 parents f68e831 + 11f31d8
commit 638d5c1
Show file tree

Hide file tree

Showing 5 changed files with 171 additions and 62 deletions.
diff --git a/docs/sources/Checkpoint/index.md b/docs/sources/Checkpoint/index.md
@@ -51,6 +51,7 @@ MSG Parse: This filter parses message content
 | SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the port number defined |
 | SC4S_ARCHIVE_CHECKPOINT_SPLUNK | no | Enable archive to disk for this specific source |
 | SC4S_DEST_CHECKPOINT_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL | no | Suppress any duplicate product+loguid pairs processed within 2 seconds of the last matching event |
 
 ### Verification
 

diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl
@@ -1,3 +1,17 @@
+
+parser checkpoint_grouping {
+    grouping-by(
+        scope(host)
+        key("${.kv.product}${.kv.loguid}")
+        aggregate(
+            value("MESSAGE" "${MESSAGE}")
+            value(".cpfinal" "1" )
+            inherit-mode(context)
+        )
+        timeout(2)
+    );
+};
+
 # Checkpoint
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
 {{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "rfc3164" }}
@@ -32,55 +46,85 @@ log {
            r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"))
         };
 
-        parser {p_add_context_splunk(key("checkpoint_splunk")); };
-
-        if  {
-            filter(f_checkpoint_splunk_NetworkTraffic);
-            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))};
-            parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); };
-        } elif  {
-            filter(f_checkpoint_splunk_Web);
-            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))};
-            parser {p_add_context_splunk(key("checkpoint_splunk_web")); };
-        } elif  {
-            filter(f_checkpoint_splunk_NetworkSessions);
-            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))};
-            parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); };
-        } elif  {
-            filter(f_checkpoint_splunk_IDS_Malware);
-            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))};
-            parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
-        } elif  {
-            filter(f_checkpoint_splunk_IDS);
-            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))};
-            parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
-        } elif  {
-            filter(f_checkpoint_splunk_email);
-            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))};
-            parser {p_add_context_splunk(key("checkpoint_splunk_email")); };
-        } elif  {
-            filter(f_checkpoint_splunk_DLP);
-            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))};
-            parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); };
-        } elif {
-            filter(f_checkpoint_splunk_syslog);
-            if {
-                    parser {
-                        syslog-parser(template("${.kv.default_device_message}") flags(guess-timezone, no-hostname));
-                        date-parser-nofilter(format("%s") template("${.kv.time}"));
-                    };
+
+        if {
+            parser {p_add_context_splunk(key("checkpoint_splunk")); };
+
+            {{- if (conv.ToBool (getenv "SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL" "no")) }}
+            parser(checkpoint_grouping);
+            filter {
+                match("1" value(".cpfinal"))
             };
-
-            rewrite {
-                    set("${.kv.hostname}", value("HOST"));
-                    set("checkpoint_splunk", value("fields.sc4s_vendor_product"));            
-                    subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
-                    set("${PROGRAM}", value(".PROGRAM"));
-                    subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            {{- end }}
+
+            if  {
+                filter(f_checkpoint_splunk_NetworkTraffic);
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))};
+                parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); };
+            } elif  {
+                filter(f_checkpoint_splunk_Web);
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))};
+                parser {p_add_context_splunk(key("checkpoint_splunk_web")); };
+            } elif  {
+                filter(f_checkpoint_splunk_NetworkSessions);
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))};
+                parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); };
+            } elif  {
+                filter(f_checkpoint_splunk_IDS_Malware);
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))};
+                parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
+            } elif  {
+                filter(f_checkpoint_splunk_IDS);
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))};
+                parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
+            } elif  {
+                filter(f_checkpoint_splunk_email);
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))};
+                parser {p_add_context_splunk(key("checkpoint_splunk_email")); };
+            } elif  {
+                filter(f_checkpoint_splunk_DLP);
+                rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))};
+                parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); };
+            } elif {
+                filter(f_checkpoint_splunk_syslog);
+                if {
+                        parser {
+                            syslog-parser(template("${.kv.default_device_message}") flags(guess-timezone, no-hostname));
+                            date-parser-nofilter(format("%s") template("${.kv.time}"));
+                        };
                 };
-            rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) };
-            parser { p_add_context_splunk(key("checkpoint_os")); };                
-
+
+                rewrite {
+                        set("${.kv.hostname}", value("HOST"));
+                        set("checkpoint_splunk", value("fields.sc4s_vendor_product"));            
+                        subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+                        set("${PROGRAM}", value(".PROGRAM"));
+                        subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+                    };
+                rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) };
+                parser { p_add_context_splunk(key("checkpoint_os")); };                
+
+            };
+
+                parser (compliance_meta_by_source);
+                rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
+
+            {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no")) }}
+                destination(d_hec);
+            {{- end}}
+
+            {{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK" "no")) }}
+                destination(d_archive);
+            {{- end}}
+
+            {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
+                {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+            {{- end }}
+
+            {{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }}
+                {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+            {{- end }}
+
         };
     } else {
         filter(f_nix_syslog);
@@ -93,25 +137,26 @@ log {
         rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) };
         parser { p_add_context_splunk(key("checkpoint_os")); };
 
-    };
-    parser (compliance_meta_by_source);
-    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
+        parser (compliance_meta_by_source);
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
 
-{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no")) }}
-    destination(d_hec);
-{{- end}}
+    {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no")) }}
+        destination(d_hec);
+    {{- end}}
 
-{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK" "no")) }}
-    destination(d_archive);
-{{- end}}
+    {{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK" "no")) }}
+        destination(d_archive);
+    {{- end}}
 
-{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
-    {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
-{{- end }}
+    {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
+        {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+    {{- end }}
 
-{{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }}
-    {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
-{{- end }}
+    {{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }}
+        {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+    {{- end }}
+
+    };
 
     flags(flow-control,final);
 };
diff --git a/tests/docker-compose-ci.yml b/tests/docker-compose-ci.yml
@@ -55,6 +55,8 @@ services:
       - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005
       - SC4S_LISTEN_PFSENSE_UDP_PORT=5006
       - SC4S_ARCHIVE_GLOBAL=no
+      - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes
+
   splunk:
     build:
       context: ../splunk

diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
@@ -48,6 +48,7 @@ services:
       - SC4S_LISTEN_F5_BIGIP_UDP_PORT=5007
       - SC4S_ARCHIVE_GLOBAL=no
       - SC4S_SOURCE_STORE_RAWMSG=yes
+      - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes
       - SC4S_SOURCE_LISTEN_UDP_SOCKETS=2
   splunk:
     build:

diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py
@@ -68,6 +68,66 @@ def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splun
 
     assert resultCount == 1
 
+
+def test_checkpoint_splunk_firewall_noise(record_property, setup_wordlist, setup_splunk, setup_sc4s):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    dt = datetime.datetime.now()
+    iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
+
+    # Tune time functions for Checkpoint
+    epoch = epoch[:-7]
+
+    mt = env.from_string(
+        "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n")
+    message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch)
+
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"")
+    search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+def test_checkpoint_splunk_firewall_noise2(record_property, setup_wordlist, setup_splunk, setup_sc4s):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    dt = datetime.datetime.now()
+    iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
+
+    # Tune time functions for Checkpoint
+    epoch = epoch[:-7]
+
+    mt = env.from_string(
+        "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n")
+    message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch)
+
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+    sleep(5)
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"")
+    search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 2
+
 def test_checkpoint_splunk_firewall2(record_property, setup_wordlist, setup_splunk, setup_sc4s):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))