Merge pull request #398 from splunk/feature/checkpoint-nested-syslog

Feature/checkpoint nested syslog
CSVD · Apr 7, 2020 · 9b07178 · 9b07178
2 parents 7568570 + 9690067
commit 9b07178
Show file tree

Hide file tree

Showing 15 changed files with 213 additions and 75 deletions.
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -103,6 +103,7 @@ and/or move them to an archival system to avoid exhaustion of disk space.
 | SC4S_SOURCE_TCP_IW_SIZE | 20000000 | Initial Window size |
 | SC4S_SOURCE_TCP_FETCH_LIMIT | 2000 | Number of events to fetch from server buffer at once |
 | SC4S_SOURCE_UDP_SO_RCVBUFF | 1703936 | UDP server buffer size in bytes. Make sure that the host OS kernel is configured [similarly](gettingstarted/index.md#prerequisites). |
+| SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of kernel sockets per active UDP port, which configures multi-threading of the UDP input buffer in the kernel to prevent packet loss.  Total UDP input buffer is the multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF |
 | SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype.  Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. |
 
 ## Syslog Source TLS Certificate Configuration
@@ -228,7 +229,7 @@ docker stack deploy --compose-file docker-compose.yml sc4s
 ## Dropping all data by ip or subnet
 
 In some cases rogue or port-probing data can be sent to SC4S from misconfigured devices or vulnerability scanners. Update
-the `vendor_product_by_source.conf` filter `f_catch_first` with one or more ip/subnet masks to drop events without
+the `vendor_product_by_source.conf` filter `f_null_queue` with one or more ip/subnet masks to drop events without
 logging. Note that drop metrics will be recorded.
 
 

diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md
@@ -97,7 +97,7 @@ index=<asconfigured> sourcetype=cisco:apic:*
 
 Verify timestamp, and host values match as expected    
 
-## Product - ASA (Pre Firepower)
+## Product - ASA AND FTD (Firepower)
 
 | Ref            | Link                                                                                                    |
 |----------------|---------------------------------------------------------------------------------------------------------|
@@ -109,7 +109,7 @@ Verify timestamp, and host values match as expected
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| cisco:asa      | None                                                                                                    |
+| cisco:asa      | cisco FTD Firepower will also use this source type                                                      |
 | cisco:pix      | Not supported                                                                                           |
 | cisco:fwsm     | Not supported                                                                                           |
 

diff --git a/package/etc/conf.d/filters/checkpoint/splunk.conf b/package/etc/conf.d/filters/checkpoint/splunk.conf
@@ -1,6 +1,8 @@
 filter f_checkpoint_splunk {
     match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("MSG") type("pcre")) or
-    match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("LEGACY_MSGHDR") type("pcre"));
+    match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("LEGACY_MSGHDR") type("pcre")) or
+    match('|product\=Syslog\|ifdir=inbound\|loguid\=' value("MSG") type("pcre")) or
+    match('|product\=Syslog\|ifdir=inbound\|loguid\=' value("LEGACY_MSGHDR") type("pcre"));
 };
 
 filter f_checkpoint_splunk_alerts {
@@ -55,4 +57,7 @@ filter f_checkpoint_splunk_NetworkTraffic {
 };
 filter f_checkpoint_splunk_Web {
     match('*Url Filtering*' value('.kv.product') type('glob'))
+};
+filter f_checkpoint_splunk_syslog {
+    match('Syslog' value('.kv.product') type('glob'))
 };
diff --git a/package/etc/conf.d/filters/cisco/asa.conf b/package/etc/conf.d/filters/cisco/asa.conf
@@ -1,8 +1,11 @@
 filter f_cisco_asa {
     message('^%ASA-\d+-\d{1,10}: ') or
-    match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR"));
+    match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR")) or
+    message('^%FTD-\d+-\d{1,10}: ') or
+    match('^%FTD-\d+-\d{1,10}:', value("LEGACY_MSGHDR"));
 };
 
 filter f_cisco_asa_nohost {
-    match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR"));
+    match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR"))
+    or match('^%FTD-\d+-\d{1,10}:', value("LEGACY_MSGHDR"));
 };
diff --git a/package/etc/conf.d/filters/misc/catchfirst.conf b/package/etc/conf.d/filters/misc/catchfirst.conf
diff --git a/package/etc/conf.d/filters/misc/null_queue.conf b/package/etc/conf.d/filters/misc/null_queue.conf
@@ -0,0 +1,5 @@
+#f_null_queue
+filter f_null_queue {
+    match("^null_queue", value("fields.sc4s_vendor_product"));
+
+};
diff --git a/....d/log_paths/lp-aaa-catch_first.conf.tmpl → ...f.d/log_paths/lp-aaa-null_queue.conf.tmpl b/....d/log_paths/lp-aaa-catch_first.conf.tmpl → ...f.d/log_paths/lp-aaa-null_queue.conf.tmpl
@@ -2,7 +2,7 @@
 # vulnerability scanners to be ignored
 log {
 
-    filter(f_catch_first);
+    filter(f_null_queue);
 
     flags(catchall,final);
 

diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl
@@ -22,61 +22,78 @@ log {
     };
 
     if {
+        parser {
+            kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}"));
+            date-parser-nofilter(format("%s") template("${.kv.time}"));
+        };
+        rewrite {
+           set("${.kv.hostname}", value("HOST"));
+           set("checkpoint_splunk", value("fields.sc4s_vendor_product"));
+           r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"))
+        };
+
+        parser {p_add_context_splunk(key("checkpoint_splunk")); };
+
+        if  {
+            filter(f_checkpoint_splunk_NetworkTraffic);
+            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))};
+            parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); };
+        } elif  {
+            filter(f_checkpoint_splunk_Web);
+            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))};
+            parser {p_add_context_splunk(key("checkpoint_splunk_web")); };
+        } elif  {
+            filter(f_checkpoint_splunk_NetworkSessions);
+            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))};
+            parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); };
+        } elif  {
+            filter(f_checkpoint_splunk_IDS_Malware);
+            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))};
+            parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
+        } elif  {
+            filter(f_checkpoint_splunk_IDS);
+            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))};
+            parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
+        } elif  {
+            filter(f_checkpoint_splunk_email);
+            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))};
+            parser {p_add_context_splunk(key("checkpoint_splunk_email")); };
+        } elif  {
+            filter(f_checkpoint_splunk_DLP);
+            rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))};
+            parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); };
+        } elif {
+            filter(f_checkpoint_splunk_syslog);
+            if {
+                    parser {
+                        syslog-parser(template("${.kv.default_device_message}") flags(guess-timezone, no-hostname));
+                        date-parser-nofilter(format("%s") template("${.kv.time}"));
+                    };
+            };
+
+            rewrite {
+                    set("${.kv.hostname}", value("HOST"));
+                    set("checkpoint_splunk", value("fields.sc4s_vendor_product"));            
+                    subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+                    set("${PROGRAM}", value(".PROGRAM"));
+                    subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+                };
+            rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) };
+            parser { p_add_context_splunk(key("checkpoint_os")); };                
+
+        };
+    } else {
         filter(f_nix_syslog);
         rewrite {
-          set("checkpoint_splunk", value("fields.sc4s_vendor_product"));
+            set("checkpoint_splunk", value("fields.sc4s_vendor_product"));
             subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
             set("${PROGRAM}", value(".PROGRAM"));
             subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
         };
         rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) };
         parser { p_add_context_splunk(key("checkpoint_os")); };
 
-    } else {
-        parser {
-            kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}"));
-            date-parser-nofilter(format("%s") template("${.kv.time}"));
-        };
-    };
-
-    rewrite {
-        set("${.kv.hostname}", value("HOST"));
-        set("checkpoint_splunk", value("fields.sc4s_vendor_product"));
-        r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"))
     };
-
-    parser {p_add_context_splunk(key("checkpoint_splunk")); };
-
-    if  {
-       filter(f_checkpoint_splunk_NetworkTraffic);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); };
-    } elif  {
-       filter(f_checkpoint_splunk_Web);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_web")); };
-    } elif  {
-       filter(f_checkpoint_splunk_NetworkSessions);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); };
-    } elif  {
-       filter(f_checkpoint_splunk_IDS_Malware);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
-    } elif  {
-       filter(f_checkpoint_splunk_IDS);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
-    } elif  {
-       filter(f_checkpoint_splunk_email);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_email")); };
-    } elif  {
-       filter(f_checkpoint_splunk_DLP);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))};
-       parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); };
-    };
-
     parser (compliance_meta_by_source);
     rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
 

diff --git a/package/etc/context_templates/vendor_product_by_source.conf.example b/package/etc/context_templates/vendor_product_by_source.conf.example
@@ -2,7 +2,7 @@ filter f_test_test {
     host("testvp-*" type(glob))
     #or netmask(xxx.xxx.xxx.xxx/xx)
 };
-filter f_catch_first {
+filter f_null_queue {
    netmask(169.254.100.0/24)
 };
 

diff --git a/package/etc/context_templates/vendor_product_by_source.csv.example b/package/etc/context_templates/vendor_product_by_source.csv.example
@@ -1,6 +1,6 @@
 f_test_test,sc4s_vendor_product,"test_test"
 f_brocade_syslog,sc4s_vendor_product,"brocade_syslog"
-f_catch_first,sc4s_vendor_product,"catch_first"
+f_null_queue,sc4s_vendor_product,"catch_first"
 f_cisco_meraki,sc4s_vendor_product,"cisco_meraki"
 f_citrix_netscaler,sc4s_vendor_product,"citrix_netscaler"
 f_dell_rsa_secureid,sc4s_vendor_product,"dell_rsa_secureid"

diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t
@@ -1,13 +1,8 @@
-{{ define "T1" }}
-
-# The following is the source port declaration for {{ (print .port_id) }}
-
-source s_{{ .port_id }} {
-    channel {
-        source {
-{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }}
-            syslog (
+{{ define "UDP" }}
+        syslog (
                 transport("udp")
+                so-reuseport(1)
+                persist-name("{{ .port_id }}{{ .instance }}")
                 port({{ getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514" }})
                 ip-protocol(4)
                 so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "1703936"}})
@@ -17,7 +12,22 @@ source s_{{ .port_id }} {
                 use-fqdn(no)
                 chain-hostnames(off)
                 flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
-            );
+            );   
+{{- end}}
+
+{{ define "T1" }}
+
+# The following is the source port declaration for {{ (print .port_id) }}
+
+source s_{{ .port_id }} {
+    channel {
+        source {
+{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }}
+{{- $port_id := .port_id }}
+{{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_SOCKETS" "5"))}}
+{{- $context := dict "instance" . "port_id" $port_id }}
+{{- template "UDP"  $context }}
+{{- end}}
 {{- end}}
 {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }}
             network (
@@ -32,7 +42,7 @@ source s_{{ .port_id }} {
                 use-dns(no)
                 use-fqdn(no)
                 chain-hostnames(off)
-                flags(no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
+                flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
             );
 {{- end}}
 {{- if (conv.ToBool (getenv "SC4S_SOURCE_TLS_ENABLE" "no")) }}
@@ -48,7 +58,7 @@ source s_{{ .port_id }} {
                 use-dns(no)
                 use-fqdn(no)
                 chain-hostnames(off)
-                flags(no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
+                flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
                 tls(allow-compress(yes)
                     key-file("/opt/syslog-ng/tls/server.key")
                     cert-file("/opt/syslog-ng/tls/server.pem")
@@ -197,4 +207,4 @@ source s_{{ .port_id }} {
 {{- end -}}
 {{- if or (or (or (getenv  (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (getenv  (print "SC4S_LISTEN_" .port_id "_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_" .port_id "_TLS_PORT"))) (eq .port_id "DEFAULT") -}}
 {{- template "T1" (.) -}}
-{{- end -}}
+{{- end -}}
diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh
@@ -13,6 +13,28 @@ if [ ${SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC} ]; then export SC4S_DEST_CEF_HEC=$SC4S
 
 cd /opt/syslog-ng
 
+# SIGTERM-handler
+term_handler() {
+  if [ $pid -ne 0 ]; then
+    echo Terminating
+    kill -SIGTERM "$pid"
+    wait "$pid"
+  fi
+  exit 143; # 128 + 15 -- SIGTERM
+}
+
+# SIGHUP-handler
+hup_handler() {
+  if [ $pid -ne 0 ]; then
+    echo Reloading
+    kill -SIGHUP "$pid"
+  fi
+}
+
+trap 'kill ${!}; hup_handler' SIGHUP
+trap 'kill ${!}; term_handler' SIGTERM
+
+
 gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/
 
 mkdir -p /opt/syslog-ng/etc/conf.d/local/context/
@@ -31,4 +53,11 @@ echo sc4s version=$(cat /VERSION) >/opt/syslog-ng/var/log/syslog-ng.out
 
 echo syslog-ng starting
 /opt/syslog-ng/bin/persist-tool add /opt/syslog-ng/etc/reset_persist -o /opt/syslog-ng/var
-exec /opt/syslog-ng/sbin/syslog-ng $@
+
+/opt/syslog-ng/sbin/syslog-ng -F $@ &
+pid="$!"
+# wait forever
+while true
+do
+  tail -f /dev/null & wait ${!}
+done
diff --git a/tests/docker-compose-ci.yml b/tests/docker-compose-ci.yml
@@ -48,6 +48,12 @@ services:
       - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004
       - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005
       - SC4S_LISTEN_PFSENSE_TCP_PORT=5006
+      - SC4S_LISTEN_CISCO_ASA_UDP_PORT=5001
+      - SC4S_LISTEN_CISCO_IOS_UDP_PORT=5002
+      - SC4S_LISTEN_CISCO_MERAKI_UDP_PORT=5003
+      - SC4S_LISTEN_JUNIPER_IDP_UDP_PORT=5004
+      - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005
+      - SC4S_LISTEN_PFSENSE_UDP_PORT=5006
       - SC4S_ARCHIVE_GLOBAL=no
   splunk:
     build:

diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
@@ -14,7 +14,7 @@ services:
       context: ../package
     hostname: sc4s
     #When this is enabled test_common will fail
-    #command: -det
+    command: -det
     ports:
       - "514"
       - "601"
@@ -38,6 +38,12 @@ services:
       - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004
       - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005
       - SC4S_LISTEN_PFSENSE_TCP_PORT=5006
+      - SC4S_LISTEN_CISCO_ASA_UDP_PORT=5001
+      - SC4S_LISTEN_CISCO_IOS_UDP_PORT=5002
+      - SC4S_LISTEN_CISCO_MERAKI_UDP_PORT=5003
+      - SC4S_LISTEN_JUNIPER_IDP_UDP_PORT=5004
+      - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005
+      - SC4S_LISTEN_PFSENSE_UDP_PORT=5006
       - SC4S_ARCHIVE_GLOBAL=no
       - SC4S_SOURCE_STORE_RAWMSG=yes
   splunk: