From 2f0ce3864c704ff1d838dc3d6975d16e502f6d39 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 30 Mar 2020 18:34:20 -0400 Subject: [PATCH 01/14] Change f_catch_first to f_null_queue --- docs/configuration.md | 2 +- package/etc/conf.d/filters/misc/catchfirst.conf | 4 ++-- package/etc/conf.d/log_paths/lp-aaa-catch_first.conf.tmpl | 2 +- .../context_templates/vendor_product_by_source.conf.example | 2 +- .../context_templates/vendor_product_by_source.csv.example | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d4e8af5..ec1569d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -228,7 +228,7 @@ docker stack deploy --compose-file docker-compose.yml sc4s ## Dropping all data by ip or subnet In some cases rogue or port-probing data can be sent to SC4S from misconfigured devices or vulnerability scanners. Update -the `vendor_product_by_source.conf` filter `f_catch_first` with one or more ip/subnet masks to drop events without +the `vendor_product_by_source.conf` filter `f_null_queue` with one or more ip/subnet masks to drop events without logging. Note that drop metrics will be recorded. diff --git a/package/etc/conf.d/filters/misc/catchfirst.conf b/package/etc/conf.d/filters/misc/catchfirst.conf index 9005f11..f967591 100644 --- a/package/etc/conf.d/filters/misc/catchfirst.conf +++ b/package/etc/conf.d/filters/misc/catchfirst.conf @@ -1,5 +1,5 @@ -#f_catch_first -filter f_catch_first { +#f_null_queue +filter f_null_queue { match("^catch_first", value("fields.sc4s_vendor_product")); }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-aaa-catch_first.conf.tmpl b/package/etc/conf.d/log_paths/lp-aaa-catch_first.conf.tmpl index b8fac16..41ce1ee 100644 --- a/package/etc/conf.d/log_paths/lp-aaa-catch_first.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-aaa-catch_first.conf.tmpl @@ -2,7 +2,7 @@ # vulnerability scanners to be ignored log { - filter(f_catch_first); + filter(f_null_queue); flags(catchall,final); diff --git a/package/etc/context_templates/vendor_product_by_source.conf.example b/package/etc/context_templates/vendor_product_by_source.conf.example index 1115e34..1eb20df 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf.example +++ b/package/etc/context_templates/vendor_product_by_source.conf.example @@ -2,7 +2,7 @@ filter f_test_test { host("testvp-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) }; -filter f_catch_first { +filter f_null_queue { netmask(169.254.100.0/24) }; diff --git a/package/etc/context_templates/vendor_product_by_source.csv.example b/package/etc/context_templates/vendor_product_by_source.csv.example index 302b8ca..0ea93b5 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv.example +++ b/package/etc/context_templates/vendor_product_by_source.csv.example @@ -1,6 +1,6 @@ f_test_test,sc4s_vendor_product,"test_test" f_brocade_syslog,sc4s_vendor_product,"brocade_syslog" -f_catch_first,sc4s_vendor_product,"catch_first" +f_null_queue,sc4s_vendor_product,"catch_first" f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" f_citrix_netscaler,sc4s_vendor_product,"citrix_netscaler" f_dell_rsa_secureid,sc4s_vendor_product,"dell_rsa_secureid" From a287d5e9c2248b5e2002dc9d89cc78f0e708ca54 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 30 Mar 2020 21:35:17 -0400 Subject: [PATCH 02/14] rename file to match --- .../{lp-aaa-catch_first.conf.tmpl => lp-aaa-null_queue.conf.tmpl} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename package/etc/conf.d/log_paths/{lp-aaa-catch_first.conf.tmpl => lp-aaa-null_queue.conf.tmpl} (100%) diff --git a/package/etc/conf.d/log_paths/lp-aaa-catch_first.conf.tmpl b/package/etc/conf.d/log_paths/lp-aaa-null_queue.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/lp-aaa-catch_first.conf.tmpl rename to package/etc/conf.d/log_paths/lp-aaa-null_queue.conf.tmpl From 8c13eab92f9eacffcc265fc6935c157ba53ef94f Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 31 Mar 2020 12:05:47 -0400 Subject: [PATCH 03/14] Support cisco FTD as Cisco ASA --- docs/sources/Cisco/index.md | 4 ++-- package/etc/conf.d/filters/cisco/asa.conf | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index ae72697..bec8942 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -97,7 +97,7 @@ index= sourcetype=cisco:apic:* Verify timestamp, and host values match as expected -## Product - ASA (Pre Firepower) +## Product - ASA AND FTD (Firepower) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| @@ -109,7 +109,7 @@ Verify timestamp, and host values match as expected | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| cisco:asa | None | +| cisco:asa | cisco FTD Firepower will also use this source type | | cisco:pix | Not supported | | cisco:fwsm | Not supported | diff --git a/package/etc/conf.d/filters/cisco/asa.conf b/package/etc/conf.d/filters/cisco/asa.conf index a7ac9b7..cda9eed 100644 --- a/package/etc/conf.d/filters/cisco/asa.conf +++ b/package/etc/conf.d/filters/cisco/asa.conf @@ -1,8 +1,11 @@ filter f_cisco_asa { message('^%ASA-\d+-\d{1,10}: ') or - match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR")); + match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR")) or + message('^%FTD-\d+-\d{1,10}: ') or + match('^%FTD-\d+-\d{1,10}:', value("LEGACY_MSGHDR")); }; filter f_cisco_asa_nohost { - match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR")); + match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR")) + or match('^%FTD-\d+-\d{1,10}:', value("LEGACY_MSGHDR")); }; From d4b90d3d691ce98b706e86e5bb0638536e73dabe Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 31 Mar 2020 12:08:49 -0400 Subject: [PATCH 04/14] fix --- package/etc/conf.d/filters/misc/catchfirst.conf | 5 ----- package/etc/conf.d/filters/misc/null_queue.conf | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 package/etc/conf.d/filters/misc/catchfirst.conf create mode 100644 package/etc/conf.d/filters/misc/null_queue.conf diff --git a/package/etc/conf.d/filters/misc/catchfirst.conf b/package/etc/conf.d/filters/misc/catchfirst.conf deleted file mode 100644 index f967591..0000000 --- a/package/etc/conf.d/filters/misc/catchfirst.conf +++ /dev/null @@ -1,5 +0,0 @@ -#f_null_queue -filter f_null_queue { - match("^catch_first", value("fields.sc4s_vendor_product")); - -}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/misc/null_queue.conf b/package/etc/conf.d/filters/misc/null_queue.conf new file mode 100644 index 0000000..561a2a8 --- /dev/null +++ b/package/etc/conf.d/filters/misc/null_queue.conf @@ -0,0 +1,5 @@ +#f_null_queue +filter f_null_queue { + match("^null_queue", value("fields.sc4s_vendor_product")); + +}; \ No newline at end of file From 73871fe3c93292fe6674b08b6f99ec01687bdea4 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Wed, 1 Apr 2020 18:15:22 -0400 Subject: [PATCH 05/14] Support a dynamic number of listeners for udp --- package/etc/go_templates/source_network.t | 33 ++++++++++++++--------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 47bfab1..b4e5353 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -1,13 +1,8 @@ -{{ define "T1" }} - -# The following is the source port declaration for {{ (print .port_id) }} - -source s_{{ .port_id }} { - channel { - source { -{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} - syslog ( +{{ define "UDP" }} + syslog ( transport("udp") + so-reuseport(1) + persist-name("udp{{.instance}}") port({{ getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514" }}) ip-protocol(4) so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "1703936"}}) @@ -17,7 +12,21 @@ source s_{{ .port_id }} { use-fqdn(no) chain-hostnames(off) flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) - ); + ); +{{- end}} + +{{ define "T1" }} + +# The following is the source port declaration for {{ (print .port_id) }} + +source s_{{ .port_id }} { + channel { + source { +{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} +{{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_INSTANCES" "5"))}} +{{- $context := dict "instance" . "port_id" "DEFAULT" }} +{{- template "UDP" $context }} +{{- end}} {{- end}} {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} network ( @@ -32,7 +41,7 @@ source s_{{ .port_id }} { use-dns(no) use-fqdn(no) chain-hostnames(off) - flags(no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) + flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); {{- end}} {{- if (conv.ToBool (getenv "SC4S_SOURCE_TLS_ENABLE" "no")) }} @@ -48,7 +57,7 @@ source s_{{ .port_id }} { use-dns(no) use-fqdn(no) chain-hostnames(off) - flags(no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) + flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) tls(allow-compress(yes) key-file("/opt/syslog-ng/tls/server.key") cert-file("/opt/syslog-ng/tls/server.pem") From ef47e252379c74386365fe029751ab7e64e6eaf2 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Wed, 1 Apr 2020 15:28:03 -0700 Subject: [PATCH 06/14] Update source_network.t Change UDP threads variable to `SC4S_SOURCE_LISTEN_UDP_SOCKETS` to match terminology in One Identity blog post and true nature of variable --- package/etc/go_templates/source_network.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index b4e5353..a2ef26f 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -23,7 +23,7 @@ source s_{{ .port_id }} { channel { source { {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} -{{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_INSTANCES" "5"))}} +{{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_SOCKETS" "5"))}} {{- $context := dict "instance" . "port_id" "DEFAULT" }} {{- template "UDP" $context }} {{- end}} @@ -206,4 +206,4 @@ source s_{{ .port_id }} { {{- end -}} {{- if or (or (or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT"))) (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT"))) (eq .port_id "DEFAULT") -}} {{- template "T1" (.) -}} -{{- end -}} \ No newline at end of file +{{- end -}} From c7142e07c63f50cadb8567941f3e9b7a47682f45 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 2 Apr 2020 07:33:12 -0400 Subject: [PATCH 07/14] Update lp-checkpoint_splunk.conf.tmpl --- .../log_paths/lp-checkpoint_splunk.conf.tmpl | 88 +++++++++---------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index 07f0aff..2a15a21 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -22,9 +22,51 @@ log { }; if { + parser { + kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); + date-parser-nofilter(format("%s") template("${.kv.time}")); + }; + rewrite { + set("${.kv.hostname}", value("HOST")); + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cp_log"), index("netops")) + }; + + parser {p_add_context_splunk(key("checkpoint_splunk")); }; + + if { + filter(f_checkpoint_splunk_NetworkTraffic); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; + } elif { + filter(f_checkpoint_splunk_Web); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; + } elif { + filter(f_checkpoint_splunk_NetworkSessions); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; + } elif { + filter(f_checkpoint_splunk_IDS_Malware); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; + } elif { + filter(f_checkpoint_splunk_IDS); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; + } elif { + filter(f_checkpoint_splunk_email); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; + } elif { + filter(f_checkpoint_splunk_DLP); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; + }; + } else { filter(f_nix_syslog); rewrite { - set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); @@ -32,51 +74,7 @@ log { rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) }; parser { p_add_context_splunk(key("checkpoint_os")); }; - } else { - parser { - kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); - date-parser-nofilter(format("%s") template("${.kv.time}")); - }; - }; - - rewrite { - set("${.kv.hostname}", value("HOST")); - set("checkpoint_splunk", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("cp_log"), index("netops")) }; - - parser {p_add_context_splunk(key("checkpoint_splunk")); }; - - if { - filter(f_checkpoint_splunk_NetworkTraffic); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; - } elif { - filter(f_checkpoint_splunk_Web); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; - } elif { - filter(f_checkpoint_splunk_NetworkSessions); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; - } elif { - filter(f_checkpoint_splunk_IDS_Malware); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; - } elif { - filter(f_checkpoint_splunk_IDS); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; - } elif { - filter(f_checkpoint_splunk_email); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; - } elif { - filter(f_checkpoint_splunk_DLP); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; - }; - parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; From d14aeba5fc7a1026389f7ec6b0e7808641ed3c24 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 2 Apr 2020 07:40:44 -0400 Subject: [PATCH 08/14] Test for field content misparse of host --- tests/test_checkpoint.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 77f8cc8..7bd9655 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -52,7 +52,33 @@ def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splun epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11_..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") + message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"") + search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_checkpoint_splunk_firewall2(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions for Checkpoint + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|severity=Medium|product=Firewall|action=Drop|ifdir=inbound|ifname=eth1|loguid={0x0,0x0,0x0,0x0}|origin=111.89.111.53|originsicname=CN\=xx-xx.xx.net,O\=cma-xx.xx.net.xx|sequencenum=64|time={{epoch}}|version=5|dst=10.11.11.11|inspection_category=anomaly|foo=bar: bat mark||\n") message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) From 37a3bed839f2d9bb009ee719ce7d48ad15c26625 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 2 Apr 2020 07:51:07 -0400 Subject: [PATCH 09/14] Add doc for SC4S_SOURCE_LISTEN_UDP_SOCKETS --- docs/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration.md b/docs/configuration.md index ec1569d..66f96ae 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,6 +103,7 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_TCP_IW_SIZE | 20000000 | Initial Window size | | SC4S_SOURCE_TCP_FETCH_LIMIT | 2000 | Number of events to fetch from server buffer at once | | SC4S_SOURCE_UDP_SO_RCVBUFF | 1703936 | UDP server buffer size in bytes. Make sure that the host OS kernel is configured [similarly](gettingstarted/index.md#prerequisites). | +| SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of virtual listeners per active UDP port this has the effect of increasing the buffer as a multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF | | SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | ## Syslog Source TLS Certificate Configuration From 56369ba99570f924cadaafd3165d2678508ca45d Mon Sep 17 00:00:00 2001 From: mbonsack Date: Thu, 2 Apr 2020 07:56:06 -0700 Subject: [PATCH 10/14] Update configuration.md Detailed description of `SC4S_SOURCE_LISTEN_UDP_SOCKETS` --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 66f96ae..16305cc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,7 +103,7 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_TCP_IW_SIZE | 20000000 | Initial Window size | | SC4S_SOURCE_TCP_FETCH_LIMIT | 2000 | Number of events to fetch from server buffer at once | | SC4S_SOURCE_UDP_SO_RCVBUFF | 1703936 | UDP server buffer size in bytes. Make sure that the host OS kernel is configured [similarly](gettingstarted/index.md#prerequisites). | -| SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of virtual listeners per active UDP port this has the effect of increasing the buffer as a multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF | +| SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of kernel sockets per active UDP port, which configures multi-threading of the UDP input buffer in the kernel to prevent packet loss. Total UDP input buffer is the multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF | | SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | ## Syslog Source TLS Certificate Configuration From 5ccc40efa0095442828ecd0ebf6409f097fac928 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 2 Apr 2020 12:22:38 -0400 Subject: [PATCH 11/14] Unique persist-names --- package/etc/go_templates/source_network.t | 2 +- tests/docker-compose-ci.yml | 6 ++++++ tests/docker-compose.yml | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index a2ef26f..5c7e8b4 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -2,7 +2,7 @@ syslog ( transport("udp") so-reuseport(1) - persist-name("udp{{.instance}}") + persist-name("{{.port_id}}{{.instance}}") port({{ getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514" }}) ip-protocol(4) so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "1703936"}}) diff --git a/tests/docker-compose-ci.yml b/tests/docker-compose-ci.yml index 3fe2010..e45bda0 100644 --- a/tests/docker-compose-ci.yml +++ b/tests/docker-compose-ci.yml @@ -48,6 +48,12 @@ services: - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 + - SC4S_LISTEN_CISCO_ASA_UDP_PORT=5001 + - SC4S_LISTEN_CISCO_IOS_UDP_PORT=5002 + - SC4S_LISTEN_CISCO_MERAKI_UDP_PORT=5003 + - SC4S_LISTEN_JUNIPER_IDP_UDP_PORT=5004 + - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005 + - SC4S_LISTEN_PFSENSE_UDP_PORT=5006 - SC4S_ARCHIVE_GLOBAL=no splunk: build: diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index d893bae..643be05 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -38,6 +38,12 @@ services: - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 + - SC4S_LISTEN_CISCO_ASA_UDP_PORT=5001 + - SC4S_LISTEN_CISCO_IOS_UDP_PORT=5002 + - SC4S_LISTEN_CISCO_MERAKI_UDP_PORT=5003 + - SC4S_LISTEN_JUNIPER_IDP_UDP_PORT=5004 + - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005 + - SC4S_LISTEN_PFSENSE_UDP_PORT=5006 - SC4S_ARCHIVE_GLOBAL=no - SC4S_SOURCE_STORE_RAWMSG=yes splunk: From ab87f12dbefd330808019cffe342d6b1a32e4b66 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 2 Apr 2020 14:01:37 -0400 Subject: [PATCH 12/14] Update source_network.t --- package/etc/go_templates/source_network.t | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 5c7e8b4..6d7a6d3 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -2,7 +2,7 @@ syslog ( transport("udp") so-reuseport(1) - persist-name("{{.port_id}}{{.instance}}") + persist-name("{{ .port_id }}{{ .instance }}") port({{ getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514" }}) ip-protocol(4) so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "1703936"}}) @@ -23,8 +23,9 @@ source s_{{ .port_id }} { channel { source { {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} +{{- $port_id := .port_id }} {{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_SOCKETS" "5"))}} -{{- $context := dict "instance" . "port_id" "DEFAULT" }} +{{- $context := dict "instance" . "port_id" $port_id }} {{- template "UDP" $context }} {{- end}} {{- end}} From 4f46409cce9a23f5c446da31db436de879d7294f Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 2 Apr 2020 14:45:14 -0400 Subject: [PATCH 13/14] Support SIGHUP and SIGTERM traps Allow developers to kill syslog-ng and not end the container --- package/sbin/entrypoint.sh | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index caaa3aa..c3bfa26 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -13,6 +13,28 @@ if [ ${SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC} ]; then export SC4S_DEST_CEF_HEC=$SC4S cd /opt/syslog-ng +# SIGTERM-handler +term_handler() { + if [ $pid -ne 0 ]; then + echo Terminating + kill -SIGTERM "$pid" + wait "$pid" + fi + exit 143; # 128 + 15 -- SIGTERM +} + +# SIGHUP-handler +hup_handler() { + if [ $pid -ne 0 ]; then + echo Reloading + kill -SIGHUP "$pid" + fi +} + +trap 'kill ${!}; hup_handler' SIGHUP +trap 'kill ${!}; term_handler' SIGTERM + + gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ @@ -31,4 +53,11 @@ echo sc4s version=$(cat /VERSION) >/opt/syslog-ng/var/log/syslog-ng.out echo syslog-ng starting /opt/syslog-ng/bin/persist-tool add /opt/syslog-ng/etc/reset_persist -o /opt/syslog-ng/var -exec /opt/syslog-ng/sbin/syslog-ng $@ + +/opt/syslog-ng/sbin/syslog-ng -F $@ & +pid="$!" +# wait forever +while true +do + tail -f /dev/null & wait ${!} +done \ No newline at end of file From 9690067ef1e769480fb1505e935aca23ef255641 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 6 Apr 2020 19:49:19 -0400 Subject: [PATCH 14/14] Support OS syslog nested in Checkpoint event format This will unwrap the proper OS syslog event from the header when origin is the logger OS however it will not support third party relay via checkpoint in this format. --- .../etc/conf.d/filters/checkpoint/splunk.conf | 7 +++- .../log_paths/lp-checkpoint_splunk.conf.tmpl | 19 ++++++++++ tests/docker-compose.yml | 2 +- tests/test_checkpoint.py | 35 +++++++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/filters/checkpoint/splunk.conf b/package/etc/conf.d/filters/checkpoint/splunk.conf index 87643d1..df1919b 100644 --- a/package/etc/conf.d/filters/checkpoint/splunk.conf +++ b/package/etc/conf.d/filters/checkpoint/splunk.conf @@ -1,6 +1,8 @@ filter f_checkpoint_splunk { match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("MSG") type("pcre")) or - match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("LEGACY_MSGHDR") type("pcre")); + match('\|(?:origin_sic_name|originsicname)\=[cC][nN]|\|product\=SmartConsole\|' value("LEGACY_MSGHDR") type("pcre")) or + match('|product\=Syslog\|ifdir=inbound\|loguid\=' value("MSG") type("pcre")) or + match('|product\=Syslog\|ifdir=inbound\|loguid\=' value("LEGACY_MSGHDR") type("pcre")); }; filter f_checkpoint_splunk_alerts { @@ -55,4 +57,7 @@ filter f_checkpoint_splunk_NetworkTraffic { }; filter f_checkpoint_splunk_Web { match('*Url Filtering*' value('.kv.product') type('glob')) +}; +filter f_checkpoint_splunk_syslog { + match('Syslog' value('.kv.product') type('glob')) }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index 2a15a21..673b11c 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -62,6 +62,25 @@ log { filter(f_checkpoint_splunk_DLP); rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; + } elif { + filter(f_checkpoint_splunk_syslog); + if { + parser { + syslog-parser(template("${.kv.default_device_message}") flags(guess-timezone, no-hostname)); + date-parser-nofilter(format("%s") template("${.kv.time}")); + }; + }; + + rewrite { + set("${.kv.hostname}", value("HOST")); + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) }; + parser { p_add_context_splunk(key("checkpoint_os")); }; + }; } else { filter(f_nix_syslog); diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 643be05..c8a8e1e 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -14,7 +14,7 @@ services: context: ../package hostname: sc4s #When this is enabled test_common will fail - #command: -det + command: -det ports: - "514" - "601" diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 7bd9655..d6ac8aa 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -223,3 +223,38 @@ def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk, set record_property("message", message) assert resultCount == 1 + + +# time=1586182935|hostname=xxxx-xxxx|product=Syslog|ifdir=inbound|loguid={0x0,0x0,0x0,0x0}|origin=10.0.0.164|sequencenum=3|time=1586182935|version=5|default_device_message=<134>ctasd[5665]: Save SenderId lists finished |facility=local use 0| +def test_checkpoint_splunk_os_nested( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions for Checkpoint + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Syslog|ifdir=inbound|loguid={0x0,0x0,0x0,0x0}|origin=10.0.0.0|sequencenum=3|time={{ epoch }}|version=5|default_device_message=<134>ctasd[5665]: Save SenderId lists finished |facility=local use 0|\n" + ) + message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="nix:syslog"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1