From 048e3b7426a7cea3a0092b4e84e810a9e4bd7fcf Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 20:24:14 -0400 Subject: [PATCH 1/4] TCP and TLS --- package/etc/go_templates/source_network.t | 8 ++++++-- tests/docker-compose.yml | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index fb59a33..adc09da 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -30,9 +30,10 @@ source s_{{ .port_id }} { {{- end}} {{- end}} {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} + {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} network ( - transport("tcp") - port({{ getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514" }}) + transport("tcp") + port({{ . }}) ip-protocol(4) max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) @@ -44,8 +45,10 @@ source s_{{ .port_id }} { chain-hostnames(off) flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); + {{- end }} {{- end}} {{- if (conv.ToBool (getenv "SC4S_SOURCE_TLS_ENABLE" "no")) }} + {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514") "," }} network( transport("tls") port({{ getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514" }}) @@ -66,6 +69,7 @@ source s_{{ .port_id }} { cipher-suite("{{- getenv "SC4S_SOURCE_TLS_CIPHER_SUITE" "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH" }}") ) ); + {{- end }} {{- end}} }; {{ if eq .parser "rfc3164" }} diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 0f070a0..c935baf 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -36,7 +36,8 @@ services: - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002 - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 - - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 + - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005,6005 + - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005,6005 - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 - SC4S_LISTEN_CISCO_ASA_UDP_PORT=5001 - SC4S_LISTEN_CISCO_IOS_UDP_PORT=5002 From f327c1157ecf12032092bd982cc6ad5c8af44cec Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 20:30:25 -0400 Subject: [PATCH 2/4] Support multiple ports per log path --- package/etc/go_templates/source_network.t | 14 +++++++++----- tests/docker-compose.yml | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index adc09da..59b9cf5 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -1,9 +1,12 @@ {{ define "UDP" }} +{{- $port_id := .port_id }} +{{- $port := .port }} + {{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_SOCKETS" "1"))}} syslog ( transport("udp") so-reuseport(1) - persist-name("{{ .port_id }}{{ .instance }}") - port({{ getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514" }}) + persist-name("{{ $port_id }}_{{ $port }}_{{ . }}") + port({{ $port }}) ip-protocol(4) so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "1703936"}}) keep-hostname(yes) @@ -13,6 +16,7 @@ chain-hostnames(off) flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); + {{- end}} {{- end}} {{ define "T1" }} @@ -24,13 +28,13 @@ source s_{{ .port_id }} { source { {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} {{- $port_id := .port_id }} -{{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_SOCKETS" "1"))}} -{{- $context := dict "instance" . "port_id" $port_id }} +{{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} +{{- $context := dict "port" . "port_id" $port_id }} {{- template "UDP" $context }} {{- end}} {{- end}} {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} - {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} + {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} network ( transport("tcp") port({{ . }}) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index c935baf..db3bad2 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -48,6 +48,7 @@ services: - SC4S_LISTEN_F5_BIGIP_UDP_PORT=5007 - SC4S_ARCHIVE_GLOBAL=no - SC4S_SOURCE_STORE_RAWMSG=yes + - SC4S_SOURCE_LISTEN_UDP_SOCKETS=2 splunk: build: context: ../splunk From fc931fa079fee91bc6d31a2539da555a2b7b37ff Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 20:33:17 -0400 Subject: [PATCH 3/4] Update configuration.md --- docs/configuration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 6fdf940..d9c5fc0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,10 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of kernel sockets per active UDP port, which configures multi-threading of the UDP input buffer in the kernel to prevent packet loss. Total UDP input buffer is the multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF | | SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | +### Multiple listening ports for the same source + +SC4S supports multiple network listening ports useful for a narrow set of compliance use cases. When configuring a source port variable use a comma ',' separated list with no spaces. + ## Syslog Source TLS Certificate Configuration * Create a folder ``/opt/sc4s/tls`` if not already done as part of the "getting started" process. From 661a4e4258902c5efb37c0ca81a8fca6457fcc46 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 8 May 2020 10:15:10 -0700 Subject: [PATCH 4/4] Refine mulitple unique port docs * Refine mulitple unique ports docs; change section it appears in --- docs/configuration.md | 4 ---- docs/sources/index.md | 10 ++++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6f707fd..38bca4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,10 +113,6 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of kernel sockets per active UDP port, which configures multi-threading of the UDP input buffer in the kernel to prevent packet loss. Total UDP input buffer is the multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF | | SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | -### Multiple listening ports for the same source - -SC4S supports multiple network listening ports useful for a narrow set of compliance use cases. When configuring a source port variable use a comma ',' separated list with no spaces. - ## Syslog Source TLS Certificate Configuration * Create a folder ``/opt/sc4s/tls`` if not already done as part of the "getting started" process. diff --git a/docs/sources/index.md b/docs/sources/index.md index 8a416bf..c709a62 100644 --- a/docs/sources/index.md +++ b/docs/sources/index.md @@ -19,3 +19,13 @@ A key aspect of SC4S is to properly set Splunk metadata prior to the data arrivi It is understood that default values will need to be changed in many installations. Each source documented in this section has a table entitled "Sourcetype and Index Configuration", which highlights the default index and sourcetype for each source. See the section "SC4S metadata configuration" in the "Configuration" page for more information on how to override the default values in this table. +## Unique listening ports + +SC4S supports unique listening ports for each source technology/log path (e.g. Cisco ASA), which is useful when the device is +sending data on a port different from the typical default syslog port (UDP port 514). In some cases, when the source device emits data that +is not able to be distinguished from other device types, a unique port is sometimes required. The specific environment variables used for +setting "unique ports" are outlined in each source document in this section. + +In most cases only one "unique port" is needed for each source. However, SC4S also supports multiple network listening ports per source, +which can be useful for a narrow set of compliance use cases. When configuring a source port variable to enable multiple ports, use a +comma-separated list with no spaces (e.g. `SC4S_LISTEN_CISCO_ASA_UDP_PORT=5005,6005`).