From 084d9364471a44681b836fd2c769d4b16f1e6c8b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 14:45:07 -0400 Subject: [PATCH 1/5] Support reverse DNS --- docs/configuration.md | 1 + package/Dockerfile | 2 +- package/etc/{syslog-ng.conf => syslog-ng.conf.tmpl} | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) rename package/etc/{syslog-ng.conf => syslog-ng.conf.tmpl} (94%) diff --git a/docs/configuration.md b/docs/configuration.md index 6fdf940..4105d17 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,6 +9,7 @@ and variables needed to properly configure SC4S for your environment. |----------|---------------|-------------| | SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list | | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | +| SC4S_GLOBAL_DNS_USE | yes/no (default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | * NOTE: Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http destination does not support this feature. Moreover, HEC Ack would significantly degrade performance for streaming data such as diff --git a/package/Dockerfile b/package/Dockerfile index bd5e477..b461c2b 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -13,7 +13,7 @@ COPY --from=hairyhenderson/gomplate:v3.5.0 /gomplate /usr/local/bin/gomplate COPY goss.yaml goss.yaml -COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf +COPY etc/syslog-ng.conf.tmpl /opt/syslog-ng/etc/syslog-ng.conf.tmpl COPY etc/conf.d /opt/syslog-ng/etc/conf.d COPY etc/go_templates /opt/syslog-ng/etc/go_templates COPY etc/context_templates /opt/syslog-ng/etc/context_templates diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf.tmpl similarity index 94% rename from package/etc/syslog-ng.conf rename to package/etc/syslog-ng.conf.tmpl index 03ba65c..8730725 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf.tmpl @@ -18,9 +18,9 @@ options { time_reopen (10); log_fifo_size (10000); chain_hostnames (off); - use_dns (no); + use_dns ({{getenv "SC4S_GLOBAL_DNS_USE" "no"}}); use_fqdn (no); - dns-cache(no); + dns-cache({{getenv "SC4S_GLOBAL_DNS_CACHE" "yes"}}); create_dirs (no); keep-hostname (yes); create_dirs(yes); From 28b41d99ce3c0206a72e14b4f093132cf7354a41 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 14:56:56 -0400 Subject: [PATCH 2/5] Update syslog-ng.conf.tmpl --- package/etc/syslog-ng.conf.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/syslog-ng.conf.tmpl b/package/etc/syslog-ng.conf.tmpl index 8730725..98db6a2 100644 --- a/package/etc/syslog-ng.conf.tmpl +++ b/package/etc/syslog-ng.conf.tmpl @@ -17,12 +17,12 @@ options { flush_lines (100); time_reopen (10); log_fifo_size (10000); - chain_hostnames (off); + chain_hostnames (yes); use_dns ({{getenv "SC4S_GLOBAL_DNS_USE" "no"}}); use_fqdn (no); dns-cache({{getenv "SC4S_GLOBAL_DNS_CACHE" "yes"}}); create_dirs (no); - keep-hostname (yes); + keep-hostname (no); create_dirs(yes); dir_perm(0750); stats-freq(30); From 9eaebb30bf16d5991cde9cf592a236ce0e679aba Mon Sep 17 00:00:00 2001 From: mbonsack Date: Thu, 7 May 2020 13:01:24 -0700 Subject: [PATCH 3/5] Update configuration.md --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4105d17..38bca4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,7 +9,7 @@ and variables needed to properly configure SC4S for your environment. |----------|---------------|-------------| | SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list | | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | -| SC4S_GLOBAL_DNS_USE | yes/no (default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | +| SC4S_GLOBAL_DNS_USE | yes or no(default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | * NOTE: Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http destination does not support this feature. Moreover, HEC Ack would significantly degrade performance for streaming data such as From dc15a9f1f2866d40818e800154ce213d24097316 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 7 May 2020 13:55:46 -0700 Subject: [PATCH 4/5] Update 5424 templates to include RAWMSG * Update 5424 output templates to include RAWMSG. Too confusing for most users without them, and there are some small parts of the message that are excluded (such as colons) that are needed to reassemble the raw message with the constituent macros. --- package/etc/conf.d/conflib/_common/templates.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index fc0c7a0..ce18953 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -80,6 +80,7 @@ template t_JSON_5424 { template('$(format-json --scope rfc5424 --pair PRI="<$PRI>" --key ISODATE + --key RAWMSG --exclude DATE --exclude FACILITY --exclude PRIORITY @@ -95,6 +96,7 @@ template t_JSON_5424_SDATA { template('$(format-json --scope rfc5424 --pair PRI="<$PRI>" --key ISODATE + --key RAWMSG --exclude DATE --exclude HOST --exclude FACILITY From c97d86af21f551de00c69ff13b3a7a9f94385a52 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 7 May 2020 14:56:10 -0700 Subject: [PATCH 5/5] Update conntrack unit file entry with required space * `ExecStartPost` command requires a space around the semicolon separating the two individual commands. --- docs/gettingstarted/podman-systemd-general.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 7bb6ee6..b0c026d 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -19,10 +19,12 @@ to install and run it each time sc4s starts. It should be available in all RHEL install conntrack ``` -After this is done, add the following entry to the unit file (and/or use the command when starting sc4s manually): +After this is done, add the following entry to the unit file (and/or use the command when starting sc4s manually). +Note that the space on either side of the semicolon in the `ExecStartPost` entry is _required_ and systemd +will error out if it is missing. ``` -ExecStartPost=sleep 2; conntrack -D -p udp +ExecStartPost=sleep 2 ; conntrack -D -p udp ``` This command will delete the old (stale) UDP entries two seconds after the container starts and allow the system to build a new table that @@ -82,7 +84,7 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ "$SC4S_TLS_DIR" \ --name SC4S \ --rm $SC4S_IMAGE -ExecStartPost=sleep 2; conntrack -D -p udp +ExecStartPost=sleep 2 ; conntrack -D -p udp ``` * Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication