From 7cae90965dfa80870a2b4fe93f8aea464df3d26b Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 4 May 2020 11:14:19 -0700 Subject: [PATCH] Document netfilter bug * Document RHEL/podman netfilter table update bug * Remove `SC4S_DEST_SPLUNK_HEC_WORKERS=6` from documentation (default in code is 10) --- docs/gettingstarted/byoe-rhel7.md | 1 - docs/gettingstarted/docker-swarm-general.md | 1 - docs/gettingstarted/docker-swarm-rhel7.md | 1 - docs/gettingstarted/docker-systemd-general.md | 1 - docs/gettingstarted/podman-systemd-general.md | 37 +++++++++++++++++-- 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 1aeea8d..47f6493 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -159,7 +159,6 @@ sudo bash /opt/sc4s/bin/preconfig.sh SYSLOGNG_OPTS=-f /etc/syslog-ng/syslog-ng.conf SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index e216dc3..dea3f12 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -107,7 +107,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index d1c795d..8c1d6f1 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -115,7 +115,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 4cc1a49..f7fde28 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -113,7 +113,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 3ebdade..7bb6ee6 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -1,8 +1,37 @@ -# WARNING: Do _not_ use Podman with RHEL/CentOS 7.x or earlier! +# UPDATE: Podman/RHEL UDP data block issue: netfilter connection table -There have been cases where UDP packet loss is noted when Podman is used with RHEL/CentOS 7.x versions. Stay tuned; the cause is -currently unkown. +We have determined the root cause for the issue with UDP data blocking and Podman/RHEL. The crux of the issue is that the netfilter +connection tables are _not_ udpdated when a new container starts _and_ there is a constant stream of UDP traffic from a given IP destined +for a given port. The table is _only_ updated if the trafffic pauses for the length of the connection table timeout (30 seconds by default). + +Therefore, if you attempt to start up sc4s on a server to which, for example, a firewall is sending a steady stream of UDP events, the kernel +will mistakenly keep trying to route the packets to the server itself rather than through the virtual network created by the new container. +Until the firewall pauses its output stream (unlikely) _or_ the workaround provided below is applied, traffic from that particular firewall +will never been seen by the container (and hence sc4s). + +## WORKAROUND + +There is a utility called `conntrack` that allows you to view/manipulate the netfilter connection tables in real time. Follow the steps below +to install and run it each time sc4s starts. It should be available in all RHEL 7/8 subscriptions. + +``` + install conntrack +``` + +After this is done, add the following entry to the unit file (and/or use the command when starting sc4s manually): + +``` +ExecStartPost=sleep 2; conntrack -D -p udp +``` + +This command will delete the old (stale) UDP entries two seconds after the container starts and allow the system to build a new table that +will properly route to the container when it sees UDP traffic. Note that this command resets the table for _all_ UDP +ports; for a purpose-built sc4s server this should not cause issues. If for any reason more granular control over _which_ UPD ports are +reset is desired, there are additional arguments to `conntrack` that can be used to select the specific UDP ports that are deleted in the +table. See the man page for `conntrack` for more information. + +The unit file entry above has been added to the example below for completeness. # Install podman @@ -53,6 +82,7 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ "$SC4S_TLS_DIR" \ --name SC4S \ --rm $SC4S_IMAGE +ExecStartPost=sleep 2; conntrack -D -p udp ``` * Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication @@ -100,7 +130,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ```