From d33feb2e4a646c8baf47da3e4ba2934853e246f8 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Wed, 25 Mar 2020 08:30:37 -0400 Subject: [PATCH 1/2] Switch to 3.26.1 base --- .circleci/config.yml | 32 +++++++++---------- docs/gettingstarted/docker-swarm-general.md | 4 +-- docs/gettingstarted/docker-swarm-rhel7.md | 4 +-- docs/gettingstarted/docker-systemd-general.md | 4 +-- docs/gettingstarted/podman-systemd-general.md | 4 +-- docs/troubleshooting.md | 2 +- package/etc/syslog-ng.conf | 2 +- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aad5ff8..f2b415d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,7 +90,7 @@ jobs: docker: - image: circleci/python:3.7 environment: - SYSLOG: "syslog-ng-3.25.1" + SYSLOG: "syslog-ng-3.26.1" SPLUNK_VERSION: "8.0.2" <<: *test test-sc4s-master-splunk-8-0: @@ -104,14 +104,14 @@ jobs: docker: - image: circleci/python:3.7 environment: - SYSLOG: "syslog-ng-3.25.1" + SYSLOG: "syslog-ng-3.26.1" SPLUNK_VERSION: "7.3.4" <<: *test test-sc4s-3-25-1-splunk-7-2: docker: - image: circleci/python:3.7 environment: - SYSLOG: "syslog-ng-3.25.1" + SYSLOG: "syslog-ng-3.26.1" SPLUNK_VERSION: "7.2.9" <<: *test @@ -195,7 +195,7 @@ jobs: docker: - image: circleci/python:3.7 environment: - SYSLOG: "syslog-ng-3.25.1" + SYSLOG: "syslog-ng-3.26.1" steps: - setup_remote_docker: docker_layer_caching: true @@ -243,21 +243,21 @@ workflows: build_test: jobs: - test-sc4s-3-25-1-splunk-8-0 -# filters: -# branches: -# only: /.*/ + # filters: + # branches: + # only: /.*/ - test-sc4s-master-splunk-8-0 -# filters: -# branches: -# only: /.*/ + # filters: + # branches: + # only: /.*/ - test-sc4s-3-25-1-splunk-7-3 - # filters: - # branches: - # only: /.*/ + # filters: + # branches: + # only: /.*/ - test-sc4s-3-25-1-splunk-7-2 - # filters: - # branches: - # only: /.*/ + # filters: + # branches: + # only: /.*/ - approval-tag-alpha: type: approval filters: diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index c2fdcd8..e216dc3 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -227,7 +227,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.25.1' +syslog-ng starting up; version='3.26.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -245,7 +245,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.26.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index cd70d13..d1c795d 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -235,7 +235,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.25.1' +syslog-ng starting up; version='3.26.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -253,7 +253,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.26.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index f3d53ba..4cc1a49 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -238,7 +238,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.25.1' +syslog-ng starting up; version='3.26.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -256,7 +256,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.26.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 8cd5ad7..3ebdade 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -225,7 +225,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.25.1' +syslog-ng starting up; version='3.26.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -243,7 +243,7 @@ podman logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.26.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 04dc417..03c4d48 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -75,7 +75,7 @@ don't expect, check to see that the index is created in Splunk, or that a `lastC cause for almost _all_ `400` errors. * If you continue to the individual log entries in these directories, you will see entries of the form ```bash -curl -k -u "sc4s HEC debug:a778f63a-5dff-4e3c-a72c-a03183659e94" "https://splunk.smg.aws:8088/services/collector/event" -d '{"time":"1584556114.271","sourcetype":"sc4s:events","source":"SC4S:s_internal","index":"main","host":"e3563b0ea5d8","fields":{"sc4s_syslog_severity":"notice","sc4s_syslog_facility":"syslog","sc4s_log_host":"e3563b0ea5d8","sc4s_fromhostip":"127.0.0.1"},"event":"syslog-ng starting up; version='3.25.1'"}' +curl -k -u "sc4s HEC debug:a778f63a-5dff-4e3c-a72c-a03183659e94" "https://splunk.smg.aws:8088/services/collector/event" -d '{"time":"1584556114.271","sourcetype":"sc4s:events","source":"SC4S:s_internal","index":"main","host":"e3563b0ea5d8","fields":{"sc4s_syslog_severity":"notice","sc4s_syslog_facility":"syslog","sc4s_log_host":"e3563b0ea5d8","sc4s_fromhostip":"127.0.0.1"},"event":"syslog-ng starting up; version='3.26.1'"}' ``` * These commands, with minimal modifications (e.g. multiple URLs specified or elements that needs shell escapes) can be run directly on the command line to determine what, exactly, the HEC endpoint is returning. This can be used to refine th index or other parameter to correct the diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 6f5e591..2966ef0 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -1,4 +1,4 @@ -@version:3.25 +@version:3.26 # syslog-ng configuration file. From 451daa2a2c85e68bfb533ff4783d44a0384b4546 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Wed, 25 Mar 2020 08:39:29 -0400 Subject: [PATCH 2/2] Drop on 400 (bad index) retry 404 (not ready) --- package/etc/conf.d/destinations/splunk_hec.conf.tmpl | 2 ++ package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl | 1 + package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl | 1 + 3 files changed, 4 insertions(+) diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 68320bc..463cba1 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -13,6 +13,8 @@ destination d_hec { headers("{{- getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}") password("{{- getenv "SPLUNK_HEC_TOKEN"}}") persist-name("splunk_hec") + response-action(400 => drop, 404 => retry) + {{- if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE" "yes") "yes"}} disk-buffer( diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl index cca8862..2e9428b 100644 --- a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -13,6 +13,7 @@ destination d_hec_internal { headers("{{- getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}") password("{{- getenv "SPLUNK_HEC_TOKEN"}}") persist-name("splunk_hec_internal") + response-action(400 => drop, 404 => retry) tls(peer-verify({{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) {{- if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}} diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl index 7c97ce8..9c2bc50 100644 --- a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl @@ -12,6 +12,7 @@ destination d_hecmetrics { headers("{{- getenv "SC4S_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}") password("{{- getenv "SPLUNK_HEC_TOKEN"}}") persist-name("splunk_hec_metrics") + response-action(400 => drop, 404 => retry) tls(peer-verify({{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) {{- if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}}