From 3c69809ede1e77884fad3f338f2cdca1e05681a1 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Fri, 8 Nov 2019 15:37:16 -0500 Subject: [PATCH] Feature/improvedmetrics (#185) * This change correct an incorrect URL issue preventing metrics flow and cleans up related documentation and filters anon metrics * Update Metrics for Splunk 8 * Resolve splunk sdk for CI unit tests move --- .env.template | 4 --- docker-compose-ci.yml | 1 - docker-compose-debug.yml | 1 - docker-compose-demo.yml | 1 - docker-compose-perf.yml | 1 - docker-compose.yml | 7 ++---- docs/gettingstarted/byoe-rhel7.md | 5 +--- docs/gettingstarted/docker-swarm-general.md | 10 ++------ docs/gettingstarted/docker-swarm-rhel7.md | 10 ++------ docs/gettingstarted/docker-systemd-general.md | 10 ++------ docs/gettingstarted/podman-systemd-general.md | 10 ++------ package/Dockerfile | 4 --- .../conf.d/destinations/splunk_hec.conf.tmpl | 2 +- .../splunk_hec_internal.conf.tmpl | 2 +- .../destinations/splunk_hec_metrics.conf.tmpl | 2 +- .../etc/conf.d/log_paths/internal.conf.tmpl | 25 +++++-------------- package/etc/syslog-ng.conf | 6 ----- .../apps/SA-syslog-ng/default/indexes.conf | 6 +++++ tests/requirements.txt | 2 +- tests/test_common.py | 12 +++++++++ 20 files changed, 39 insertions(+), 82 deletions(-) diff --git a/.env.template b/.env.template index c1f7af2..f360cd6 100644 --- a/.env.template +++ b/.env.template @@ -12,10 +12,6 @@ SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SPLUNK_PASSWORD=Changed@11 SPLUNK_START_ARGS=--accept-license SPLUNK_HEC_URL=https://splunk:8088/services/collector/event -SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download #SPLUNKBASE_USERNAME=username #SPLUNKBASE_PASSWORD=password diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 6028e6e..a699498 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -33,7 +33,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml index 9b885f3..bb22763 100644 --- a/docker-compose-debug.yml +++ b/docker-compose-debug.yml @@ -33,7 +33,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml index c4fb88f..3934fe1 100644 --- a/docker-compose-demo.yml +++ b/docker-compose-demo.yml @@ -34,7 +34,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml index 9831278..e1bd795 100644 --- a/docker-compose-perf.yml +++ b/docker-compose-perf.yml @@ -26,7 +26,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose.yml b/docker-compose.yml index 7be17aa..082040f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,8 @@ services: RH_ORG: ${RH_ORG} RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s - command: -det +#When this is enabled test_common will fail +# command: -det ports: - "514:514" - "601:601" @@ -43,11 +44,7 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - SC4S_SOURCE_TLS_ENABLE=no - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - SC4S_LISTEN_DEFAULT_TCP_PORT=514 diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 73b7b33..d970dcd 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -129,12 +129,9 @@ sudo bash /opt/sc4s/bin/preconfig.sh ```dotenv SYSLOGNG_OPTS=-f /opt/syslog-ng/etc/syslog-ng.conf -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 8f536d1..6fd894f 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -65,12 +65,9 @@ of events in the event of network failure to the Splunk infrastructure. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -179,12 +176,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index ec93e82..5ef0e66 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -92,12 +92,9 @@ again upon restart. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -208,12 +205,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 27fa688..94b164f 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -71,12 +71,9 @@ unit file above. Failure to do this will cause SC4S to abort at startup. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -182,12 +179,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 1dea7bd..1d0ddc3 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -71,12 +71,9 @@ unit file above. Failure to do this will cause SC4S to abort at startup. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -182,12 +179,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/package/Dockerfile b/package/Dockerfile index 7008a31..a5cbb4b 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -72,8 +72,6 @@ RUN cd /tmp ;\ source scl_source enable rh-python36 ENV DEBCONF_NONINTERACTIVE_SEEN=true -ENV SPLUNK_CONNECT_METHOD=hec -ENV SYSLOGNG_HEC_WORKERS=10 RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh COPY goss.yaml /etc/goss.yaml @@ -98,8 +96,6 @@ EXPOSE 514 EXPOSE 601/tcp EXPOSE 6514/tcp -ENV SPLUNK_CONNECT_METHOD=UF - ENTRYPOINT ["/entrypoint.sh", "-F"] HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 78a9246..8769667 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec { http( - url("{{- getenv "SPLUNK_HEC_URL"}}") + url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers({{- getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}}) diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl index b20109b..550063c 100644 --- a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec_internal { http( - url("{{- getenv "SPLUNK_HEC_URL"}}") + url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers(10) diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl index a473a04..2593b8c 100644 --- a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl @@ -1,6 +1,6 @@ destination d_hecmetrics { http( - url("{{- getenv "SPLUNK_HEC_URL"}}") + url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector") method("POST") batch-lines(50) batch-bytes(1024Kb) diff --git a/package/etc/conf.d/log_paths/internal.conf.tmpl b/package/etc/conf.d/log_paths/internal.conf.tmpl index 3c210ff..c751b9e 100644 --- a/package/etc/conf.d/log_paths/internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/internal.conf.tmpl @@ -8,26 +8,13 @@ log { parser {p_add_context_splunk(key("sc4s_metrics")); }; rewrite { - subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', - ' - {"time": "$S_UNIXTIME.$S_MSEC", - "event": "metric", - "host": "$HOST", - "index": "${.splunk.index}", - "source": "internal", - "sourcetype": "${.splunk.sourcetype}", - "fields": { - "source_name": "${SourceName}", - "source_instance": "${SourceInstance}", - "state": "${State}", - "type": "${Type}", - "_value": ${Number}, - "metric_name": "syslogng.${SourceId}" - } - } - ', + subst('Log statistics; ', '', value("MESSAGE"), flags("utf8" "global")); + subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global")); + subst('(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27,? ?', +'{"time": "$S_UNIXTIME.$S_MSEC","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}} +', value("MESSAGE") flags("utf8" "global") - ); + ); }; destination(d_hecmetrics); #--HEC-- } else { diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index e3e33e9..4f19083 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -39,12 +39,6 @@ options { # =============================================================================================== @define splunk-template "t_standard" -# =============================================================================================== -# Data collection parameters, buffers, and Timezone -# =============================================================================================== -#TODO: Remove once release with this PR is produced by upstream -#https://github.com/balabit/syslog-ng/pull/2932 -@define syslog-ng-sysconfdir "/opt/syslog-ng/etc" # =============================================================================================== # Global modules and includes. All device-specific filters and destinations exist in conf.d diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index 31e5f1e..04c2df6 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -4,6 +4,12 @@ homePath = $SPLUNK_DB/syslogng_metrics/db coldPath = $SPLUNK_DB/syslogng_metrics/colddb thawedPath = $SPLUNK_DB/syslogng_metrics/thaweddb +[em_metrics] +datatype=metric +homePath = $SPLUNK_DB/em_metrics/db +coldPath = $SPLUNK_DB/em_metrics/colddb +thawedPath = $SPLUNK_DB/em_metrics/thaweddb + [syslogng_fallback] homePath = $SPLUNK_DB/syslogng_fallback/db coldPath = $SPLUNK_DB/syslogng_fallback/colddb diff --git a/tests/requirements.txt b/tests/requirements.txt index cc706aa..758c531 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,5 +9,5 @@ pytest jinja2 jinja2-time -http://dev.splunk.com/goto/sdk-python +splunk-sdk flake8 diff --git a/tests/test_common.py b/tests/test_common.py index e026893..8325b52 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -67,4 +67,16 @@ def test_tag(record_property, setup_wordlist, setup_splunk): record_property("resultCount", resultCount) record_property("message", message) + assert resultCount == 1 + +# +def test_metrics(record_property, setup_wordlist, setup_splunk): + + st = env.from_string('mcatalog values(metric_name) WHERE metric_name="syslogng.d_*#0" AND ("index"="*" OR "index"="_*") BY index | fields index') + search = st.render() + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("resultCount", resultCount) + assert resultCount == 1 \ No newline at end of file