Feature/improvedmetrics (#185)

* This change correct an incorrect URL issue preventing metrics flow and cleans up related documentation and filters anon metrics

* Update Metrics for Splunk 8

* Resolve splunk sdk for CI unit tests move

.env.template

@@ -12,10 +12,6 @@ SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SPLUNK_PASSWORD=Changed@11
 SPLUNK_START_ARGS=--accept-license
 SPLUNK_HEC_URL=https://splunk:8088/services/collector/event
-SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 #SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download
 #SPLUNKBASE_USERNAME=username
 #SPLUNKBASE_PASSWORD=password

docker-compose-ci.yml

@@ -33,7 +33,6 @@ services:
       - splunk
     environment:
       - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
       - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}

docker-compose-debug.yml

@@ -33,7 +33,6 @@ services:
       - splunk
     environment:
       - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
       - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}

docker-compose-demo.yml

@@ -34,7 +34,6 @@ services:
       - splunk
     environment:
       - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
       - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}

docker-compose-perf.yml

@@ -26,7 +26,6 @@ services:
       - splunk
     environment:
       - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
       - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}

docker-compose.yml

@@ -29,7 +29,8 @@ services:
         RH_ORG: ${RH_ORG}
         RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
-    command: -det
+#When this is enabled test_common will fail
+#    command: -det
     ports:
       - "514:514"
       - "601:601"
@@ -43,11 +44,7 @@ services:
       - splunk
     environment:
       - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
       - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
-      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
-      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SC4S_SOURCE_TLS_ENABLE=no
       - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
       - SC4S_LISTEN_DEFAULT_TCP_PORT=514

docs/gettingstarted/byoe-rhel7.md

@@ -129,12 +129,9 @@ sudo bash /opt/sc4s/bin/preconfig.sh
 
 ```dotenv
 SYSLOGNG_OPTS=-f /opt/syslog-ng/etc/syslog-ng.conf 
-SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

docs/gettingstarted/docker-swarm-general.md

@@ -65,12 +65,9 @@ of events in the event of network failure to the Splunk infrastructure.
 Create a file named ``/opt/sc4s/env_file`` and add the following environment variables:
 
 ```dotenv
-SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```
@@ -179,12 +176,9 @@ match this value to the total number of indexers behind the load balancer.
 uncomment the last line in the example below.
 
 ```dotenv
-SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no

docs/gettingstarted/docker-swarm-rhel7.md

@@ -92,12 +92,9 @@ again upon restart.
 Create a file named ``/opt/sc4s/env_file`` and add the following environment variables:
 
 ```dotenv
-SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```
@@ -208,12 +205,9 @@ match this value to the total number of indexers behind the load balancer.
 uncomment the last line in the example below.
 
 ```dotenv
-SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no

docs/gettingstarted/docker-systemd-general.md

@@ -71,12 +71,9 @@ unit file above.  Failure to do this will cause SC4S to abort at startup.
 Create a file named ``/opt/sc4s/env_file`` and add the following environment variables:
 
 ```dotenv
-SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```
@@ -182,12 +179,9 @@ match this value to the total number of indexers behind the load balancer.
 uncomment the last line in the example below.
 
 ```dotenv
-SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no

docs/gettingstarted/podman-systemd-general.md

@@ -71,12 +71,9 @@ unit file above.  Failure to do this will cause SC4S to abort at startup.
 Create a file named ``/opt/sc4s/env_file`` and add the following environment variables:
 
 ```dotenv
-SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```
@@ -182,12 +179,9 @@ match this value to the total number of indexers behind the load balancer.
 uncomment the last line in the example below.
 
 ```dotenv
-SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6
-SPLUNK_CONNECT_METHOD=hec
-SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=em_metrics
 SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no

package/Dockerfile

@@ -72,8 +72,6 @@ RUN cd /tmp ;\
     source scl_source enable rh-python36
 
 ENV DEBCONF_NONINTERACTIVE_SEEN=true
-ENV SPLUNK_CONNECT_METHOD=hec
-ENV SYSLOGNG_HEC_WORKERS=10
 
 RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh
 COPY goss.yaml /etc/goss.yaml
@@ -98,8 +96,6 @@ EXPOSE 514
 EXPOSE 601/tcp
 EXPOSE 6514/tcp
 
-ENV SPLUNK_CONNECT_METHOD=UF
-
 ENTRYPOINT ["/entrypoint.sh", "-F"]
 
 HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate

package/etc/conf.d/destinations/splunk_hec.conf.tmpl

@@ -1,6 +1,6 @@
 destination d_hec {
     http(
-         url("{{- getenv "SPLUNK_HEC_URL"}}")
+         url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event")
          method("POST")
          log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}})
          workers({{- getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}})

package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl

@@ -1,6 +1,6 @@
 destination d_hec_internal {
     http(
-         url("{{- getenv "SPLUNK_HEC_URL"}}")
+         url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event")
          method("POST")
          log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}})
          workers(10)

package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl

@@ -1,6 +1,6 @@
 destination d_hecmetrics {
     http(
-         url("{{- getenv "SPLUNK_HEC_URL"}}")
+         url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector")
          method("POST")
          batch-lines(50)
          batch-bytes(1024Kb)

package/etc/conf.d/log_paths/internal.conf.tmpl

@@ -8,26 +8,13 @@ log {
 
         parser {p_add_context_splunk(key("sc4s_metrics")); };
         rewrite {
-            subst('(?:Log statistics; )?(?<Type>[^= ]+)=\x27(?<SourceName>[^\(]+)\((?<SourceId>[^,\)]+)(?:,(?<SourceInstance>[^,]+),(?<State>[^\)]+))?\)\=(?<Number>\d+)\x27(?:, )?',
-                    '
-                    {"time": "$S_UNIXTIME.$S_MSEC",
-                      "event": "metric",
-                      "host": "$HOST",
-                      "index": "${.splunk.index}",
-                      "source": "internal",
-                      "sourcetype": "${.splunk.sourcetype}",
-                      "fields": {
-                         "source_name": "${SourceName}",
-                         "source_instance": "${SourceInstance}",
-                         "state": "${State}",
-                         "type": "${Type}",
-                         "_value": ${Number},
-                         "metric_name": "syslogng.${SourceId}"
-                       }
-                     }
-                     ',
+            subst('Log statistics; ', '', value("MESSAGE"), flags("utf8" "global"));
+            subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global"));
+            subst('(?<Type>[^= ]+)=\x27(?<SourceName>[^\(]+)\((?<SourceId>[^,\)]+)(?:,(?<SourceInstance>[^,]+),(?<State>[^\)]+))?\)\=(?<Number>\d+)\x27,? ?',
+'{"time": "$S_UNIXTIME.$S_MSEC","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}}
+',
                      value("MESSAGE") flags("utf8" "global")
-                  );
+            );
         };
         destination(d_hecmetrics);  #--HEC--
     } else {

package/etc/syslog-ng.conf

@@ -39,12 +39,6 @@ options {
 # ===============================================================================================
 @define splunk-template "t_standard"
 
-# ===============================================================================================
-# Data collection parameters, buffers, and Timezone
-# ===============================================================================================
-#TODO: Remove once release with this PR is produced by upstream
-#https://github.com/balabit/syslog-ng/pull/2932
-@define syslog-ng-sysconfdir "/opt/syslog-ng/etc"
 
 # ===============================================================================================
 # Global modules and includes.  All device-specific filters and destinations exist in conf.d

splunk/etc/apps/SA-syslog-ng/default/indexes.conf

@@ -4,6 +4,12 @@ homePath   = $SPLUNK_DB/syslogng_metrics/db
 coldPath   = $SPLUNK_DB/syslogng_metrics/colddb
 thawedPath = $SPLUNK_DB/syslogng_metrics/thaweddb
 
+[em_metrics]
+datatype=metric
+homePath   = $SPLUNK_DB/em_metrics/db
+coldPath   = $SPLUNK_DB/em_metrics/colddb
+thawedPath = $SPLUNK_DB/em_metrics/thaweddb
+
 [syslogng_fallback]
 homePath   = $SPLUNK_DB/syslogng_fallback/db
 coldPath   = $SPLUNK_DB/syslogng_fallback/colddb

tests/requirements.txt

@@ -9,5 +9,5 @@
 pytest
 jinja2
 jinja2-time
-http://dev.splunk.com/goto/sdk-python
+splunk-sdk
 flake8

tests/test_common.py

@@ -67,4 +67,16 @@ def test_tag(record_property, setup_wordlist, setup_splunk):
     record_property("resultCount", resultCount)
     record_property("message", message)
 
+    assert resultCount == 1
+
+#
+def test_metrics(record_property, setup_wordlist, setup_splunk):
+
+    st = env.from_string('mcatalog values(metric_name) WHERE metric_name="syslogng.d_*#0" AND ("index"="*" OR "index"="_*") BY index | fields index')
+    search = st.render()
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("resultCount", resultCount)
+
     assert resultCount == 1