Merge pull request #518 from splunk/metrics/remove_hec_default

Remove default HEC delivery of SC4S metrics
CSVD · Jun 13, 2020 · f467d63 · f467d63
2 parents 2363cca + 451f1f5
commit f467d63
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 14 deletions.
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -44,7 +44,7 @@ separately from that of the alternates below.
 | Variable | Values        | Description |
 |----------|---------------|-------------|
 | SC4S_DEST_GLOBAL_ALTERNATES | Comma or space-separated list of syslog-ng destinations | Send all sources to alternate destinations |
-| SC4S_DEST_&lt;SOURCE&gt;_ALTERNATES | Comma or space-separated list of syslog-ng destiinations  | Send specific sources to alternate syslog-ng destinations, e.g. SC4S_DEST_CISCO_ASA_ALTERNATES |
+| SC4S_DEST_&lt;VENDOR_PRODUCT&gt;_ALTERNATES | Comma or space-separated list of syslog-ng destinations  | Send specific sources to alternate syslog-ng destinations using the VENDOR_PRODUCT syntax, e.g. SC4S_DEST_CISCO_ASA_ALTERNATES |
 
 ## SC4S Disk Buffer Configuration
 

diff --git a/docs/sources/Citrix/index.md b/docs/sources/Citrix/index.md
@@ -35,11 +35,9 @@ MSG Parse: This filter parses message content
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_CITRIX_NETSCALER_SPLUNK_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the port number defined |
-| SC4S_LISTEN_CITRIX_NETSCALERSPLUNK_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the port number defined |
-| SC4S_ARCHIVE_CITRIX_NETSCALER_SPLUNK | no | Enable archive to disk for this specific source |
-| SC4S_DEST_CITRIX_NETSCALER_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
-| SC4S_DEST_CITRIX_NETSCALER_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_CITRIX_NETSCALER_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the port number defined |
+| SC4S_LISTEN_CITRIX_NETSCALER_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the port number defined |
+| SC4S_DEST_CITRIX_NETSCALER_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 | SC4S_SOURCE_CITRIX_NETSCALER_USEALT_DATE_FORMAT | no | Use "DDMMYYYY" format rather than "MMDDYYYY" |
 
 ### Verification

diff --git a/docs/sources/Splunk/index.md b/docs/sources/Splunk/index.md
@@ -0,0 +1,51 @@
+# Vendor - Splunk
+
+
+## Product - Splunk Connect for Syslog (SC4S)
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/4740/                                                                 |
+| Product Manual | https://splunk-connect-for-syslog.readthedocs.io/en/master/  |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| sc4s:events    | Internal events from the SC4S container and underlying syslog-ng process                                |
+| sc4s:metrics   | syslog-ng operational metrics that will be delivered directly to a metrics index in Splunk              |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| sc4s_events    | all            | main           | none           |
+| sc4s_metrics   | all            | em_metrics     | none           |
+
+### Filter type
+
+SC4S events and metrics are generated automatically and no specific ports or filters need to be configured for the collection of this data.
+
+### Setup and Configuration
+
+* No specific requirements are required for the collection of sc4s internal events.
+* Metrics data is _not_ collected by default; it is an opt-in set by the variable `SC4S_DEST_SC4S_METRICS_HEC`. See the "Options"
+section below for details.
+
+### Options
+
+| Variable                          | default   | description    |
+|-----------------------------------|-----------|----------------|
+| SC4S_DEST_SPLUNK_SC4S_EVENTS_HEC  | no        | When Splunk HEC is disabled globally set to "yes" to enable this specific source |
+| SC4S_DEST_SPLUNK_SC4S_METRICS_HEC | no        | Set to "yes" to send metrics via HEC to Splunk (opt-in).  Metrics are _not_ enabled by default when HEC is enabled globally. |
+
+### Verification
+
+SC4S will generate versioning events at startup. These startup events can be used to validate HEC is set up properly on the Splunk side.
+
+```
+index=<asconfigured> sourcetype=sc4s:events | stats count by host
+```
+Metrics can be observed via the "Analytics-->Metrics" navigation in the Search and Reporting app in Splunk.
+* NOTE:  The presentation of metrics is undergoing active development; the delivery of metrics is currently considered an experimental feature.
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -33,6 +33,7 @@ nav:
       - "pfSense": sources/Pfsense/index.md
       - Proofpoint: sources/Proofpoint/index.md
       - Schneider: sources/Schneider/index.md
+      - Splunk: sources/Splunk/index.md
       - Symantec: sources/Symantec/index.md
       - Ubiquiti: sources/Ubiquiti/index.md
       - VMware: sources/VMWare/index.md

diff --git a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl
@@ -16,21 +16,21 @@ log {
             );
         };
 
-        {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_METRICS_HEC" "no")) }}
+        {{- if eq (getenv "SC4S_DEST_SPLUNK_SC4S_METRICS_HEC" "no") "yes" }}
         destination(d_hecmetrics);
-        {{- end}}
 
-        {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "no"}}
+        {{- if eq (getenv "SC4S_DEBUG_STDOUT" "no") "yes" }}
         destination(d_stdout);
         {{- end}}
 
         {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
         {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n        destination(" }});
         {{- end }}
 
-        {{- if (print (getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES")) }}
+        {{- if (print (getenv "SC4S_DEST_SPLUNK_SC4S_METRICS_ALTERNATES")) }}
         {{ getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n        destination(" }});
         {{- end }}
+        {{- end }}
 
     } else {
 
@@ -42,20 +42,20 @@ log {
         not match("Syslog connection closed; fd=" value("MESSAGE")) and
         not match("Syslog connection accepted; fd=" value("MESSAGE"))) {
 
-            {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no")) }}
+            {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_SPLUNK_SC4S_EVENTS_HEC" "no")) }}
             destination(d_hec_internal);
             {{- end}}
 
             {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
             {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n        destination(" }});
             {{- end }}
 
-            {{- if (print (getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES")) }}
-            {{ getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n        destination(" }});
+            {{- if (print (getenv "SC4S_DEST_SPLUNK_SC4S_EVENTS_ALTERNATES")) }}
+            {{ getenv "SC4S_DEST_SPLUNK_SC4S_EVENTS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n        destination(" }});
             {{- end }}
             };
 
-        {{- if eq (getenv "SC4S_DEBUG_STDOUT" "no") "yes"}}
+        {{- if eq (getenv "SC4S_DEBUG_STDOUT" "no") "yes" }}
         destination(d_stdout);
         {{- end}}
     };

diff --git a/tests/docker-compose-ci.yml b/tests/docker-compose-ci.yml
@@ -39,6 +39,7 @@ services:
     environment:
       - SPLUNK_HEC_URL=https://splunk:8088
       - SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630
+      - SC4S_DEST_SPLUNK_SC4S_METRICS_HEC=yes
       - SC4S_SOURCE_TLS_ENABLE=no
       - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
       - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000

diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
@@ -29,6 +29,7 @@ services:
     environment:
       - SPLUNK_HEC_URL=https://splunk:8088
       - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+      - SC4S_DEST_SPLUNK_SC4S_METRICS_HEC=yes
       - SC4S_SOURCE_TLS_ENABLE=no
       - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
       - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000