Merge pull request #446 from splunk/feature/optional-indexed-fields

Fixes #273 make indexed fields optional

docs/configuration.md

@@ -10,6 +10,7 @@ and variables needed to properly configure SC4S for your environment.
 | SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list |
 | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token |
 | SC4S_GLOBAL_DNS_USE | yes or no(default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header |
+| SC4S_CONTAINER_HOST | string | variable passed to the container to identify the actual log host for container implementations |
 
 * NOTE:  Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http
 destination does not support this feature.  Moreover, HEC Ack would significantly degrade performance for streaming data such as
@@ -26,6 +27,7 @@ syslog.
 | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file |
 | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate |
 | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads).  This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. |
+| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,loghost,dport,fromhostip,proto | list of sc4s indexed fields default list is all fields )
 
 ## Alternate Destination Configuration
 

docs/gettingstarted/docker-systemd-general.md

@@ -59,6 +59,7 @@ ExecStartPre=/usr/bin/docker run \
         --name SC4S_preflight \
         --rm $SC4S_IMAGE -s
 ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \
+        -e "SC4S_CONTAINER_HOST=$(hostname -s)" \
         --env-file=/opt/sc4s/env_file \
         "$SC4S_PERSIST_VOLUME" \
         "$SC4S_LOCAL_CONFIG_MOUNT" \

docs/gettingstarted/podman-systemd-general.md

@@ -77,6 +77,7 @@ ExecStartPre=/usr/bin/podman run \
         --name SC4S_preflight \
         --rm $SC4S_IMAGE -s
 ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \
+        -e "SC4S_CONTAINER_HOST=$(hostname -s)" \
         --env-file=/opt/sc4s/env_file \
         "$SC4S_PERSIST_VOLUME" \
         "$SC4S_LOCAL_CONFIG_MOUNT" \

docs/troubleshooting.md

@@ -75,7 +75,7 @@ don't expect, check to see that the index is created in Splunk, or that a `lastC
 cause for almost _all_ `400` errors.
 * If you continue to the individual log entries in these directories, you will see entries of the form
 ```bash
-curl -k -u "sc4s HEC debug:a778f63a-5dff-4e3c-a72c-a03183659e94" "https://splunk.smg.aws:8088/services/collector/event" -d '{"time":"1584556114.271","sourcetype":"sc4s:events","source":"SC4S:s_internal","index":"main","host":"e3563b0ea5d8","fields":{"sc4s_syslog_severity":"notice","sc4s_syslog_facility":"syslog","sc4s_log_host":"e3563b0ea5d8","sc4s_fromhostip":"127.0.0.1"},"event":"syslog-ng starting up; version='3.26.1'"}'
+curl -k -u "sc4s HEC debug:a778f63a-5dff-4e3c-a72c-a03183659e94" "https://splunk.smg.aws:8088/services/collector/event" -d '{"time":"1584556114.271","sourcetype":"sc4s:events","source":"SC4S:s_internal","index":"main","host":"e3563b0ea5d8","fields":{"sc4s_syslog_severity":"notice","sc4s_syslog_facility":"syslog","sc4s_loghost":"e3563b0ea5d8","sc4s_fromhostip":"127.0.0.1"},"event":"syslog-ng starting up; version='3.26.1'"}'
 ```
 * These commands, with minimal modifications (e.g. multiple URLs specified or elements that needs shell escapes) can be run directly on the
 command line to determine what, exactly, the HEC endpoint is returning.  This can be used to refine th index or other parameter to correct the

package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl

@@ -0,0 +1,42 @@
+#Used to set indexed fields we will always use to global defaults
+rewrite r_set_splunk_default {
+      set("SC4S:$SOURCE", value(".splunk.source"));
+{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "facility" }}
+      set($FACILITY, value("fields.sc4s_syslog_facility"));
+{{- end}}      
+{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "severity" }}
+      set($LEVEL, value("fields.sc4s_syslog_severity"));
+{{- end}}      
+{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "loghost" }}
+{{- if (getenv "SC4S_CONTAINER_HOST") }}
+      set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost"));
+{{- end}}      
+{{- end}}      
+{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "container" }}
+      set($LOGHOST, value("fields.sc4s_container"));
+{{- end}}      
+{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "fromhostip" }}
+      set($SOURCEIP, value("fields.sc4s_fromhostip"));
+{{- end}}      
+{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,destport,fromhostip,proto") ",") has "destport" }}
+      set($DESTPORT, value("fields.sc4s_destport"));
+{{- end}}    
+{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,destport,fromhostip,proto") ",") has "proto" }}
+      set($PROTO, value("fields.sc4s_proto"));
+{{- end}}    
+};
+#used by each log-path to set index and sourcetype which may be
+#overridden by user defined values
+block rewrite r_set_splunk_dest_default(
+                              index()
+                              source("${.splunk.source}")
+                              sourcetype()
+                              template(`splunk-template`)
+                           ) {
+      set("`index`", value(".splunk.index"));
+      set("`source`", value(".splunk.source"));
+      set("`sourcetype`", value(".splunk.sourcetype"));
+};
+
+
+