From 7585981eb42372846b3990b3f41f6ca9b3a323aa Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 08:07:15 -0400 Subject: [PATCH] Fixes #273 make indexed fields optional The fields sc4s_syslog_product and sc4s_syslog_format are not made optional in this change as the current use is intergral to the logic tree a future change could change the internal field name and conditional include the field in the json payload. This change also adds new optional indexed fields for dest port when and container host. --- docs/configuration.md | 2 + docs/gettingstarted/docker-systemd-general.md | 1 + docs/gettingstarted/podman-systemd-general.md | 1 + .../conf.d/conflib/_splunk/splunkfields.conf | 23 ----------- .../conflib/_splunk/splunkfields.conf.tmpl | 41 +++++++++++++++++++ 5 files changed, 45 insertions(+), 23 deletions(-) delete mode 100644 package/etc/conf.d/conflib/_splunk/splunkfields.conf create mode 100644 package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl diff --git a/docs/configuration.md b/docs/configuration.md index 38bca4b..a30ec11 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,6 +10,7 @@ and variables needed to properly configure SC4S for your environment. | SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list | | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | | SC4S_GLOBAL_DNS_USE | yes or no(default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | +| SC4S_CONTAINER_HOST | string | variable passed to the container to identify the actual log host for container implementations | * NOTE: Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http destination does not support this feature. Moreover, HEC Ack would significantly degrade performance for streaming data such as @@ -26,6 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | +| SC4S_DEST_SPLUNK_INDEX_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is (container,log_host,dport,fromhostip,proto) ) ## Alternate Destination Configuration diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index f7fde28..5614607 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -59,6 +59,7 @@ ExecStartPre=/usr/bin/docker run \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \ + -e "SC4S_CONTAINER_HOST=$(hostname -s)" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index b0c026d..2b2269a 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -77,6 +77,7 @@ ExecStartPre=/usr/bin/podman run \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ + -e "SC4S_CONTAINER_HOST=$(hostname -s)" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf deleted file mode 100644 index bf911a6..0000000 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf +++ /dev/null @@ -1,23 +0,0 @@ -#Used to set indexed fields we will always use to global defaults -rewrite r_set_splunk_default { - set("SC4S:$SOURCE", value(".splunk.source")); - set($FACILITY, value("fields.sc4s_syslog_facility")); - set($LEVEL, value("fields.sc4s_syslog_severity")); - set($LOGHOST, value("fields.sc4s_log_host")); - set($SOURCEIP, value("fields.sc4s_fromhostip")); -}; -#used by each log-path to set index and sourcetype which may be -#overridden by user defined values -block rewrite r_set_splunk_dest_default( - index() - source("${.splunk.source}") - sourcetype() - template(`splunk-template`) - ) { - set("`index`", value(".splunk.index")); - set("`source`", value(".splunk.source")); - set("`sourcetype`", value(".splunk.sourcetype")); -}; - - - diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl new file mode 100644 index 0000000..9f43664 --- /dev/null +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -0,0 +1,41 @@ +#Used to set indexed fields we will always use to global defaults +rewrite r_set_splunk_default { + set("SC4S:$SOURCE", value(".splunk.source")); +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "facility" }} + set($FACILITY, value("fields.sc4s_syslog_facility")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "severity" }} + set($LEVEL, value("fields.sc4s_syslog_severity")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "log_host" }} +{{- if (getenv "SC4S_CONTAINER_HOST") }} + set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "container" }} + set($LOGHOST, value("fields.sc4s_container")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} + set($SOURCEIP, value("fields.sc4s_fromhostip")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "destport" }} + set($DESTPORT, value("fields.sc4s_destport")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "proto" }} + set($PROTO, value("fields.sc4s_proto")); +{{- end}} +}; +#used by each log-path to set index and sourcetype which may be +#overridden by user defined values +block rewrite r_set_splunk_dest_default( + index() + source("${.splunk.source}") + sourcetype() + template(`splunk-template`) + ) { + set("`index`", value(".splunk.index")); + set("`source`", value(".splunk.source")); + set("`sourcetype`", value(".splunk.sourcetype")); +}; + + +