From ca5f646a59292a0fc08244555bc54fb52b55348a Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 14:47:49 -0400 Subject: [PATCH 01/10] Update entrypoint.sh --- package/sbin/entrypoint.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 0ae034e..64ffe4c 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -53,16 +53,15 @@ touch /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then LEGACY_SPLUNK_INDEX_FILE=/opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv fi -sed -i 's/^#//' # Add new entries -awk '{print $0}' ${LEGACY_SPLUNK_INDEX_FILE} /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv.example | sort -b -t ',' -k1,2 -u +awk '{print $0}' ${LEGACY_SPLUNK_INDEX_FILE} /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv.example | grep -v '^#' | sort -b -t ',' -k1,2 -u #We don't need this file anylonger rm -f /opt/syslog-ng/etc/context_templates/splunk_index.csv.example || true rm -f /opt/syslog-ng/etc/context_templates/splunk_metadata.csv.example || true if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then mv /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_index.deprecated fi -cp --verbose -R /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ +cp --verbose -R -f /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ mkdir -p /opt/syslog-ng/var/log #Test HEC Connectivity From 47351d0a357d7a8547faea59d47032ab5f2d8c5e Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 15:03:46 -0400 Subject: [PATCH 02/10] Update splunkfields.conf.tmpl --- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 56e3017..d5dddb1 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -36,7 +36,9 @@ rewrite r_set_splunk_default { #used by each log-path to set source and sourcetype which may be #overridden by user defined values block rewrite r_set_splunk_dest_default( - source("${.splunk.source}") + #While the following is not used it remains to prevent breaking changes in content + index('') + source("${.splunk.source}") sourcetype() template(`splunk-template`) ) { From 3f8e58febcb7f89d2f2adbf853ece1a7b2e3a44a Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 15:59:08 -0400 Subject: [PATCH 03/10] Implement a default if splunk_metadata does not match --- .../conflib/_splunk/splunkfields.conf.tmpl | 1 + package/sbin/entrypoint.sh | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index d5dddb1..2d0b30c 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -42,6 +42,7 @@ block rewrite r_set_splunk_dest_default( sourcetype() template(`splunk-template`) ) { + set("{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}", value(".splunk.index")); set("`source`", value(".splunk.source")); set("`sourcetype`", value(".splunk.sourcetype")); }; diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 64ffe4c..ada3b2e 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -34,14 +34,6 @@ hup_handler() { trap 'kill ${!}; hup_handler' SIGHUP trap 'kill ${!}; term_handler' SIGTERM -# Run gomplate to create config from templates if the command errors this is fatal -# Stop the container. Errors in this step should only happen with user provided -#Templates -if ! gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/; then - echo "Error in Gomplate template; unable to continue, exiting..." - exit 800 -fi - mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context @@ -68,15 +60,24 @@ mkdir -p /opt/syslog-ng/var/log if [ "$SC4S_DEST_SPLUNK_HEC_GLOBAL" != "no" ] then HEC=$(echo '{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event' | gomplate | cut -d' ' -f 1) - index=$(cat /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3) - if ! curl -k "${HEC}?/index=${index}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}' + SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX=$(cat /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3) + export SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX + if ! curl -k "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}' then echo SC4S_ENV_CHECK_HEC: Splunk unreachable startup will continue to prevent data loss if this is a transient failure else echo SC4S_ENV_CHECK_INDEX: Splunk connection succesfull checking indexes - cat /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv | grep -v sc4s_metrics | grep ',index,' | cut -d, -f 3 | sort -u | while read index ; do export index; echo -e "\nSC4S_ENV_CHECK_INDEX: Checking $index" $(curl -s -S -k "${HEC}?index=${index}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}') ; done + cat /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv | grep -v sc4s_metrics | grep ',index,' | cut -d, -f 3 | sort -u | while read index ; do export index; echo -e "\nSC4S_ENV_CHECK_INDEX: Checking $index" $(curl -s -S -k "${HEC}?index=${index}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}') ; done fi fi + +# Run gomplate to create config from templates if the command errors this is fatal +# Stop the container. Errors in this step should only happen with user provided +#Templates +if ! gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/; then + echo "Error in Gomplate template; unable to continue, exiting..." + exit 800 +fi #Setup SNMPD /opt/net-snmp/sbin/snmptrapd -Lf /opt/syslog-ng/var/log/snmptrapd.log From bbc5201b9b105d71795f703ec146c6c2aeaaebb6 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 16:16:29 -0400 Subject: [PATCH 04/10] Update splunkfields.conf.tmpl --- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 2d0b30c..8f94271 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -37,12 +37,12 @@ rewrite r_set_splunk_default { #overridden by user defined values block rewrite r_set_splunk_dest_default( #While the following is not used it remains to prevent breaking changes in content - index('') + index('{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}') source("${.splunk.source}") sourcetype() template(`splunk-template`) ) { - set("{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}", value(".splunk.index")); + set("`index`", value(".splunk.index")); set("`source`", value(".splunk.source")); set("`sourcetype`", value(".splunk.sourcetype")); }; From abd84fc0ab553fd6efe7e6a46edee7c551c17130 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 16:21:38 -0400 Subject: [PATCH 05/10] Update splunkfields.conf.tmpl --- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 8f94271..45811d7 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -37,7 +37,7 @@ rewrite r_set_splunk_default { #overridden by user defined values block rewrite r_set_splunk_dest_default( #While the following is not used it remains to prevent breaking changes in content - index('{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}') + index("{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}") source("${.splunk.source}") sourcetype() template(`splunk-template`) From 603214b278d3d3ea62a5e3da4dba53b614b34393 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 19:58:14 -0400 Subject: [PATCH 06/10] Correct key --- docs/sources/Checkpoint/index.md | 1 + package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl | 4 ++-- package/etc/context_templates/splunk_metadata.csv.example | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/sources/Checkpoint/index.md b/docs/sources/Checkpoint/index.md index ffe44c5..2ce5d96 100644 --- a/docs/sources/Checkpoint/index.md +++ b/docs/sources/Checkpoint/index.md @@ -30,6 +30,7 @@ to allow routing to appropriate indexes. All other source meta data is left at d | checkpoint_splunk_dlp | dlp | netdlp | none | | checkpoint_splunk_email | email | email | none | | checkpoint_splunk_firewall | firewall | netfw | none | +| checkpoint_splunk_os | program:${program} | netops | none | | checkpoint_splunk_sessions | sessions | netops | none | | checkpoint_splunk_web | web | netproxy | none | diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index f08ee05..1fbfe9e 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -131,7 +131,7 @@ log { subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); }; rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), source("program:${.PROGRAM}")) }; - parser { p_add_context_splunk(key("checkpoint_os")); }; + parser { p_add_context_splunk(key("checkpoint_splunk_os")); }; }; @@ -164,7 +164,7 @@ log { subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); }; rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), source("program:${.PROGRAM}")) }; - parser { p_add_context_splunk(key("checkpoint_os")); }; + parser { p_add_context_splunk(key("checkpoint_splunk_os")); }; parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; diff --git a/package/etc/context_templates/splunk_metadata.csv.example b/package/etc/context_templates/splunk_metadata.csv.example index 7b8e14e..f88c5ae 100644 --- a/package/etc/context_templates/splunk_metadata.csv.example +++ b/package/etc/context_templates/splunk_metadata.csv.example @@ -11,6 +11,7 @@ checkpoint_splunk_dlp,index,netdlp checkpoint_splunk_email,index,email checkpoint_splunk_firewall,index,netfw checkpoint_splunk_ids,index,netids +checkpoint_splunk_os,index,netops checkpoint_splunk_sessions,index,netops checkpoint_splunk_web,index,netproxy checkpoint_splunk,index,netops From 8bd59d9f536bebb52a70483f01a7be89cd8da65e Mon Sep 17 00:00:00 2001 From: Nandini Vij Date: Mon, 15 Jun 2020 17:20:43 -0700 Subject: [PATCH 07/10] Docs: Upgrade process updated --- docs/upgrade.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index a0a3dde..1033fbd 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -4,16 +4,18 @@ Splunk Connect for Syslog is updated regularly using a CI/CD development process must be taken into account prior and after an upgrade. Ensure to follow specific instructions below to ensure a smooth transition to a new version of SC4S in production. -## Version 1.9.0 +###Upgrade process +Check the current version of SC4S by running ```sudo docker logs SC4S```. For latest version choose the latest SC4S image in the sc4s.service file +``` +[Service] +Environment="SC4S_IMAGE=splunk/scs:latest" +``` +Restart the service +```sudo systemctl restart sc4s``` -* Example context files have been added to the local mount `context` directory. These example files will be updated at each release -to outline support for new data sources, which can be added to existing context files (those without the `.example` extension). -Existing context files will _not_ be overwritten on subsequent SC4S starts/upgrades, so ensure that any new content from these example -files is incorporated into existing context files. - -* UNIT FILE CHANGES: Make sure to update the unit file used to start the sc4s service with the changes included in this release. It -includes updates for proper operation with RHEL 8, and is backward-compatible with RHEL 7.7. - -## Version 1.10.0 - -* The "Development" section outlines new instructions for operation with the vscode IDE. \ No newline at end of file +Having latest is recommended but if some other version is required specify in the service file. For eg: +``` +[Service] +Environment="SC4S_IMAGE=splunk/scs:v1.20.0" +``` +Follow the link for release information https://github.com/splunk/splunk-connect-for-syslog/releases \ No newline at end of file From e4d5571b038f64505652ed180ebfa64be89dfb32 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Mon, 15 Jun 2020 17:53:38 -0700 Subject: [PATCH 08/10] Minor clarification for upgrade docs * Minor clarification of proper tag for sc4s unit file --- docs/upgrade.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 1033fbd..71f472c 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -1,11 +1,12 @@ # Upgrading Splunk Connect for Syslog Splunk Connect for Syslog is updated regularly using a CI/CD development process. The notes below outline significant changes that -must be taken into account prior and after an upgrade. Ensure to follow specific instructions below to ensure a smooth transition to -a new version of SC4S in production. +must be taken into account prior and after an upgrade. Ensure to follow specific instructions below to ensure a smooth +transition to a new version of SC4S in production. ###Upgrade process -Check the current version of SC4S by running ```sudo docker logs SC4S```. For latest version choose the latest SC4S image in the sc4s.service file +Check the current version of SC4S by running ```sudo logs SC4S```. For the latest version, use the +`latest` tag for the SC4S image in the sc4s.service unit file: ``` [Service] Environment="SC4S_IMAGE=splunk/scs:latest" @@ -18,4 +19,4 @@ Having latest is recommended but if some other version is required specify in th [Service] Environment="SC4S_IMAGE=splunk/scs:v1.20.0" ``` -Follow the link for release information https://github.com/splunk/splunk-connect-for-syslog/releases \ No newline at end of file +Follow the link for release information https://github.com/splunk/splunk-connect-for-syslog/releases From d107dc034fe269fbbe8ffb65f1af890e993744ae Mon Sep 17 00:00:00 2001 From: mbonsack Date: Mon, 15 Jun 2020 17:54:57 -0700 Subject: [PATCH 09/10] Markdown fix * Markdown fix --- docs/upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 71f472c..0c16ff1 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -4,7 +4,7 @@ Splunk Connect for Syslog is updated regularly using a CI/CD development process must be taken into account prior and after an upgrade. Ensure to follow specific instructions below to ensure a smooth transition to a new version of SC4S in production. -###Upgrade process +## Upgrade process Check the current version of SC4S by running ```sudo logs SC4S```. For the latest version, use the `latest` tag for the SC4S image in the sc4s.service unit file: ``` From bcbf7fe143e20defac58fb326177be9796c145d3 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Mon, 15 Jun 2020 18:01:02 -0700 Subject: [PATCH 10/10] Update markdown for URL * Update markdown for URL --- docs/upgrade.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 0c16ff1..ea15acb 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -14,9 +14,9 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Restart the service ```sudo systemctl restart sc4s``` -Having latest is recommended but if some other version is required specify in the service file. For eg: +Using the latest version is recommended, but a specific version can be specified in the unit file if desired: ``` [Service] Environment="SC4S_IMAGE=splunk/scs:v1.20.0" ``` -Follow the link for release information https://github.com/splunk/splunk-connect-for-syslog/releases +See the [release information](https://github.com/splunk/splunk-connect-for-syslog/releases) for more detail.