diff --git a/docs/sources/Checkpoint/index.md b/docs/sources/Checkpoint/index.md index ffe44c5..2ce5d96 100644 --- a/docs/sources/Checkpoint/index.md +++ b/docs/sources/Checkpoint/index.md @@ -30,6 +30,7 @@ to allow routing to appropriate indexes. All other source meta data is left at d | checkpoint_splunk_dlp | dlp | netdlp | none | | checkpoint_splunk_email | email | email | none | | checkpoint_splunk_firewall | firewall | netfw | none | +| checkpoint_splunk_os | program:${program} | netops | none | | checkpoint_splunk_sessions | sessions | netops | none | | checkpoint_splunk_web | web | netproxy | none | diff --git a/docs/upgrade.md b/docs/upgrade.md index a0a3dde..ea15acb 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -1,19 +1,22 @@ # Upgrading Splunk Connect for Syslog Splunk Connect for Syslog is updated regularly using a CI/CD development process. The notes below outline significant changes that -must be taken into account prior and after an upgrade. Ensure to follow specific instructions below to ensure a smooth transition to -a new version of SC4S in production. +must be taken into account prior and after an upgrade. Ensure to follow specific instructions below to ensure a smooth +transition to a new version of SC4S in production. -## Version 1.9.0 +## Upgrade process +Check the current version of SC4S by running ```sudo logs SC4S```. For the latest version, use the +`latest` tag for the SC4S image in the sc4s.service unit file: +``` +[Service] +Environment="SC4S_IMAGE=splunk/scs:latest" +``` +Restart the service +```sudo systemctl restart sc4s``` -* Example context files have been added to the local mount `context` directory. These example files will be updated at each release -to outline support for new data sources, which can be added to existing context files (those without the `.example` extension). -Existing context files will _not_ be overwritten on subsequent SC4S starts/upgrades, so ensure that any new content from these example -files is incorporated into existing context files. - -* UNIT FILE CHANGES: Make sure to update the unit file used to start the sc4s service with the changes included in this release. It -includes updates for proper operation with RHEL 8, and is backward-compatible with RHEL 7.7. - -## Version 1.10.0 - -* The "Development" section outlines new instructions for operation with the vscode IDE. \ No newline at end of file +Using the latest version is recommended, but a specific version can be specified in the unit file if desired: +``` +[Service] +Environment="SC4S_IMAGE=splunk/scs:v1.20.0" +``` +See the [release information](https://github.com/splunk/splunk-connect-for-syslog/releases) for more detail. diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 56e3017..45811d7 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -36,10 +36,13 @@ rewrite r_set_splunk_default { #used by each log-path to set source and sourcetype which may be #overridden by user defined values block rewrite r_set_splunk_dest_default( - source("${.splunk.source}") + #While the following is not used it remains to prevent breaking changes in content + index("{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}") + source("${.splunk.source}") sourcetype() template(`splunk-template`) ) { + set("`index`", value(".splunk.index")); set("`source`", value(".splunk.source")); set("`sourcetype`", value(".splunk.sourcetype")); }; diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index f08ee05..1fbfe9e 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -131,7 +131,7 @@ log { subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); }; rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), source("program:${.PROGRAM}")) }; - parser { p_add_context_splunk(key("checkpoint_os")); }; + parser { p_add_context_splunk(key("checkpoint_splunk_os")); }; }; @@ -164,7 +164,7 @@ log { subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); }; rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), source("program:${.PROGRAM}")) }; - parser { p_add_context_splunk(key("checkpoint_os")); }; + parser { p_add_context_splunk(key("checkpoint_splunk_os")); }; parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; diff --git a/package/etc/context_templates/splunk_metadata.csv.example b/package/etc/context_templates/splunk_metadata.csv.example index 7b8e14e..f88c5ae 100644 --- a/package/etc/context_templates/splunk_metadata.csv.example +++ b/package/etc/context_templates/splunk_metadata.csv.example @@ -11,6 +11,7 @@ checkpoint_splunk_dlp,index,netdlp checkpoint_splunk_email,index,email checkpoint_splunk_firewall,index,netfw checkpoint_splunk_ids,index,netids +checkpoint_splunk_os,index,netops checkpoint_splunk_sessions,index,netops checkpoint_splunk_web,index,netproxy checkpoint_splunk,index,netops diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index e544b42..53a7756 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -34,14 +34,6 @@ hup_handler() { trap 'kill ${!}; hup_handler' SIGHUP trap 'kill ${!}; term_handler' SIGTERM -# Run gomplate to create config from templates if the command errors this is fatal -# Stop the container. Errors in this step should only happen with user provided -#Templates -if ! gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/; then - echo "Error in Gomplate template; unable to continue, exiting..." - exit 800 -fi - mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context @@ -53,31 +45,39 @@ touch /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then LEGACY_SPLUNK_INDEX_FILE=/opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv fi -sed -i 's/^#//' # Add new entries -awk '{print $0}' ${LEGACY_SPLUNK_INDEX_FILE} /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv.example | sort -b -t ',' -k1,2 -u +awk '{print $0}' ${LEGACY_SPLUNK_INDEX_FILE} /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv.example | grep -v '^#' | sort -b -t ',' -k1,2 -u #We don't need this file anylonger rm -f /opt/syslog-ng/etc/context_templates/splunk_index.csv.example || true rm -f /opt/syslog-ng/etc/context_templates/splunk_metadata.csv.example || true if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then mv /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_index.deprecated fi -cp --verbose -R /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ +cp --verbose -R -f /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ mkdir -p /opt/syslog-ng/var/log #Test HEC Connectivity if [ "$SC4S_DEST_SPLUNK_HEC_GLOBAL" != "no" ] then HEC=$(echo '{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event' | gomplate | cut -d' ' -f 1) - index=$(cat /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3) - if ! curl -k "${HEC}?/index=${index}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}' + SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX=$(cat /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3) + export SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX + if ! curl -k "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}' then echo "SC4S_ENV_CHECK_HEC: Splunk HEC endpoint is unreachable; startup will continue to prevent data loss if this is a transient failure" else echo "SC4S_ENV_CHECK_INDEX: Splunk HEC connection successfull; checking indexes" - cat /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv | grep -v sc4s_metrics | grep ',index,' | cut -d, -f 3 | sort -u | while read index ; do export index; echo -e "\nSC4S_ENV_CHECK_INDEX: Checking $index" $(curl -s -S -k "${HEC}?index=${index}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}') ; done + cat /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv | grep -v sc4s_metrics | grep ',index,' | cut -d, -f 3 | sort -u | while read index ; do export index; echo -e "\nSC4S_ENV_CHECK_INDEX: Checking $index" $(curl -s -S -k "${HEC}?index=${index}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}') ; done fi fi + +# Run gomplate to create config from templates if the command errors this is fatal +# Stop the container. Errors in this step should only happen with user provided +#Templates +if ! gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/; then + echo "Error in Gomplate template; unable to continue, exiting..." + exit 800 +fi #Setup SNMPD /opt/net-snmp/sbin/snmptrapd -Lf /opt/syslog-ng/var/log/snmptrapd.log