From ca5f646a59292a0fc08244555bc54fb52b55348a Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 14:47:49 -0400 Subject: [PATCH 1/6] Update entrypoint.sh --- package/sbin/entrypoint.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 0ae034e..64ffe4c 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -53,16 +53,15 @@ touch /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then LEGACY_SPLUNK_INDEX_FILE=/opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv fi -sed -i 's/^#//' # Add new entries -awk '{print $0}' ${LEGACY_SPLUNK_INDEX_FILE} /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv.example | sort -b -t ',' -k1,2 -u +awk '{print $0}' ${LEGACY_SPLUNK_INDEX_FILE} /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv.example | grep -v '^#' | sort -b -t ',' -k1,2 -u #We don't need this file anylonger rm -f /opt/syslog-ng/etc/context_templates/splunk_index.csv.example || true rm -f /opt/syslog-ng/etc/context_templates/splunk_metadata.csv.example || true if [ -f /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv ]; then mv /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv /opt/syslog-ng/etc/conf.d/local/context/splunk_index.deprecated fi -cp --verbose -R /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ +cp --verbose -R -f /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ mkdir -p /opt/syslog-ng/var/log #Test HEC Connectivity From 47351d0a357d7a8547faea59d47032ab5f2d8c5e Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 15:03:46 -0400 Subject: [PATCH 2/6] Update splunkfields.conf.tmpl --- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 56e3017..d5dddb1 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -36,7 +36,9 @@ rewrite r_set_splunk_default { #used by each log-path to set source and sourcetype which may be #overridden by user defined values block rewrite r_set_splunk_dest_default( - source("${.splunk.source}") + #While the following is not used it remains to prevent breaking changes in content + index('') + source("${.splunk.source}") sourcetype() template(`splunk-template`) ) { From 3f8e58febcb7f89d2f2adbf853ece1a7b2e3a44a Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 15:59:08 -0400 Subject: [PATCH 3/6] Implement a default if splunk_metadata does not match --- .../conflib/_splunk/splunkfields.conf.tmpl | 1 + package/sbin/entrypoint.sh | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index d5dddb1..2d0b30c 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -42,6 +42,7 @@ block rewrite r_set_splunk_dest_default( sourcetype() template(`splunk-template`) ) { + set("{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}", value(".splunk.index")); set("`source`", value(".splunk.source")); set("`sourcetype`", value(".splunk.sourcetype")); }; diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 64ffe4c..ada3b2e 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -34,14 +34,6 @@ hup_handler() { trap 'kill ${!}; hup_handler' SIGHUP trap 'kill ${!}; term_handler' SIGTERM -# Run gomplate to create config from templates if the command errors this is fatal -# Stop the container. Errors in this step should only happen with user provided -#Templates -if ! gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/; then - echo "Error in Gomplate template; unable to continue, exiting..." - exit 800 -fi - mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context @@ -68,15 +60,24 @@ mkdir -p /opt/syslog-ng/var/log if [ "$SC4S_DEST_SPLUNK_HEC_GLOBAL" != "no" ] then HEC=$(echo '{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event' | gomplate | cut -d' ' -f 1) - index=$(cat /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3) - if ! curl -k "${HEC}?/index=${index}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}' + SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX=$(cat /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3) + export SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX + if ! curl -k "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}' then echo SC4S_ENV_CHECK_HEC: Splunk unreachable startup will continue to prevent data loss if this is a transient failure else echo SC4S_ENV_CHECK_INDEX: Splunk connection succesfull checking indexes - cat /opt/syslog-ng/etc/conf.d/local/context/splunk_index.csv | grep -v sc4s_metrics | grep ',index,' | cut -d, -f 3 | sort -u | while read index ; do export index; echo -e "\nSC4S_ENV_CHECK_INDEX: Checking $index" $(curl -s -S -k "${HEC}?index=${index}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}') ; done + cat /opt/syslog-ng/etc/conf.d/local/context/splunk_metadata.csv | grep -v sc4s_metrics | grep ',index,' | cut -d, -f 3 | sort -u | while read index ; do export index; echo -e "\nSC4S_ENV_CHECK_INDEX: Checking $index" $(curl -s -S -k "${HEC}?index=${index}" -H "Authorization: Splunk ${SPLUNK_HEC_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "SC4S:PROBE"}') ; done fi fi + +# Run gomplate to create config from templates if the command errors this is fatal +# Stop the container. Errors in this step should only happen with user provided +#Templates +if ! gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/; then + echo "Error in Gomplate template; unable to continue, exiting..." + exit 800 +fi #Setup SNMPD /opt/net-snmp/sbin/snmptrapd -Lf /opt/syslog-ng/var/log/snmptrapd.log From bbc5201b9b105d71795f703ec146c6c2aeaaebb6 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 16:16:29 -0400 Subject: [PATCH 4/6] Update splunkfields.conf.tmpl --- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 2d0b30c..8f94271 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -37,12 +37,12 @@ rewrite r_set_splunk_default { #overridden by user defined values block rewrite r_set_splunk_dest_default( #While the following is not used it remains to prevent breaking changes in content - index('') + index('{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}') source("${.splunk.source}") sourcetype() template(`splunk-template`) ) { - set("{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}", value(".splunk.index")); + set("`index`", value(".splunk.index")); set("`source`", value(".splunk.source")); set("`sourcetype`", value(".splunk.sourcetype")); }; From abd84fc0ab553fd6efe7e6a46edee7c551c17130 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 16:21:38 -0400 Subject: [PATCH 5/6] Update splunkfields.conf.tmpl --- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 8f94271..45811d7 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -37,7 +37,7 @@ rewrite r_set_splunk_default { #overridden by user defined values block rewrite r_set_splunk_dest_default( #While the following is not used it remains to prevent breaking changes in content - index('{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}') + index("{{- getenv "SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX" }}") source("${.splunk.source}") sourcetype() template(`splunk-template`) From 603214b278d3d3ea62a5e3da4dba53b614b34393 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 15 Jun 2020 19:58:14 -0400 Subject: [PATCH 6/6] Correct key --- docs/sources/Checkpoint/index.md | 1 + package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl | 4 ++-- package/etc/context_templates/splunk_metadata.csv.example | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/sources/Checkpoint/index.md b/docs/sources/Checkpoint/index.md index ffe44c5..2ce5d96 100644 --- a/docs/sources/Checkpoint/index.md +++ b/docs/sources/Checkpoint/index.md @@ -30,6 +30,7 @@ to allow routing to appropriate indexes. All other source meta data is left at d | checkpoint_splunk_dlp | dlp | netdlp | none | | checkpoint_splunk_email | email | email | none | | checkpoint_splunk_firewall | firewall | netfw | none | +| checkpoint_splunk_os | program:${program} | netops | none | | checkpoint_splunk_sessions | sessions | netops | none | | checkpoint_splunk_web | web | netproxy | none | diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index f08ee05..1fbfe9e 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -131,7 +131,7 @@ log { subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); }; rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), source("program:${.PROGRAM}")) }; - parser { p_add_context_splunk(key("checkpoint_os")); }; + parser { p_add_context_splunk(key("checkpoint_splunk_os")); }; }; @@ -164,7 +164,7 @@ log { subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); }; rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), source("program:${.PROGRAM}")) }; - parser { p_add_context_splunk(key("checkpoint_os")); }; + parser { p_add_context_splunk(key("checkpoint_splunk_os")); }; parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; diff --git a/package/etc/context_templates/splunk_metadata.csv.example b/package/etc/context_templates/splunk_metadata.csv.example index 7b8e14e..f88c5ae 100644 --- a/package/etc/context_templates/splunk_metadata.csv.example +++ b/package/etc/context_templates/splunk_metadata.csv.example @@ -11,6 +11,7 @@ checkpoint_splunk_dlp,index,netdlp checkpoint_splunk_email,index,email checkpoint_splunk_firewall,index,netfw checkpoint_splunk_ids,index,netids +checkpoint_splunk_os,index,netops checkpoint_splunk_sessions,index,netops checkpoint_splunk_web,index,netproxy checkpoint_splunk,index,netops