From e9cc6f961f3d7696c414594b8f75c7353a74c929 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 20 Dec 2019 11:13:38 -0500 Subject: [PATCH] Muliple startup fixes Log previously silent startup config errors as sc4s:events:startup:err and sc4s:events:startup:out. Bump syslog-ng config to 3.25.1 remove version string from templates no longer required add tests to ensure version is bumbed in the future --- .../etc/conf.d/log_paths/internal.conf.tmpl | 8 ++-- .../etc/conf.d/log_paths/startup.conf.tmpl | 37 +++++++++++++++++++ package/etc/conf.d/sources/startup.conf | 13 +++++++ .../compliance_meta_by_source.conf | 1 - .../vendor_product_by_source.conf | 2 - package/etc/syslog-ng.conf | 2 +- package/sbin/entrypoint.sh | 3 ++ tests/test_common.py | 23 ++++++++++++ 8 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 package/etc/conf.d/log_paths/startup.conf.tmpl create mode 100644 package/etc/conf.d/sources/startup.conf diff --git a/package/etc/conf.d/log_paths/internal.conf.tmpl b/package/etc/conf.d/log_paths/internal.conf.tmpl index 4373f5c..d5955e3 100644 --- a/package/etc/conf.d/log_paths/internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/internal.conf.tmpl @@ -23,9 +23,6 @@ log { } else { - {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}} - destination(d_stdout); - {{- end}} rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events")); }; @@ -33,7 +30,12 @@ log { destination(d_hec_internal); {{- end}} + {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}} + destination(d_stdout); + {{- end}} + }; + flags(flow-control,final); }; {{- end}} {{- tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/startup.conf.tmpl b/package/etc/conf.d/log_paths/startup.conf.tmpl new file mode 100644 index 0000000..3924cf9 --- /dev/null +++ b/package/etc/conf.d/log_paths/startup.conf.tmpl @@ -0,0 +1,37 @@ +{{- define "log_path"}} +log { + source(s_startup_out); + + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))}; + parser {p_add_context_splunk(key("sc4s_events:startup:out")); }; + + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no") | conv.ToBool) }} + destination(d_hec_internal); + {{- end}} + + {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}} + destination(d_stdout); + {{- end}} + + flags(flow-control,final); + }; +log { + source(s_startup_err); + + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events:startup:err"), index("main"))}; + parser {p_add_context_splunk(key("sc4s_events")); }; + + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no") | conv.ToBool) }} + destination(d_hec_internal); + {{- end}} + + {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}} + destination(d_stdout); + {{- end}} + + flags(flow-control,final); + }; +{{- end}} +{{- tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/sources/startup.conf b/package/etc/conf.d/sources/startup.conf new file mode 100644 index 0000000..00fbce7 --- /dev/null +++ b/package/etc/conf.d/sources/startup.conf @@ -0,0 +1,13 @@ +source s_startup_out { + file("/var/log/syslog-ng.out" + program-override("syslog-ng-config") + flags(no-hostname,no-parse,assume-utf8) + ); + +}; +source s_startup_err { + file("/var/log/syslog-ng.err" + program-override("syslog-ng-config") + flags(no-hostname,no-parse,assume-utf8) + ); +}; \ No newline at end of file diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf index 90b67f4..e74da82 100644 --- a/package/etc/context_templates/compliance_meta_by_source.conf +++ b/package/etc/context_templates/compliance_meta_by_source.conf @@ -1,4 +1,3 @@ -@version: 3.24 filter f_test_test { # host("something-*" type(glob)) or # netmask(169.254.100.0/24) diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index ac87c2e..1f5f13f 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -1,5 +1,3 @@ -@version: 3.24 - filter f_test_test { host("testvp-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 84c177b..fd96543 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -1,4 +1,4 @@ -@version:3.24 +@version:3.25 # syslog-ng configuration file. # diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 9abad88..aa01301 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -14,5 +14,8 @@ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ cp --verbose -R /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ +echo syslog-ng checking config +/opt/syslog-ng/sbin/syslog-ng -s >/var/log/syslog-ng.out 2>/var/log/syslog-ng.err + echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file diff --git a/tests/test_common.py b/tests/test_common.py index 9855cc9..1ed4c97 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -149,3 +149,26 @@ def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk): record_property("message", message) assert resultCount == 1 + + +def test_check_config_version(record_property, setup_wordlist, setup_splunk): + + st = env.from_string("search index=main sourcetype=\"sc4s:events:startup:err\" \"Configuration file format is too old\" ") + search = st.render() + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("resultCount", resultCount) + + assert resultCount == 0 + +def test_check_config_version_multiple(record_property, setup_wordlist, setup_splunk): + + st = env.from_string("search index=main sourcetype=\"sc4s:events:startup:err\" \"you have multiple @version directives\" ") + search = st.render() + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("resultCount", resultCount) + + assert resultCount == 0