diff --git a/docs/configuration.md b/docs/configuration.md index 79fbb94..a0fe48c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -85,10 +85,14 @@ total buffer size needed. To determine the proper size of the disk buffer, consu ## Archive File Configuration -This feature is designed to support "compliance" archival of all messages. Instructions for enabling this feature are included -in each "getting started" runtime document. The files will be stored in a folder structure using the naming pattern -``${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"``. -This pattern will create one file per minute for each "vendor_product", with records formatted using syslog-ng's EWMM template. +This feature is designed to support compliance or "diode mode" archival of all messages. Instructions for enabling this feature are included +in each "getting started" runtime document. The files will be stored in a folder structure using the pattern shwon in the table below +depending on the value of the `SC4S_GLOBAL_ARCHIVE_MODE` variable. All events for both modes are formatted using syslog-ng's EWMM template. + +| Variable | Values/Default | Location/Pattern | +|----------|------------------|------------------| +| SC4S_GLOBAL_ARCHIVE_MODE | compliance(default) | ``/${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"`` | +| SC4S_GLOBAL_ARCHIVE_MODE | diode | ``/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-archive.log`` | **WARNING POTENTIAL OUTAGE CAUSING CONSEQUENCE** @@ -246,17 +250,20 @@ logging. Note that drop metrics will be recorded. Splunk Connect for Syslog utilizes the syslog-ng template mechanism to format the output payload (event) that will be sent to Splunk. These templates can format the messages in a number of ways (straight text, JSON, etc.) as well as utilize the many syslog-ng "macros" (fields) to specify what gets placed in the payload that is delivered to the destination. Here is a list of the templates used in SC4S, which can be used in the metadata override section immediately above. New templates can also be added by the administrator in the "local" section for local destinations; pay careful attention to the syntax as the templates are "live" syslog-ng config code. -| Template name | Template contents | Notes | -|------------------|------------------------------------------|------------------------------------------------------------------| -| t_standard | ${DATE} ${HOST} ${MSGHDR}${MESSAGE} | Standard template for most RFC3164 (standard syslog) traffic | -| t_msg_only | ${MSGONLY} | syslog-ng $MSG is sent, no headers (host, timestamp, etc.) | -| t_msg_trim | $(strip $MSGONLY) | As above with whitespace stripped | -| t_everything | ${ISODATE} ${HOST} ${MSGHDR}${MESSAGE} | Standard template with ISO date format | -| t_hdr_msg | ${MSGHDR}${MESSAGE} | Useful for non-compliant syslog messages | -| t_legacy_hdr_msg | ${LEGACY_MSGHDR}${MESSAGE} | Useful for non-compliant syslog messages | -| t_hdr_sdata_msg | ${MSGHDR}${MSGID} ${SDATA} ${MESSAGE} | Text-based representation of RFC5424-compliant syslog messages | -| t_JSON_3164 | $(format-json --scope rfc3164
--pair PRI="<$PRI>"
--key LEGACY_MSGHDR
--exclude FACILITY
--exclude PRIORITY) | JSON output of all RFC3164-based syslog-ng macros. Useful with the "fallback" sourcetype to aid in new filter development. | -| t_JSON_5424 | $(format-json --scope rfc5424
--pair PRI="<$PRI>"
--key ISODATE
--exclude DATE
--exclude FACILITY
--exclude PRIORITY) | JSON output of all RFC5424-based syslog-ng macros; for use with RFC5424-compliant traffic. | +| Template name | Template contents | Notes | +|---------------------|------------------------------------------|------------------------------------------------------------------| +| t_standard | ${DATE} ${HOST} ${MSGHDR}${MESSAGE} | Standard template for most RFC3164 (standard syslog) traffic | +| t_msg_only | ${MSGONLY} | syslog-ng $MSG is sent, no headers (host, timestamp, etc.) | +| t_msg_trim | $(strip $MSGONLY) | As above with whitespace stripped | +| t_everything | ${ISODATE} ${HOST} ${MSGHDR}${MESSAGE} | Standard template with ISO date format | +| t_hdr_msg | ${MSGHDR}${MESSAGE} | Useful for non-compliant syslog messages | +| t_legacy_hdr_msg | ${LEGACY_MSGHDR}${MESSAGE} | Useful for non-compliant syslog messages | +| t_hdr_sdata_msg | ${MSGHDR}${MSGID} ${SDATA} ${MESSAGE} | Useful for non-compliant syslog messages | +| t_program_msg | ${PROGRAM}[${PID}]: ${MESSAGE} | Useful for non-compliant syslog messages | +| t_program_nopid_msg | ${PROGRAM}: ${MESSAGE} | Useful for non-compliant syslog messages | +| t_JSON_3164 | $(format-json --scope rfc3164
--pair PRI="<$PRI>"
--key LEGACY_MSGHDR
--exclude FACILITY
--exclude PRIORITY) | JSON output of all RFC3164-based syslog-ng macros. Useful with the "fallback" sourcetype to aid in new filter development. | +| t_JSON_5424 | $(format-json --scope rfc5424
--pair PRI="<$PRI>"
--key ISODATE
--exclude DATE
--exclude FACILITY
--exclude PRIORITY) | JSON output of all RFC5424-based syslog-ng macros; for use with RFC5424-compliant traffic. | +| t_JSON_5424_SDATA | $(format-json --scope rfc5424
--pair PRI="<$PRI>"
--key ISODATE
--exclude DATE
--exclude FACILITY
--exclude PRIORITY)
--exclude MESSAGE | JSON output of all RFC5424-based syslog-ng macros except for MESSAGE; for use with RFC5424-compliant traffic. | ## Data Resilience - Local Disk Buffer Configuration diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index f033343..83fd6cd 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -10,7 +10,7 @@ will mistakenly keep trying to route the packets to the server itself rather tha Until the firewall pauses its output stream (unlikely) _or_ the workaround provided below is applied, traffic from that particular firewall will never been seen by the container (and hence sc4s). -## WORKAROUND +## Workaround There is a utility called `conntrack` that allows you to view/manipulate the netfilter connection tables in real time. Follow the steps below to install and run it each time sc4s starts. It should be available in all RHEL 7/8 subscriptions. diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index a5e0197..f9afd04 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -50,10 +50,6 @@ template t_hdr_msg { template("${MSGHDR}${MESSAGE}"); }; -template t_program_nopid_msg { - template("${PROGRAM}: ${MESSAGE}"); - }; - template t_legacy_hdr_msg { template("${LEGACY_MSGHDR}${MESSAGE}"); }; @@ -62,6 +58,10 @@ template t_program_msg { template("${PROGRAM}[${PID}]: ${MESSAGE}"); }; +template t_program_nopid_msg { + template("${PROGRAM}: ${MESSAGE}"); + }; + # =============================================================================================== # Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper # =============================================================================================== diff --git a/package/etc/conf.d/destinations/archive_file.conf b/package/etc/conf.d/destinations/archive_file.conf deleted file mode 100644 index efd7990..0000000 --- a/package/etc/conf.d/destinations/archive_file.conf +++ /dev/null @@ -1,5 +0,0 @@ -destination d_archive { - file("/opt/syslog-ng/var/archive/${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log" - template('$(format-ewmm)') - ); -}; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/archive_file.conf.tmpl b/package/etc/conf.d/destinations/archive_file.conf.tmpl new file mode 100644 index 0000000..5cc4d79 --- /dev/null +++ b/package/etc/conf.d/destinations/archive_file.conf.tmpl @@ -0,0 +1,10 @@ +destination d_archive { + {{- if eq (getenv "SC4S_GLOBAL_ARCHIVE_MODE" "compliance") "diode"}} + file("/opt/syslog-ng/var/archive/${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log" + template('$(format-ewmm)') + {{- else }} + file("/opt/syslog-ng/var/archive/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-archive.log" + template('$(format-ewmm)') + {{- end }} + ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/rawmsg_file.conf b/package/etc/conf.d/destinations/rawmsg_file.conf index 8ca2f79..2e2f826 100644 --- a/package/etc/conf.d/destinations/rawmsg_file.conf +++ b/package/etc/conf.d/destinations/rawmsg_file.conf @@ -2,7 +2,7 @@ destination d_rawmsg { channel { if ("${RAWMSG}" ne "") { destination { - file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" + file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-rawmsg.log" template("${RAWMSG}\n") ); }; diff --git a/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl index d02db92..a1f7e36 100644 --- a/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl @@ -1,5 +1,5 @@ destination d_hec_debug { - file("/opt/syslog-ng/var/archive/debug/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" + file("/opt/syslog-ng/var/archive/debug/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-hec.log" template("curl -k -u \"sc4s HEC debug:{{- getenv "SPLUNK_HEC_TOKEN"}}\" \"{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event\" -d '$(format-json time=$S_UNIXTIME host=${HOST} diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl index 6d62c36..6352a76 100644 --- a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -30,6 +30,9 @@ destination d_hec_internal { sourcetype=${.splunk.sourcetype} index=${.splunk.index} event=$MSG - fields.*)') + {{- if ne (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS") "none" }} + fields.* + {{- end }} + )') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/stdout.conf.tmpl b/package/etc/conf.d/destinations/stdout.conf similarity index 100% rename from package/etc/conf.d/destinations/stdout.conf.tmpl rename to package/etc/conf.d/destinations/stdout.conf diff --git a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl index f015298..f1f50aa 100644 --- a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl @@ -37,22 +37,27 @@ log { rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events")); }; - {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no")) }} - destination(d_hec_internal); - {{- end}} + if (not match("Destination timeout has elapsed, closing connection; fd=" value("MESSAGE")) and + not match("Input is valid utf8, but the log message is not tagged as such," value("MESSAGE")) and + not match("Syslog connection closed; fd=" value("MESSAGE")) and + not match("Syslog connection accepted; fd=" value("MESSAGE"))) { - {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "no"}} - destination(d_stdout); - {{- end}} + {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no")) }} + destination(d_hec_internal); + {{- end}} - {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} - {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); - {{- end }} + {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} - {{- if (print (getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES")) }} - {{ getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); - {{- end }} + {{- if (print (getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} + }; + {{- if eq (getenv "SC4S_DEBUG_STDOUT" "no") "yes"}} + destination(d_stdout); + {{- end}} }; - flags(flow-control,final); + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-sc4s_startup.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_startup.conf.tmpl index d72fd52..c0dedf6 100644 --- a/package/etc/conf.d/log_paths/lp-sc4s_startup.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-sc4s_startup.conf.tmpl @@ -10,7 +10,7 @@ log { destination(d_hec_internal); {{- end}} -{{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "no"}} +{{- if eq (getenv "SC4S_DEBUG_STDOUT" "no") "yes"}} destination(d_stdout); {{- end}} diff --git a/package/etc/conf.d/sources/startup.conf b/package/etc/conf.d/sources/startup.conf index e4de003..ea86dd9 100644 --- a/package/etc/conf.d/sources/startup.conf +++ b/package/etc/conf.d/sources/startup.conf @@ -1,13 +1,23 @@ source s_startup_out { - file("/opt/syslog-ng/var/log/syslog-ng.out" - program-override("syslog-ng-config") - flags(no-hostname,no-parse,assume-utf8) - ); - + channel { + source { + file("/opt/syslog-ng/var/log/syslog-ng.out" + program-override("syslog-ng-config") + flags(no-hostname,no-parse,assume-utf8) + ); + }; + rewrite(r_set_splunk_default); + }; }; + source s_startup_err { - file("/opt/syslog-ng/var/log/syslog-ng.err" - program-override("syslog-ng-config") - flags(no-hostname,no-parse,assume-utf8) - ); + channel { + source { + file("/opt/syslog-ng/var/log/syslog-ng.err" + program-override("syslog-ng-config") + flags(no-hostname,no-parse,assume-utf8) + ); + }; + rewrite(r_set_splunk_default); + }; }; \ No newline at end of file