Merge pull request #480 from splunk/destinations/overall_fixup

Destination fixup

docs/configuration.md

@@ -85,10 +85,14 @@ total buffer size needed. To determine the proper size of the disk buffer, consu
 
 ## Archive File Configuration
 
-This feature is designed to support "compliance" archival of all messages. Instructions for enabling this feature are included
-in each "getting started" runtime document. The files will be stored in a folder structure using the naming pattern
-``${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"``.
-This pattern will create one file per minute for each "vendor_product", with records formatted using syslog-ng's EWMM template.
+This feature is designed to support compliance or "diode mode" archival of all messages. Instructions for enabling this feature are included
+in each "getting started" runtime document. The files will be stored in a folder structure using the pattern shwon in the table below
+depending on the value of the `SC4S_GLOBAL_ARCHIVE_MODE` variable. All events for both modes are formatted using syslog-ng's EWMM template.
+
+| Variable | Values/Default   | Location/Pattern |
+|----------|------------------|------------------|
+| SC4S_GLOBAL_ARCHIVE_MODE | compliance(default) | ``<archive mount>/${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"`` |
+| SC4S_GLOBAL_ARCHIVE_MODE | diode | ``<archive mount>/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-archive.log`` |
 
 **WARNING POTENTIAL OUTAGE CAUSING CONSEQUENCE**
 
@@ -246,17 +250,20 @@ logging. Note that drop metrics will be recorded.
 
 Splunk Connect for Syslog utilizes the syslog-ng template mechanism to format the output payload (event) that will be sent to Splunk.  These templates can format the messages in a number of ways (straight text, JSON, etc.) as well as utilize the many syslog-ng "macros" (fields) to specify what gets placed in the payload that is delivered to the destination.  Here is a list of the templates used in SC4S, which can be used in the metadata override section immediately above.  New templates can also be added by the administrator in the "local" section for local destinations; pay careful attention to the syntax as the templates are "live" syslog-ng config code.
 
-| Template name    | Template contents                        |  Notes                                                           |
-|------------------|------------------------------------------|------------------------------------------------------------------|
-| t_standard       | ${DATE} ${HOST} ${MSGHDR}${MESSAGE}      |  Standard template for most RFC3164 (standard syslog) traffic    |
-| t_msg_only       | ${MSGONLY}                               |  syslog-ng $MSG is sent, no headers (host, timestamp, etc.)      |
-| t_msg_trim       | $(strip $MSGONLY)                        |  As above with whitespace stripped                               |
-| t_everything     | ${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}   |  Standard template with ISO date format                          |
-| t_hdr_msg        | ${MSGHDR}${MESSAGE}                      |  Useful for non-compliant syslog messages                        |
-| t_legacy_hdr_msg | ${LEGACY_MSGHDR}${MESSAGE}               |  Useful for non-compliant syslog messages                        |
-| t_hdr_sdata_msg  | ${MSGHDR}${MSGID} ${SDATA} ${MESSAGE}    |  Text-based representation of RFC5424-compliant syslog messages  |
-| t_JSON_3164      | $(format-json --scope rfc3164<br>--pair PRI="<$PRI>"<br>--key LEGACY_MSGHDR<br>--exclude FACILITY<br>--exclude PRIORITY)   |  JSON output of all RFC3164-based syslog-ng macros.  Useful with the "fallback" sourcetype to aid in new filter development. |
-| t_JSON_5424      | $(format-json --scope rfc5424<br>--pair PRI="<$PRI>"<br>--key ISODATE<br>--exclude DATE<br>--exclude FACILITY<br>--exclude PRIORITY)  |  JSON output of all RFC5424-based syslog-ng macros; for use with RFC5424-compliant traffic. |
+| Template name       | Template contents                        |  Notes                                                           |
+|---------------------|------------------------------------------|------------------------------------------------------------------|
+| t_standard          | ${DATE} ${HOST} ${MSGHDR}${MESSAGE}      |  Standard template for most RFC3164 (standard syslog) traffic    |
+| t_msg_only          | ${MSGONLY}                               |  syslog-ng $MSG is sent, no headers (host, timestamp, etc.)      |
+| t_msg_trim          | $(strip $MSGONLY)                        |  As above with whitespace stripped                               |
+| t_everything        | ${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}   |  Standard template with ISO date format                          |
+| t_hdr_msg           | ${MSGHDR}${MESSAGE}                      |  Useful for non-compliant syslog messages                        |
+| t_legacy_hdr_msg    | ${LEGACY_MSGHDR}${MESSAGE}               |  Useful for non-compliant syslog messages                        |
+| t_hdr_sdata_msg     | ${MSGHDR}${MSGID} ${SDATA} ${MESSAGE}    |  Useful for non-compliant syslog messages                        |
+| t_program_msg       | ${PROGRAM}[${PID}]: ${MESSAGE}           |  Useful for non-compliant syslog messages                        |
+| t_program_nopid_msg | ${PROGRAM}: ${MESSAGE}                   |  Useful for non-compliant syslog messages                        |
+| t_JSON_3164         | $(format-json --scope rfc3164<br>--pair PRI="<$PRI>"<br>--key LEGACY_MSGHDR<br>--exclude FACILITY<br>--exclude PRIORITY)   |  JSON output of all RFC3164-based syslog-ng macros.  Useful with the "fallback" sourcetype to aid in new filter development. |
+| t_JSON_5424         | $(format-json --scope rfc5424<br>--pair PRI="<$PRI>"<br>--key ISODATE<br>--exclude DATE<br>--exclude FACILITY<br>--exclude PRIORITY)  |  JSON output of all RFC5424-based syslog-ng macros; for use with RFC5424-compliant traffic. |
+| t_JSON_5424_SDATA   | $(format-json --scope rfc5424<br>--pair PRI="<$PRI>"<br>--key ISODATE<br>--exclude DATE<br>--exclude FACILITY<br>--exclude PRIORITY)<br>--exclude MESSAGE  |  JSON output of all RFC5424-based syslog-ng macros except for MESSAGE; for use with RFC5424-compliant traffic. |
 
 ## Data Resilience - Local Disk Buffer Configuration
 

docs/gettingstarted/podman-systemd-general.md

@@ -10,7 +10,7 @@ will mistakenly keep trying to route the packets to the server itself rather tha
 Until the firewall pauses its output stream (unlikely) _or_ the workaround provided below is applied, traffic from that particular firewall
 will never been seen by the container (and hence sc4s).
 
-## WORKAROUND
+## Workaround
 
 There is a utility called `conntrack` that allows you to view/manipulate the netfilter connection tables in real time. Follow the steps below
 to install and run it each time sc4s starts.  It should be available in all RHEL 7/8 subscriptions.

package/etc/conf.d/conflib/_common/templates.conf

@@ -50,10 +50,6 @@ template t_hdr_msg {
         template("${MSGHDR}${MESSAGE}");
         };
 
-template t_program_nopid_msg {
-        template("${PROGRAM}: ${MESSAGE}");
-        };
-
 template t_legacy_hdr_msg {
         template("${LEGACY_MSGHDR}${MESSAGE}");
         };
@@ -62,6 +58,10 @@ template t_program_msg {
         template("${PROGRAM}[${PID}]: ${MESSAGE}");
         };
 
+template t_program_nopid_msg {
+        template("${PROGRAM}: ${MESSAGE}");
+        };
+
 # ===============================================================================================
 # Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper
 # ===============================================================================================

package/etc/conf.d/destinations/archive_file.conf.tmpl

@@ -0,0 +1,10 @@
+destination d_archive {
+    {{- if eq (getenv "SC4S_GLOBAL_ARCHIVE_MODE" "compliance") "diode"}}
+    file("/opt/syslog-ng/var/archive/${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"
+         template('$(format-ewmm)')
+    {{- else }}
+    file("/opt/syslog-ng/var/archive/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-archive.log"
+        template('$(format-ewmm)')
+    {{- end }}
+     );
+};

package/etc/conf.d/destinations/rawmsg_file.conf

@@ -2,7 +2,7 @@ destination d_rawmsg {
     channel {
         if ("${RAWMSG}" ne "") {
             destination {
-                file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log"
+                file("/opt/syslog-ng/var/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-rawmsg.log"
                 template("${RAWMSG}\n")
                 );
             };

package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl

@@ -1,5 +1,5 @@
 destination d_hec_debug {
-    file("/opt/syslog-ng/var/archive/debug/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log"
+    file("/opt/syslog-ng/var/archive/debug/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-hec.log"
         template("curl -k -u \"sc4s HEC debug:{{- getenv "SPLUNK_HEC_TOKEN"}}\" \"{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event\" -d '$(format-json
                  time=$S_UNIXTIME
                  host=${HOST}

package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl

@@ -30,6 +30,9 @@ destination d_hec_internal {
                  sourcetype=${.splunk.sourcetype}
                  index=${.splunk.index}
                  event=$MSG
-                 fields.*)')
+                 {{- if ne (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS") "none" }}
+                 fields.*
+                 {{- end }}
+                 )')
         );
 };

package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl

@@ -37,22 +37,27 @@ log {
         rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))};
         parser {p_add_context_splunk(key("sc4s_events")); };
 
-        {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no")) }}
-        destination(d_hec_internal);
-        {{- end}}
+        if (not match("Destination timeout has elapsed, closing connection; fd=" value("MESSAGE")) and
+        not match("Input is valid utf8, but the log message is not tagged as such," value("MESSAGE")) and
+        not match("Syslog connection closed; fd=" value("MESSAGE")) and
+        not match("Syslog connection accepted; fd=" value("MESSAGE"))) {
 
-        {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "no"}}
-        destination(d_stdout);
-        {{- end}}
+            {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no")) }}
+            destination(d_hec_internal);
+            {{- end}}
 
-        {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
-        {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n        destination(" }});
-        {{- end }}
+            {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
+            {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n        destination(" }});
+            {{- end }}
 
-        {{- if (print (getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES")) }}
-        {{ getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n        destination(" }});
-        {{- end }}
+            {{- if (print (getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES")) }}
+            {{ getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n        destination(" }});
+            {{- end }}
+            };
 
+        {{- if eq (getenv "SC4S_DEBUG_STDOUT" "no") "yes"}}
+        destination(d_stdout);
+        {{- end}}
     };
-     flags(flow-control,final);
+    flags(flow-control,final);
 };

package/etc/conf.d/log_paths/lp-sc4s_startup.conf.tmpl

@@ -10,7 +10,7 @@ log {
     destination(d_hec_internal);
 {{- end}}
 
-{{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "no"}}
+{{- if eq (getenv "SC4S_DEBUG_STDOUT" "no") "yes"}}
     destination(d_stdout);
 {{- end}}
 

package/etc/conf.d/sources/startup.conf

@@ -1,13 +1,23 @@
 source s_startup_out {
-    file("/opt/syslog-ng/var/log/syslog-ng.out"
-    program-override("syslog-ng-config")
-    flags(no-hostname,no-parse,assume-utf8)
-    );
-
+    channel {
+        source {
+            file("/opt/syslog-ng/var/log/syslog-ng.out"
+            program-override("syslog-ng-config")
+            flags(no-hostname,no-parse,assume-utf8)
+            );
+        };
+        rewrite(r_set_splunk_default);
+    };
 };
+
 source s_startup_err {
-    file("/opt/syslog-ng/var/log/syslog-ng.err"
-    program-override("syslog-ng-config")
-    flags(no-hostname,no-parse,assume-utf8)
-    );
+    channel {
+        source {
+            file("/opt/syslog-ng/var/log/syslog-ng.err"
+            program-override("syslog-ng-config")
+            flags(no-hostname,no-parse,assume-utf8)
+            );
+        };
+        rewrite(r_set_splunk_default);
+    };
 };