Add RAWMSG storage for debugging

* Add an environment variable `SC4S_SOURCE_STORE_RAWMSG` to enable the collection/storage of the "on the wire" message for use in the 3164 fallback template. * Useful for development and debugging of log paths and filters. NOT FOR USE IN PRODUCTION due to memory/disk overhead.
CSVD · Feb 2, 2020 · 3348868 · 3348868
1 parent 9b48d9d
commit 3348868
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 5 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -55,6 +55,7 @@ services:
       - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005
       - SC4S_LISTEN_PFSENSE_TCP_PORT=5006
       - SC4S_ARCHIVE_GLOBAL=yes
+#     - SC4S_SOURCE_STORE_RAWMSG=yes
     volumes:
       - ./tls:/opt/syslog-ng/tls
   splunk:

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -73,7 +73,7 @@ and/or move them to an archival system to avoid exhaustion of disk space.
 | SC4S_SOURCE_TCP_IW_SIZE | 20000000 | Initial Window size |
 | SC4S_SOURCE_TCP_FETCH_LIMIT | 2000 | Number of events to fetch from server buffer at once |
 | SC4S_SOURCE_UDP_SO_RCVBUFF | 425984 | UDP server buffer size in bytes |
-
+| SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype.  Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. |
 
 ## Syslog Source TLS Certificate Configuration
 

diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf
@@ -61,6 +61,7 @@ template t_JSON_3164 {
     template('$(format-json --scope rfc3164
                 --pair PRI="<$PRI>"
                 --key LEGACY_MSGHDR
+                --key RAWMSG
                 --exclude FACILITY
                 --exclude PRIORITY
                 )');

diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t
@@ -60,14 +60,14 @@ source s_{{ .port_id }} {
         };
 {{ if eq .parser "rfc3164" }}
         parser {
-            syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
+            syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}));
         };
         rewrite(set_rfc3164);
 {{ else if eq .parser "rfc3164_version" }}
 #       filter(f_rfc3164_version);
         rewrite(set_rfc3164_no_version_string);
         parser {
-            syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
+            syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}));
         };
         rewrite(set_rfc3164_version);
 {{ else if eq .parser "rfc5424_strict" }}
@@ -104,7 +104,7 @@ source s_{{ .port_id }} {
             filter(f_rfc3164_version);
             rewrite(set_rfc3164_no_version_string);
             parser {
-                    syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
+                    syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}));
                 };
             rewrite(set_rfc3164_version);
         } elif {
@@ -118,7 +118,7 @@ source s_{{ .port_id }} {
             rewrite(set_cisco_ios);
         } else {
             parser {
-                syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
+                syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}));
             };
             rewrite(set_rfc3164);
         };