Update templates and syslog formats

* Update JSON templates for simplicity and reduced archive disk space * Remove RAWMSG from 3164 JSON template in preparation for complete removal * Rename t_JSON to t_JSON_3164 to match that of 5424 * Rename the syslog format rfc5424_bsdtime to rfc3164_version to reflect it is actually RFC3164 with an extra "version" string appended rather than RFC5424 with a BSD timestamp * Unrelated -- fix Dockerfile to include the "-y" flag on Devlopment Tools installation
CSVD · Dec 31, 2019 · c46eb8c · c46eb8c
1 parent f5beba3
commit c46eb8c
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 31 deletions.
diff --git a/package/Dockerfile b/package/Dockerfile
@@ -18,7 +18,7 @@ RUN dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noar
     dnf config-manager --set-enabled PowerTools -y; \
     dnf update -y
 
-RUN dnf group install "Development Tools" ;\
+RUN dnf group install "Development Tools" -y ;\
     dnf install findutils autoconf \
     autoconf automake ca-certificates git libtool pkgconfig bison byacc file \
     flex pcre-devel glib2-devel openssl-devel libcurl-devel \

diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf
@@ -7,7 +7,7 @@ filter f_rfc5424_noversion{
 filter f_rfc5424_epochtime{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>(?<EPOCH>\d{10})(?:.(?<TIMESECFRAC>\d{1,9})?)) (?<FROMHOST>[^ ]+) ))');
 };
-filter f_rfc5424_bsdtime{
+filter f_rfc3164_version{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?<FROMHOST>[^ ]+) ))');
 };
 rewrite set_rfcnonconformant{
@@ -31,11 +31,11 @@ rewrite set_rfc5424_epochtime{
 filter f_is_rfc5424_epochtime{
     match("rfc5424_epochtime" value("fields.sc4s_syslog_format"))
 };
-rewrite set_rfc5424_bsdtime{
-    set("rfc5424_bsdtime" value("fields.sc4s_syslog_format"));
+rewrite set_rfc3164_version{
+    set("rfc3164_version" value("fields.sc4s_syslog_format"));
 };
-filter f_is_rfc5424_bsdtime{
-    match("rfc5424_bsdtime" value("fields.sc4s_syslog_format"))
+filter f_is_rfc3164_version{
+    match("rfc3164_version" value("fields.sc4s_syslog_format"))
 };
 rewrite set_rfc3164{
     set("rfc3164" value("fields.sc4s_syslog_format"));
@@ -56,6 +56,6 @@ filter f_is_no_parse{
     match("no_parse" value("fields.sc4s_syslog_format"))
 };
 
-rewrite set_rfc3894_bsdtime{
-    subst('^<\d+>(\d)', "", value("MESSAGE"));
+rewrite set_rfc3164_no_version_string{
+    subst('(^<\d+>)\d', $1, value("MESSAGE"));
 };
diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf
@@ -57,27 +57,25 @@ template t_hdr_sdata_msg {
 # JSON; for JSON pretty-printing (for debugging)
 # ===============================================================================================
 
-template t_JSON {
-	template("$(format-json --scope all-nv-pairs
-	            --exclude fields.*
-	            --exclude .splunk.*
-	            --exclude HOST
-	            --exclude HOST_FROM
-	            --exclude SOURCE
-	            )");
-	};
+template t_JSON_3164 {
+    template('$(format-json --scope rfc3164
+                --pair PRI="<$PRI>"
+                --key LEGACY_MSGHDR
+                --exclude FACILITY
+                --exclude PRIORITY
+                )');
+    };
 
 # ===============================================================================================
 # JSON; for JSON pretty-printing (for RFC5424)
 # ===============================================================================================
 
 template t_JSON_5424 {
-	template("$(format-json --scope all-nv-pairs
-	            --exclude fields.*
-	            --exclude .splunk.*
-	            --exclude HOST
-	            --exclude HOST_FROM
-	            --exclude RAWMSG
-	            --exclude SOURCE
-	            )");
-	};
+    template('$(format-json --scope rfc5424
+                --pair PRI="<$PRI>"
+                --key ISODATE
+                --exclude DATE
+                --exclude FACILITY
+                --exclude PRIORITY
+                )');
+    };
diff --git a/package/etc/conf.d/filters/paloalto/panos.conf b/package/etc/conf.d/filters/paloalto/panos.conf
@@ -1,5 +1,5 @@
 filter f_is_palalto_format{
-    match("rfc5424_bsdtime" value("fields.sc4s_syslog_format"))
+    match("rfc3164_version" value("fields.sc4s_syslog_format"))
     or
     match("rfc3164" value("fields.sc4s_syslog_format"))
 };

diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl
@@ -24,7 +24,7 @@ log {
 
         rewrite {
             r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main") );
-            set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG"));
+            set("$(template ${.splunk.sc4s_template} $(template t_JSON_3164))" value("MSG"));
             unset(value("RAWMSG"));
             unset(value("PROGRAM"));
             unset(value("LEGACY_MSGHDR"));

diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t
@@ -100,12 +100,12 @@ source s_{{ .port_id}} {
             parser {cisco-parser()};
             rewrite(set_cisco_ios);
         } elif {
-            filter(f_rfc5424_bsdtime);
-            rewrite(set_rfc3894_bsdtime);
+            filter(f_rfc3164_version);
+            rewrite(set_rfc3164_no_version_string);
             parser {
                     syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone));
                 };
-            rewrite(set_rfc5424_bsdtime);
+            rewrite(set_rfc3164_version);
         } elif {
             parser (p_cisco_meraki);
             rewrite(set_rfc5424_epochtime);