Add separate f5 iRule parsers

* Add separate f5 iRule parser to log path and filter * Add `frac-digits(3)` to global syslog-ng.conf options so that ISO date-ralated macros render with ms resolution. * Remove the `$S_MSEC` macro from the time element of all destination JSON payloads
CSVD · Apr 12, 2020 · bef1f15 · bef1f15
1 parent 44aaf60
commit bef1f15
Show file tree

Hide file tree

Showing 9 changed files with 35 additions and 13 deletions.
diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl
@@ -41,7 +41,7 @@ destination d_hec {
          {{- end}}
          ca-file("{{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}"))
          body('$(format-json
-                 time=$S_UNIXTIME.$S_MSEC
+                 time=$S_UNIXTIME
                  host=${HOST}
                  source=${.splunk.source}
                  sourcetype=${.splunk.sourcetype}

diff --git a/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl
@@ -1,7 +1,7 @@
 destination d_hec_debug {
     file("/opt/syslog-ng/var/archive/debug/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log"
         template("curl -k -u \"sc4s HEC debug:{{- getenv "SPLUNK_HEC_TOKEN"}}\" \"{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event\" -d '$(format-json
-                 time=$S_UNIXTIME.$S_MSEC
+                 time=$S_UNIXTIME
                  host=${HOST}
                  source=${.splunk.source}
                  sourcetype=${.splunk.sourcetype}

diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl
@@ -24,7 +24,7 @@ destination d_hec_internal {
          {{- end}}
          ca-file("{{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}"))
          body('$(format-json
-                 time=$S_UNIXTIME.$S_MSEC
+                 time=$S_UNIXTIME
                  host=${HOST}
                  source=${.splunk.source}
                  sourcetype=${.splunk.sourcetype}

diff --git a/package/etc/conf.d/filters/f5/bigip.conf.tmpl b/package/etc/conf.d/filters/f5/bigip.conf.tmpl
@@ -5,17 +5,34 @@ filter f_f5_bigip {
     or program("mcpd")
     or program("apmd")
     or program("tmm\d?")
-    or program('^,f5_irule');
+    or program('^f5_irule=');
 };
+
+filter f_f5_bigip_irule {
+    message("*,f5_irule=*" type(glob));
+};
+
 filter f_f5_bigip_message {
     message(
         '^(?i)(<\d+> ?[[:alpha:]]+\s{1,2}\d{1,2} \d\d:\d\d:\d\d +[^ ]+) +(?:notice|err|error|warning|info) +?(.*)'
         flags(store-matches)
     );
 };
-parser f_f5_bigip_message {
+parser p_f5_bigip_message {
     syslog-parser(
         template("$1 $2")
         flags(guess-timezone,assume-utf8,{{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
     );
 };
+
+parser p_f5_bigip_irule {
+    channel {
+       rewrite { subst(",f5_irule=", " f5_irule=", value("MESSAGE")); };
+        parser {
+            syslog-parser(
+            template("$MESSAGE")
+            flags(guess-timezone,assume-utf8,{{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
+            );
+        };
+    };
+};
diff --git a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl
@@ -35,7 +35,7 @@ log {
         };
         parser { p_add_context_splunk(key("f5_bigip")); };
         parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); };
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };
 #    } elif {
 #        filter {
 #            program('apmd')
@@ -48,10 +48,10 @@ log {
 #        };
 #        parser { p_add_context_splunk(key("f5_bigip")); };
 #        parser (compliance_meta_by_source);
-#        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); };
+#        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };
     } elif {
         filter {
-            program('^,f5_irule')
+            program('f5_irule=')
         };
         rewrite {
             set("f5_bigip_irule", value("fields.sc4s_vendor_product"));
@@ -71,7 +71,7 @@ log {
         };
         parser { p_add_context_splunk(key("f5_bigip")); };
         parser (compliance_meta_by_source);
-        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); };
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };
     } else {
         rewrite {
             set("f5_bigip_rogue_message", value("fields.sc4s_vendor_product"));

diff --git a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl
@@ -10,7 +10,7 @@ log {
             subst('Log statistics; ', '', value("MESSAGE"), flags("utf8" "global"));
             subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global"));
             subst('(?<Type>[^= ]+)=\x27(?<SourceName>[^\(]+)\((?<SourceId>[^,\)]+)(?:,(?<SourceInstance>[^,]+),(?<State>[^\)]+))?\)\=(?<Number>\d+)\x27,? ?',
-'{"time": "$S_UNIXTIME.$S_MSEC","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}}
+'{"time": "$S_UNIXTIME","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}}
 ',
                      value("MESSAGE") flags("utf8" "global")
             );

diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t
@@ -121,7 +121,11 @@ source s_{{ .port_id }} {
             rewrite(r_citrix_netscaler_message);
         } elif {
             filter(f_f5_bigip_message);
-            parser(f_f5_bigip_message);
+            parser(p_f5_bigip_message);
+            rewrite(set_rfc3164);
+        } elif {
+            filter(f_f5_bigip_irule);
+            parser(p_f5_bigip_irule);
             rewrite(set_rfc3164);
         } elif {
             #JSON over IP its not syslog but it can work

diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf
@@ -30,6 +30,7 @@ options {
         stats-max-dynamics(2000);
         normalize-hostnames(yes);
         on-error(fallback-to-string);
+        frac-digits(3);
 };
 
 

diff --git a/tests/test_f5_bigip.py b/tests/test_f5_bigip.py
@@ -24,7 +24,7 @@
 #Jan 17 04:03:37 SV5-F5-5600-2 warning tmm1[23068]: 01260009:4: Connection error: ssl_passthru:5234: not SSL (40)
 #Jan 17 04:42:37 SV5-F5-5600-2.splunk.com notice mcpd[10653]: 01070638:5: Pool /Common/infra-docs-pool member /Common/go_web3:4000 monitor status down. [ /Common/tcp_half_open: down; last error:  ]  [ was up for 837hrs:31mins:36sec ]
 #Jan 17 04:42:37 SV5-F5-5600-2 notice apmd[11023]: 01490248:5: /Common/Network_Access_02:Common:8c6be305: Received client info - Hostname:  Type: IE Version: 8 Platform: Win7 CPU: WOW64 UI Mode: Full Javascript Support: 1 ActiveX Support: 1 Plugin Support: 0
-#2019-12-12T15:54:12.972208-08:00 10.160.21.242 ,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.32.30.21,vip=10.156.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015
+#2019-12-12T15:54:12.972208-08:00 10.160.21.242,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.32.30.21,vip=10.156.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015
 
 testdata_nix = [
 '{{ mark }}{{ bsd }} {{ host }} info httpd(pam_audit)[16784]: 01070417:6: AUDIT - user aesguerra - RAW: httpd(pam_audit): user=aesguerra(aesguerra) partition=[All] level=Administrator tty=(unknown) host=10.77.112.22 attempts=1 start="Fri Jan 17 09:44:10 2020" end="Fri Jan 17 09:44:14 2020".',
@@ -42,7 +42,7 @@
 '{{ mark }}{{ bsd }} {{ host }} notice apmd[11023]: 01490248:5: /Common/Network_Access_02:Common:8c6be305: Received client info - Hostname:  Type: IE Version: 8 Platform: Win7 CPU: WOW64 UI Mode: Full Javascript Support: 1 ActiveX Support: 1 Plugin Support: 0',
 ]
 testdata_irule = [
-'{{ mark }}{{ iso }} {{ host }} ,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.111.30.21,vip=10.1111.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015'
+'{{ mark }}{{ iso }} {{ host }},f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.111.30.21,vip=10.1111.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015'
 ]
 @pytest.mark.parametrize("event", testdata_nix)
 def test_f5_bigip_nix(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event):