From bef1f15d013ac1d2a9f892dcd36dc3da9bb4e066 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sun, 12 Apr 2020 11:51:27 -0700 Subject: [PATCH] Add separate f5 iRule parsers * Add separate f5 iRule parser to log path and filter * Add `frac-digits(3)` to global syslog-ng.conf options so that ISO date-ralated macros render with ms resolution. * Remove the `$S_MSEC` macro from the time element of all destination JSON payloads --- .../conf.d/destinations/splunk_hec.conf.tmpl | 2 +- .../destinations/splunk_hec_debug.conf.tmpl | 2 +- .../splunk_hec_internal.conf.tmpl | 2 +- package/etc/conf.d/filters/f5/bigip.conf.tmpl | 21 +++++++++++++++++-- .../conf.d/log_paths/lp-f5_bigip.conf.tmpl | 8 +++---- .../log_paths/lp-sc4s_internal.conf.tmpl | 2 +- package/etc/go_templates/source_network.t | 6 +++++- package/etc/syslog-ng.conf | 1 + tests/test_f5_bigip.py | 4 ++-- 9 files changed, 35 insertions(+), 13 deletions(-) diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 65f8dfc..0b2b349 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -41,7 +41,7 @@ destination d_hec { {{- end}} ca-file("{{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}")) body('$(format-json - time=$S_UNIXTIME.$S_MSEC + time=$S_UNIXTIME host=${HOST} source=${.splunk.source} sourcetype=${.splunk.sourcetype} diff --git a/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl index c7f32dd..d02db92 100644 --- a/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_debug.conf.tmpl @@ -1,7 +1,7 @@ destination d_hec_debug { file("/opt/syslog-ng/var/archive/debug/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-message.log" template("curl -k -u \"sc4s HEC debug:{{- getenv "SPLUNK_HEC_TOKEN"}}\" \"{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event\" -d '$(format-json - time=$S_UNIXTIME.$S_MSEC + time=$S_UNIXTIME host=${HOST} source=${.splunk.source} sourcetype=${.splunk.sourcetype} diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl index 2e9428b..6d62c36 100644 --- a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -24,7 +24,7 @@ destination d_hec_internal { {{- end}} ca-file("{{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}")) body('$(format-json - time=$S_UNIXTIME.$S_MSEC + time=$S_UNIXTIME host=${HOST} source=${.splunk.source} sourcetype=${.splunk.sourcetype} diff --git a/package/etc/conf.d/filters/f5/bigip.conf.tmpl b/package/etc/conf.d/filters/f5/bigip.conf.tmpl index c39f24c..fdf7984 100644 --- a/package/etc/conf.d/filters/f5/bigip.conf.tmpl +++ b/package/etc/conf.d/filters/f5/bigip.conf.tmpl @@ -5,17 +5,34 @@ filter f_f5_bigip { or program("mcpd") or program("apmd") or program("tmm\d?") - or program('^,f5_irule'); + or program('^f5_irule='); }; + +filter f_f5_bigip_irule { + message("*,f5_irule=*" type(glob)); +}; + filter f_f5_bigip_message { message( '^(?i)(<\d+> ?[[:alpha:]]+\s{1,2}\d{1,2} \d\d:\d\d:\d\d +[^ ]+) +(?:notice|err|error|warning|info) +?(.*)' flags(store-matches) ); }; -parser f_f5_bigip_message { +parser p_f5_bigip_message { syslog-parser( template("$1 $2") flags(guess-timezone,assume-utf8,{{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); }; + +parser p_f5_bigip_irule { + channel { + rewrite { subst(",f5_irule=", " f5_irule=", value("MESSAGE")); }; + parser { + syslog-parser( + template("$MESSAGE") + flags(guess-timezone,assume-utf8,{{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) + ); + }; + }; +}; diff --git a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl index 4efb271..934f71c 100644 --- a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl @@ -35,7 +35,7 @@ log { }; parser { p_add_context_splunk(key("f5_bigip")); }; parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; # } elif { # filter { # program('apmd') @@ -48,10 +48,10 @@ log { # }; # parser { p_add_context_splunk(key("f5_bigip")); }; # parser (compliance_meta_by_source); -# rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); }; +# rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; } elif { filter { - program('^,f5_irule') + program('f5_irule=') }; rewrite { set("f5_bigip_irule", value("fields.sc4s_vendor_product")); @@ -71,7 +71,7 @@ log { }; parser { p_add_context_splunk(key("f5_bigip")); }; parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; } else { rewrite { set("f5_bigip_rogue_message", value("fields.sc4s_vendor_product")); diff --git a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl index fafde80..1ce291f 100644 --- a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl @@ -10,7 +10,7 @@ log { subst('Log statistics; ', '', value("MESSAGE"), flags("utf8" "global")); subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global")); subst('(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27,? ?', -'{"time": "$S_UNIXTIME.$S_MSEC","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}} +'{"time": "$S_UNIXTIME","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}} ', value("MESSAGE") flags("utf8" "global") ); diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 63ec329..7995659 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -121,7 +121,11 @@ source s_{{ .port_id }} { rewrite(r_citrix_netscaler_message); } elif { filter(f_f5_bigip_message); - parser(f_f5_bigip_message); + parser(p_f5_bigip_message); + rewrite(set_rfc3164); + } elif { + filter(f_f5_bigip_irule); + parser(p_f5_bigip_irule); rewrite(set_rfc3164); } elif { #JSON over IP its not syslog but it can work diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 2966ef0..03ba65c 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -30,6 +30,7 @@ options { stats-max-dynamics(2000); normalize-hostnames(yes); on-error(fallback-to-string); + frac-digits(3); }; diff --git a/tests/test_f5_bigip.py b/tests/test_f5_bigip.py index 57d22c3..bc06fc0 100644 --- a/tests/test_f5_bigip.py +++ b/tests/test_f5_bigip.py @@ -24,7 +24,7 @@ #Jan 17 04:03:37 SV5-F5-5600-2 warning tmm1[23068]: 01260009:4: Connection error: ssl_passthru:5234: not SSL (40) #Jan 17 04:42:37 SV5-F5-5600-2.splunk.com notice mcpd[10653]: 01070638:5: Pool /Common/infra-docs-pool member /Common/go_web3:4000 monitor status down. [ /Common/tcp_half_open: down; last error: ] [ was up for 837hrs:31mins:36sec ] #Jan 17 04:42:37 SV5-F5-5600-2 notice apmd[11023]: 01490248:5: /Common/Network_Access_02:Common:8c6be305: Received client info - Hostname: Type: IE Version: 8 Platform: Win7 CPU: WOW64 UI Mode: Full Javascript Support: 1 ActiveX Support: 1 Plugin Support: 0 -#2019-12-12T15:54:12.972208-08:00 10.160.21.242 ,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.32.30.21,vip=10.156.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015 +#2019-12-12T15:54:12.972208-08:00 10.160.21.242,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.32.30.21,vip=10.156.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015 testdata_nix = [ '{{ mark }}{{ bsd }} {{ host }} info httpd(pam_audit)[16784]: 01070417:6: AUDIT - user aesguerra - RAW: httpd(pam_audit): user=aesguerra(aesguerra) partition=[All] level=Administrator tty=(unknown) host=10.77.112.22 attempts=1 start="Fri Jan 17 09:44:10 2020" end="Fri Jan 17 09:44:14 2020".', @@ -42,7 +42,7 @@ '{{ mark }}{{ bsd }} {{ host }} notice apmd[11023]: 01490248:5: /Common/Network_Access_02:Common:8c6be305: Received client info - Hostname: Type: IE Version: 8 Platform: Win7 CPU: WOW64 UI Mode: Full Javascript Support: 1 ActiveX Support: 1 Plugin Support: 0', ] testdata_irule = [ -'{{ mark }}{{ iso }} {{ host }} ,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.111.30.21,vip=10.1111.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015' +'{{ mark }}{{ iso }} {{ host }},f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.111.30.21,vip=10.1111.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015' ] @pytest.mark.parametrize("event", testdata_nix) def test_f5_bigip_nix(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event):