From 6961185baae6650a0f8894451de9b4ac184b910d Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 26 Mar 2020 17:00:51 -0400 Subject: [PATCH 1/5] Support F5 and fix vmware docs --- docs/sources/F5/index.md | 53 +++++ docs/sources/VMWare/index.md | 10 +- mkdocs.yml | 1 + package/etc/conf.d/filters/f5/bigip.conf | 15 ++ .../conf.d/log_paths/lp-f5_bigip.conf.tmpl | 80 +++++++ .../splunk_index.csv.example | 3 + .../vendor_product_by_source.conf.example | 4 + .../vendor_product_by_source.csv.example | 1 + package/etc/go_templates/source_network.t | 6 +- tests/test_f5_bigip.py | 205 ++++++++++++++++++ 10 files changed, 372 insertions(+), 6 deletions(-) create mode 100644 docs/sources/F5/index.md create mode 100644 package/etc/conf.d/filters/f5/bigip.conf create mode 100644 package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl create mode 100644 tests/test_f5_bigip.py diff --git a/docs/sources/F5/index.md b/docs/sources/F5/index.md new file mode 100644 index 0000000..7ec2c86 --- /dev/null +++ b/docs/sources/F5/index.md @@ -0,0 +1,53 @@ +# Vendor - F5 + + +## Product - BigIP + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2680/ | +| Product Manual | unknown | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| f5:bigip:syslog | None | +| f5:bigip:irule | None | +| nix:syslog | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| f5_bigip | f5:bigip:syslog | netops | none | +| f5_bigip_irule | f5:bigip:syslog | netops | none | +| f5_bigip_nix | nix:syslog | netops | if `f_f5_bigip` is not set the index osnix will be used | + +### Filter type + +Must be identified by host or ip assignment. Update the filter `f_f5_bigip` or configure a dedicated port as required + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the admin manual for specific details of configuration + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_F5_BIGIP_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_F5_BIGIP_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_F5_BIGIP | no | Enable archive to disk for this specific source | +| SC4S_DEST_F5_BIGIP_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active device will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=f5:bigip:*| stats count by host +``` diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md index 9463ae5..4942e35 100644 --- a/docs/sources/VMWare/index.md +++ b/docs/sources/VMWare/index.md @@ -38,11 +38,11 @@ MSG Parse: This filter parses message content when using the default configurati | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_VMWARE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_VMWARE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_LISTEN_VMWARE_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source | -| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_VMWARE_VSPHERE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_VSPHERE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_VSPHERE_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_VMWARE_VSPHERE | no | Enable archive to disk for this specific source | +| SC4S_DEST_VMWARE_VSPHERE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification diff --git a/mkdocs.yml b/mkdocs.yml index dc8acd7..2fc6e08 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,6 +19,7 @@ nav: - Citrix: sources/Citrix/index.md - "Common Event Format": sources/CommonEventFormat/index.md - CyberArk: sources/CyberArk/index.md + - F5: sources/F5/index.md - Forcepoint: sources/Forcepoint/index.md - Fortinet: sources/Fortinet/index.md - Imperva: sources/Imperva/index.md diff --git a/package/etc/conf.d/filters/f5/bigip.conf b/package/etc/conf.d/filters/f5/bigip.conf new file mode 100644 index 0000000..6faa939 --- /dev/null +++ b/package/etc/conf.d/filters/f5/bigip.conf @@ -0,0 +1,15 @@ +filter f_f5_bigip { + match("^f5_bigip", value("fields.sc4s_vendor_product")); +}; +filter f_f5_bigip_message { + message( + '^(?i)(<\d+> ?[[:alpha:]]+ \d\d \d\d:\d\d:\d\d +[^ ]+) +(?:notice|err|error|warning|info) +?(.*)' + flags(store-matches) + ); +}; +parser f_f5_bigip_message { + syslog-parser( + template("$1 $2") + flags(guess-timezone,assume-utf8,store-raw-message) + ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl new file mode 100644 index 0000000..1f393c3 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl @@ -0,0 +1,80 @@ +# F5_BIGIP +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "F5_BIGIP" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log { + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_F5_BIGIP_TCP_PORT")) (getenv (print "SC4S_LISTEN_F5_BIGIP_UDP_PORT"))) (getenv (print "SC4S_LISTEN_F5_BIGIP_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for F5_BIGIP traffic + source (s_F5_BIGIP); + flags (final); + }; +{{- end}} + channel { + # Listen on the default port (typically 514) for F5_BIGIP traffic + source (s_DEFAULT); + flags(final); + }; + }; + + if { + filter{ + program("tmsh") + or program("mcpd") + or program("tmm\d?") + }; + rewrite { + set("f5_bigip", value("fields.sc4s_vendor_product")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("f5:bigip:syslog"), index("netops"), source("program:${.PROGRAM}")) + }; + parser { p_add_context_splunk(key("f5_bigip")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); }; + } elif { + filter { + program('^,f5_irule') + }; + rewrite { + set("f5_bigip_irule", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("f5:bigip:irule"), index("netops")) + }; + parser { p_add_context_splunk(key("f5_bigip_irule")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + } elif { + filter(f_f5_bigip); + rewrite { + set("f5_bigip_nix", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) + }; + parser { p_add_context_splunk(key("f5_bigip")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); }; + }; + + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_F5_BIGIP_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_F5_BIGIP" "no")) }} + destination(d_archive); +{{- end}} + +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_F5_BIGIP_ALTERNATES")) }} + {{ getenv "SC4S_DEST_F5_BIGIP_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + + flags(flow-control,final); +}; diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 5b598ff..7cf43f3 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -24,6 +24,9 @@ #citrix_netscaler,index,netfw #local_example,index,main #forcepoint_webprotect,index,netproxy +#f5_bigip,index,netops +#f5_bigip_irule,index,netops +#f5_bigip_nix,index,netops #fortinet_fortios_event,index,netops #fortinet_fortios_log,index,netops #fortinet_fortios_traffic,index,netfw diff --git a/package/etc/context_templates/vendor_product_by_source.conf.example b/package/etc/context_templates/vendor_product_by_source.conf.example index 7504c41..7df43d2 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf.example +++ b/package/etc/context_templates/vendor_product_by_source.conf.example @@ -39,6 +39,10 @@ filter f_cisco_nx_os { host("csconx-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) }; +filter f_f5_bigip { + host("test_f5-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) +}; filter f_infoblox { host("vib-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) diff --git a/package/etc/context_templates/vendor_product_by_source.csv.example b/package/etc/context_templates/vendor_product_by_source.csv.example index 6edbb8c..9c18cb3 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv.example +++ b/package/etc/context_templates/vendor_product_by_source.csv.example @@ -3,6 +3,7 @@ f_brocade_syslog,sc4s_vendor_product,"brocade_syslog" f_catch_first,sc4s_vendor_product,"catch_first" f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" f_citrix_netscaler,sc4s_vendor_product,"citrix_netscaler" +f_f5_bigip,sc4s_vendor_product,"f5_bigip" f_infoblox,sc4s_vendor_product,"infoblox" f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index a2894bd..47bfab1 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -16,7 +16,7 @@ source s_{{ .port_id }} { use-dns(no) use-fqdn(no) chain-hostnames(off) - flags(no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) + flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); {{- end}} {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} @@ -109,6 +109,10 @@ source s_{{ .port_id }} { filter(f_citrix_netscaler_message); parser(p_citrix_netscaler_date); rewrite(r_citrix_netscaler_message); + } elif { + filter(f_f5_bigip_message); + parser(f_f5_bigip_message); + rewrite(set_rfc3164); } elif { #JSON over IP its not syslog but it can work filter { message('^{') and message('}$') }; diff --git a/tests/test_f5_bigip.py b/tests/test_f5_bigip.py new file mode 100644 index 0000000..7fc9d3d --- /dev/null +++ b/tests/test_f5_bigip.py @@ -0,0 +1,205 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * +from .timeutils import * + +import pytest +env = Environment() + + +#Jan 17 03:35:18 SV5-F5-5600-2.splunk.com notice tmsh[16322]: 01420002:5: AUDIT - pid=16322 user=root folder=/ module=(tmos)# status=[Command OK] cmd_data=cd / ; +#Jan 17 09:44:14 SV5-F5-5600-2.splunk.com info httpd(pam_audit)[16784]: 01070417:6: AUDIT - user aesguerra - RAW: httpd(pam_audit): user=aesguerra(aesguerra) partition=[All] level=Administrator tty=(unknown) host=10.77.112.22 attempts=1 start=""Fri Jan 17 09:44:10 2020"" end=""Fri Jan 17 09:44:14 2020"". +#Jan 17 09:52:46 SV5-F5-5600-2.splunk.com info sshd(pam_audit)[21398]: 01070417:6: AUDIT - user aesguerra - RAW: sshd(pam_audit): user=aesguerra(aesguerra) partition=[All] level=Administrator tty=ssh host=10.77.112.22 attempts=1 start="Fri Jan 17 09:52:46 2020". +#Jan 17 03:35:04 SV5-F5-5600-2.splunk.com info audit_forwarder[16218]: audit_forwarder started. +#Jan 17 03:35:35 SV5-F5-5600-2.splunk.com info systemd-journal[626]: Suppressed 1255 messages from /system.slice/runit.service +#Jan 17 03:37:54 SV5-F5-5600-2 warning tmm[23068]: 01260013:4: SSL Handshake failed for TCP 10.160.23.133:50365 -> 10.156.1.150:443 +#Jan 17 03:38:00 SV5-F5-5600-2 err tmm[23068]: 01220001:3: TCL error: /Common/stg-Artifactory-iRule - ERR_NOT_SUPPORTED (line 8) invoked from within "HTTP::method" +#Jan 17 04:03:37 SV5-F5-5600-2 warning tmm1[23068]: 01260009:4: Connection error: ssl_passthru:5234: not SSL (40) +#Jan 17 04:42:37 SV5-F5-5600-2.splunk.com notice mcpd[10653]: 01070638:5: Pool /Common/infra-docs-pool member /Common/go_web3:4000 monitor status down. [ /Common/tcp_half_open: down; last error: ] [ was up for 837hrs:31mins:36sec ] +#2019-12-12T15:54:12.972208-08:00 10.160.21.242 ,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.32.30.21,vip=10.156.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015 + +testdata_nix = [ +'{{ mark }}{{ bsd }} {{ host }} info httpd(pam_audit)[16784]: 01070417:6: AUDIT - user aesguerra - RAW: httpd(pam_audit): user=aesguerra(aesguerra) partition=[All] level=Administrator tty=(unknown) host=10.77.112.22 attempts=1 start="Fri Jan 17 09:44:10 2020" end="Fri Jan 17 09:44:14 2020".', +'{{ mark }}{{ bsd }} {{ host }} info sshd(pam_audit)[21398]: 01070417:6: AUDIT - user aesguerra - RAW: sshd(pam_audit): user=aesguerra(aesguerra) partition=[All] level=Administrator tty=ssh host=10.77.112.22 attempts=1 start="Fri Jan 17 09:52:46 2020".', +'{{ mark }}{{ bsd }} {{ host }} info audit_forwarder[16218]: audit_forwarder started.', +'{{ mark }}{{ bsd }} {{ host }} info systemd-journal[626]: Suppressed 1255 messages from /system.slice/runit.service', +] + +testdata_app = [ +'{{ mark }}{{ bsd }} {{ host }} notice tmsh[16322]: 01420002:5: AUDIT - pid=16322 user=root folder=/ module=(tmos)# status=[Command OK] cmd_data=cd / ;', +'{{ mark }}{{ bsd }} {{ host }} warning tmm[23068]: 01260013:4: SSL Handshake failed for TCP 10.160.23.133:50365 -> 10.156.1.150:443', +'{{ mark }}{{ bsd }} {{ host }} err tmm[23068]: 01220001:3: TCL error: /Common/stg-Artifactory-iRule - ERR_NOT_SUPPORTED (line 8) invoked from within "HTTP::method"', +'{{ mark }}{{ bsd }} {{ host }} warning tmm1[23068]: 01260009:4: Connection error: ssl_passthru:5234: not SSL (40)', +'{{ mark }}{{ bsd }} {{ host }} notice mcpd[10653]: 01070638:5: Pool /Common/infra-docs-pool member /Common/go_web3:4000 monitor status down. [ /Common/tcp_half_open: down; last error: ] [ was up for 837hrs:31mins:36sec ]', +] +testdata_irule = [ +'{{ mark }}{{ iso }}{{ tzoffset }} {{ host }} ,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.111.30.21,vip=10.1111.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015' +] +@pytest.mark.parametrize("event", testdata_nix) +def test_f5_bigip_nix(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): + host = "test_f5-" + get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + time = time[:-7] + millisec = iso[20:23] + microsec = iso[20:26] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, + millisec=millisec, microsec=microsec, tzname=tzname, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops (_time={{ epoch }}) sourcetype=\"nix:syslog\" (host=\"{{ host }}\")") + search = st.render(epoch=epoch, millisec=millisec, + microsec=microsec, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +@pytest.mark.parametrize("event", testdata_app) +def test_f5_bigip_app(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): + host = "test_f5-" + get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + time = time[:-7] + millisec = iso[20:23] + microsec = iso[20:26] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, + millisec=millisec, microsec=microsec, tzname=tzname, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"f5:bigip:syslog\" (host=\"{{ host }}\")") + search = st.render(epoch=epoch, millisec=millisec, + microsec=microsec, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +@pytest.mark.parametrize("event", testdata_irule) +def test_f5_bigip_irule(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): + host = "test_f5-" + get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + time = time[:-7] + millisec = iso[20:23] + microsec = iso[20:26] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, iso=iso, + millisec=millisec, microsec=microsec, tzname=tzname, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"f5:bigip:irule\" (host=\"{{ host }}\")") + search = st.render(epoch=epoch, millisec=millisec, + microsec=microsec, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +@pytest.mark.parametrize("event", testdata_app) +def test_f5_bigip_app_default(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + time = time[:-7] + millisec = iso[20:23] + microsec = iso[20:26] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, + millisec=millisec, microsec=microsec, tzname=tzname, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"f5:bigip:syslog\" (host=\"{{ host }}\")") + search = st.render(epoch=epoch, millisec=millisec, + microsec=microsec, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +@pytest.mark.parametrize("event", testdata_irule) +def test_f5_bigip_irule_default(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + time = time[:-7] + millisec = iso[20:23] + microsec = iso[20:26] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, iso=iso, + millisec=millisec, microsec=microsec, tzname=tzname, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"f5:bigip:irule\" (host=\"{{ host }}\")") + search = st.render(epoch=epoch, millisec=millisec, + microsec=microsec, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file From 91f2cfc697938346bc6a8bbdb1411d1897fa3735 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Thu, 26 Mar 2020 21:39:24 -0700 Subject: [PATCH 2/5] Update lp-f5_bigip.conf.tmpl --- package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl index 1f393c3..90fb26a 100644 --- a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl @@ -1,6 +1,6 @@ # F5_BIGIP {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "F5_BIGIP" "parser" "rfc3164" }} +{{- $context := dict "port_id" "F5_BIGIP" "parser" "common" }} {{- tmpl.Exec "t/source_network.t" $context }} log { From cdc1f9c486f9f5cd8b7eb394cda4c61e4a68e9c4 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Thu, 26 Mar 2020 21:42:25 -0700 Subject: [PATCH 3/5] Update bigip.conf Add template logic for `store_raw_message` flag --- package/etc/conf.d/filters/f5/bigip.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/filters/f5/bigip.conf b/package/etc/conf.d/filters/f5/bigip.conf index 6faa939..7d509a2 100644 --- a/package/etc/conf.d/filters/f5/bigip.conf +++ b/package/etc/conf.d/filters/f5/bigip.conf @@ -10,6 +10,6 @@ filter f_f5_bigip_message { parser f_f5_bigip_message { syslog-parser( template("$1 $2") - flags(guess-timezone,assume-utf8,store-raw-message) + flags(guess-timezone,assume-utf8,{{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); -}; \ No newline at end of file +}; From 2eb94cd5c0526e89deb16fce30ce90953cbaff06 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Thu, 26 Mar 2020 22:01:05 -0700 Subject: [PATCH 4/5] Update test_f5_bigip.py Simplify test from copied IOS test --- tests/test_f5_bigip.py | 63 +++++++++++++----------------------------- 1 file changed, 19 insertions(+), 44 deletions(-) diff --git a/tests/test_f5_bigip.py b/tests/test_f5_bigip.py index 7fc9d3d..e7bf36f 100644 --- a/tests/test_f5_bigip.py +++ b/tests/test_f5_bigip.py @@ -40,7 +40,7 @@ '{{ mark }}{{ bsd }} {{ host }} notice mcpd[10653]: 01070638:5: Pool /Common/infra-docs-pool member /Common/go_web3:4000 monitor status down. [ /Common/tcp_half_open: down; last error: ] [ was up for 837hrs:31mins:36sec ]', ] testdata_irule = [ -'{{ mark }}{{ iso }}{{ tzoffset }} {{ host }} ,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.111.30.21,vip=10.1111.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015' +'{{ mark }}{{ iso }} {{ host }} ,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.111.30.21,vip=10.1111.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015' ] @pytest.mark.parametrize("event", testdata_nix) def test_f5_bigip_nix(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): @@ -51,20 +51,15 @@ def test_f5_bigip_nix(record_property, setup_wordlist, get_host_key, setup_splun # Tune time functions epoch = epoch[:-7] - time = time[:-7] - millisec = iso[20:23] - microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, - millisec=millisec, microsec=microsec, tzname=tzname, host=host) + message = mt.render(mark="<166>", bsd=bsd, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops (_time={{ epoch }}) sourcetype=\"nix:syslog\" (host=\"{{ host }}\")") - search = st.render(epoch=epoch, millisec=millisec, - microsec=microsec, host=host) + "search index=netops _time={{ epoch }} sourcetype=\"nix:syslog\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -83,20 +78,15 @@ def test_f5_bigip_app(record_property, setup_wordlist, get_host_key, setup_splun # Tune time functions epoch = epoch[:-7] - time = time[:-7] - millisec = iso[20:23] - microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, - millisec=millisec, microsec=microsec, tzname=tzname, host=host) + message = mt.render(mark="<166>", bsd=bsd, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"f5:bigip:syslog\" (host=\"{{ host }}\")") - search = st.render(epoch=epoch, millisec=millisec, - microsec=microsec, host=host) + "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:syslog\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -116,20 +106,15 @@ def test_f5_bigip_irule(record_property, setup_wordlist, get_host_key, setup_spl # Tune time functions epoch = epoch[:-7] - time = time[:-7] - millisec = iso[20:23] - microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, iso=iso, - millisec=millisec, microsec=microsec, tzname=tzname, host=host) + message = mt.render(mark="<166>", bsd=bsd, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"f5:bigip:irule\" (host=\"{{ host }}\")") - search = st.render(epoch=epoch, millisec=millisec, - microsec=microsec, host=host) + "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:irule\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -148,20 +133,15 @@ def test_f5_bigip_app_default(record_property, setup_wordlist, get_host_key, set # Tune time functions epoch = epoch[:-7] - time = time[:-7] - millisec = iso[20:23] - microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, - millisec=millisec, microsec=microsec, tzname=tzname, host=host) + message = mt.render(mark="<166>", bsd=bsd, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"f5:bigip:syslog\" (host=\"{{ host }}\")") - search = st.render(epoch=epoch, millisec=millisec, - microsec=microsec, host=host) + "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:syslog\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -179,22 +159,17 @@ def test_f5_bigip_irule_default(record_property, setup_wordlist, get_host_key, s dt = datetime.datetime.now() iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) - # Tune time functions - epoch = epoch[:-7] - time = time[:-7] - millisec = iso[20:23] - microsec = iso[20:26] + # Tune time functions + epoch = epoch[:-3] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, iso=iso, - millisec=millisec, microsec=microsec, tzname=tzname, host=host) + message = mt.render(mark="<166>", iso=iso, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"f5:bigip:irule\" (host=\"{{ host }}\")") - search = st.render(epoch=epoch, millisec=millisec, - microsec=microsec, host=host) + "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:irule\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -202,4 +177,4 @@ def test_f5_bigip_irule_default(record_property, setup_wordlist, get_host_key, s record_property("resultCount", resultCount) record_property("message", message) - assert resultCount == 1 \ No newline at end of file + assert resultCount == 1 From f7d50d22531eab220784402c213a4b90ec5b8d1e Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 26 Mar 2020 22:16:54 -0700 Subject: [PATCH 5/5] Simplify test and templatize filter * Simplify test (from IOS copy/paste) * Add `tmpl` extension to filter file for gomplate --- package/etc/conf.d/filters/f5/{bigip.conf => bigip.conf.tmpl} | 0 tests/test_f5_bigip.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename package/etc/conf.d/filters/f5/{bigip.conf => bigip.conf.tmpl} (100%) diff --git a/package/etc/conf.d/filters/f5/bigip.conf b/package/etc/conf.d/filters/f5/bigip.conf.tmpl similarity index 100% rename from package/etc/conf.d/filters/f5/bigip.conf rename to package/etc/conf.d/filters/f5/bigip.conf.tmpl diff --git a/tests/test_f5_bigip.py b/tests/test_f5_bigip.py index e7bf36f..ad38d69 100644 --- a/tests/test_f5_bigip.py +++ b/tests/test_f5_bigip.py @@ -105,10 +105,10 @@ def test_f5_bigip_irule(record_property, setup_wordlist, get_host_key, setup_spl iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) # Tune time functions - epoch = epoch[:-7] + epoch = epoch[:-3] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", bsd=bsd, host=host) + message = mt.render(mark="<166>", iso=iso, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])