From 802d853a8db1be83821c1156cb80c4b9d1f8d587 Mon Sep 17 00:00:00 2001 From: kenny Date: Thu, 13 Feb 2020 00:50:11 +0400 Subject: [PATCH 01/11] FortiWeb initial commit --- docs/sources/Fortinet/index.md | 92 ++++++++++++++++++- .../etc/conf.d/filters/fortinet/fortiweb.conf | 3 + .../log_paths/lp-fortinet_fortiweb.conf.tmpl | 71 ++++++++++++++ .../splunk_index.csv.example | 3 + tests/test_fortinet_web.py | 73 +++++++++++++++ 5 files changed, 241 insertions(+), 1 deletion(-) create mode 100755 package/etc/conf.d/filters/fortinet/fortiweb.conf create mode 100644 package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl create mode 100644 tests/test_fortinet_web.py diff --git a/docs/sources/Fortinet/index.md b/docs/sources/Fortinet/index.md index a13bc1d..31f5f1f 100644 --- a/docs/sources/Fortinet/index.md +++ b/docs/sources/Fortinet/index.md @@ -22,7 +22,7 @@ | key | sourcetype | index | notes | |----------------|----------------|----------------|----------------| -| fortinet_fortios_traffic | fgt_traffic | netops | none | +| fortinet_fortios_traffic | fgt_traffic | netfw | none | | fortinet_fortios_utm | fgt_utm | netids | none | | fortinet_fortios_event | fgt_event | netops | none | | fortinet_fortios_log | fgt_log | netops | none | @@ -105,4 +105,94 @@ index= (sourcetype=fgt_log OR sourcetype=fgt_traffic OR sourcetype ###Event Message Type ![FortiGate Event message](FortiGate_event.png) +Verify timestamp, and host values match as expected + +## Product - FortiWeb + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/4679/ | +| Product Manual | https://docs.fortinet.com/product/fortiweb/6.3 | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| fweb_log | The catch all sourcetype is not used | +| fwb_traffic | None | +| fwb_attack | None | +| fwb_event | None + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| fortinet_fortiweb_traffic | fwb_traffic | netfw | none | +| fortinet_fortiweb_attack | fwb_attack | netids | none | +| fortinet_fortiweb_event | fwb_event | netops | none | +| fortinet_fortiweb_log | fwb_log | netops | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the admin manual for specific details of configuration to send Reliable syslog using RFC 3195 format, a typical logging configuration will include the following features. + +``` +config log syslog-policy + +edit splunk + +config syslog-server-list + +edit 1 + +set server x.x.x.x + +set port 514 (Example. Should be the same as in data input of Splunk server) + +end + +end + +config log syslogd + +set policy splunk + +set status enable + +end + +``` + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_FORTINET_FORTIWEB | no | Enable archive to disk for this specific source | +| SC4S_DEST_FORTINET_FORTIWEB_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active firewall will generate frequent events, in addition fortigate has the ability to test logging functionality using a built in command + +``` +diag log test +``` + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=fwb_log OR sourcetype=fwb_traffic OR sourcetype=fwb_attack OR sourcetype=fwb_event) +``` + Verify timestamp, and host values match as expected \ No newline at end of file diff --git a/package/etc/conf.d/filters/fortinet/fortiweb.conf b/package/etc/conf.d/filters/fortinet/fortiweb.conf new file mode 100755 index 0000000..8a3654e --- /dev/null +++ b/package/etc/conf.d/filters/fortinet/fortiweb.conf @@ -0,0 +1,3 @@ +filter f_fortinet_fortiweb { + message('device_id=\"?FV.+type=\"?(traffic|attack|event)'); +}; diff --git a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl new file mode 100644 index 0000000..8bcc8c3 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl @@ -0,0 +1,71 @@ +# Fortinet FortiWeb +{{ $context := dict "port_id" "FORTINET_FORTIWEB" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_fortinet_fortiweb); +{{- end}} +{{- if eq (.) "no"}} + source (s_FORTINET_FORTIWEB); +{{- end}} + + parser { + kv-parser(prefix(".kv.") pair-separator(",") template("${MSGHDR} ${MSG}")); + }; + + rewrite { + set("${.kv.devname}", value("HOST")); + set("fortigate_fortiweb", value("fields.sc4s_vendor_product")); + }; + + if (match("traffic" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fwb_traffic"), index("netfw"))}; + parser {p_add_context_splunk(key("fortinet_fortiweb_traffic")); }; + } elif (match("attack" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fwb_attack"), index("netids"))}; + parser {p_add_context_splunk(key("fortinet_fortiweb_attack")); }; + } elif (match("event" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fwb_event"), index("netops"))}; + parser {p_add_context_splunk(key("fortinet_fortiweb_event")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("fwb_log"), index("netops"))}; + parser {p_add_context_splunk(key("fortinet_fortiweb_log")); }; + }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORTINET_FORTIWEB_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORTINET_FORTIWEB") }} + destination(d_archive); +{{- end}} + + flags(flow-control); +}; +{{- end}} + +{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for FORTINET_FORTIWEB traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for FORTINET_FORTIWEB traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index bf8e3bb..5b598ff 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -28,6 +28,9 @@ #fortinet_fortios_log,index,netops #fortinet_fortios_traffic,index,netfw #fortinet_fortios_utm,index,netids +#fortinet_fortweb_log,index,netops +#fortinet_fortweb_traffic,index,netfw +#fortinet_fortweb_attack,index,netids #infoblox_dns,index,netdns #infoblox_dhcp,index,netipam #infoblox_threat,index,netids diff --git a/tests/test_fortinet_web.py b/tests/test_fortinet_web.py new file mode 100644 index 0000000..aa3757a --- /dev/null +++ b/tests/test_fortinet_web.py @@ -0,0 +1,73 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<111> 2020-02-12,23:13:33,devname=FortiWeb-A,log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\" +def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\"\n") + message = mt.render(mark="<13>", host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"fwb_event\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +#<111> 2020-02-12,23:16:41,devname=FortiWeb-A,log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd="root","timezone=""(GMT+3:00)Kuwait,Riyadh""",type=traffic,subtype="https",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,"http_url=""/publish/templates/images/bluebottom.jpg""","http_host=""splunk.infigo.hr""","http_agent=""Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36""",http_retcode=200,"msg=""HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80""",original_srccountry="Saudi Arabia",srccountry="Saudi Arabia",content_switch_name="none",server_pool_name="PHOME","user_name=""Unknown""","http_refer=""https://splunk.infigo.hr/publish/templates/CSS/sc4s.css""",http_version="1.x",dev_id=none,cipher_suite="TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" +def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=traffic,subtype=\"https\",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,\"http_url=\"\"/publish/templates/images/bluebottom.jpg\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36\"\"\",http_retcode=200,\"msg=\"\"HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80\"\"\",original_srccountry=\"Saudi Arabia\",srccountry=\"Saudi Arabia\",content_switch_name=\"none\",server_pool_name=\"PHOME\",\"user_name=\"\"Unknown\"\"\",\"http_refer=\"\"https://splunk.infigo.hr/publish/templates/CSS/sc4s.css\"\"\",http_version=\"1.x\",dev_id=none,cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\"\n") + message = mt.render(mark="<13>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"fwb_traffic\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<111> 2020-02-12,23:16:41,devname=FortiWeb-A,log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd="root","timezone=""(GMT+3:00)Kuwait,Riyadh""",type=attack,pri=alert,main_type="Signature Detection",sub_type="Information Disclosure",trigger_policy="",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy="MobApp_policy",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,"http_url=""/mfp/api/abc""","http_host=""splunk.infigo.hr""","http_agent=""WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)""",http_session_id=ASDSADSA,"msg=""HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only""",signature_subclass="HTTP Header Leakage",signature_id="080200004",signature_cve_id="N/A",srccountry="Kuwait",content_switch_name="none",server_pool_name="MObApp_pool",false_positive_mitigation="none","user_name=""Unknown""",monitor_status="Enabled","http_refer=""none""",http_version="1.x",dev_id="none",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode="N/A",ftp_cmd="N/A",cipher_suite="TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types="none",ml_svm_accuracy="none",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method="none",owasp_top10="A3:2017-Sensitive Data Exposure",bot_info="none",matched_field="header","matched_pattern=""X-Powered-By: Servlet/3.1""" +def test_fortinet_fwb_attack(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=attack,pri=alert,main_type=\"Signature Detection\",sub_type=\"Information Disclosure\",trigger_policy=\"\",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy=\"MobApp_policy\",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,\"http_url=\"\"/mfp/api/abc\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)\"\"\",http_session_id=ASDSADSA,\"msg=\"\"HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only\"\"\",signature_subclass=\"HTTP Header Leakage\",signature_id=\"080200004\",signature_cve_id=\"N/A\",srccountry=\"Kuwait\",content_switch_name=\"none\",server_pool_name=\"MObApp_pool\",false_positive_mitigation=\"none\",\"user_name=\"\"Unknown\"\"\",monitor_status=\"Enabled\",\"http_refer=\"\"none\"\"\",http_version=\"1.x\",dev_id=\"none\",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode=\"N/A\",ftp_cmd=\"N/A\",cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types=\"none\",ml_svm_accuracy=\"none\",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method=\"none\",owasp_top10=\"A3:2017-Sensitive Data Exposure\",bot_info=\"none\",matched_field=\"header\",\"matched_pattern=\"\"X-Powered-By: Servlet/3.1\"\"\"\n") + message = mt.render(mark="<13>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"fwb_attack\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From f80877c7b0c1c780f7924a91807b39782a4a6d2f Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 13 Feb 2020 07:07:39 -0500 Subject: [PATCH 02/11] replace splunk with sc4s --- docs/sources/Fortinet/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/Fortinet/index.md b/docs/sources/Fortinet/index.md index 31f5f1f..eae3613 100644 --- a/docs/sources/Fortinet/index.md +++ b/docs/sources/Fortinet/index.md @@ -156,7 +156,7 @@ edit 1 set server x.x.x.x -set port 514 (Example. Should be the same as in data input of Splunk server) +set port 514 (Example. Should be the same as default or dedicated port selected for sc4s) end From 572f51fda2c9cfd6dff38ff721db6cbeafc3e02f Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 13 Feb 2020 07:16:18 -0500 Subject: [PATCH 03/11] update time tests --- tests/test_fortinet_ngfw.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index 892f87d..c50cef4 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -12,17 +12,21 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 +# <111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 + + def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") + "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") + st = env.from_string( + "search earliest=-1m@m latest=+1m@m index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -33,16 +37,20 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk, setup assert resultCount == 1 -#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 +# <111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 + + def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") + "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") + st = env.from_string( + "search earliest=-1m@m latest=+1m@m index=netfw host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -53,16 +61,20 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk, set assert resultCount == 1 -#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:21:40 logver=52 devname=US-Corp_Main1 devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user="" srcip=172.30.16.119 srcport=53235 srcintf="Internal" dstip=114.112.67.75 dstport=80 dstintf="External-SDC" proto=6 service=HTTP hostname="popo.wan.ijinshan.com" profile="scan" action=passthrough reqtype=direct url="/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl" sentbyte=525 rcvdbyte=325 direction=outgoing msg="URL belongs to an allowed category in policy" method=domain cat=52 catdesc="Information Technology" +# <111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:21:40 logver=52 devname=US-Corp_Main1 devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user="" srcip=172.30.16.119 srcport=53235 srcintf="Internal" dstip=114.112.67.75 dstport=80 dstintf="External-SDC" proto=6 service=HTTP hostname="popo.wan.ijinshan.com" profile="scan" action=passthrough reqtype=direct url="/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl" sentbyte=525 rcvdbyte=325 direction=outgoing msg="URL belongs to an allowed category in policy" method=domain cat=52 catdesc="Information Technology" + + def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") + "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") + st = env.from_string( + "search earliest=-1m@m latest=+1m@m index=netids host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) From b057805930b00ff037004196e35254ed2f115aeb Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 13 Feb 2020 22:30:40 -0800 Subject: [PATCH 04/11] Update/fwb_log_path * Update Fortiweb log path to properly parse timestamp * Update Fortigate FW test to emit local time * Update Fortiweb log path to emit hard-coded `GMT-8:00` format to match that of the device TZ output --- .../log_paths/lp-fortinet_fortiweb.conf.tmpl | 67 ++++++++++--------- tests/test_fortinet_ngfw.py | 2 +- tests/test_fortinet_web.py | 6 +- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl index 8bcc8c3..bf3bb74 100644 --- a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl @@ -1,22 +1,47 @@ # Fortinet FortiWeb -{{ $context := dict "port_id" "FORTINET_FORTIWEB" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "FORTINET_FORTIWEB" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} log { -{{- if eq (.) "yes"}} - source(s_DEFAULT); - filter(f_fortinet_fortiweb); -{{- end}} -{{- if eq (.) "no"}} - source (s_FORTINET_FORTIWEB); + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for FORTINET_FORTIWEB traffic + source (s_FORTINET_FORTIWEB); + flags (final); + }; {{- end}} + channel { + # Listen on the default port (typically 514) for FORTINET_FORTIWEB traffic + source (s_DEFAULT); + filter(f_is_rfc3164); + filter(f_fortinet_fortiweb); + flags(final); + }; + }; parser { kv-parser(prefix(".kv.") pair-separator(",") template("${MSGHDR} ${MSG}")); }; + rewrite { + subst('.*([\+-]\d+:\d+).*', $1, value(".kv.timezone")); + subst('([\+-])(\d)(?=:):(\d+)', "${1}0${2}${3}", value(".kv.timezone")); + subst('([\+-])(\d+):(\d+)', "$1$2$3", value(".kv.timezone")); + }; + + #2020-02-12,23:13:33 + #parse the date + parser { + date-parser( + format("%Y-%m-%d,%H:%M:%S%z") + template('$(substr "$LEGACY_MSGHDR$MSG" "0" "19")${.kv.timezone}') + time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) + flags(guess-timezone) + ); + }; + rewrite { set("${.kv.devname}", value("HOST")); set("fortigate_fortiweb", value("fields.sc4s_vendor_product")); @@ -37,17 +62,7 @@ log { }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_3164))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORTINET_FORTIWEB_HEC" "no") | conv.ToBool) }} destination(d_hec); @@ -58,14 +73,6 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; -{{- end}} - -{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TLS_PORT")) }} -# Listen on the specified dedicated port(s) for FORTINET_FORTIWEB traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} -# Listen on the default port (typically 514) for FORTINET_FORTIWEB traffic -{{ tmpl.Exec "log_path" "yes" }} diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index 6a8cd3c..bdfeb88 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -67,7 +67,7 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk, setup_s random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") + "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'local', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) diff --git a/tests/test_fortinet_web.py b/tests/test_fortinet_web.py index aa3757a..97bd21d 100644 --- a/tests/test_fortinet_web.py +++ b/tests/test_fortinet_web.py @@ -17,7 +17,7 @@ def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\"\n") + "{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) @@ -37,7 +37,7 @@ def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, set host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=traffic,subtype=\"https\",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,\"http_url=\"\"/publish/templates/images/bluebottom.jpg\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36\"\"\",http_retcode=200,\"msg=\"\"HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80\"\"\",original_srccountry=\"Saudi Arabia\",srccountry=\"Saudi Arabia\",content_switch_name=\"none\",server_pool_name=\"PHOME\",\"user_name=\"\"Unknown\"\"\",\"http_refer=\"\"https://splunk.infigo.hr/publish/templates/CSS/sc4s.css\"\"\",http_version=\"1.x\",dev_id=none,cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\"\n") + "{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT-8:00)Pacific Time(US&Canada)\"\"\",type=traffic,subtype=\"https\",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,\"http_url=\"\"/publish/templates/images/bluebottom.jpg\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36\"\"\",http_retcode=200,\"msg=\"\"HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80\"\"\",original_srccountry=\"Saudi Arabia\",srccountry=\"Saudi Arabia\",content_switch_name=\"none\",server_pool_name=\"PHOME\",\"user_name=\"\"Unknown\"\"\",\"http_refer=\"\"https://splunk.infigo.hr/publish/templates/CSS/sc4s.css\"\"\",http_version=\"1.x\",dev_id=none,cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) @@ -57,7 +57,7 @@ def test_fortinet_fwb_attack(record_property, setup_wordlist, setup_splunk, setu host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=attack,pri=alert,main_type=\"Signature Detection\",sub_type=\"Information Disclosure\",trigger_policy=\"\",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy=\"MobApp_policy\",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,\"http_url=\"\"/mfp/api/abc\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)\"\"\",http_session_id=ASDSADSA,\"msg=\"\"HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only\"\"\",signature_subclass=\"HTTP Header Leakage\",signature_id=\"080200004\",signature_cve_id=\"N/A\",srccountry=\"Kuwait\",content_switch_name=\"none\",server_pool_name=\"MObApp_pool\",false_positive_mitigation=\"none\",\"user_name=\"\"Unknown\"\"\",monitor_status=\"Enabled\",\"http_refer=\"\"none\"\"\",http_version=\"1.x\",dev_id=\"none\",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode=\"N/A\",ftp_cmd=\"N/A\",cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types=\"none\",ml_svm_accuracy=\"none\",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method=\"none\",owasp_top10=\"A3:2017-Sensitive Data Exposure\",bot_info=\"none\",matched_field=\"header\",\"matched_pattern=\"\"X-Powered-By: Servlet/3.1\"\"\"\n") + "{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=attack,pri=alert,main_type=\"Signature Detection\",sub_type=\"Information Disclosure\",trigger_policy=\"\",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy=\"MobApp_policy\",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,\"http_url=\"\"/mfp/api/abc\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)\"\"\",http_session_id=ASDSADSA,\"msg=\"\"HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only\"\"\",signature_subclass=\"HTTP Header Leakage\",signature_id=\"080200004\",signature_cve_id=\"N/A\",srccountry=\"Kuwait\",content_switch_name=\"none\",server_pool_name=\"MObApp_pool\",false_positive_mitigation=\"none\",\"user_name=\"\"Unknown\"\"\",monitor_status=\"Enabled\",\"http_refer=\"\"none\"\"\",http_version=\"1.x\",dev_id=\"none\",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode=\"N/A\",ftp_cmd=\"N/A\",cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types=\"none\",ml_svm_accuracy=\"none\",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method=\"none\",owasp_top10=\"A3:2017-Sensitive Data Exposure\",bot_info=\"none\",matched_field=\"header\",\"matched_pattern=\"\"X-Powered-By: Servlet/3.1\"\"\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) From 3a2da357aed79a86cfd037897297f6e15959d341 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 13 Feb 2020 22:34:57 -0800 Subject: [PATCH 05/11] Update comment on TZ parser * Update comment on TZ parser --- package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl index bf3bb74..564f4d9 100644 --- a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl @@ -25,13 +25,15 @@ log { kv-parser(prefix(".kv.") pair-separator(",") template("${MSGHDR} ${MSG}")); }; +# Fetch timezone from timezone nv pair and parse unique format (no zero padding, e.g. "-8:00" rather than "-08:00" +# Reformat to "-0800" rewrite { subst('.*([\+-]\d+:\d+).*', $1, value(".kv.timezone")); subst('([\+-])(\d)(?=:):(\d+)', "${1}0${2}${3}", value(".kv.timezone")); subst('([\+-])(\d+):(\d+)', "$1$2$3", value(".kv.timezone")); }; - #2020-02-12,23:13:33 + #2020-02-12,23:13:33-0800 #parse the date parser { date-parser( From 600361c864fc8e6b717f99376f644349a273e640 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 14 Feb 2020 08:43:20 -0800 Subject: [PATCH 06/11] Update subst regex to allow for colon in TZ * Simplify subst regex as date-parser() allows for both `-08:00` and `-0800` TZ offsets --- package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl index 564f4d9..4b146f1 100644 --- a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl @@ -26,11 +26,10 @@ log { }; # Fetch timezone from timezone nv pair and parse unique format (no zero padding, e.g. "-8:00" rather than "-08:00" -# Reformat to "-0800" +# Reformat to "-08:00" rewrite { subst('.*([\+-]\d+:\d+).*', $1, value(".kv.timezone")); - subst('([\+-])(\d)(?=:):(\d+)', "${1}0${2}${3}", value(".kv.timezone")); - subst('([\+-])(\d+):(\d+)', "$1$2$3", value(".kv.timezone")); + subst('([\+-])(\d)(?=:)(:\d+)', "${1}0${2}${3}", value(".kv.timezone")); }; #2020-02-12,23:13:33-0800 From 092c6521dff91990667a3436cfc81405fbf03973 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 20 Feb 2020 13:47:00 -0800 Subject: [PATCH 07/11] Consoldate log paths for Fortniet; bug fixes * Consolidate two log paths for FortiOS and Fortiweb into one * Add microsecond time parsing to Meraki * Update Cisco ACS test with new epoch test (included due to previous commit --- docs/gettingstarted/byoe-rhel7.md | 10 +- docs/sources/Fortinet/index.md | 76 ++++++++--- package/etc/conf.d/filters/cisco/meraki.conf | 4 +- .../etc/conf.d/filters/fortinet/fortinet.conf | 8 ++ .../etc/conf.d/filters/fortinet/fortios.conf | 3 - .../etc/conf.d/filters/fortinet/fortiweb.conf | 3 - .../conf.d/log_paths/lp-fortinet.conf.tmpl | 118 ++++++++++++++++++ .../log_paths/lp-fortinet_fortios.conf.tmpl | 61 --------- .../log_paths/lp-fortinet_fortiweb.conf.tmpl | 79 ------------ tests/test_cisco_acs.py | 70 ++++++++--- tests/test_fortinet_ngfw.py | 76 ++++++++--- tests/test_fortinet_web.py | 80 +++++++++--- 12 files changed, 370 insertions(+), 218 deletions(-) create mode 100644 package/etc/conf.d/filters/fortinet/fortinet.conf delete mode 100644 package/etc/conf.d/filters/fortinet/fortios.conf delete mode 100755 package/etc/conf.d/filters/fortinet/fortiweb.conf create mode 100644 package/etc/conf.d/log_paths/lp-fortinet.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/lp-fortinet_fortios.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 544178d..1aeea8d 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -188,4 +188,12 @@ the data. In other cases, a unique listening port is required for certain devic For collection of such sources we provide a means of dedicating a unique listening port to a specific source. Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology -in use. \ No newline at end of file +in use. + +## Unique Ports for Device "Families" + +Certain technology "families", such as CEF and Fortinet, are handled by a single log path in SC4S. To set unique ports for individual +devices in a family (e.g. one each for Fortiweb and FortiOS), the container version of SC4S uses "container networking" (detailed +in the source document for the respective device families). This, of course, is not avaialble in BYOE. For this reason, the syslog-ng source +configuration for the extra ports that need to be mapped will need to be added manually to either the template or final "conf" version of the +respective log path file. \ No newline at end of file diff --git a/docs/sources/Fortinet/index.md b/docs/sources/Fortinet/index.md index eae3613..147d1d6 100644 --- a/docs/sources/Fortinet/index.md +++ b/docs/sources/Fortinet/index.md @@ -1,5 +1,41 @@ # Vendor - Fortinet +There are two Fortinet device flavors (FortiOS and Fortiweb) that are supported by a single log path +in SC4S. Therefore, both Fortinet variants use "FORTINET" as the core of the unique port and +archive environment variable settings (rather than a unique one per product), as the Fortinet log path +handles either variant sending events to SC4S. Therefore, the FORTINET environment variables for unique +port, archive, etc. should be set only _once_, regardless of how many unique ports or Fortinet appliance +variants are in use. + +If your deployment has multiple Fortinet devices that send to more than one port, +set the FORTINET unique port variable(s) to just one of the ports in use. Then, map the others with +container networking to the port chosen, similar to the way default ports are configured (see the +"Getting Started" runtime documents for more details). + +Example: If you have three Fortinet devices, sending on TCP ports 2000,2001, and 2002, set +`SC4S_LISTEN_FORTINET_TCP_PORT=2000`. Then, change the unit/compose files to route the three external +ports to the single port 2000 on the container. Here is the example for podman/systemd: + +``` +ExecStart=/usr/bin/podman -p 514:514 -p 514:514/udp -p 6514:6514 -p 2000-2002:2000 \ +``` + +or this, for docker-compose/swarm installations: + +``` +# Comment the following line out if using docker-compose + mode: host + - target: 2000 + published: 2000-2002 + protocol: tcp +``` + +These changes will route all three ports to TCP port 2000 inside the container, and the single Fortinet log +path will properly process data from all three devices. + +The source documentation included below includes settings for both appliance types (FortiOS and Fortigate) +supported by SC4S. + ## Product - Fortigate | Ref | Link | @@ -12,10 +48,10 @@ | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| fgt_log | The catch all sourcetype is not used | -| fgt_traffic | None | -| fgt_utm | None | -| fgt_event | None +| fgt_log | Catch-all sourcetype; not used by the TA | +| fgt_traffic | None | +| fgt_utm | None | +| fgt_event | None | ### Sourcetype and Index Configuration @@ -73,12 +109,15 @@ end ### Options +* NOTE: Remember to set the variable(s) below only _once_, regardless of how many unique ports and/or Fortinet device types +are in use. See the introductory note above for more details. + | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_FORTINET_FORTIOS | no | Enable archive to disk for this specific source | -| SC4S_DEST_FORTINET_FORTIOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_FORTINET_TCP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_FORTINET_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_FORTINET | no | Enable archive to disk for this specific source | +| SC4S_DEST_FORTINET_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -119,10 +158,10 @@ Verify timestamp, and host values match as expected | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| fweb_log | The catch all sourcetype is not used | -| fwb_traffic | None | -| fwb_attack | None | -| fwb_event | None +| fgt_log | Catch-all sourcetype; not used by the TA | +| fwb_traffic | None | +| fwb_attack | None | +| fwb_event | None | ### Sourcetype and Index Configuration @@ -174,12 +213,15 @@ end ### Options +* NOTE: Remember to set the variable(s) below only _once_, regardless of how many unique ports and/or Fortinet device types +are in use. See the introductory note above for more details. + | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_FORTINET_FORTIWEB | no | Enable archive to disk for this specific source | -| SC4S_DEST_FORTINET_FORTIWEB_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_FORTINET_TCP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_FORTINET_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_FORTINET | no | Enable archive to disk for this specific source | +| SC4S_DEST_FORTINET_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -195,4 +237,4 @@ Verify timestamp, and host values match as expected index= (sourcetype=fwb_log OR sourcetype=fwb_traffic OR sourcetype=fwb_attack OR sourcetype=fwb_event) ``` -Verify timestamp, and host values match as expected \ No newline at end of file +Verify timestamp, and host values match as expected diff --git a/package/etc/conf.d/filters/cisco/meraki.conf b/package/etc/conf.d/filters/cisco/meraki.conf index bf1eec3..f0c9c13 100644 --- a/package/etc/conf.d/filters/cisco/meraki.conf +++ b/package/etc/conf.d/filters/cisco/meraki.conf @@ -14,8 +14,8 @@ parser p_cisco_meraki { ); }; parser { - date-parser(format('%s') - template("${EPOCH}") + date-parser(format('%s.%f') + template("${EPOCH}.${TIMESECFRAC}") flags(guess-timezone) ); }; diff --git a/package/etc/conf.d/filters/fortinet/fortinet.conf b/package/etc/conf.d/filters/fortinet/fortinet.conf new file mode 100644 index 0000000..e74833f --- /dev/null +++ b/package/etc/conf.d/filters/fortinet/fortinet.conf @@ -0,0 +1,8 @@ +filter f_fortinet { + message('devid=\"?F[G|W|6K].+type=\"?(traffic|utm|event)') or + message('device_id=\"?FV.+type=\"?(traffic|attack|event)'); +}; + +filter f_fortinet_fortiweb { + message('device_id=\"?FV.+type=\"?(traffic|attack|event)'); +}; diff --git a/package/etc/conf.d/filters/fortinet/fortios.conf b/package/etc/conf.d/filters/fortinet/fortios.conf deleted file mode 100644 index 6c08a87..0000000 --- a/package/etc/conf.d/filters/fortinet/fortios.conf +++ /dev/null @@ -1,3 +0,0 @@ -filter f_fortinet_fortios { - message('devid=\"?F[G|W|6K].+type=\"?(traffic|utm|event)'); -}; diff --git a/package/etc/conf.d/filters/fortinet/fortiweb.conf b/package/etc/conf.d/filters/fortinet/fortiweb.conf deleted file mode 100755 index 8a3654e..0000000 --- a/package/etc/conf.d/filters/fortinet/fortiweb.conf +++ /dev/null @@ -1,3 +0,0 @@ -filter f_fortinet_fortiweb { - message('device_id=\"?FV.+type=\"?(traffic|attack|event)'); -}; diff --git a/package/etc/conf.d/log_paths/lp-fortinet.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet.conf.tmpl new file mode 100644 index 0000000..7435657 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-fortinet.conf.tmpl @@ -0,0 +1,118 @@ +# Fortinet Fortigate and Fortiweb +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "FORTINET" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log { + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for FORTINET traffic + source (s_FORTINET); + flags (final); + }; +{{- end}} + channel { + # Listen on the default port (typically 514) for FORTINET traffic + source (s_DEFAULT); + filter(f_is_rfc3164); + filter(f_fortinet); + flags(final); + }; + }; + + parser { + kv-parser(prefix(".kv.") template("${LEGACY_MSGHDR} ${MSG}")); + }; + if { + filter(f_fortinet_fortiweb); + # Fetch timezone from timezone nv pair and parse unique format (no zero padding, e.g. "-8:00" rather than "-08:00" + # Reformat to "-08:00" + rewrite { + subst('.*([\+-]\d+:\d+).*', $1, value(".kv.timezone")); + subst('([\+-])(\d)(?=:)(:\d+)', "${1}0${2}${3}", value(".kv.timezone")); + }; + parser { + date-parser( + format("%Y-%m-%d:%H:%M:%S%z") + template('${.kv.date}:${.kv.time}${.kv.timezone}') + flags(guess-timezone) + ); + }; + } elif { + filter { match('.{5}' value (".kv.tz")) }; + parser { + date-parser( + format("%Y-%m-%d:%H:%M:%S%z") + template("${.kv.date}:${.kv.time}${.kv.tz}") + flags(guess-timezone) + ); + }; + } elif { + parser { + date-parser( + format("%Y-%m-%d:%H:%M:%S") + template("${.kv.date}:${.kv.time}") + time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) + flags(guess-timezone) + ); + }; + } else { + rewrite { set("date/time parser failed", value("fields.sc4s_error")); }; + }; + +# Fortiweb + if { + filter(f_fortinet_fortiweb); + rewrite { + set("fortigate_fortiweb", value("fields.sc4s_vendor_product")); + set("${.kv.devname}", value("HOST")); + }; + if (match("traffic" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fwb_traffic"), index("netfw"))}; + parser {p_add_context_splunk(key("fortinet_fortiweb_traffic")); }; + } elif (match("attack" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fwb_attack"), index("netids"))}; + parser {p_add_context_splunk(key("fortinet_fortiweb_attack")); }; + } elif (match("event" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fwb_event"), index("netops"))}; + parser {p_add_context_splunk(key("fortinet_fortiweb_event")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("fwb_log"), index("netops"))}; + parser {p_add_context_splunk(key("fortinet_fortiweb_log")); }; + }; +#FortiOS + } else { + rewrite { + set("fortigate_fortios", value("fields.sc4s_vendor_product")); + set("${.kv.devname}", value("HOST")); + }; + if (match("traffic" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"))}; + parser {p_add_context_splunk(key("fortinet_fortios_traffic")); }; + } elif (match("utm" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"))}; + parser {p_add_context_splunk(key("fortinet_fortios_utm")); }; + } elif (match("event" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"))}; + parser {p_add_context_splunk(key("fortinet_fortios_event")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"))}; + parser {p_add_context_splunk(key("fortinet_fortios_log")); }; + }; + }; + + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_FORTINET_HEC" "no")) }} + destination(d_hec); +{{- end}} + + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_FORTINET" "no")) }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; diff --git a/package/etc/conf.d/log_paths/lp-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet_fortios.conf.tmpl deleted file mode 100644 index aba6936..0000000 --- a/package/etc/conf.d/log_paths/lp-fortinet_fortios.conf.tmpl +++ /dev/null @@ -1,61 +0,0 @@ -# Fortinet Fortios -{{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "FORTINET_FORTIOS" "parser" "rfc3164" }} -{{- tmpl.Exec "t/source_network.t" $context }} - -log { - junction { -{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT")) }} - channel { - # Listen on the specified dedicated port(s) for FORTINET_FORTIOS traffic - source (s_FORTINET_FORTIOS); - flags (final); - }; -{{- end}} - channel { - # Listen on the default port (typically 514) for FORTINET_FORTIOS traffic - source (s_DEFAULT); - filter(f_is_rfc3164); - filter(f_fortinet_fortios); - flags(final); - }; - }; - - parser { - kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); - date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); - }; - - rewrite { - set("${.kv.devname}", value("HOST")); - set("fortigate_fortios", value("fields.sc4s_vendor_product")); - }; - - if (match("traffic" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"))}; - parser {p_add_context_splunk(key("fortinet_fortios_traffic")); }; - } elif (match("utm" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"))}; - parser {p_add_context_splunk(key("fortinet_fortios_utm")); }; - } elif (match("event" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"))}; - parser {p_add_context_splunk(key("fortinet_fortios_event")); }; - } else { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"))}; - parser {p_add_context_splunk(key("fortinet_fortios_log")); }; - }; - - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; - -{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_FORTINET_FORTIOS_HEC" "no")) }} - destination(d_hec); -{{- end}} - - -{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_FORTINET_FORTIOS" "no")) }} - destination(d_archive); -{{- end}} - - flags(flow-control,final); -}; diff --git a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl deleted file mode 100644 index 4b146f1..0000000 --- a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl +++ /dev/null @@ -1,79 +0,0 @@ -# Fortinet FortiWeb -{{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "FORTINET_FORTIWEB" "parser" "rfc3164" }} -{{- tmpl.Exec "t/source_network.t" $context }} - -log { - junction { -{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TLS_PORT")) }} - channel { - # Listen on the specified dedicated port(s) for FORTINET_FORTIWEB traffic - source (s_FORTINET_FORTIWEB); - flags (final); - }; -{{- end}} - channel { - # Listen on the default port (typically 514) for FORTINET_FORTIWEB traffic - source (s_DEFAULT); - filter(f_is_rfc3164); - filter(f_fortinet_fortiweb); - flags(final); - }; - }; - - parser { - kv-parser(prefix(".kv.") pair-separator(",") template("${MSGHDR} ${MSG}")); - }; - -# Fetch timezone from timezone nv pair and parse unique format (no zero padding, e.g. "-8:00" rather than "-08:00" -# Reformat to "-08:00" - rewrite { - subst('.*([\+-]\d+:\d+).*', $1, value(".kv.timezone")); - subst('([\+-])(\d)(?=:)(:\d+)', "${1}0${2}${3}", value(".kv.timezone")); - }; - - #2020-02-12,23:13:33-0800 - #parse the date - parser { - date-parser( - format("%Y-%m-%d,%H:%M:%S%z") - template('$(substr "$LEGACY_MSGHDR$MSG" "0" "19")${.kv.timezone}') - time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) - flags(guess-timezone) - ); - }; - - rewrite { - set("${.kv.devname}", value("HOST")); - set("fortigate_fortiweb", value("fields.sc4s_vendor_product")); - }; - - if (match("traffic" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fwb_traffic"), index("netfw"))}; - parser {p_add_context_splunk(key("fortinet_fortiweb_traffic")); }; - } elif (match("attack" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fwb_attack"), index("netids"))}; - parser {p_add_context_splunk(key("fortinet_fortiweb_attack")); }; - } elif (match("event" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fwb_event"), index("netops"))}; - parser {p_add_context_splunk(key("fortinet_fortiweb_event")); }; - } else { - rewrite { r_set_splunk_dest_default(sourcetype("fwb_log"), index("netops"))}; - parser {p_add_context_splunk(key("fortinet_fortiweb_log")); }; - }; - - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_3164))" value("MSG")); }; - -{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORTINET_FORTIWEB_HEC" "no") | conv.ToBool) }} - destination(d_hec); -{{- end}} - - -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORTINET_FORTIWEB") }} - destination(d_archive); -{{- end}} - - flags(flow-control,final); -}; - diff --git a/tests/test_cisco_acs.py b/tests/test_cisco_acs.py index 283c696..7c5e3d0 100644 --- a/tests/test_cisco_acs.py +++ b/tests/test_cisco_acs.py @@ -11,25 +11,55 @@ from .sendmessage import * from .splunkutils import * -env = Environment(extensions=['jinja2_time.TimeExtension']) +env = Environment() -def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) +# Helper functions +# Insert a character at given location in string (e.g space between ms and TZ offset 2020-02-12 12:46:39.323 -08:00) +def insert_char(string, char, integer): + return string[0:integer] + char + string[integer:] -# Generate an ISO 8601 compliant timestamp with local timezone offset (2020-02-12 12:46:39.323-08:00) - dt = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).astimezone().isoformat(sep=' ', timespec='milliseconds') +# Generate current datetime structure +dt = datetime.datetime.now() -# Function to insert a space between the time and TZ offset (2020-02-12 12:46:39.323 -08:00) - def insert_space(string, integer): - return string[0:integer] + ' ' + string[integer:] +# Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) +# See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python +iso = dt.astimezone().isoformat(sep='T', timespec='milliseconds') + +# Generate an BSD-style (RFC 3164) compliant timestamp with no timezone (Oct 25 13:08:00) +bsd = dt.strftime("%b %d %H:%M:%S") + +# Other variants of timestamps needed for this log sample +time = dt.strftime("%H:%M:%S.%f")[:-3] +date = dt.strftime("%Y-%m-%d") +# Insert colon in tzoffset string; normally just 'tzoffset = dt.astimezone().strftime("%z")' +# Could use helper function above; e.g. 'tzoffset = insert_char(dt.astimezone().strftime("%z"), ":", 3)' +tzoffset = dt.astimezone().strftime("%z")[0:3] + ":" + dt.astimezone().strftime("%z")[3:] + +# Derive epoch timestamp for use in search string +# NOTE: There are caveats with 'strftime("%s")', see references below + +# See https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime +# See https://docs.python.org/3/library/datetime.html#datetime-objects +# Basically: Don't use 'utcnow()' + +# Strict way to get epoch as a string (rather than float) avoiding naive objects +# epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') + +# Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK +# Trim last 3 digits of microsecond resolution to obtain milliseconds +epoch = dt.astimezone().strftime("%s.%f")[:-3] + +def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'local', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0765855540 1 0 {{ dt }} 0178632943 5202 NOTICE Device-Administration: Command Authorization succeeded, ACSVersion=acs-5.8.1.4-B.462.x86_64, ConfigVersionId=16489, Device IP Address=10.0.0.93, DestinationIPAddress=10.0.0.10, DestinationPort=49, UserName=nsdevman, CmdSet=[ CmdAV=show CmdArgAV=vpn-sessiondb CmdArgAV=full CmdArgAV=ra-ikev2-ipsec ], Protocol=Tacacs, MatchedCommandSet=fw3, RequestLatency=11, Type=Authorization, Privilege-Level=15, Authen-Type=ASCII, Service=None, User=nsdevman, Port=443, Remote-Address=10.0.0.15, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=mnsvdcfpiuac03/359448835/9871764, AuthenticationIdentityStore=AD1, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedCommandSet=fw3, IdentityGroup=IdentityGroup:All Groups:SystemID, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15004 , Step=15013 , Step=24210 , Step=24212 , Step=24432 , Step=24325 , Step=24313 , Step=24319 , Step=24323 , Step=24420 , Step=24355 , Step=24416 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15018 , Step=13024 , Step=13034 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceName=devicenamehere, NetworkDeviceGroups=Device Type:All Device Types:Firewall:Cisco Systems:Firewall:ASA5545, NetworkDeviceGroups=Location:All Locations:MN, ServiceSelectionMatchedRule=TACACS, IdentityPolicyMatchedRule=Firewall, AuthorizationPolicyMatchedRule=nsdevman, AD-User-Candidate-Identities=nsdevman@ent.example.corp, AD-User-DNS-Domain=ent.example.corp, AD-User-NetBios-Name=AD-ENT, AD-User-Resolved-Identities=nsdevman@ent.example.corp, AD-User-Join-Point=ENT.example.CORP, AD-User-Resolved-DNs=CN=nsdevman\,OU=Service Accounts\,OU=CAO\,OU=ENT\,DC=ent\,DC=wfb\,DC=example\,DC=corp, StepData=10=nsdevman, StepData=11=ent.example.corp, StepData=12=example.corp, StepData=15=ent.example.corp, AD-Domain=ent.example.corp, IdentityAccessRestricted=false, UserIdentityGroup=IdentityGroup:All Groups:SystemID, Cisco-Firewall=Superuser, Firewall=Superuser, NetSec-CSM=User, NetSec-Logging=Engineer, Response={Type=Authorization; Author-Reply-Status=PassAdd; ExternalIdentityStoreName=AD1; }\n") - message = mt.render(mark="<165>", host=host, dt=insert_space(dt,23)) + "{{ mark }} {{ bsd }} {{ host }} CSCOacs_Passed_Authentications 0765855540 1 0 {{ date }} {{ time }} {{ tzoffset }} 0178632943 5202 NOTICE Device-Administration: Command Authorization succeeded, ACSVersion=acs-5.8.1.4-B.462.x86_64, ConfigVersionId=16489, Device IP Address=10.0.0.93, DestinationIPAddress=10.0.0.10, DestinationPort=49, UserName=nsdevman, CmdSet=[ CmdAV=show CmdArgAV=vpn-sessiondb CmdArgAV=full CmdArgAV=ra-ikev2-ipsec ], Protocol=Tacacs, MatchedCommandSet=fw3, RequestLatency=11, Type=Authorization, Privilege-Level=15, Authen-Type=ASCII, Service=None, User=nsdevman, Port=443, Remote-Address=10.0.0.15, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=mnsvdcfpiuac03/359448835/9871764, AuthenticationIdentityStore=AD1, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedCommandSet=fw3, IdentityGroup=IdentityGroup:All Groups:SystemID, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15004 , Step=15013 , Step=24210 , Step=24212 , Step=24432 , Step=24325 , Step=24313 , Step=24319 , Step=24323 , Step=24420 , Step=24355 , Step=24416 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15018 , Step=13024 , Step=13034 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceName=devicenamehere, NetworkDeviceGroups=Device Type:All Device Types:Firewall:Cisco Systems:Firewall:ASA5545, NetworkDeviceGroups=Location:All Locations:MN, ServiceSelectionMatchedRule=TACACS, IdentityPolicyMatchedRule=Firewall, AuthorizationPolicyMatchedRule=nsdevman, AD-User-Candidate-Identities=nsdevman@ent.example.corp, AD-User-DNS-Domain=ent.example.corp, AD-User-NetBios-Name=AD-ENT, AD-User-Resolved-Identities=nsdevman@ent.example.corp, AD-User-Join-Point=ENT.example.CORP, AD-User-Resolved-DNs=CN=nsdevman\,OU=Service Accounts\,OU=CAO\,OU=ENT\,DC=ent\,DC=wfb\,DC=example\,DC=corp, StepData=10=nsdevman, StepData=11=ent.example.corp, StepData=12=example.corp, StepData=15=ent.example.corp, AD-Domain=ent.example.corp, IdentityAccessRestricted=false, UserIdentityGroup=IdentityGroup:All Groups:SystemID, Cisco-Firewall=Superuser, Firewall=Superuser, NetSec-CSM=User, NetSec-Logging=Engineer, Response={Type=Authorization; Author-Reply-Status=PassAdd; ExternalIdentityStoreName=AD1; }\n") + message = mt.render(mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search earliest=-1m@m latest=+1m@m index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" | head 11") - search = st.render(host=host) + st = env.from_string("search _time={{ epoch }} index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" | head 11") + search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -50,17 +80,21 @@ def insert_space(string, integer): return string[0:integer] + ' ' + string[integer:] mt = env.from_string( - "{{ mark }} {% now 'local', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 0 {{ dt }} 0000008450 5203 NOTICE Device-Administration: Session Authorization succeeded, ACSVersion=acs-5.2.0.26-B.3075, ConfigVersionId=117, Device IP Address=192.168.26.137, UserName=edward, CmdSet=[ CmdAV= ], Protocol=Tacacs, RequestLatency=10, NetworkDeviceName=switch, Type=Authorization, Privilege-Level=1, Authen-Type=ASCII, Service=Login, User=edward, Port=tty2, Remote-Address=10.78.167.190, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=ACS41/101085887/112, AuthenticationIdentityStore=Internal Users, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedShellProfile=Permit Access, IdentityGroup=IdentityGroup:All Groups, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15006 , Step=15013 , Step=24210 , Step=24212 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15017 , Step=13034 ,\n") - message = mt.render(mark="<165>", host=host, dt=insert_space(dt,23)) + "{{ mark }} {{ bsd }} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 0 {{ date }} {{ time }} {{ tzoffset }} 0000008450 5203 NOTICE Device-Administration: Session Authorization succeeded, ACSVersion=acs-5.2.0.26-B.3075, ConfigVersionId=117, Device IP Address=192.168.26.137, UserName=edward, CmdSet=[ CmdAV= ], Protocol=Tacacs, RequestLatency=10, NetworkDeviceName=switch, Type=Authorization, Privilege-Level=1, Authen-Type=ASCII, Service=Login, User=edward, Port=tty2, Remote-Address=10.78.167.190, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=ACS41/101085887/112, AuthenticationIdentityStore=Internal Users, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedShellProfile=Permit Access, IdentityGroup=IdentityGroup:All Groups, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15006 , Step=15013 , Step=24210 , Step=24212 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15017 , Step=13034 ,\n") + message = mt.render(mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + # Generate new datetime for second message; not used in log path parser so actually could be anything + dt = datetime.datetime.now() + second_bsd = dt.strftime("%b %d %H:%M:%S") + mt = env.from_string( - "{{ mark }} {% now 'local', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 1 Step=13015 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceGroups=s1Migrated_NDGs:All s1Migrated_NDGs, NetworkDeviceGroups=Device Type:All Device Types, NetworkDeviceGroups=Location:All Locations, ServiceSelectionMatchedRule=Rule-2, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Rule-0, Action=Login, Privilege-Level=1, Authen-Type=ASCII, Service=Login, Remote-Address=10.78.167.190, UserIdentityGroup=IdentityGroup:All\n") - message = mt.render(mark="<165>", host=host) + "{{ mark }} {{ second_bsd }} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 1 Step=13015 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceGroups=s1Migrated_NDGs:All s1Migrated_NDGs, NetworkDeviceGroups=Device Type:All Device Types, NetworkDeviceGroups=Location:All Locations, ServiceSelectionMatchedRule=Rule-2, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Rule-0, Action=Login, Privilege-Level=1, Authen-Type=ASCII, Service=Login, Remote-Address=10.78.167.190, UserIdentityGroup=IdentityGroup:All\n") + message = mt.render(mark="<165>", second_bsd=second_bsd, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search earliest=-1m@m latest=+1m@m index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" \"Step=13015\" | head 11") - search = st.render(host=host) + st = env.from_string("search _time={{ epoch }} index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" \"Step=13015\"") + search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index bdfeb88..95a8d39 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -4,29 +4,72 @@ # license that can be found in the LICENSE-BSD2 file or at # https://opensource.org/licenses/BSD-2-Clause import random +import datetime from jinja2 import Environment from .sendmessage import * from .splunkutils import * -env = Environment(extensions=['jinja2_time.TimeExtension']) +# env = Environment(extensions=['jinja2_time.TimeExtension']) +env = Environment() -# <111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 +# Helper functions +# Insert a character at given location in string (e.g space between ms and TZ offset 2020-02-12 12:46:39.323 -08:00) +def insert_char(string, char, integer): + return string[0:integer] + char + string[integer:] + +# Function to remove leading zero from TZ offsets less than 10 hours +def removeZero(tz): + return re.sub(r'\b0+(\d)(?=:)', r'\1', tz) + +# Generate current datetime structure +dt = datetime.datetime.now() + +# Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) +# See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python +iso = dt.astimezone().isoformat(sep='T', timespec='milliseconds') +# Generate an BSD-style (RFC 3164) compliant timestamp with no timezone (Oct 25 13:08:00) +bsd = dt.strftime("%b %d %H:%M:%S") + +# Other variants of timestamps needed for this log sample +time = dt.strftime("%H:%M:%S") +date = dt.strftime("%Y-%m-%d") +# Insert colon in tzoffset string; normally just 'tzoffset = dt.astimezone().strftime("%z")' +# Could use helper function above; e.g. 'tzoffset = insert_char(dt.astimezone().strftime("%z"), ":", 3)' +# tzoffset = dt.astimezone().strftime("%z")[0:3] + ":" + dt.astimezone().strftime("%z")[3:] +tzoffset = dt.astimezone().strftime("%z") + +# Derive epoch timestamp for use in search string +# NOTE: There are caveats with 'strftime("%s")', see references below + +# See https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime +# See https://docs.python.org/3/library/datetime.html#datetime-objects +# Basically: Don't use 'utcnow()' + +# Strict way to get epoch as a string (rather than float) avoiding naive objects +# epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') +# Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK +# Trim last 3 or 7 digits of microsecond resolution to obtain milliseconds or whole seconds respectively +epoch = dt.astimezone().strftime("%s.%f")[:-7] +# <111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'local', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") - message = mt.render(mark="<13>", host=host) +# "{{ mark }} {{ bsd }} fortigate date={{ date }} time={{ time }} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice tz=\"{{ tzoffset }}\" vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") + "{{ mark }} {{ bsd }} fortigate date={{ date }} time={{ time }} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") + message = mt.render(mark="<111>", bsd=bsd, date=date, time=time, tzoffset=tzoffset, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search earliest=-1m@m latest=+1m@m index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") - search = st.render(host=host) + st = env.from_string( +# "search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\"") + "search index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\"") + search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -37,19 +80,18 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk, setup assert resultCount == 1 # <111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 - - def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'local', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") - message = mt.render(mark="<13>", host=host) + "{{ mark }} {{ bsd }} fortigate date={{ date }} time={{ time }} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") + message = mt.render(mark="<111>", bsd=bsd, date=date, time=time, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search earliest=-1m@m latest=+1m@m index=netfw host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") - search = st.render(host=host) + st = env.from_string( + "search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"fgt_traffic\"") + search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -60,20 +102,18 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk, set assert resultCount == 1 # <111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:21:40 logver=52 devname=US-Corp_Main1 devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user="" srcip=172.30.16.119 srcport=53235 srcintf="Internal" dstip=114.112.67.75 dstport=80 dstintf="External-SDC" proto=6 service=HTTP hostname="popo.wan.ijinshan.com" profile="scan" action=passthrough reqtype=direct url="/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl" sentbyte=525 rcvdbyte=325 direction=outgoing msg="URL belongs to an allowed category in policy" method=domain cat=52 catdesc="Information Technology" - - def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'local', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") - message = mt.render(mark="<13>", host=host) + "{{ mark }} {{ bsd }} fortigate date={{ date }} time={{ time }} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") + message = mt.render(mark="<111>", bsd=bsd, date=date, time=time, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search earliest=-1m@m latest=+1m@m index=netids host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") - search = st.render(host=host) + "search _time={{ epoch }} index=netids host=\"{{ host }}\" sourcetype=\"fgt_utm\"") + search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_fortinet_web.py b/tests/test_fortinet_web.py index 97bd21d..24ed1ac 100644 --- a/tests/test_fortinet_web.py +++ b/tests/test_fortinet_web.py @@ -4,26 +4,71 @@ # license that can be found in the LICENSE-BSD2 file or at # https://opensource.org/licenses/BSD-2-Clause import random +import datetime +import re from jinja2 import Environment from .sendmessage import * from .splunkutils import * -env = Environment(extensions=['jinja2_time.TimeExtension']) +env = Environment() -#<111> 2020-02-12,23:13:33,devname=FortiWeb-A,log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\" +# Function to insert a colon between the time and TZ offset (from -0800 to -08:00) +def insert_colon(string, integer): + return string[0:integer] + ':' + string[integer:] + +# Function to remove leading zero from TZ offsets less than 10 hours +def removeZero(tz): + return re.sub(r'\b0+(\d)(?=:)', r'\1', tz) + + +# Generate current datetime structure +dt = datetime.datetime.now() + +# Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) +# See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python +iso = dt.astimezone().isoformat(sep='T', timespec='milliseconds') +# Generate an BSD-style (RFC 3164) compliant timestamp with no timezone (Oct 25 13:08:00) +bsd = dt.strftime("%b %d %H:%M:%S") + +# Other variants of timestamps needed for this log sample +time = dt.strftime("%H:%M:%S") +date = dt.strftime("%Y-%m-%d") +# Insert colon in tzoffset string; normally just 'tzoffset = dt.astimezone().strftime("%z")' +# Could use helper function above; e.g. 'tzoffset = insert_char(dt.astimezone().strftime("%z"), ":", 3)' +# tzoffset = dt.astimezone().strftime("%z")[0:3] + ":" + dt.astimezone().strftime("%z")[3:] +tzoffset = dt.astimezone().strftime("%z") + +# Derive epoch timestamp for use in search string +# NOTE: There are caveats with 'strftime("%s")', see references below + +# See https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime +# See https://docs.python.org/3/library/datetime.html#datetime-objects +# Basically: Don't use 'utcnow()' + +# Strict way to get epoch as a string (rather than float) avoiding naive objects +# epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') + +# Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK +# Trim last 3 or 7 digits of microsecond resolution to obtain milliseconds or whole seconds respectively +epoch = dt.astimezone().strftime("%s.%f")[:-7] + +# Generate Timezone in "-08:00" format (with HH:MM colon; no leading zero for hour < 10) +tzoffset = removeZero(insert_colon((dt.astimezone().strftime("%z")), 3)) + +# <111> Oct 25 13:08:00 fortiweb date=2013-10-07 time=11:30:53 devname=FortiWeb-A log_id=10000017 msg_id=000000001117 device_id=FVVM040000010871 vd="root" timezone="(GMT-5:00)Eastern Time(US & Canada)" type=event subtype="system" pri=information trigger_policy="" user=admin ui=GUI action=login status=success msg="User admin login successfully from GUI(172.20.120.47)" def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\"\n") - message = mt.render(mark="<13>", host=host) + "{{ mark }} {{ bsd }} fortiweb date={{ date }} time={{ time }} devname={{ host }} log_id=10000017 msg_id=000000001117 device_id=FVVM040000010871 vd=\"root\" timezone=\"(GMT{{ tzoffset }})Region,City\" type=event subtype=\"system\" pri=information trigger_policy=\"\" user=admin ui=GUI action=login status=success msg=\"User admin login successfully from GUI(172.20.120.47)\"") + message = mt.render(mark="<111>", bsd=bsd, host=host, time=time, date=date, tzoffset=tzoffset) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"fwb_event\" | head 2") - search = st.render(host=host) + st = env.from_string("search _time={{epoch}} index=netops sourcetype=\"fwb_event\"") + search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -32,17 +77,19 @@ def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup record_property("message", message) assert resultCount == 1 -#<111> 2020-02-12,23:16:41,devname=FortiWeb-A,log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd="root","timezone=""(GMT+3:00)Kuwait,Riyadh""",type=traffic,subtype="https",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,"http_url=""/publish/templates/images/bluebottom.jpg""","http_host=""splunk.infigo.hr""","http_agent=""Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36""",http_retcode=200,"msg=""HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80""",original_srccountry="Saudi Arabia",srccountry="Saudi Arabia",content_switch_name="none",server_pool_name="PHOME","user_name=""Unknown""","http_refer=""https://splunk.infigo.hr/publish/templates/CSS/sc4s.css""",http_version="1.x",dev_id=none,cipher_suite="TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" + +# <111> Oct 25 13:08:00 fortiweb date=2013-10-07 time=11:30:53 devname=FortiWeb-A log_id=30000000 msg_id=000001351251 device_id=FV-1KD3A14800059 vd="root" timezone="(GMT-8:00)Pacific Time(US&Canada)" type=traffic subtype="http" pri=notice proto=tcp service=http status=success reason=none policy=Auto-policy src=10.0.8.103 src_port=8142 dst=10.20.8.22 dst_port=80 http_request_time=0 http_response_time=0 http_request_bytes=444 http_response_bytes=401 http_method=get http_url="/" http_host="10.0.8.22" http_agent="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; " http_retcode=200 msg="HTTP GET request from 10.0.8.103:8142 to 10.20.8.22:80" srccountry="Reserved" content_switch_name="testa" server_pool_name="Auto-ServerFarm" def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT-8:00)Pacific Time(US&Canada)\"\"\",type=traffic,subtype=\"https\",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,\"http_url=\"\"/publish/templates/images/bluebottom.jpg\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36\"\"\",http_retcode=200,\"msg=\"\"HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80\"\"\",original_srccountry=\"Saudi Arabia\",srccountry=\"Saudi Arabia\",content_switch_name=\"none\",server_pool_name=\"PHOME\",\"user_name=\"\"Unknown\"\"\",\"http_refer=\"\"https://splunk.infigo.hr/publish/templates/CSS/sc4s.css\"\"\",http_version=\"1.x\",dev_id=none,cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\"\n") - message = mt.render(mark="<13>", host=host) + "{{ mark }} {{ bsd }} fortiweb date={{ date }} time={{ time }} devname={{ host }} log_id=30000000 msg_id=000001351251 device_id=FV-1KD3A14800059 vd=\"root\" timezone=\"(GMT{{ tzoffset }})Region,City\" type=traffic subtype=\"http\" pri=notice proto=tcp service=http status=success reason=none policy=Auto-policy src=10.0.8.103 src_port=8142 dst=10.20.8.22 dst_port=80 http_request_time=0 http_response_time=0 http_request_bytes=444 http_response_bytes=401 http_method=get http_url=\"/\" http_host=\"10.0.8.22\" http_agent=\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; \" http_retcode=200 msg=\"HTTP GET request from 10.0.8.103:8142 to 10.20.8.22:80\" srccountry=\"Reserved\" content_switch_name=\"testa\" server_pool_name=\"Auto-ServerFarm\"") + message = mt.render(mark="<111>", bsd=bsd, host=host, time=time, date=date, tzoffset=tzoffset) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"fwb_traffic\" | head 2") - search = st.render(host=host) + st = env.from_string("search _time={{epoch}} index=netfw sourcetype=\"fwb_traffic\"") + search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -52,17 +99,18 @@ def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, set assert resultCount == 1 -#<111> 2020-02-12,23:16:41,devname=FortiWeb-A,log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd="root","timezone=""(GMT+3:00)Kuwait,Riyadh""",type=attack,pri=alert,main_type="Signature Detection",sub_type="Information Disclosure",trigger_policy="",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy="MobApp_policy",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,"http_url=""/mfp/api/abc""","http_host=""splunk.infigo.hr""","http_agent=""WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)""",http_session_id=ASDSADSA,"msg=""HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only""",signature_subclass="HTTP Header Leakage",signature_id="080200004",signature_cve_id="N/A",srccountry="Kuwait",content_switch_name="none",server_pool_name="MObApp_pool",false_positive_mitigation="none","user_name=""Unknown""",monitor_status="Enabled","http_refer=""none""",http_version="1.x",dev_id="none",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode="N/A",ftp_cmd="N/A",cipher_suite="TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types="none",ml_svm_accuracy="none",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method="none",owasp_top10="A3:2017-Sensitive Data Exposure",bot_info="none",matched_field="header","matched_pattern=""X-Powered-By: Servlet/3.1""" +# <111> Oct 25 13:08:00 fortiweb date=2013-10-07 time=11:30:53 devname=FortiWeb-A log_id=20000010 msg_id=000139289631 device_id=FV-1KD3A15800072 vd="root" timezone="(GMT+8:00)Beijing,ChongQing,HongKong,Urumgi" type=attack subtype="waf_signature_detection" pri=alert trigger_policy="" severity_level=Medium proto=tcp service=http action=Alert policy="123" src=172.22.6.234 src_port=60554 dst=10.0.9.13 dst_port=80 http_method=get http_url="/preview.php?file==../" http_host="10.0.9.123" http_agent="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0" http_session_id=3B9864AEKNQSLLODNTILCG37M2FZ6A88 msg="[Signatures name: 123] [main class name: Generic Attacks(Extended)] [sub class name: Directory Traversal]: 060150002" signature_subclass="Directory Traversal" signature_id="060150002" srccountry="Reserved" content_switch_name="none" server_pool_name="123" false_positive_mitigation="none" log_type=LOG_TYPE_SCORE_SUM event_score=3 score_message="[score_type: total_score] [score_scope: TCP Session] [score_threshold: 5] [score_sum: 7]" entry_sequence="000139289630" def test_fortinet_fwb_attack(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=attack,pri=alert,main_type=\"Signature Detection\",sub_type=\"Information Disclosure\",trigger_policy=\"\",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy=\"MobApp_policy\",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,\"http_url=\"\"/mfp/api/abc\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)\"\"\",http_session_id=ASDSADSA,\"msg=\"\"HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only\"\"\",signature_subclass=\"HTTP Header Leakage\",signature_id=\"080200004\",signature_cve_id=\"N/A\",srccountry=\"Kuwait\",content_switch_name=\"none\",server_pool_name=\"MObApp_pool\",false_positive_mitigation=\"none\",\"user_name=\"\"Unknown\"\"\",monitor_status=\"Enabled\",\"http_refer=\"\"none\"\"\",http_version=\"1.x\",dev_id=\"none\",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode=\"N/A\",ftp_cmd=\"N/A\",cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types=\"none\",ml_svm_accuracy=\"none\",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method=\"none\",owasp_top10=\"A3:2017-Sensitive Data Exposure\",bot_info=\"none\",matched_field=\"header\",\"matched_pattern=\"\"X-Powered-By: Servlet/3.1\"\"\"\n") - message = mt.render(mark="<13>", host=host) + "{{ mark }} {{ bsd }} fortiweb date={{ date }} time={{ time }} devname={{ host }} log_id=20000010 msg_id=000139289631 device_id=FV-1KD3A15800072 vd=\"root\" timezone=\"(GMT{{ tzoffset }})Region,City\" type=attack subtype=\"waf_signature_detection\" pri=alert trigger_policy=\"\" severity_level=Medium proto=tcp service=http action=Alert policy=\"123\" src=172.22.6.234 src_port=60554 dst=10.0.9.13 dst_port=80 http_method=get http_url=\"/preview.php?file==../\" http_host=\"10.0.9.123\" http_agent=\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0\" http_session_id=3B9864AEKNQSLLODNTILCG37M2FZ6A88 msg=\"[Signatures name: 123] [main class name: Generic Attacks(Extended)] [sub class name: Directory Traversal]: 060150002\" signature_subclass=\"Directory Traversal\" signature_id=\"060150002\" srccountry=\"Reserved\" content_switch_name=\"none\" server_pool_name=\"123\" false_positive_mitigation=\"none\" log_type=LOG_TYPE_SCORE_SUM event_score=3 score_message=\"[score_type: total_score] [score_scope: TCP Session] [score_threshold: 5] [score_sum: 7]\" entry_sequence=\"000139289630\"") + message = mt.render(mark="<111>", bsd=bsd, host=host, time=time, date=date, tzoffset=tzoffset) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"fwb_attack\" | head 2") - search = st.render(host=host) + st = env.from_string("search _time={{epoch}} index=netids sourcetype=\"fwb_attack\"") + search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) From 146c12b10cf0afab8820baab30afa144d6755e68 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 20 Feb 2020 17:05:00 -0800 Subject: [PATCH 08/11] Fix time generation in tests * Perform all time operations from within each test; issue surfaced when tests were run in parallel --- tests/test_cisco_acs.py | 58 +++++++++++++++++------------------ tests/test_fortinet_ngfw.py | 60 +++++++++++++++++++++---------------- tests/test_fortinet_web.py | 60 ++++++++++++++++++++----------------- 3 files changed, 94 insertions(+), 84 deletions(-) diff --git a/tests/test_cisco_acs.py b/tests/test_cisco_acs.py index 7c5e3d0..f83e4ea 100644 --- a/tests/test_cisco_acs.py +++ b/tests/test_cisco_acs.py @@ -18,40 +18,42 @@ def insert_char(string, char, integer): return string[0:integer] + char + string[integer:] -# Generate current datetime structure -dt = datetime.datetime.now() +def time_operations(dt): + # Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) + # See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python + iso = dt.astimezone().isoformat(sep='T', timespec='milliseconds') + # Generate an BSD-style (RFC 3164) compliant timestamp with no timezone (Oct 25 13:08:00) + bsd = dt.strftime("%b %d %H:%M:%S") -# Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) -# See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python -iso = dt.astimezone().isoformat(sep='T', timespec='milliseconds') + # Other variants of timestamps needed for this log sample + time = dt.strftime("%H:%M:%S.%f")[:-3] + date = dt.strftime("%Y-%m-%d") + # Insert colon in tzoffset string; normally just 'tzoffset = dt.astimezone().strftime("%z")' + # Could use helper function above; e.g. 'tzoffset = insert_char(dt.astimezone().strftime("%z"), ":", 3)' + tzoffset = dt.astimezone().strftime("%z")[0:3] + ":" + dt.astimezone().strftime("%z")[3:] -# Generate an BSD-style (RFC 3164) compliant timestamp with no timezone (Oct 25 13:08:00) -bsd = dt.strftime("%b %d %H:%M:%S") + # Derive epoch timestamp for use in search string + # NOTE: There are caveats with 'strftime("%s")', see references below -# Other variants of timestamps needed for this log sample -time = dt.strftime("%H:%M:%S.%f")[:-3] -date = dt.strftime("%Y-%m-%d") -# Insert colon in tzoffset string; normally just 'tzoffset = dt.astimezone().strftime("%z")' -# Could use helper function above; e.g. 'tzoffset = insert_char(dt.astimezone().strftime("%z"), ":", 3)' -tzoffset = dt.astimezone().strftime("%z")[0:3] + ":" + dt.astimezone().strftime("%z")[3:] + # See https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime + # See https://docs.python.org/3/library/datetime.html#datetime-objects + # Basically: Don't use 'utcnow()' -# Derive epoch timestamp for use in search string -# NOTE: There are caveats with 'strftime("%s")', see references below + # Strict way to get epoch as a string (rather than float) avoiding naive objects + # epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') -# See https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime -# See https://docs.python.org/3/library/datetime.html#datetime-objects -# Basically: Don't use 'utcnow()' + # Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK + # Trim last 3 digits of microsecond resolution to obtain milliseconds + epoch = dt.astimezone().strftime("%s.%f")[:-3] -# Strict way to get epoch as a string (rather than float) avoiding naive objects -# epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') - -# Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK -# Trim last 3 digits of microsecond resolution to obtain milliseconds -epoch = dt.astimezone().strftime("%s.%f")[:-3] + return iso, bsd, time, date, tzoffset, epoch def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, epoch = time_operations(dt) + mt = env.from_string( "{{ mark }} {{ bsd }} {{ host }} CSCOacs_Passed_Authentications 0765855540 1 0 {{ date }} {{ time }} {{ tzoffset }} 0178632943 5202 NOTICE Device-Administration: Command Authorization succeeded, ACSVersion=acs-5.8.1.4-B.462.x86_64, ConfigVersionId=16489, Device IP Address=10.0.0.93, DestinationIPAddress=10.0.0.10, DestinationPort=49, UserName=nsdevman, CmdSet=[ CmdAV=show CmdArgAV=vpn-sessiondb CmdArgAV=full CmdArgAV=ra-ikev2-ipsec ], Protocol=Tacacs, MatchedCommandSet=fw3, RequestLatency=11, Type=Authorization, Privilege-Level=15, Authen-Type=ASCII, Service=None, User=nsdevman, Port=443, Remote-Address=10.0.0.15, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=mnsvdcfpiuac03/359448835/9871764, AuthenticationIdentityStore=AD1, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedCommandSet=fw3, IdentityGroup=IdentityGroup:All Groups:SystemID, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15004 , Step=15013 , Step=24210 , Step=24212 , Step=24432 , Step=24325 , Step=24313 , Step=24319 , Step=24323 , Step=24420 , Step=24355 , Step=24416 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15018 , Step=13024 , Step=13034 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceName=devicenamehere, NetworkDeviceGroups=Device Type:All Device Types:Firewall:Cisco Systems:Firewall:ASA5545, NetworkDeviceGroups=Location:All Locations:MN, ServiceSelectionMatchedRule=TACACS, IdentityPolicyMatchedRule=Firewall, AuthorizationPolicyMatchedRule=nsdevman, AD-User-Candidate-Identities=nsdevman@ent.example.corp, AD-User-DNS-Domain=ent.example.corp, AD-User-NetBios-Name=AD-ENT, AD-User-Resolved-Identities=nsdevman@ent.example.corp, AD-User-Join-Point=ENT.example.CORP, AD-User-Resolved-DNs=CN=nsdevman\,OU=Service Accounts\,OU=CAO\,OU=ENT\,DC=ent\,DC=wfb\,DC=example\,DC=corp, StepData=10=nsdevman, StepData=11=ent.example.corp, StepData=12=example.corp, StepData=15=ent.example.corp, AD-Domain=ent.example.corp, IdentityAccessRestricted=false, UserIdentityGroup=IdentityGroup:All Groups:SystemID, Cisco-Firewall=Superuser, Firewall=Superuser, NetSec-CSM=User, NetSec-Logging=Engineer, Response={Type=Authorization; Author-Reply-Status=PassAdd; ExternalIdentityStoreName=AD1; }\n") message = mt.render(mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) @@ -72,12 +74,8 @@ def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk, setup_s def test_cisco_acs_multi(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) -# Generate an ISO 8601 compliant timestamp with local timezone offset (2020-02-12 12:46:39.323-08:00) - dt = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).astimezone().isoformat(sep=' ', timespec='milliseconds') - -# Function to insert a space between the time and TZ offset (2020-02-12 12:46:39.323 -08:00) - def insert_space(string, integer): - return string[0:integer] + ' ' + string[integer:] + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, epoch = time_operations(dt) mt = env.from_string( "{{ mark }} {{ bsd }} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 0 {{ date }} {{ time }} {{ tzoffset }} 0000008450 5203 NOTICE Device-Administration: Session Authorization succeeded, ACSVersion=acs-5.2.0.26-B.3075, ConfigVersionId=117, Device IP Address=192.168.26.137, UserName=edward, CmdSet=[ CmdAV= ], Protocol=Tacacs, RequestLatency=10, NetworkDeviceName=switch, Type=Authorization, Privilege-Level=1, Authen-Type=ASCII, Service=Login, User=edward, Port=tty2, Remote-Address=10.78.167.190, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=ACS41/101085887/112, AuthenticationIdentityStore=Internal Users, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedShellProfile=Permit Access, IdentityGroup=IdentityGroup:All Groups, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15006 , Step=15013 , Step=24210 , Step=24212 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15017 , Step=13034 ,\n") diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index 95a8d39..6d63bc7 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -23,42 +23,44 @@ def insert_char(string, char, integer): def removeZero(tz): return re.sub(r'\b0+(\d)(?=:)', r'\1', tz) -# Generate current datetime structure -dt = datetime.datetime.now() +def time_operations(dt): + # Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) + # See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python + iso = dt.astimezone().isoformat(sep='T', timespec='milliseconds') + # Generate an BSD-style (RFC 3164) compliant timestamp with no timezone (Oct 25 13:08:00) + bsd = dt.strftime("%b %d %H:%M:%S") -# Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) -# See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python -iso = dt.astimezone().isoformat(sep='T', timespec='milliseconds') -# Generate an BSD-style (RFC 3164) compliant timestamp with no timezone (Oct 25 13:08:00) -bsd = dt.strftime("%b %d %H:%M:%S") + # Other variants of timestamps needed for this log sample + time = dt.strftime("%H:%M:%S") + date = dt.strftime("%Y-%m-%d") + # Insert colon in tzoffset string; normally just 'tzoffset = dt.astimezone().strftime("%z")' + # Could use helper function above; e.g. 'tzoffset = insert_char(dt.astimezone().strftime("%z"), ":", 3)' + tzoffset = dt.astimezone().strftime("%z") -# Other variants of timestamps needed for this log sample -time = dt.strftime("%H:%M:%S") -date = dt.strftime("%Y-%m-%d") -# Insert colon in tzoffset string; normally just 'tzoffset = dt.astimezone().strftime("%z")' -# Could use helper function above; e.g. 'tzoffset = insert_char(dt.astimezone().strftime("%z"), ":", 3)' -# tzoffset = dt.astimezone().strftime("%z")[0:3] + ":" + dt.astimezone().strftime("%z")[3:] -tzoffset = dt.astimezone().strftime("%z") + # Derive epoch timestamp for use in search string + # NOTE: There are caveats with 'strftime("%s")', see references below -# Derive epoch timestamp for use in search string -# NOTE: There are caveats with 'strftime("%s")', see references below + # See https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime + # See https://docs.python.org/3/library/datetime.html#datetime-objects + # Basically: Don't use 'utcnow()' -# See https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime -# See https://docs.python.org/3/library/datetime.html#datetime-objects -# Basically: Don't use 'utcnow()' + # Strict way to get epoch as a string (rather than float) avoiding naive objects + # epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') -# Strict way to get epoch as a string (rather than float) avoiding naive objects -# epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') + # Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK + # Trim last 3 digits of microsecond resolution to obtain milliseconds + epoch = dt.astimezone().strftime("%s.%f")[:-7] -# Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK -# Trim last 3 or 7 digits of microsecond resolution to obtain milliseconds or whole seconds respectively -epoch = dt.astimezone().strftime("%s.%f")[:-7] + return iso, bsd, time, date, tzoffset, epoch # <111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, epoch = time_operations(dt) + mt = env.from_string( # "{{ mark }} {{ bsd }} fortigate date={{ date }} time={{ time }} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice tz=\"{{ tzoffset }}\" vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") "{{ mark }} {{ bsd }} fortigate date={{ date }} time={{ time }} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") @@ -67,8 +69,7 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk, setup sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( -# "search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\"") - "search index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\"") + "search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\"") search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -84,6 +85,9 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk, set host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, epoch = time_operations(dt) + mt = env.from_string( "{{ mark }} {{ bsd }} fortigate date={{ date }} time={{ time }} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") message = mt.render(mark="<111>", bsd=bsd, date=date, time=time, host=host) @@ -106,6 +110,9 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk, setup_s host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, epoch = time_operations(dt) + mt = env.from_string( "{{ mark }} {{ bsd }} fortigate date={{ date }} time={{ time }} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") message = mt.render(mark="<111>", bsd=bsd, date=date, time=time, host=host) @@ -122,3 +129,4 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk, setup_s record_property("message", message) assert resultCount == 1 + diff --git a/tests/test_fortinet_web.py b/tests/test_fortinet_web.py index 24ed1ac..abc92b0 100644 --- a/tests/test_fortinet_web.py +++ b/tests/test_fortinet_web.py @@ -22,45 +22,43 @@ def insert_colon(string, integer): def removeZero(tz): return re.sub(r'\b0+(\d)(?=:)', r'\1', tz) +def time_operations(dt): + # Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) + # See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python + iso = dt.astimezone().isoformat(sep='T', timespec='milliseconds') + # Generate an BSD-style (RFC 3164) compliant timestamp with no timezone (Oct 25 13:08:00) + bsd = dt.strftime("%b %d %H:%M:%S") -# Generate current datetime structure -dt = datetime.datetime.now() + # Other variants of timestamps needed for this log sample + time = dt.strftime("%H:%M:%S") + date = dt.strftime("%Y-%m-%d") + # Insert colon in tzoffset string; normally just 'tzoffset = dt.astimezone().strftime("%z")' + # Could use helper function above; e.g. 'tzoffset = insert_char(dt.astimezone().strftime("%z"), ":", 3)' + tzoffset = removeZero(insert_colon((dt.astimezone().strftime("%z")), 3)) -# Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) -# See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python -iso = dt.astimezone().isoformat(sep='T', timespec='milliseconds') -# Generate an BSD-style (RFC 3164) compliant timestamp with no timezone (Oct 25 13:08:00) -bsd = dt.strftime("%b %d %H:%M:%S") + # Derive epoch timestamp for use in search string + # NOTE: There are caveats with 'strftime("%s")', see references below -# Other variants of timestamps needed for this log sample -time = dt.strftime("%H:%M:%S") -date = dt.strftime("%Y-%m-%d") -# Insert colon in tzoffset string; normally just 'tzoffset = dt.astimezone().strftime("%z")' -# Could use helper function above; e.g. 'tzoffset = insert_char(dt.astimezone().strftime("%z"), ":", 3)' -# tzoffset = dt.astimezone().strftime("%z")[0:3] + ":" + dt.astimezone().strftime("%z")[3:] -tzoffset = dt.astimezone().strftime("%z") + # See https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime + # See https://docs.python.org/3/library/datetime.html#datetime-objects + # Basically: Don't use 'utcnow()' -# Derive epoch timestamp for use in search string -# NOTE: There are caveats with 'strftime("%s")', see references below + # Strict way to get epoch as a string (rather than float) avoiding naive objects + # epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') -# See https://stackoverflow.com/questions/11743019/convert-python-datetime-to-epoch-with-strftime -# See https://docs.python.org/3/library/datetime.html#datetime-objects -# Basically: Don't use 'utcnow()' + # Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK + # Trim last 3 digits of microsecond resolution to obtain milliseconds + epoch = dt.astimezone().strftime("%s.%f")[:-7] -# Strict way to get epoch as a string (rather than float) avoiding naive objects -# epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') - -# Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK -# Trim last 3 or 7 digits of microsecond resolution to obtain milliseconds or whole seconds respectively -epoch = dt.astimezone().strftime("%s.%f")[:-7] - -# Generate Timezone in "-08:00" format (with HH:MM colon; no leading zero for hour < 10) -tzoffset = removeZero(insert_colon((dt.astimezone().strftime("%z")), 3)) + return iso, bsd, time, date, tzoffset, epoch # <111> Oct 25 13:08:00 fortiweb date=2013-10-07 time=11:30:53 devname=FortiWeb-A log_id=10000017 msg_id=000000001117 device_id=FVVM040000010871 vd="root" timezone="(GMT-5:00)Eastern Time(US & Canada)" type=event subtype="system" pri=information trigger_policy="" user=admin ui=GUI action=login status=success msg="User admin login successfully from GUI(172.20.120.47)" def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, epoch = time_operations(dt) + mt = env.from_string( "{{ mark }} {{ bsd }} fortiweb date={{ date }} time={{ time }} devname={{ host }} log_id=10000017 msg_id=000000001117 device_id=FVVM040000010871 vd=\"root\" timezone=\"(GMT{{ tzoffset }})Region,City\" type=event subtype=\"system\" pri=information trigger_policy=\"\" user=admin ui=GUI action=login status=success msg=\"User admin login successfully from GUI(172.20.120.47)\"") message = mt.render(mark="<111>", bsd=bsd, host=host, time=time, date=date, tzoffset=tzoffset) @@ -82,6 +80,9 @@ def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, epoch = time_operations(dt) + mt = env.from_string( "{{ mark }} {{ bsd }} fortiweb date={{ date }} time={{ time }} devname={{ host }} log_id=30000000 msg_id=000001351251 device_id=FV-1KD3A14800059 vd=\"root\" timezone=\"(GMT{{ tzoffset }})Region,City\" type=traffic subtype=\"http\" pri=notice proto=tcp service=http status=success reason=none policy=Auto-policy src=10.0.8.103 src_port=8142 dst=10.20.8.22 dst_port=80 http_request_time=0 http_response_time=0 http_request_bytes=444 http_response_bytes=401 http_method=get http_url=\"/\" http_host=\"10.0.8.22\" http_agent=\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; \" http_retcode=200 msg=\"HTTP GET request from 10.0.8.103:8142 to 10.20.8.22:80\" srccountry=\"Reserved\" content_switch_name=\"testa\" server_pool_name=\"Auto-ServerFarm\"") message = mt.render(mark="<111>", bsd=bsd, host=host, time=time, date=date, tzoffset=tzoffset) @@ -103,6 +104,9 @@ def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, set def test_fortinet_fwb_attack(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, epoch = time_operations(dt) + mt = env.from_string( "{{ mark }} {{ bsd }} fortiweb date={{ date }} time={{ time }} devname={{ host }} log_id=20000010 msg_id=000139289631 device_id=FV-1KD3A15800072 vd=\"root\" timezone=\"(GMT{{ tzoffset }})Region,City\" type=attack subtype=\"waf_signature_detection\" pri=alert trigger_policy=\"\" severity_level=Medium proto=tcp service=http action=Alert policy=\"123\" src=172.22.6.234 src_port=60554 dst=10.0.9.13 dst_port=80 http_method=get http_url=\"/preview.php?file==../\" http_host=\"10.0.9.123\" http_agent=\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0\" http_session_id=3B9864AEKNQSLLODNTILCG37M2FZ6A88 msg=\"[Signatures name: 123] [main class name: Generic Attacks(Extended)] [sub class name: Directory Traversal]: 060150002\" signature_subclass=\"Directory Traversal\" signature_id=\"060150002\" srccountry=\"Reserved\" content_switch_name=\"none\" server_pool_name=\"123\" false_positive_mitigation=\"none\" log_type=LOG_TYPE_SCORE_SUM event_score=3 score_message=\"[score_type: total_score] [score_scope: TCP Session] [score_threshold: 5] [score_sum: 7]\" entry_sequence=\"000139289630\"") message = mt.render(mark="<111>", bsd=bsd, host=host, time=time, date=date, tzoffset=tzoffset) From b2253747dbc41034e512084c6d4947d60823fdb3 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 20 Feb 2020 17:21:14 -0800 Subject: [PATCH 09/11] Minor fixup to Cisco ACS single message search * Remove "| head 11" from ACS single message search --- tests/test_cisco_acs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cisco_acs.py b/tests/test_cisco_acs.py index f83e4ea..19cf8b9 100644 --- a/tests/test_cisco_acs.py +++ b/tests/test_cisco_acs.py @@ -60,7 +60,7 @@ def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk, setup_s sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" | head 11") + st = env.from_string("search _time={{ epoch }} index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\"") search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) From 020468316b63e59e2de2298602977dc813be4af0 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 20 Feb 2020 17:25:54 -0800 Subject: [PATCH 10/11] Update epoch string generation comment * Update epoch string generation comment --- tests/test_cisco_acs.py | 2 +- tests/test_fortinet_ngfw.py | 2 +- tests/test_fortinet_web.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_cisco_acs.py b/tests/test_cisco_acs.py index 19cf8b9..a26e19d 100644 --- a/tests/test_cisco_acs.py +++ b/tests/test_cisco_acs.py @@ -43,7 +43,7 @@ def time_operations(dt): # epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') # Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK - # Trim last 3 digits of microsecond resolution to obtain milliseconds + # Trim last 3 or 7 characters of microsecond resolution to obtain milliseconds or whole seconds, respectively epoch = dt.astimezone().strftime("%s.%f")[:-3] return iso, bsd, time, date, tzoffset, epoch diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index 6d63bc7..725934f 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -48,7 +48,7 @@ def time_operations(dt): # epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') # Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK - # Trim last 3 digits of microsecond resolution to obtain milliseconds + # Trim last 3 or 7 characters of microsecond resolution to obtain milliseconds or whole seconds, respectively epoch = dt.astimezone().strftime("%s.%f")[:-7] return iso, bsd, time, date, tzoffset, epoch diff --git a/tests/test_fortinet_web.py b/tests/test_fortinet_web.py index abc92b0..f62a298 100644 --- a/tests/test_fortinet_web.py +++ b/tests/test_fortinet_web.py @@ -47,7 +47,7 @@ def time_operations(dt): # epoch = dt.fromtimestamp(dt.timestamp()).strftime('%s') # Since datetime.now().astimezone() is aware, strftime() should be safe and form below OK - # Trim last 3 digits of microsecond resolution to obtain milliseconds + # Trim last 3 or 7 characters of microsecond resolution to obtain milliseconds or whole seconds, respectively epoch = dt.astimezone().strftime("%s.%f")[:-7] return iso, bsd, time, date, tzoffset, epoch From 1ceb5a11503492ba0d9d5c190ffc8bf02f8c62d9 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 20 Feb 2020 20:32:21 -0500 Subject: [PATCH 11/11] Update test_fortinet_web.py --- tests/test_fortinet_web.py | 39 +++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/test_fortinet_web.py b/tests/test_fortinet_web.py index f62a298..cc59a5d 100644 --- a/tests/test_fortinet_web.py +++ b/tests/test_fortinet_web.py @@ -7,6 +7,7 @@ import datetime import re + from jinja2 import Environment from .sendmessage import * @@ -15,13 +16,18 @@ env = Environment() # Function to insert a colon between the time and TZ offset (from -0800 to -08:00) + + def insert_colon(string, integer): return string[0:integer] + ':' + string[integer:] # Function to remove leading zero from TZ offsets less than 10 hours + + def removeZero(tz): return re.sub(r'\b0+(\d)(?=:)', r'\1', tz) + def time_operations(dt): # Generate an ISO 8601 (RFC 5424) compliant timestamp with local timezone offset (2020-02-12T12:46:39.323-08:00) # See https://stackoverflow.com/questions/2150739/iso-time-iso-8601-in-python @@ -53,19 +59,24 @@ def time_operations(dt): return iso, bsd, time, date, tzoffset, epoch # <111> Oct 25 13:08:00 fortiweb date=2013-10-07 time=11:30:53 devname=FortiWeb-A log_id=10000017 msg_id=000000001117 device_id=FVVM040000010871 vd="root" timezone="(GMT-5:00)Eastern Time(US & Canada)" type=event subtype="system" pri=information trigger_policy="" user=admin ui=GUI action=login status=success msg="User admin login successfully from GUI(172.20.120.47)" + + def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) dt = datetime.datetime.now() iso, bsd, time, date, tzoffset, epoch = time_operations(dt) mt = env.from_string( "{{ mark }} {{ bsd }} fortiweb date={{ date }} time={{ time }} devname={{ host }} log_id=10000017 msg_id=000000001117 device_id=FVVM040000010871 vd=\"root\" timezone=\"(GMT{{ tzoffset }})Region,City\" type=event subtype=\"system\" pri=information trigger_policy=\"\" user=admin ui=GUI action=login status=success msg=\"User admin login successfully from GUI(172.20.120.47)\"") - message = mt.render(mark="<111>", bsd=bsd, host=host, time=time, date=date, tzoffset=tzoffset) + message = mt.render(mark="<111>", bsd=bsd, host=host, + time=time, date=date, tzoffset=tzoffset) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{epoch}} index=netops sourcetype=\"fwb_event\"") + st = env.from_string( + "search _time={{epoch}} index=netops sourcetype=\"fwb_event\"") search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -77,19 +88,24 @@ def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup assert resultCount == 1 # <111> Oct 25 13:08:00 fortiweb date=2013-10-07 time=11:30:53 devname=FortiWeb-A log_id=30000000 msg_id=000001351251 device_id=FV-1KD3A14800059 vd="root" timezone="(GMT-8:00)Pacific Time(US&Canada)" type=traffic subtype="http" pri=notice proto=tcp service=http status=success reason=none policy=Auto-policy src=10.0.8.103 src_port=8142 dst=10.20.8.22 dst_port=80 http_request_time=0 http_response_time=0 http_request_bytes=444 http_response_bytes=401 http_method=get http_url="/" http_host="10.0.8.22" http_agent="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; " http_retcode=200 msg="HTTP GET request from 10.0.8.103:8142 to 10.20.8.22:80" srccountry="Reserved" content_switch_name="testa" server_pool_name="Auto-ServerFarm" + + def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) dt = datetime.datetime.now() iso, bsd, time, date, tzoffset, epoch = time_operations(dt) mt = env.from_string( "{{ mark }} {{ bsd }} fortiweb date={{ date }} time={{ time }} devname={{ host }} log_id=30000000 msg_id=000001351251 device_id=FV-1KD3A14800059 vd=\"root\" timezone=\"(GMT{{ tzoffset }})Region,City\" type=traffic subtype=\"http\" pri=notice proto=tcp service=http status=success reason=none policy=Auto-policy src=10.0.8.103 src_port=8142 dst=10.20.8.22 dst_port=80 http_request_time=0 http_response_time=0 http_request_bytes=444 http_response_bytes=401 http_method=get http_url=\"/\" http_host=\"10.0.8.22\" http_agent=\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; \" http_retcode=200 msg=\"HTTP GET request from 10.0.8.103:8142 to 10.20.8.22:80\" srccountry=\"Reserved\" content_switch_name=\"testa\" server_pool_name=\"Auto-ServerFarm\"") - message = mt.render(mark="<111>", bsd=bsd, host=host, time=time, date=date, tzoffset=tzoffset) + message = mt.render(mark="<111>", bsd=bsd, host=host, + time=time, date=date, tzoffset=tzoffset) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{epoch}} index=netfw sourcetype=\"fwb_traffic\"") + st = env.from_string( + "search _time={{epoch}} index=netfw sourcetype=\"fwb_traffic\"") search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -101,19 +117,24 @@ def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, set assert resultCount == 1 # <111> Oct 25 13:08:00 fortiweb date=2013-10-07 time=11:30:53 devname=FortiWeb-A log_id=20000010 msg_id=000139289631 device_id=FV-1KD3A15800072 vd="root" timezone="(GMT+8:00)Beijing,ChongQing,HongKong,Urumgi" type=attack subtype="waf_signature_detection" pri=alert trigger_policy="" severity_level=Medium proto=tcp service=http action=Alert policy="123" src=172.22.6.234 src_port=60554 dst=10.0.9.13 dst_port=80 http_method=get http_url="/preview.php?file==../" http_host="10.0.9.123" http_agent="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0" http_session_id=3B9864AEKNQSLLODNTILCG37M2FZ6A88 msg="[Signatures name: 123] [main class name: Generic Attacks(Extended)] [sub class name: Directory Traversal]: 060150002" signature_subclass="Directory Traversal" signature_id="060150002" srccountry="Reserved" content_switch_name="none" server_pool_name="123" false_positive_mitigation="none" log_type=LOG_TYPE_SCORE_SUM event_score=3 score_message="[score_type: total_score] [score_scope: TCP Session] [score_threshold: 5] [score_sum: 7]" entry_sequence="000139289630" + + def test_fortinet_fwb_attack(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) dt = datetime.datetime.now() iso, bsd, time, date, tzoffset, epoch = time_operations(dt) mt = env.from_string( "{{ mark }} {{ bsd }} fortiweb date={{ date }} time={{ time }} devname={{ host }} log_id=20000010 msg_id=000139289631 device_id=FV-1KD3A15800072 vd=\"root\" timezone=\"(GMT{{ tzoffset }})Region,City\" type=attack subtype=\"waf_signature_detection\" pri=alert trigger_policy=\"\" severity_level=Medium proto=tcp service=http action=Alert policy=\"123\" src=172.22.6.234 src_port=60554 dst=10.0.9.13 dst_port=80 http_method=get http_url=\"/preview.php?file==../\" http_host=\"10.0.9.123\" http_agent=\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0\" http_session_id=3B9864AEKNQSLLODNTILCG37M2FZ6A88 msg=\"[Signatures name: 123] [main class name: Generic Attacks(Extended)] [sub class name: Directory Traversal]: 060150002\" signature_subclass=\"Directory Traversal\" signature_id=\"060150002\" srccountry=\"Reserved\" content_switch_name=\"none\" server_pool_name=\"123\" false_positive_mitigation=\"none\" log_type=LOG_TYPE_SCORE_SUM event_score=3 score_message=\"[score_type: total_score] [score_scope: TCP Session] [score_threshold: 5] [score_sum: 7]\" entry_sequence=\"000139289630\"") - message = mt.render(mark="<111>", bsd=bsd, host=host, time=time, date=date, tzoffset=tzoffset) + message = mt.render(mark="<111>", bsd=bsd, host=host, + time=time, date=date, tzoffset=tzoffset) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{epoch}} index=netids sourcetype=\"fwb_attack\"") + st = env.from_string( + "search _time={{epoch}} index=netids sourcetype=\"fwb_attack\"") search = st.render(host=host, epoch=epoch) resultCount, eventCount = splunk_single(setup_splunk, search)