From b057805930b00ff037004196e35254ed2f115aeb Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 13 Feb 2020 22:30:40 -0800 Subject: [PATCH] Update/fwb_log_path * Update Fortiweb log path to properly parse timestamp * Update Fortigate FW test to emit local time * Update Fortiweb log path to emit hard-coded `GMT-8:00` format to match that of the device TZ output --- .../log_paths/lp-fortinet_fortiweb.conf.tmpl | 67 ++++++++++--------- tests/test_fortinet_ngfw.py | 2 +- tests/test_fortinet_web.py | 6 +- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl index 8bcc8c3..bf3bb74 100644 --- a/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl @@ -1,22 +1,47 @@ # Fortinet FortiWeb -{{ $context := dict "port_id" "FORTINET_FORTIWEB" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "FORTINET_FORTIWEB" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} log { -{{- if eq (.) "yes"}} - source(s_DEFAULT); - filter(f_fortinet_fortiweb); -{{- end}} -{{- if eq (.) "no"}} - source (s_FORTINET_FORTIWEB); + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for FORTINET_FORTIWEB traffic + source (s_FORTINET_FORTIWEB); + flags (final); + }; {{- end}} + channel { + # Listen on the default port (typically 514) for FORTINET_FORTIWEB traffic + source (s_DEFAULT); + filter(f_is_rfc3164); + filter(f_fortinet_fortiweb); + flags(final); + }; + }; parser { kv-parser(prefix(".kv.") pair-separator(",") template("${MSGHDR} ${MSG}")); }; + rewrite { + subst('.*([\+-]\d+:\d+).*', $1, value(".kv.timezone")); + subst('([\+-])(\d)(?=:):(\d+)', "${1}0${2}${3}", value(".kv.timezone")); + subst('([\+-])(\d+):(\d+)', "$1$2$3", value(".kv.timezone")); + }; + + #2020-02-12,23:13:33 + #parse the date + parser { + date-parser( + format("%Y-%m-%d,%H:%M:%S%z") + template('$(substr "$LEGACY_MSGHDR$MSG" "0" "19")${.kv.timezone}') + time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) + flags(guess-timezone) + ); + }; + rewrite { set("${.kv.devname}", value("HOST")); set("fortigate_fortiweb", value("fields.sc4s_vendor_product")); @@ -37,17 +62,7 @@ log { }; parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_3164))" value("MSG")); }; {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORTINET_FORTIWEB_HEC" "no") | conv.ToBool) }} destination(d_hec); @@ -58,14 +73,6 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; -{{- end}} - -{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TLS_PORT")) }} -# Listen on the specified dedicated port(s) for FORTINET_FORTIWEB traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} -# Listen on the default port (typically 514) for FORTINET_FORTIWEB traffic -{{ tmpl.Exec "log_path" "yes" }} diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index 6a8cd3c..bdfeb88 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -67,7 +67,7 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk, setup_s random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") + "{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'local', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) diff --git a/tests/test_fortinet_web.py b/tests/test_fortinet_web.py index aa3757a..97bd21d 100644 --- a/tests/test_fortinet_web.py +++ b/tests/test_fortinet_web.py @@ -17,7 +17,7 @@ def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\"\n") + "{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) @@ -37,7 +37,7 @@ def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, set host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=traffic,subtype=\"https\",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,\"http_url=\"\"/publish/templates/images/bluebottom.jpg\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36\"\"\",http_retcode=200,\"msg=\"\"HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80\"\"\",original_srccountry=\"Saudi Arabia\",srccountry=\"Saudi Arabia\",content_switch_name=\"none\",server_pool_name=\"PHOME\",\"user_name=\"\"Unknown\"\"\",\"http_refer=\"\"https://splunk.infigo.hr/publish/templates/CSS/sc4s.css\"\"\",http_version=\"1.x\",dev_id=none,cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\"\n") + "{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT-8:00)Pacific Time(US&Canada)\"\"\",type=traffic,subtype=\"https\",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,\"http_url=\"\"/publish/templates/images/bluebottom.jpg\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36\"\"\",http_retcode=200,\"msg=\"\"HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80\"\"\",original_srccountry=\"Saudi Arabia\",srccountry=\"Saudi Arabia\",content_switch_name=\"none\",server_pool_name=\"PHOME\",\"user_name=\"\"Unknown\"\"\",\"http_refer=\"\"https://splunk.infigo.hr/publish/templates/CSS/sc4s.css\"\"\",http_version=\"1.x\",dev_id=none,cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) @@ -57,7 +57,7 @@ def test_fortinet_fwb_attack(record_property, setup_wordlist, setup_splunk, setu host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=attack,pri=alert,main_type=\"Signature Detection\",sub_type=\"Information Disclosure\",trigger_policy=\"\",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy=\"MobApp_policy\",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,\"http_url=\"\"/mfp/api/abc\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)\"\"\",http_session_id=ASDSADSA,\"msg=\"\"HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only\"\"\",signature_subclass=\"HTTP Header Leakage\",signature_id=\"080200004\",signature_cve_id=\"N/A\",srccountry=\"Kuwait\",content_switch_name=\"none\",server_pool_name=\"MObApp_pool\",false_positive_mitigation=\"none\",\"user_name=\"\"Unknown\"\"\",monitor_status=\"Enabled\",\"http_refer=\"\"none\"\"\",http_version=\"1.x\",dev_id=\"none\",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode=\"N/A\",ftp_cmd=\"N/A\",cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types=\"none\",ml_svm_accuracy=\"none\",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method=\"none\",owasp_top10=\"A3:2017-Sensitive Data Exposure\",bot_info=\"none\",matched_field=\"header\",\"matched_pattern=\"\"X-Powered-By: Servlet/3.1\"\"\"\n") + "{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=attack,pri=alert,main_type=\"Signature Detection\",sub_type=\"Information Disclosure\",trigger_policy=\"\",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy=\"MobApp_policy\",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,\"http_url=\"\"/mfp/api/abc\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)\"\"\",http_session_id=ASDSADSA,\"msg=\"\"HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only\"\"\",signature_subclass=\"HTTP Header Leakage\",signature_id=\"080200004\",signature_cve_id=\"N/A\",srccountry=\"Kuwait\",content_switch_name=\"none\",server_pool_name=\"MObApp_pool\",false_positive_mitigation=\"none\",\"user_name=\"\"Unknown\"\"\",monitor_status=\"Enabled\",\"http_refer=\"\"none\"\"\",http_version=\"1.x\",dev_id=\"none\",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode=\"N/A\",ftp_cmd=\"N/A\",cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types=\"none\",ml_svm_accuracy=\"none\",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method=\"none\",owasp_top10=\"A3:2017-Sensitive Data Exposure\",bot_info=\"none\",matched_field=\"header\",\"matched_pattern=\"\"X-Powered-By: Servlet/3.1\"\"\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])