Skip to content

Commit

Permalink
Update/fwb_log_path
Browse files Browse the repository at this point in the history 
* Update Fortiweb log path to properly parse timestamp
* Update Fortigate FW test to emit local time
* Update Fortiweb log path to emit hard-coded `GMT-8:00` format to match that of the device TZ output
  • Loading branch information
Mark Bonsack committed Feb 14, 2020
1 parent 09b1b14 commit b057805
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 34 deletions.
67 changes: 37 additions & 30 deletions package/etc/conf.d/log_paths/lp-fortinet_fortiweb.conf.tmpl
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,47 @@
# Fortinet FortiWeb
{{ $context := dict "port_id" "FORTINET_FORTIWEB" "parser" "common" }}
{{ tmpl.Exec "t/source_network.t" $context }}
{{- /* The following provides a unique port source configuration if env var(s) are set */}}
{{- $context := dict "port_id" "FORTINET_FORTIWEB" "parser" "rfc3164" }}
{{- tmpl.Exec "t/source_network.t" $context }}

# The following is an inline template; we will use this to generate the actual log path
{{ define "log_path" }}
log {
{{- if eq (.) "yes"}}
source(s_DEFAULT);
filter(f_fortinet_fortiweb);
{{- end}}
{{- if eq (.) "no"}}
source (s_FORTINET_FORTIWEB);
junction {
{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TLS_PORT")) }}
channel {
# Listen on the specified dedicated port(s) for FORTINET_FORTIWEB traffic
source (s_FORTINET_FORTIWEB);
flags (final);
};
{{- end}}
channel {
# Listen on the default port (typically 514) for FORTINET_FORTIWEB traffic
source (s_DEFAULT);
filter(f_is_rfc3164);
filter(f_fortinet_fortiweb);
flags(final);
};
};

parser {
kv-parser(prefix(".kv.") pair-separator(",") template("${MSGHDR} ${MSG}"));
};

rewrite {
subst('.*([\+-]\d+:\d+).*', $1, value(".kv.timezone"));
subst('([\+-])(\d)(?=:):(\d+)', "${1}0${2}${3}", value(".kv.timezone"));
subst('([\+-])(\d+):(\d+)', "$1$2$3", value(".kv.timezone"));
};

#2020-02-12,23:13:33
#parse the date
parser {
date-parser(
format("%Y-%m-%d,%H:%M:%S%z")
template('$(substr "$LEGACY_MSGHDR$MSG" "0" "19")${.kv.timezone}')
time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})
flags(guess-timezone)
);
};

rewrite {
set("${.kv.devname}", value("HOST"));
set("fortigate_fortiweb", value("fields.sc4s_vendor_product"));
Expand All @@ -37,17 +62,7 @@ log {
};

parser (compliance_meta_by_source);

#We want to unset the fields we won't need, as this is copied into the
#disk queue for network destinations. This can be very disk expensive
#if we don't
rewrite {
set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
unset(value("RAWMSG"));
unset(value("PROGRAM"));
unset(value("LEGACY_MSGHDR"));
groupunset(values(".kv.*"));
};
rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_3164))" value("MSG")); };

{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_FORTINET_FORTIWEB_HEC" "no") | conv.ToBool) }}
destination(d_hec);
Expand All @@ -58,14 +73,6 @@ log {
destination(d_archive);
{{- end}}

flags(flow-control);
flags(flow-control,final);
};
{{- end}}

{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIWEB_TLS_PORT")) }}
# Listen on the specified dedicated port(s) for FORTINET_FORTIWEB traffic
{{ tmpl.Exec "log_path" "no" }}
{{- end}}

# Listen on the default port (typically 514) for FORTINET_FORTIWEB traffic
{{ tmpl.Exec "log_path" "yes" }}
2 changes: 1 addition & 1 deletion tests/test_fortinet_ngfw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk, setup_s
random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n")
"{{ mark }}date={% now 'local', '%Y-%m-%d' %} time={% now 'local', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

Expand Down
6 changes: 3 additions & 3 deletions tests/test_fortinet_web.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def test_fortinet_fwb_event(record_property, setup_wordlist, setup_splunk, setup
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\"\n")
"{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=11005607,msg_id=000377260939,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=event,subtype=\"system\",pri=notice,trigger_policy=\"Splunk_policy\",user=daemon,ui=daemon,action=check-resource,status=success,\"msg=\"\"The logdisk usage is too high\"\"\"\n")
message = mt.render(mark="<13>", host=host)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
Expand All @@ -37,7 +37,7 @@ def test_fortinet_fwb_traffic(record_property, setup_wordlist, setup_splunk, set
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=traffic,subtype=\"https\",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,\"http_url=\"\"/publish/templates/images/bluebottom.jpg\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36\"\"\",http_retcode=200,\"msg=\"\"HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80\"\"\",original_srccountry=\"Saudi Arabia\",srccountry=\"Saudi Arabia\",content_switch_name=\"none\",server_pool_name=\"PHOME\",\"user_name=\"\"Unknown\"\"\",\"http_refer=\"\"https://splunk.infigo.hr/publish/templates/CSS/sc4s.css\"\"\",http_version=\"1.x\",dev_id=none,cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\"\n")
"{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=30001000,msg_id=000377262759,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT-8:00)Pacific Time(US&Canada)\"\"\",type=traffic,subtype=\"https\",pri=notice,proto=tcp,service=https/tls1.2,status=success,reason=none,policy=Phome_Policy,original_src=1.107.71.90,src=1.107.71.90,src_port=28799,dst=1.16.16.11,dst_port=80,http_request_time=0,http_response_time=0,http_request_bytes=623,http_response_bytes=15660,http_method=get,\"http_url=\"\"/publish/templates/images/bluebottom.jpg\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"Mozilla/5.0 (Linux; Android 9; SM-J415F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Mobile Safari/537.36\"\"\",http_retcode=200,\"msg=\"\"HTTPS get request from 1.107.71.90:28799 to 1.16.16.11:80\"\"\",original_srccountry=\"Saudi Arabia\",srccountry=\"Saudi Arabia\",content_switch_name=\"none\",server_pool_name=\"PHOME\",\"user_name=\"\"Unknown\"\"\",\"http_refer=\"\"https://splunk.infigo.hr/publish/templates/CSS/sc4s.css\"\"\",http_version=\"1.x\",dev_id=none,cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\"\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

Expand All @@ -57,7 +57,7 @@ def test_fortinet_fwb_attack(record_property, setup_wordlist, setup_splunk, setu
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }}{% now 'utc', '%Y-%m-%d' %},{% now 'utc', '%H:%M:%S' %},devname={{ host }},log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=attack,pri=alert,main_type=\"Signature Detection\",sub_type=\"Information Disclosure\",trigger_policy=\"\",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy=\"MobApp_policy\",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,\"http_url=\"\"/mfp/api/abc\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)\"\"\",http_session_id=ASDSADSA,\"msg=\"\"HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only\"\"\",signature_subclass=\"HTTP Header Leakage\",signature_id=\"080200004\",signature_cve_id=\"N/A\",srccountry=\"Kuwait\",content_switch_name=\"none\",server_pool_name=\"MObApp_pool\",false_positive_mitigation=\"none\",\"user_name=\"\"Unknown\"\"\",monitor_status=\"Enabled\",\"http_refer=\"\"none\"\"\",http_version=\"1.x\",dev_id=\"none\",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode=\"N/A\",ftp_cmd=\"N/A\",cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types=\"none\",ml_svm_accuracy=\"none\",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method=\"none\",owasp_top10=\"A3:2017-Sensitive Data Exposure\",bot_info=\"none\",matched_field=\"header\",\"matched_pattern=\"\"X-Powered-By: Servlet/3.1\"\"\"\n")
"{{ mark }}{% now 'local', '%Y-%m-%d' %},{% now 'local', '%H:%M:%S' %},devname={{ host }},log_id=20000008,msg_id=000377262743,device_id=FV-1111111800222,vd=\"root\",\"timezone=\"\"(GMT+3:00)Kuwait,Riyadh\"\"\",type=attack,pri=alert,main_type=\"Signature Detection\",sub_type=\"Information Disclosure\",trigger_policy=\"\",severity_level=Low,proto=tcp,service=https/tls1.2,backend_service=https/tls1.2,action=Alert,policy=\"MobApp_policy\",src=1.70.8.51,src_port=20894,dst=1.16.220.15,dst_port=443,http_method=post,\"http_url=\"\"/mfp/api/abc\"\"\",\"http_host=\"\"splunk.infigo.hr\"\"\",\"http_agent=\"\"WLNativeAPI(HWSTK-HF; STK-L21MDV 9.1.0.336(C185E3R2P1); STK-L21; SDK 28; Android 9)\"\"\",http_session_id=ASDSADSA,\"msg=\"\"HTTP Header triggered signature ID 080200004 of Signatures policy Alert Only\"\"\",signature_subclass=\"HTTP Header Leakage\",signature_id=\"080200004\",signature_cve_id=\"N/A\",srccountry=\"Kuwait\",content_switch_name=\"none\",server_pool_name=\"MObApp_pool\",false_positive_mitigation=\"none\",\"user_name=\"\"Unknown\"\"\",monitor_status=\"Enabled\",\"http_refer=\"\"none\"\"\",http_version=\"1.x\",dev_id=\"none\",es=1,threat_weight=5,history_threat_weight=0,threat_level=Low,ftp_mode=\"N/A\",ftp_cmd=\"N/A\",cipher_suite=\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\"ml_log_hmm_probability=0.000000,ml_log_sample_prob_mean=0.000000,ml_log_sample_arglen_mean=0.000000,ml_log_arglen=0,ml_svm_log_main_types=0,ml_svm_log_match_types=\"none\",ml_svm_accuracy=\"none\",ml_domain_index=0,ml_url_dbid=0,ml_arg_dbid=0,ml_allow_method=\"none\",owasp_top10=\"A3:2017-Sensitive Data Exposure\",bot_info=\"none\",matched_field=\"header\",\"matched_pattern=\"\"X-Powered-By: Servlet/3.1\"\"\"\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

Expand Down

0 comments on commit b057805

Please sign in to comment.