Skip to content

Commit

Permalink
Refine filter (#40)
Browse files Browse the repository at this point in the history 
Refine filter for performance and simplicity; add full payload to pycharm tests Fixes #38
  • Loading branch information
mbonsack authored and Ryan Faircloth committed Aug 7, 2019
1 parent f472608 commit e3fbe1f
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 14 deletions.
15 changes: 4 additions & 11 deletions package/etc/conf.d/filters/fortinet_fgt.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ log {
};

parser {
kv-parser(prefix(".kv."));
date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}"));
syslog-parser();
kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}"));
date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone(`default-timezone`));
};

rewrite { set("${.kv.devname}", value("HOST")); };
rewrite { subst('<\d+>(.*)' "$1" value("MSG")); };

#set the source type based on program field and lookup index from the splunk context csv
if (match("traffic" value(".kv.type"))) {
Expand All @@ -24,16 +24,9 @@ log {
parser {p_add_context_splunk(key("fgt_log")); };
};

rewrite {
#drop the header
subst('date=[^ ]+ time=[^ ]+ devname=.+ (devid.*)', '$1', value("MESSAGE"));
#Strip empty fields because SEDCMD can't when we use event endpoint
subst('([^\= ]+=(?: |\"\"|-|\"-") ?)', '', value("MESSAGE"));
};

#rewrite the final message for splunk json
#sourcetype and index are set in the filter defaults won't be used
rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC--
rewrite {r_set_splunk_basic(template("t_standard")) }; #--HEC--
destination(d_hec); #--HEC--

flags(flow-control,final);
Expand Down
6 changes: 3 additions & 3 deletions tests/test_fortinet_ngfw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n")
"{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n")
message = mt.render(mark="<13>", host=host)

sendsingle(message)
Expand All @@ -41,7 +41,7 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n")
"{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message)

Expand All @@ -62,7 +62,7 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n")
"{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message)

Expand Down

0 comments on commit e3fbe1f

Please sign in to comment.