From e3fbe1fadc9ddd66adbfc155c32ef9f2330a352e Mon Sep 17 00:00:00 2001 From: mbonsack Date: Tue, 6 Aug 2019 17:39:04 -0700 Subject: [PATCH] Refine filter (#40) Refine filter for performance and simplicity; add full payload to pycharm tests Fixes #38 --- package/etc/conf.d/filters/fortinet_fgt.conf | 15 ++++----------- tests/test_fortinet_ngfw.py | 6 +++--- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/package/etc/conf.d/filters/fortinet_fgt.conf b/package/etc/conf.d/filters/fortinet_fgt.conf index f862fa9..d6205b5 100644 --- a/package/etc/conf.d/filters/fortinet_fgt.conf +++ b/package/etc/conf.d/filters/fortinet_fgt.conf @@ -6,12 +6,12 @@ log { }; parser { - kv-parser(prefix(".kv.")); - date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}")); + syslog-parser(); + kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); + date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone(`default-timezone`)); }; rewrite { set("${.kv.devname}", value("HOST")); }; - rewrite { subst('<\d+>(.*)' "$1" value("MSG")); }; #set the source type based on program field and lookup index from the splunk context csv if (match("traffic" value(".kv.type"))) { @@ -24,16 +24,9 @@ log { parser {p_add_context_splunk(key("fgt_log")); }; }; - rewrite { - #drop the header - subst('date=[^ ]+ time=[^ ]+ devname=.+ (devid.*)', '$1', value("MESSAGE")); - #Strip empty fields because SEDCMD can't when we use event endpoint - subst('([^\= ]+=(?: |\"\"|-|\"-") ?)', '', value("MESSAGE")); - }; - #rewrite the final message for splunk json #sourcetype and index are set in the filter defaults won't be used - rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- + rewrite {r_set_splunk_basic(template("t_standard")) }; #--HEC-- destination(d_hec); #--HEC-- flags(flow-control,final); diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index e7f982d..60fdae7 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -19,7 +19,7 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message) @@ -41,7 +41,7 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") message = mt.render(mark="<13>", host=host) sendsingle(message) @@ -62,7 +62,7 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message)