Skip to content

Commit

Permalink
Update filter based on wire samples (#31)
Browse files Browse the repository at this point in the history 
* Add filter with tests for Fortinet Fortigate
  • Loading branch information
Ryan Faircloth authored and GitHub committed Jul 18, 2019
1 parent 6301769 commit 2ffdec5
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 54 deletions.
41 changes: 21 additions & 20 deletions package/etc/conf.d/filters/fortinet_fgt.conf
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,39 @@

log {
source(s_default-ports);

filter {tags("fgt_log")
or message('devid=\"?F[G|W|6K].+type=\"?(traffic|utm|event)')
;};

# the message does not include a program value in the header unfortunatly
# the use of colon in the message tricks the RFC3164 parser we will re-write the message
# so the parser will not incorrectly break it apart.
# while we are at it we will save the message type into the program field so parser can find it
rewrite {
subst('([A-Za-z]+ \d\d \d\d:\d\d:\d\d [^ ]+)(.*devid=\"?F[G|W|6K].+type=\"?)(traffic|utm|event)(.*)', '$1 $3:$2$3$4', value("MESSAGE") );
or message('devid=\"?F[G|W|6K].+type=\"?(traffic|utm|event)');
};

parser {
#basic parsing
syslog-parser(time-zone(`default-timezone`));
kv-parser(prefix(".kv."));
date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}"));
};

rewrite { set("${.kv.devname}", value("HOST")); };
rewrite { subst('<\d+>(.*)' "$1" value("MSG")); };

#set the source type based on program field and lookup index from the splunk context csv
if (program('traffic')) {
parser {p_add_context_splunk(key("fgt_traffic")); };
} elif (program('utm')) {
parser {p_add_context_splunk(key("fgt_utm")); };
} elif (program('event')) {
parser {p_add_context_splunk(key("fgt_event")); };
if (match("traffic" value(".kv.type"))) {
parser {p_add_context_splunk(key("fgt_traffic")); };
} elif (match("utm" value(".kv.type"))) {
parser {p_add_context_splunk(key("fgt_utm")); };
} elif (match("event" value(".kv.type"))) {
parser {p_add_context_splunk(key("fgt_event")); };
} else {
parser {p_add_context_splunk(key("fgt_log")); };
parser {p_add_context_splunk(key("fgt_log")); };
};

rewrite {
#drop the header
subst('date=[^ ]+ time=[^ ]+ devname=.+ (devid.*)', '$1', value("MESSAGE"));
#Strip empty fields because SEDCMD can't when we use event endpoint
subst('([^\= ]+=(?: |\"\"|-|\"-") ?)', '', value("MESSAGE"));
};

#rewrite the final message for splunk json
#sourcetype and index are set in the filter defaults won't be used
rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC--
# rewrite { r_set_splunk(template("t_JSON"))}; #--HEC--
destination(d_hec); #--HEC--

flags(final);
Expand Down
43 changes: 9 additions & 34 deletions tests/test_fortinet_ngfw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,18 @@

env = Environment(extensions=['jinja2_time.TimeExtension'])


@flaky(max_runs=3, min_passes=2)
#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1
def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk):
def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date=2015-08-11 time=19:19:43 devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n")
message = mt.render(mark="<111>", host=host)
"{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n")
message = mt.render(mark="<13>", host=host)

sendsingle(message)

st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2")
st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2")
search = st.render(host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -38,13 +37,12 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk):

@flaky(max_runs=3, min_passes=2)
#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1
def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk):
def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date=2015-08-11 time=19:19:43 devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n")
message = mt.render(mark="<111>", host=host)

"{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message)

st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2")
Expand All @@ -64,9 +62,8 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date=2015-08-11 time=19:21:40 logver=52 devname=US-Corp_Main1 devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n")
message = mt.render(mark="<111>", host=host)

"{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message)

st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2")
Expand All @@ -79,25 +76,3 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk):
record_property("message", message)

assert resultCount == 1

@flaky(max_runs=3, min_passes=2)
#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=17:49:37 clusterid=FGHA001500704701_CID logver=52 devname=FGT-FortiToken1 devid=FGHA001500704701_CID logid=0100044547 type=event subtype=system level=information vd=root logdesc="Object attribute configured" user="charlihchen" ui="GUI(96.45.36.160)" action=Edit cfgtid=2760243 cfgpath="user.local" cfgobj="gzhang" cfgattr="fortitoken[FTKMOB5374362440->FTKMOB47ED6DD69D]" msg="Edit user.local gzhang"
def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

mt = env.from_string(
"{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date=2015-08-11 time=17:49:37 clusterid=FGHA001500704701_CID logver=52 devname=FGT-FortiToken1 devid=FGHA001500704701_CID logid=0100044547 type=event subtype=system level=information vd=root logdesc=\"Object attribute configured\" user=\"charlihchen\" ui=\"GUI(96.45.36.160)\" action=Edit cfgtid=2760243 cfgpath=\"user.local\" cfgobj=\"gzhang\" cfgattr=\"fortitoken[FTKMOB5374362440->FTKMOB47ED6DD69D]\" msg=\"Edit user.local gzhang\"\n")
message = mt.render(mark="<111>", host=host)

sendsingle(message)

st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2")
search = st.render(host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)

record_property("host", host)
record_property("resultCount", resultCount)
record_property("message", message)

assert resultCount == 1

0 comments on commit 2ffdec5

Please sign in to comment.