From b2232c6c73f0b6f40526c65024b3f42935a63b5e Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 17 Jul 2019 12:04:59 -0400 Subject: [PATCH] Feature/add fortinet filter (#24) * Add filter with tests for Fortinet Fortigate * CI CD CLEANUP --- .circleci/config.yml | 10 +- .env.template | 2 +- docker-compose.yml | 30 +++--- package/etc/conf.d/filters/fortinet_fgt.conf | 39 +++++++ package/etc/context/splunk_index.csv | 8 ++ test-with-compose.sh | 2 +- tests/conftest.py | 2 +- tests/test_cisco_asa.py | 1 + tests/test_cisco_ios.py | 3 +- tests/test_fortinet_ngfw.py | 103 +++++++++++++++++++ 10 files changed, 174 insertions(+), 26 deletions(-) create mode 100644 package/etc/conf.d/filters/fortinet_fgt.conf create mode 100644 tests/test_fortinet_ngfw.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 4786b41..a811e2a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,9 +103,7 @@ jobs: - run: name: Docker Compose build command: | - docker-compose build \ - --build-arg RH_ACTIVATION=$RH_ACTIVATION \ - --build-arg RH_ORG=$RH_ORG + docker-compose build - run: name: Docker Compose up command: docker-compose up --abort-on-container-exit @@ -136,10 +134,8 @@ jobs: pip install -r requirements.txt python clair_to_junit_parser.py "/clair-reports/$IMAGE_NAME:$CIRCLE_SHA1.json" --output test-results/results.xml when: on_fail -# - store_test_results: -# path: test-results/results.xml - - store_artifacts: - path: test-results + - store_test_results: + path: test-results/results.xml - store_artifacts: path: /clair-reports diff --git a/.env.template b/.env.template index 8950cae..5a9bda1 100644 --- a/.env.template +++ b/.env.template @@ -16,6 +16,6 @@ SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=metrics -SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download +SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download SPLUNKBASE_USERNAME=username SPLUNKBASE_PASSWORD=password diff --git a/docker-compose.yml b/docker-compose.yml index 94a8320..f13c181 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,15 +17,15 @@ services: - sc4s-tests:/work/tests - sc4s-results:/work/test-results environment: - - SPLUNK_PASSWORD + - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} sc4s: image: rfaircloth/scs:latest build: context: ./package args: - RH_ORG: $RH_ORG - RH_ACTIVATION: $RH_ACTIVATION + RH_ORG: ${RH_ORG} + RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s ports: - "514" @@ -38,12 +38,12 @@ services: links: - splunk environment: - - SPLUNK_HEC_URL - - SPLUNK_HEC_STATSURL - - SPLUNK_HEC_TOKEN - - SPLUNK_CONNECT_METHOD - - SPLUNK_DEFAULT_INDEX - - SPLUNK_METRICS_INDEX + - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} + - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} + - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} # logging: # driver: splunk # options: @@ -60,12 +60,12 @@ services: - "8088:8088" - "8089:8089" environment: - - SPLUNK_HEC_TOKEN - - SPLUNK_PASSWORD - - SPLUNK_START_ARGS - - SPLUNK_APPS_URL - - SPLUNKBASE_USERNAME - - SPLUNKBASE_PASSWORD + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} + - SPLUNK_START_ARGS=${SPLUNK_START_ARGS} + - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} + - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} + - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng # logging: diff --git a/package/etc/conf.d/filters/fortinet_fgt.conf b/package/etc/conf.d/filters/fortinet_fgt.conf new file mode 100644 index 0000000..94678df --- /dev/null +++ b/package/etc/conf.d/filters/fortinet_fgt.conf @@ -0,0 +1,39 @@ + +log { + source(s_default-ports); + + filter {tags("fgt_log") + or message('devid=\"?F[G|W|6K].+type=\"?(traffic|utm|event)') + ;}; + + # the message does not include a program value in the header unfortunatly + # the use of colon in the message tricks the RFC3164 parser we will re-write the message + # so the parser will not incorrectly break it apart. + # while we are at it we will save the message type into the program field so parser can find it + rewrite { + subst('([A-Za-z]+ \d\d \d\d:\d\d:\d\d [^ ]+)(.*devid=\"?F[G|W|6K].+type=\"?)(traffic|utm|event)(.*)', '$1 $3:$2$3$4', value("MESSAGE") ); + }; + parser { + #basic parsing + syslog-parser(time-zone(`default-timezone`)); + }; + + #set the source type based on program field and lookup index from the splunk context csv + if (program('traffic')) { + parser {p_add_context_splunk(key("fgt_traffic")); }; + } elif (program('utm')) { + parser {p_add_context_splunk(key("fgt_utm")); }; + } elif (program('event')) { + parser {p_add_context_splunk(key("fgt_event")); }; + } else { + parser {p_add_context_splunk(key("fgt_log")); }; + }; + + #rewrite the final message for splunk json + #sourcetype and index are set in the filter defaults won't be used + rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- +# rewrite { r_set_splunk(template("t_JSON"))}; #--HEC-- + destination(d_hec); #--HEC-- + + flags(final); +}; diff --git a/package/etc/context/splunk_index.csv b/package/etc/context/splunk_index.csv index 75b98d6..eae8cf3 100644 --- a/package/etc/context/splunk_index.csv +++ b/package/etc/context/splunk_index.csv @@ -2,6 +2,14 @@ cisco:asa,index,main cisco:asa,sourcetype,cisco:asa cisco:ios,index,main cisco:ios,sourcetype,cisco:ios +fgt_event,sourcetype,fgt_event +fgt_event,index,main +fgt_log,sourcetype,fgt_log +fgt_log,index,main +fgt_traffic,sourcetype,fgt_traffic +fgt_traffic,index,main +fgt_utm,sourcetype,fgt_utm +fgt_utm,index,main pan:traffic,index,main pan:threat,index,main pan:system,index,main diff --git a/test-with-compose.sh b/test-with-compose.sh index 400c3c8..3f08cbf 100755 --- a/test-with-compose.sh +++ b/test-with-compose.sh @@ -16,7 +16,7 @@ docker container create --name dummy \ docker cp tests/ dummy:/work/tests/ docker rm dummy -docker-compose build --build-arg RH_ACTIVATION=$RH_ACTIVATION --build-arg RH_ORG=$RH_ORG +docker-compose build docker-compose up --abort-on-container-exit --exit-code-from test docker container create --name dummy \ diff --git a/tests/conftest.py b/tests/conftest.py index cc16563..ab4fd96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,7 +39,7 @@ def setup_splunk(): break except ConnectionRefusedError: tried += 1 - if tried > 180: + if tried > 600: raise sleep(1) return c diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index a908fc4..308356a 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -40,6 +40,7 @@ def test_cisco_asa_tradditional(record_property, setup_wordlist, setup_splunk): # <166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 +@flaky(max_runs=3, min_passes=2) def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index 19746af..f729df9 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -5,7 +5,7 @@ # https://opensource.org/licenses/BSD-2-Clause from jinja2 import Environment -from jinja2 import Environment +from flaky import flaky from .sendmessage import * from .splunkutils import * @@ -14,6 +14,7 @@ # <190>30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated +@flaky(max_runs=3, min_passes=2) def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py new file mode 100644 index 0000000..fe5873d --- /dev/null +++ b/tests/test_fortinet_ngfw.py @@ -0,0 +1,103 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from flaky import flaky +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +@flaky(max_runs=3, min_passes=2) +#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 +def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date=2015-08-11 time=19:19:43 devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +@flaky(max_runs=3, min_passes=2) +#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 +def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date=2015-08-11 time=19:19:43 devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +@flaky(max_runs=3, min_passes=2) +#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:21:40 logver=52 devname=US-Corp_Main1 devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user="" srcip=172.30.16.119 srcport=53235 srcintf="Internal" dstip=114.112.67.75 dstport=80 dstintf="External-SDC" proto=6 service=HTTP hostname="popo.wan.ijinshan.com" profile="scan" action=passthrough reqtype=direct url="/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl" sentbyte=525 rcvdbyte=325 direction=outgoing msg="URL belongs to an allowed category in policy" method=domain cat=52 catdesc="Information Technology" +def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date=2015-08-11 time=19:21:40 logver=52 devname=US-Corp_Main1 devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +@flaky(max_runs=3, min_passes=2) +#<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=17:49:37 clusterid=FGHA001500704701_CID logver=52 devname=FGT-FortiToken1 devid=FGHA001500704701_CID logid=0100044547 type=event subtype=system level=information vd=root logdesc="Object attribute configured" user="charlihchen" ui="GUI(96.45.36.160)" action=Edit cfgtid=2760243 cfgpath="user.local" cfgobj="gzhang" cfgattr="fortitoken[FTKMOB5374362440->FTKMOB47ED6DD69D]" msg="Edit user.local gzhang" +def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date=2015-08-11 time=17:49:37 clusterid=FGHA001500704701_CID logver=52 devname=FGT-FortiToken1 devid=FGHA001500704701_CID logid=0100044547 type=event subtype=system level=information vd=root logdesc=\"Object attribute configured\" user=\"charlihchen\" ui=\"GUI(96.45.36.160)\" action=Edit cfgtid=2760243 cfgpath=\"user.local\" cfgobj=\"gzhang\" cfgattr=\"fortitoken[FTKMOB5374362440->FTKMOB47ED6DD69D]\" msg=\"Edit user.local gzhang\"\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1