From 2fb4009d215ee36f648549dd4a17dbbce0b871ad Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Tue, 15 Oct 2019 15:48:55 -0400 Subject: [PATCH] Support Cisco Meraki (#150) --- docker-compose.yml | 1 + docs/sources.md | 57 ++++++++++++++++--- .../conf.d/conflib/_common/syslog_format.conf | 9 +++ package/etc/conf.d/filters/cisco/meraki.conf | 22 +++++++ package/etc/conf.d/local/context/README.md | 1 + .../context/compliance_meta_by_source.conf | 5 -- .../context/compliance_meta_by_source.csv | 2 - .../etc/conf.d/local/context/splunk_index.csv | 40 ------------- .../context/vendor_product_by_source.conf | 37 ------------ .../context/vendor_product_by_source.csv | 8 --- .../p_rfc5424_epoch-cisco_merkai.conf.tmpl | 42 ++++++++++++++ package/etc/conf.d/sources/network.conf.tmpl | 3 + .../vendor_product_by_source.conf | 4 ++ .../vendor_product_by_source.csv | 1 + package/etc/go_templates/source_network.t | 6 ++ tests/test_cisco_meraki.py | 35 ++++++++++++ 16 files changed, 174 insertions(+), 99 deletions(-) create mode 100644 package/etc/conf.d/filters/cisco/meraki.conf create mode 100644 package/etc/conf.d/local/context/README.md delete mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.conf delete mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.csv delete mode 100644 package/etc/conf.d/local/context/splunk_index.csv delete mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.conf delete mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.csv create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl create mode 100644 tests/test_cisco_meraki.py diff --git a/docker-compose.yml b/docker-compose.yml index 1765168..a9e5e74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,7 @@ services: RH_ORG: ${RH_ORG} RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s + command: -det ports: - "514:514" - "601:601" diff --git a/docs/sources.md b/docs/sources.md index 12f4f3c..7ec8f00 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -41,12 +41,7 @@ MSG Parse: This filter parses message content * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. -* Follow vendor configuration steps per Product Manual above ensure: - * Log Level is 6 "Informational" - * Protocol is TCP/IP - * permit-hostdown is on - * device-id is hostname and included - * timestamp is included +* Follow vendor configuration steps per Product Manual above ### Options @@ -107,7 +102,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_JUNIPER_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | | SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format | ### Verification @@ -189,6 +184,54 @@ Use the following search to validate events are present, for NX-OS, WLC and ACI index= sourcetype=cisco:ios | stats count by host ``` +## Product - Meraki Product Line MR, MS, MX, MV + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3018/ | +| Product Manual | https://documentation.meraki.com/zGeneral_Administration/Monitoring_and_Reporting/Syslog_Server_Overview_and_Configuration | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| merkai | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_meraki | meraki | netfw | The current TA does not sub sourcetype or utilize source preventing segmenation into more appropriate indexes | + + +### Filter type + +IP, Netmask, Host or Port + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Follow vendor configuration steps per Product Manual above + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_MERAKI_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_MERAKI_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=merkai +``` + +Verify timestamp, and host values match as expected + Verify timestamp, and host values match as expected diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 8c5794c..5b69e71 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -4,6 +4,9 @@ filter f_rfc5424_strict{ filter f_rfc5424_noversion{ message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); }; +filter f_rfc5424_epochtime{ + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) ))'); +}; rewrite set_rfcnonconformant{ set("rfc5424_nonconform" value("fields.sc4s_syslog_format")); }; @@ -19,6 +22,12 @@ rewrite set_rfc5424_noversion{ filter f_is_rfc5424_noversion{ match("rfc5424_noversion" value("fields.sc4s_syslog_format")) }; +rewrite set_rfc5424_epochtime{ + set("rfc5424_epochtime" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_epochtime{ + match("rfc5424_epochtime" value("fields.sc4s_syslog_format")) +}; rewrite set_rfc3164{ set("rfc3164" value("fields.sc4s_syslog_format")); }; diff --git a/package/etc/conf.d/filters/cisco/meraki.conf b/package/etc/conf.d/filters/cisco/meraki.conf new file mode 100644 index 0000000..3d93cb6 --- /dev/null +++ b/package/etc/conf.d/filters/cisco/meraki.conf @@ -0,0 +1,22 @@ +# Meraki + +filter f_cisco_meraki { + match("cisco_meraki", value("fields.sc4s_vendor_product") type(glob)) +}; + +parser p_cisco_meraki { + channel { + filter { + match( + #'(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+ )(?.*))' + '(?:(?:<(?\d{1,3})>(?[1-9][0-9]?) (?:(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) )(?.*))' + flags(store-matches) + ); + }; + parser { + date-parser(format('%s') + template("${EPOCH}")); + }; + }; + +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/README.md b/package/etc/conf.d/local/context/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/context/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.conf b/package/etc/conf.d/local/context/compliance_meta_by_source.conf deleted file mode 100644 index f325b4f..0000000 --- a/package/etc/conf.d/local/context/compliance_meta_by_source.conf +++ /dev/null @@ -1,5 +0,0 @@ -@version: 3.24 -filter f_test_test { - host("something-*" type(glob)) or - netmask(192.168.100.1/24) -}; diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.csv b/package/etc/conf.d/local/context/compliance_meta_by_source.csv deleted file mode 100644 index 6608db0..0000000 --- a/package/etc/conf.d/local/context/compliance_meta_by_source.csv +++ /dev/null @@ -1,2 +0,0 @@ -#f_test_test,.splunk.index,"badindex" -#f_test_test,fields.compliance,"pci" diff --git a/package/etc/conf.d/local/context/splunk_index.csv b/package/etc/conf.d/local/context/splunk_index.csv deleted file mode 100644 index e93911a..0000000 --- a/package/etc/conf.d/local/context/splunk_index.csv +++ /dev/null @@ -1,40 +0,0 @@ -#bluecoat_proxy,index,netproxy -#cef_ArcSight_ArcSight,index,netwaf -#cef_Incapsula_SIEMintegration,index,netwaf -#cef_Microsoft_Microsoft Windows,index,oswinsec -#cef_Microsoft_System or Application Event,index,oswin -#cisco_asa,index,netfw -#cisco_ios,index,netops -#cisco_nx_os,index,netops -#local_example,index,main -#fortinet_fortios_event,index,netops -#fortinet_fortios_log,index,netops -#fortinet_fortios_traffic,index,netfw -#fortinet_fortios_utm,index,netids -#juniper_idp,index,netids -#juniper_structured,index,netops -#juniper_idp_structured,index,netids -#juniper_junos_fw_structured,index,netfw -#juniper_junos_ids_structured,index,netids -#juniper_junos_utm_structured,index,netfw -#juniper_junos_fw,index,netfw -#juniper_junos_ids,index,netids -#juniper_junos_utm,index,netfw -#juniper_sslvpn,index,netfw -#juniper_netscreen,index,netfw -#juniper_nsm,index,netfw -#juniper_nsm_idp,index,netids -#juniper_legacy,index,netops -#pan_traffic,index,netfw -#pan_threat,index,netproxy -#pan_system,index,netops -#pan_config,index,netops -#pan_hipwatch,index,main -#pan_correlation,index,main -#pan_userid,index,netauth -#pan_unknown,index,netops -#proofpoint_pps_filter,index,email -#proofpoint_pps_sendmail,index,email -#sc4s_events,index,main -#sc4s_fallback,index,main -#sc4s_metrics,index,em_metrics diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.conf b/package/etc/conf.d/local/context/vendor_product_by_source.conf deleted file mode 100644 index 37e3412..0000000 --- a/package/etc/conf.d/local/context/vendor_product_by_source.conf +++ /dev/null @@ -1,37 +0,0 @@ -@version: 3.22 -#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a -#bug/limitation of selector files. The better syntax should be as follows -#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; - -filter f_test_test { - host("testvp-*" type(glob)) or - netmask(192.168.100.1/24) -}; -filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) or - netmask(192.168.1.0/24) -}; -filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) or - netmask(192.168.2.0/24) -}; -filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) -}; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or - netmask(192.168.4.0/24) -}; -filter f_cisco_nx_os { - host("csconx-*" type(glob)) or - netmask(192.168.5.0/24) -}; -filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) -}; -filter f_proofpoint_pps_filter { - host("pps-*" type(glob)) or - netmask(192.168.7.0/24) -}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.csv b/package/etc/conf.d/local/context/vendor_product_by_source.csv deleted file mode 100644 index 3f90603..0000000 --- a/package/etc/conf.d/local/context/vendor_product_by_source.csv +++ /dev/null @@ -1,8 +0,0 @@ -f_test_test,sc4s_vendor_product,"test_test" -f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" -f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" -f_juniper_idp,sc4s_vendor_product,"juniper_idp" -f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" -f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" -f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" -f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl new file mode 100644 index 0000000..dbbb1f6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl @@ -0,0 +1,42 @@ +# Checkpoint Splunk format +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_default-ports); + filter(f_cisco_meraki); +{{- end}} +{{- if eq (.) "no"}} + source (s_dedicated_port_CISCO_MERAKI); +{{- end}} + + #parser { + # kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); + # date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); + # + # }; + + #rewrite { set("${.kv.hostname}", value("HOST")); }; + + rewrite { r_set_splunk_dest_default(sourcetype("meraki"), index("netfw"), template("t_hdr_msg"))}; + parser {p_add_context_splunk(key("cisco_meraki")); }; + + + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_MERAKI traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index a9e8b78..234e87c 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -82,6 +82,9 @@ source s_default-ports { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 0903ca3..b736ac3 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -1,5 +1,9 @@ @version: 3.24 +filter f_cisco_meraki { + host("testcm-*" type(glob)) or + netmask(192.168.100.1/24) +}; filter f_test_test { host("testvp-*" type(glob)) or netmask(192.168.100.1/24) diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 3f90603..2f85bc4 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -1,4 +1,5 @@ f_test_test,sc4s_vendor_product,"test_test" +f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index f804816..09ea120 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -74,6 +74,9 @@ source s_dedicated_port_{{ .port_id}} { {{- else if eq .parser "cisco_parser" }} parser {cisco-parser()}; rewrite(set_cisco_ios); +{{- else if eq .parser "cisco_meraki_parser" }} + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); {{- else if eq .parser "rfc3164" }} parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); @@ -96,6 +99,9 @@ source s_dedicated_port_{{ .port_id}} { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/tests/test_cisco_meraki.py b/tests/test_cisco_meraki.py new file mode 100644 index 0000000..920714a --- /dev/null +++ b/tests/test_cisco_meraki.py @@ -0,0 +1,35 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<134>1 1563249630.774247467 devicename security_event ids_alerted signature=1:28423:1 priority=1 timestamp=1468531589.810079 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection +def test_cisco_meraki_security_event(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}1 {% now 'utc', '%s' %}.123456789 testcm-{{ host }} security_event ids_alerted signature=1:28423:1 priority=1 timestamp={% now 'utc', '%s' %}.123456 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection\n") + message = mt.render(mark="<134>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"testcm-{{ host }}\" sourcetype=\"meraki\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +#<134>1 Dec 6 08:41:44 192.168.1.1 1 1386337316.207232138 MX84 events Cellular connection up