From 906e609dff1239c13757d75a3097037d20183b30 Mon Sep 17 00:00:00 2001 From: themrkeys Date: Mon, 11 Nov 2019 13:03:27 -0500 Subject: [PATCH] Add Ubiquiti Networks Unifi product range (#188) * Resolve fallback events in msg rather than JSON mode * Add support for Ubiquiti networks unifi product line AP, Switch, USG (firewalls) --- docs/sources.md | 74 ++++++++++ .../etc/conf.d/filters/Ubiquiti/unifi.conf | 7 + .../p_rfc3164-ubiquiti_unifi.conf.tmpl | 137 ++++++++++++++++++ package/etc/conf.d/log_paths/zfallback.conf | 5 +- .../vendor_product_by_source.conf | 4 + .../vendor_product_by_source.csv | 3 +- package/etc/syslog-ng.conf | 2 +- tests/test_ubiquiti_unifi.py | 93 ++++++++++++ 8 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 package/etc/conf.d/filters/Ubiquiti/unifi.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl create mode 100644 tests/test_ubiquiti_unifi.py diff --git a/docs/sources.md b/docs/sources.md index bb9d552..22c3aab 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -948,6 +948,80 @@ index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` +# Vendor - Ubiquiti - Unifi + +All Ubiquity Unfi firewalls, switches, and access points share a common syslog configuration via the NMS. + + +* Login to NMS +* Navigate to settings +* Navigate to Site +* Enable Remote syslog server +* Enter hostname and port +* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf `` update the host or ip mask for ``f_ubiquiti_unifi_fw`` to identify USG firewalls + +## Product - Unifi Switch and Access Points + +Unifi devices are managed using the Network Management Controller + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/4107/ | +| Product Manual | https://https://help.ubnt.com/ | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| ubnt | Used when no sub source type is required by add on | +| ubnt:fw | USG events | +| ubnt:threat | USG IDS events | +| ubnt:switch | Unifi Switches | +| ubnt:wireless | Access Point logs | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| ubiquiti_unifi | ubnt | netops | none | +| ubiquiti_unifi_fw | ubnt:fw | netfw | none | +| ubiquiti_unifi_link | ubnt:link | netops | none | +| ubiquiti_unifi_sudo | ubnt:sudo | netops | none | +| ubiquiti_unifi_switch | ubnt:switch | netops | none | +| ubiquiti_unifi_threat | ubnt:threat | netids | none | +| ubiquiti_unifi_wireless | ubnt:wireless | netops | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_ZSCALER_NSS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=zscalernss-* | stats count by host +``` + + # Vendor - Zscaler ## Product - All Products diff --git a/package/etc/conf.d/filters/Ubiquiti/unifi.conf b/package/etc/conf.d/filters/Ubiquiti/unifi.conf new file mode 100644 index 0000000..49740eb --- /dev/null +++ b/package/etc/conf.d/filters/Ubiquiti/unifi.conf @@ -0,0 +1,7 @@ +filter f_ubiquiti_unifi { + host('^U[^,]{1,10},[a-z0-9]{9,16},v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6}') + or + program('^U[^,]{1,10},[a-z0-9]{9,16},v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6}') + or + match("ubiquiti_unifi_*", value("fields.sc4s_vendor_product") type(glob) ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl new file mode 100644 index 0000000..0962cc6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -0,0 +1,137 @@ +#Ubiquiti unifi +{{ $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_is_rfc3164); + filter(f_ubiquiti_unifi); +{{- end}} +{{- if eq (.) "no"}} + source (s_UBIQUITI_UNIFI); +{{- end}} + + + parser {p_add_context_splunk(key("ubiquiti_unifi")); }; + + #Fiewall + if { + filter { + match("ubiquiti_unifi_fw", value("fields.sc4s_vendor_product") type(glob) ); + }; + + if (match("[^)]\s\S+\skernel:\s[^ll\sheader][^\[\d+.\d+\]]\S+\s\w+:" value("RAWMSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:threat"), index("netids")) + set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + parser {p_add_context_splunk(key("ubiquiti_unifi_threat")); }; + } elif (match("\S+\slinkcheck:" value("RAWMSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:link"), index("netops")) + set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + parser {p_add_context_splunk(key("ubiquiti_unifi_link")); }; + } elif (match("\d+:\d+:\d+\s\S+\ssudo:" value("RAWMSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:sudo"), index("netops")) + set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + parser {p_add_context_splunk(key("ubiquiti_unifi_sudo")); }; + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:fw"), index("netfw")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_fw")); }; + }; + #Switch + } elif { + filter { + host('^(?US[^,]{1,10}),(?[a-z0-9]{9,16}),(?v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches")); + }; + if (match("hostapd:\s+ath" value("RAWMSG"))) { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:hostapd"), index("netops")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${HOST_FROM}", value("HOST")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; + } elif (match("\d+:\d+:\d+\s\S+\smcad:" value("RAWMSG"))) { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:mcad"), index("netops")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${HOST_FROM}", value("HOST")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:switch"), index("netops")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${FROM_HOST}",value("HOST")); + set("${model}", value("fields.model")); + set("${serial}", value("fields.serial")); + set("${firmware}", value("fields.firmware")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_switch")); }; + + }; + + } elif { + filter { + program('^(?U\d[^,]{1,10}),(?[a-z0-9]{9,16}),(?v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches")); + }; + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:wireless"), index("netops")); + set("${FROM_HOST}",value("HOST")); + set("${model}", value("fields.model")); + set("${serial}", value("fields.serial")); + set("${firmware}", value("fields.firmware")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; + + } elif (match("traputil.c\(696\) " value("RAWMSG"))) { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:edgeswitch"), index("netops")); + set("${HOST_FROM}", value("HOST")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_edge_switch")); }; + + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt"), index("netops")); + set("${HOST_FROM}", value("HOST")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi")); }; + }; + + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_UBIQUITI_UNIFI") }} + destination(d_archive); +{{- end}} + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for UBIQUITI_UNIFI traffic + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for UBIQUITI_UNIFI traffic +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/zfallback.conf b/package/etc/conf.d/log_paths/zfallback.conf index 4902bd1..478ff31 100644 --- a/package/etc/conf.d/log_paths/zfallback.conf +++ b/package/etc/conf.d/log_paths/zfallback.conf @@ -1,7 +1,10 @@ log { source(s_DEFAULT); - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")) }; + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); + set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); + }; parser { p_add_context_splunk(key("sc4s_fallback")); }; diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 57c73ac..ec20e82 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -36,3 +36,7 @@ filter f_proofpoint_pps_sendmail { host("pps-*" type(glob)) or netmask(192.168.6.0/24) }; +filter f_ubiquiti_unifi_fw { + host("usg-*" type(glob)) or + netmask(192.168.6.0/24) +}; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 2f85bc4..698f672 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -6,4 +6,5 @@ f_juniper_idp,sc4s_vendor_product,"juniper_idp" f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" -f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" +f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" \ No newline at end of file diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 4f19083..88897f1 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -21,7 +21,7 @@ options { chain_hostnames (off); use_dns (no); use_fqdn (no); - dns-cache(no); + dns-cache(yes); create_dirs (no); keep-hostname (yes); create_dirs(yes); diff --git a/tests/test_ubiquiti_unifi.py b/tests/test_ubiquiti_unifi.py new file mode 100644 index 0000000..4eed2b2 --- /dev/null +++ b/tests/test_ubiquiti_unifi.py @@ -0,0 +1,93 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +#<27>Nov 8 17:28:43 US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type + +def test_ubiquiti_unifi_us8p60(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<29>Nov 10 20:46:02 US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0 +def test_ubiquiti_unifi_switch_us24p250(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<30>Nov 10 11:49:46 U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514 +def test_ubiquiti_unifi_ap_u7pg2(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netops sourcetype=ubnt:wireless earliest=-2m | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<4>Nov 10 23:04:06 USG kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328 +def test_ubiquiti_unifi_usg(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} usg-{{host}} kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netfw sourcetype=ubnt:fw host=usg-{{host}} | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1