Add Ubiquiti Networks Unifi product range (#188)

* Resolve fallback events in msg rather than JSON mode
* Add support for Ubiquiti networks unifi product line AP, Switch, USG (firewalls)

docs/sources.md

@@ -948,6 +948,80 @@ index=<asconfigured> sourcetype=bluecoat:proxysg:access:kv | stats count by host
 ```
 
 
+# Vendor - Ubiquiti - Unifi
+
+All Ubiquity Unfi firewalls, switches, and access points share a common syslog configuration via the NMS.
+
+
+* Login to NMS
+* Navigate to settings
+* Navigate to Site 
+* Enable Remote syslog server
+* Enter hostname and port
+* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf `` update the host or ip mask for ``f_ubiquiti_unifi_fw`` to identify USG firewalls
+
+## Product - Unifi Switch and Access Points 
+
+Unifi devices are managed using the Network Management Controller
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/4107/                                                                 |
+| Product Manual | https://https://help.ubnt.com/    |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| ubnt  | Used when no sub source type is required by add on |
+| ubnt:fw  | USG events |
+| ubnt:threat | USG IDS events    |
+| ubnt:switch  | Unifi Switches |
+| ubnt:wireless  | Access Point logs |
+
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| ubiquiti_unifi      | ubnt     | netops          | none          |
+| ubiquiti_unifi_fw      | ubnt:fw       | netfw          | none          |
+| ubiquiti_unifi_link      | ubnt:link     | netops          | none          |
+| ubiquiti_unifi_sudo      | ubnt:sudo     | netops          | none          |
+| ubiquiti_unifi_switch | ubnt:switch | netops          | none          |
+| ubiquiti_unifi_threat      | ubnt:threat      | netids          | none          |
+| ubiquiti_unifi_wireless | ubnt:wireless | netops          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_ZSCALER_NSS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=zscalernss-* | stats count by host
+```
+
+
 # Vendor - Zscaler
 
 ## Product - All Products

package/etc/conf.d/filters/Ubiquiti/unifi.conf

@@ -0,0 +1,7 @@
+filter f_ubiquiti_unifi {
+    host('^U[^,]{1,10},[a-z0-9]{9,16},v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6}')
+    or
+    program('^U[^,]{1,10},[a-z0-9]{9,16},v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6}')
+    or
+    match("ubiquiti_unifi_*", value("fields.sc4s_vendor_product") type(glob) );
+};

package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl

@@ -0,0 +1,137 @@
+#Ubiquiti unifi
+{{ $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common"}}
+{{ tmpl.Exec "t/source_network.t" $context }}
+
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes"}}
+    source(s_DEFAULT);
+    filter(f_is_rfc3164);
+    filter(f_ubiquiti_unifi);
+{{- end}}
+{{- if eq (.) "no"}}
+    source (s_UBIQUITI_UNIFI);
+{{- end}}
+
+
+    parser {p_add_context_splunk(key("ubiquiti_unifi")); };
+
+    #Fiewall
+    if {
+        filter {
+            match("ubiquiti_unifi_fw", value("fields.sc4s_vendor_product") type(glob) );
+        };
+
+        if (match("[^)]\s\S+\skernel:\s[^ll\sheader][^\[\d+.\d+\]]\S+\s\w+:" value("RAWMSG"))) {
+            rewrite { r_set_splunk_dest_default(sourcetype("ubnt:threat"), index("netids"))
+            set("${LEGACY_MSGHDR}${MSG}" value("MSG"));};
+            parser {p_add_context_splunk(key("ubiquiti_unifi_threat")); };
+        } elif (match("\S+\slinkcheck:" value("RAWMSG"))) {
+            rewrite { r_set_splunk_dest_default(sourcetype("ubnt:link"), index("netops"))
+            set("${LEGACY_MSGHDR}${MSG}" value("MSG"));};
+            parser {p_add_context_splunk(key("ubiquiti_unifi_link")); };
+        } elif (match("\d+:\d+:\d+\s\S+\ssudo:" value("RAWMSG"))) {
+            rewrite { r_set_splunk_dest_default(sourcetype("ubnt:sudo"), index("netops"))
+            set("${LEGACY_MSGHDR}${MSG}" value("MSG"));};
+            parser {p_add_context_splunk(key("ubiquiti_unifi_sudo")); };
+        } else {
+            rewrite {
+                r_set_splunk_dest_default(sourcetype("ubnt:fw"), index("netfw"));
+                set("${LEGACY_MSGHDR}${MSG}" value("MSG"));
+            };
+            parser {p_add_context_splunk(key("ubiquiti_unifi_fw")); };
+        };
+    #Switch
+    } elif {
+        filter {
+            host('^(?<model>US[^,]{1,10}),(?<serial>[a-z0-9]{9,16}),(?<firmware>v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches"));
+        };
+        if (match("hostapd:\s+ath" value("RAWMSG"))) {
+            rewrite {
+                r_set_splunk_dest_default(sourcetype("ubnt:hostapd"), index("netops"));
+                set("${LEGACY_MSGHDR}${MSG}" value("MSG"));
+                set("${HOST_FROM}", value("HOST"));
+            };
+            parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); };
+        } elif (match("\d+:\d+:\d+\s\S+\smcad:" value("RAWMSG"))) {
+            rewrite {
+                r_set_splunk_dest_default(sourcetype("ubnt:mcad"), index("netops"));
+                set("${LEGACY_MSGHDR}${MSG}" value("MSG"));
+                set("${HOST_FROM}", value("HOST"));
+            };
+            parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); };
+        } else {
+            rewrite {
+                r_set_splunk_dest_default(sourcetype("ubnt:switch"), index("netops"));
+                set("${LEGACY_MSGHDR}${MSG}" value("MSG"));
+                set("${FROM_HOST}",value("HOST"));
+                set("${model}", value("fields.model"));
+                set("${serial}", value("fields.serial"));
+                set("${firmware}", value("fields.firmware"));
+            };
+            parser {p_add_context_splunk(key("ubiquiti_unifi_switch")); };
+
+        };
+
+    } elif {
+        filter {
+            program('^(?<model>U\d[^,]{1,10}),(?<serial>[a-z0-9]{9,16}),(?<firmware>v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches"));
+        };
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("ubnt:wireless"), index("netops"));
+            set("${FROM_HOST}",value("HOST"));
+            set("${model}", value("fields.model"));
+            set("${serial}", value("fields.serial"));
+            set("${firmware}", value("fields.firmware"));
+        };
+       parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); };
+
+    } elif (match("traputil.c\(696\) " value("RAWMSG"))) {
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("ubnt:edgeswitch"), index("netops"));
+            set("${HOST_FROM}", value("HOST"));
+            set("${LEGACY_MSGHDR}${MSG}" value("MSG"));
+        };
+       parser {p_add_context_splunk(key("ubiquiti_unifi_edge_switch")); };
+
+    } else {
+       rewrite {
+        r_set_splunk_dest_default(sourcetype("ubnt"), index("netops"));
+        set("${HOST_FROM}", value("HOST"));
+        set("${LEGACY_MSGHDR}${MSG}" value("MSG"));
+       };
+       parser {p_add_context_splunk(key("ubiquiti_unifi")); };
+    };
+
+
+    parser (compliance_meta_by_source);
+
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+    };
+
+    destination(d_hec);
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_UBIQUITI_UNIFI") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control);
+
+};
+{{- end}}
+{{- if (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT") "no") "no") }}
+
+# Listen on the specified dedicated port(s) for UBIQUITI_UNIFI traffic
+    {{tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for UBIQUITI_UNIFI traffic
+{{tmpl.Exec "log_path" "yes" }}

package/etc/conf.d/log_paths/zfallback.conf

@@ -1,7 +1,10 @@
 log {
     source(s_DEFAULT);
 
-    rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")) };
+    rewrite {
+        r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
+        set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG"));
+    };
     parser {
         p_add_context_splunk(key("sc4s_fallback"));
     };

package/etc/context_templates/vendor_product_by_source.conf

@@ -36,3 +36,7 @@ filter f_proofpoint_pps_sendmail {
     host("pps-*" type(glob)) or
     netmask(192.168.6.0/24)
 };
+filter f_ubiquiti_unifi_fw {
+        host("usg-*" type(glob)) or
+        netmask(192.168.6.0/24)
+};

package/etc/context_templates/vendor_product_by_source.csv

@@ -6,4 +6,5 @@ f_juniper_idp,sc4s_vendor_product,"juniper_idp"
 f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen"
 f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os"
 f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail"
-f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter"
+f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter"
+f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw"

package/etc/syslog-ng.conf

@@ -21,7 +21,7 @@ options {
         chain_hostnames (off);
         use_dns (no);
         use_fqdn (no);
-        dns-cache(no);
+        dns-cache(yes);
         create_dirs (no);
         keep-hostname (yes);
         create_dirs(yes);

tests/test_ubiquiti_unifi.py

@@ -0,0 +1,93 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import random
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+#<27>Nov  8 17:28:43 US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type
+
+def test_ubiquiti_unifi_us8p60(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type")
+    message = mt.render(mark="<27>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#<29>Nov 10 20:46:02 US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0
+def test_ubiquiti_unifi_switch_us24p250(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0")
+    message = mt.render(mark="<27>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#<30>Nov 10 11:49:46 U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514
+def test_ubiquiti_unifi_ap_u7pg2(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514")
+    message = mt.render(mark="<27>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=netops sourcetype=ubnt:wireless earliest=-2m | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#<4>Nov 10 23:04:06 USG kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328
+def test_ubiquiti_unifi_usg(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} usg-{{host}} kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328")
+    message = mt.render(mark="<27>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=netfw sourcetype=ubnt:fw host=usg-{{host}} | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1