Merge pull request #289 from splunk/feature/pfsense

Support pfSense

docker-compose-ci.yml

@@ -45,6 +45,8 @@ services:
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
       - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
+      - SC4S_LISTEN_PFSENSE_TCP_PORT=5006
+
   splunk:
     image: splunk/splunk:latest
     hostname: splunk

docker-compose.yml

@@ -53,6 +53,7 @@ services:
       - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003
       - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004
       - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005
+      - SC4S_LISTEN_PFSENSE_TCP_PORT=5006
       - SC4S_ARCHIVE_GLOBAL=yes
     volumes:
       - ./tls:/opt/syslog-ng/tls

docs/sources/Pfsense/index.md

@@ -0,0 +1,57 @@
+# Vendor - pfSense
+
+All pfSense based firewalls
+
+
+## Product
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/1527/                                                                 |
+| Product Manual | https://docs.netgate.com/pfsense/en/latest/monitoring/copying-logs-to-a-remote-host-with-syslog.html?highlight=syslog |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| pfsense:filterlog  | None |
+| pfsense:* | All programs other than filterlog |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| pfsense      | pfsense     | netops          | none          |
+| pfsense_filterlog      | pfsense:filterlog      | netfw          | none          |
+
+### Filter type
+
+Source does not provide a hostname, port or IP based filter is required
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Configure a dedicated SC4S port OR configure IP filter 
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_PFSENSE_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_PFSENSE_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_PFSENSE | no | Enable archive to disk for this specific source |
+| SC4S_DEST_PFSENSE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=pfsense:filterlog | stats count by host
+```

mkdocs.yml

@@ -23,6 +23,7 @@ nav:
       - Microfocus: sources/Microfocus/index.md
       - Nix: sources/nix/index.md
       - 'Palo Alto Networks': sources/PaloaltoNetworks/index.md
+      - 'pfSense': sources/pfSense/index.md
       - Proofpoint: sources/Proofpoint/index.md
       - Symantec: sources/Symantec/index.md
       - Ubiquiti: sources/Ubiquiti/index.md

package/etc/conf.d/filters/pfsense/syslog.conf

@@ -0,0 +1,4 @@
+filter f_pfsense {
+    match("^pfsense", value("fields.sc4s_vendor_product"));
+
+};

package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl

@@ -0,0 +1,58 @@
+# pfSense
+{{- /* The following provides a unique port source configuration if env var(s) are set */}}
+{{- $context := dict "port_id" "PFSENSE" "parser" "rfc3164" }}
+{{- tmpl.Exec "t/source_network.t" $context }}
+
+log {
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_PFSENSE_TCP_PORT")) (getenv  (print "SC4S_LISTEN_PFSENSE_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_PFSENSE_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for PFSENSE traffic
+            source (s_PFSENSE);
+            flags (final);
+	    };
+{{- end}}
+        channel {
+        # Listen on the default port (typically 514) for PFSENSE traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_pfsense);
+            flags(final);
+        };
+    };
+
+    if {
+        filter{program("filterlog")};
+        rewrite {
+            set("pfsense_filterlog", value("fields.sc4s_vendor_product"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("pfsense:filterlog"), index("netfw"), source("program:${.PROGRAM}"))
+        };
+        parser { p_add_context_splunk(key("pfsense_filterlog")); };
+        parser (compliance_meta_by_source);
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); };
+    } else {
+        rewrite {
+            set("pfsense", value("fields.sc4s_vendor_product"));
+            subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("pfsense:${.PROGRAM}"), index("netops"), source("program:${.PROGRAM}"))
+        };
+        parser { p_add_context_splunk(key("pfsense")); };
+        parser (compliance_meta_by_source);
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };
+    };
+
+
+{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_PFSENSE_HEC" "no")) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_PFSENSE" "no")) }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};

package/etc/context_templates/splunk_index.csv

@@ -50,6 +50,8 @@
 #pan_correlation,index,main
 #pan_userid,index,netauth
 #pan_unknown,index,netops
+#pfsense,index,netops
+#pfsense_filterlog,index,netfw
 #proofpoint_pps_filter,index,email
 #proofpoint_pps_sendmail,index,email
 #sc4s_events,index,main

package/etc/context_templates/vendor_product_by_source.conf

@@ -30,6 +30,10 @@ filter f_infoblox {
     host("vib-*" type(glob))
     #or netmask(xxx.xxx.xxx.xxx/xx)
 };
+filter f_pfsense {
+    host("pfsense-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
+};
 filter f_proofpoint_pps_filter {
     host("pps-*" type(glob))
     #or netmask(xxx.xxx.xxx.xxx/xx)

package/etc/context_templates/vendor_product_by_source.csv

@@ -6,6 +6,7 @@ f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp"
 f_juniper_idp,sc4s_vendor_product,"juniper_idp"
 f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen"
 f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os"
+f_pfsense,sc4s_vendor_product,"pfsense"
 f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail"
 f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter"
 f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw"

tests/test_pfsense.py

@@ -0,0 +1,72 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import random
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+#<27>Jan 25 01:58:06 filterlog: 82,,,1000002666,mvneta2,match,pass,out,6,0x00,0x00000,64,ICMPv6,58,8,fe80::208:a2ff:fe0f:cb66,fe80::56a6:5cff:fe7d:1d43,
+def test_pfsense_filterlog(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} filterlog: 82,,,1000002666,mvneta2,match,pass,out,6,0x00,0x00000,64,ICMPv6,58,8,{{key}},\n")
+    message = mt.render(mark="<27>", key=host)
+    sendsingle(message, port=5006)
+
+    st = env.from_string("search index=netfw sourcetype=pfsense:filterlog \"{{key}}\" earliest=-2m | head 2")
+    search = st.render(key=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#<27>Jan 25 01:58:06 kqueue error: unknown
+def test_pfsense_other(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} kqueue error: {{key}}\n")
+    message = mt.render(mark="<27>", key=host)
+    sendsingle(message, port=5006)
+
+    st = env.from_string("search index=netops sourcetype=pfsense:* \"{{key}}\" earliest=-2m | head 2")
+    search = st.render(key=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#<27>Jan 25 01:58:06 syslogd: restart
+def test_pfsense_syslogd(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} syslogd: restart {{key}}\n")
+    message = mt.render(mark="<27>", key=host)
+    sendsingle(message, port=5006)
+
+    st = env.from_string("search index=netops sourcetype=pfsense:syslogd \"{{key}}\" earliest=-2m | head 2")
+    search = st.render(key=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1