Feature/proofpoint (#97)

* Add Proofpoint PPS filter support
CSVD · Sep 28, 2019 · 991b77e · 991b77e
1 parent 57e7243
commit 991b77e
Show file tree

Hide file tree

Showing 9 changed files with 210 additions and 3 deletions.
diff --git a/docs/sources.md b/docs/sources.md
@@ -493,6 +493,58 @@ An active firewall will generate frequent events. Use the following search to va
 index=<asconfigured> sourcetype=pan:*| stats count by host
 ```
 
+# Vendor - Proofpoint
+
+## Product - Proofpoint Protection Server
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/3080/                                                                 |
+| Product Manual | https://proofpointcommunities.force.com/community/s/article/Remote-Syslog-Forwarding                    |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| pps_filter_log |                                                                               |
+|  pps_mail_log  | This sourcetype will conflict with sendmail itself, so will require that the PPS send syslog on a dedicated port or be uniquely identifiable with a hostname glob or CIDR block if this sourcetype is desired for PPS.   |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| proofpoint_pps_filter        | pps_filter_log       | email          | none          |
+| proofpoint_pps_sendmail      | pps_mail_log       | email          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+* NOTE:  This filter will simply parse the syslog message itself, and will _not_ perform the (required) re-assembly of related
+messages to create meaningful final output.  This will require follow-on processing in Splunk.
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Follow vendor configuration steps per referenced Product Manual
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined.   If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT variable immediately below.|
+| SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT variable immediately above. |
+
+### Verification
+
+One or two sourcetypes are included in Proofpoint PPS logs.  The search below will surface both of them:
+
+```
+index=<asconfigured> sourcetype=pps_*_log | stats count by host
+```
+
 # Vendor - Symantec
 
 ## Product - ProxySG/ASG (Bluecoat)

diff --git a/package/etc/conf.d/filters/proofpoint/pps.conf b/package/etc/conf.d/filters/proofpoint/pps.conf
@@ -0,0 +1,18 @@
+# Proofpoint
+
+filter f_proofpoint_pps_filter {
+    match("proofpoint_pps_filter", value("fields.sc4s_vendor_product") type(glob)) or
+    (
+        (
+        match('^(background|cvt|filter|pps)_instance\d+$' value("PROGRAM") type("pcre")) or
+        match('^\/opt\/proofpoint\/pps-\d\.\d\.\d\.\d+\/\S' value("PROGRAM") type("pcre")) or
+        match('^queued-(alert|default|reinject|released)$' value("PROGRAM") type("pcre"))
+        ) and
+        match('^rprt\s' value(MESSAGE) type("pcre"))
+    );
+};
+
+filter f_proofpoint_pps_sendmail {
+    match('sendmail' value("PROGRAM") type("pcre")) and
+    match("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product") type(glob));
+};
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl
@@ -0,0 +1,32 @@
+# Proofpoint
+{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }}
+{{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+{{- end -}}
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_default-ports);
+    filter(f_proofpoint_pps_filter);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_dedicated_port_PROOFPOINT_PPS_FILTER);
+{{- end }}
+
+    rewrite { r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"), template("t_msg_only"))};
+    parser {
+        p_add_context_splunk(key("proofpoint_pps_filter"));
+    };
+
+    destination(d_hec);  #--HEC--
+
+    flags(flow-control);
+};
+{{- end}}
+{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }}
+# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for PROOFPOINT_PPS_FILTER traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl
@@ -0,0 +1,32 @@
+# Proofpoint
+{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }}
+{{ $context := dict "port_id" "PROOFPOINT_PPS_SENDMAIL" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+{{- end -}}
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_default-ports);
+    filter(f_proofpoint_pps_sendmail);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_dedicated_port_PROOFPOINT_PPS_SENDMAIL);
+{{- end }}
+
+    rewrite { r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"), template("t_msg_only"))};
+    parser {
+        p_add_context_splunk(key("proofpoint_pps_sendmail"));
+    };
+
+    destination(d_hec);  #--HEC--
+
+    flags(flow-control);
+};
+{{- end}}
+{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }}
+# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_SENDMAIL traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for PROOFPOINT_PPS_SENDMAIL traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv
@@ -28,6 +28,8 @@
 #pan_correlation,index,main
 #pan_userid,index,netauth
 #pan_unknown,index,netops
-#sc4s_events,index,_internal
+#proofpoint_pps_filter,index,email
+#proofpoint_pps_sendmail,index,email
+#sc4s_events,index,main
 #sc4s_fallback,index,main
 #sc4s_metrics,index,em_metrics
diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/context-local/vendor_product_by_source.conf
@@ -26,4 +26,12 @@ filter f_juniper_netscreen {
 filter f_cisco_nx_os {
     host("csconx-*" type(glob) )
     or match("cisco_nx_os" value("fields.sc4s_presume"))
+};
+filter f_proofpoint_pps_sendmail {
+    host("pps-*" type(glob)) or
+    netmask(192.168.1.0/24)
+};
+filter f_proofpoint_pps_filter {
+    host("pps-*" type(glob)) or
+    netmask(192.168.1.0/24)
 };
diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/context-local/vendor_product_by_source.csv
@@ -3,4 +3,6 @@ f_juniper_nsm,sc4s_vendor_product,"juniper_nsm"
 f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp"
 f_juniper_idp,sc4s_vendor_product,"juniper_idp"
 f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen"
-f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os"
+f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os"
+f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail"
+f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter"
diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
@@ -39,4 +39,9 @@ thawedPath = $SPLUNK_DB/netops/thaweddb
 [netproxy]
 homePath   = $SPLUNK_DB/netproxy/db
 coldPath   = $SPLUNK_DB/netproxy/colddb
-thawedPath = $SPLUNK_DB/netproxy/thaweddb
+thawedPath = $SPLUNK_DB/netproxy/thaweddb
+
+[email]
+homePath   = $SPLUNK_DB/email/db
+coldPath   = $SPLUNK_DB/email/colddb
+thawedPath = $SPLUNK_DB/email/thaweddb
diff --git a/tests/test_proofpoint.py b/tests/test_proofpoint.py
@@ -0,0 +1,56 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+
+# Apr 17 18:33:26 aplegw01 filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com
+def test_proofpoint_pps_filter(record_property, setup_wordlist, get_host_key, setup_splunk):
+    host = get_host_key
+
+    mt = env.from_string(
+        "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com\n")
+    message = mt.render(mark="<166>", host=host)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"pps_filter_log\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+# Apr 17 18:35:26 aplegw02 sendmail[56106]: w3HMZPVT056101: to=<rfaircloth@splunk.com>, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok:  Message 22675962 accepted)
+def test_proofpoint_pps_mail(record_property, setup_wordlist, get_host_key, setup_splunk):
+    host = get_host_key
+
+    mt = env.from_string(
+        "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} pps-{{ host }} sendmail[195529]: w3HMZPVT056101: to=<rfaircloth@splunk.com>, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok:  Message 22675962 accepted)\n")
+    message = mt.render(mark="<166>", host=host)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=email host=\"pps-{{ host }}\" sourcetype=\"pps_mail_log\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+