From 991b77eaac748cd68f3633a235ca28cbeeb9072a Mon Sep 17 00:00:00 2001 From: mbonsack Date: Sat, 28 Sep 2019 10:43:32 -0700 Subject: [PATCH] Feature/proofpoint (#97) * Add Proofpoint PPS filter support --- docs/sources.md | 52 +++++++++++++++++ .../etc/conf.d/filters/proofpoint/pps.conf | 18 ++++++ .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 32 +++++++++++ ..._rfc3164-proofpoint_pps_sendmail.conf.tmpl | 32 +++++++++++ package/etc/context-local/splunk_index.csv | 4 +- .../vendor_product_by_source.conf | 8 +++ .../vendor_product_by_source.csv | 4 +- .../apps/SA-syslog-ng/default/indexes.conf | 7 ++- tests/test_proofpoint.py | 56 +++++++++++++++++++ 9 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 package/etc/conf.d/filters/proofpoint/pps.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl create mode 100644 tests/test_proofpoint.py diff --git a/docs/sources.md b/docs/sources.md index 363bbe5..19f26d1 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -493,6 +493,58 @@ An active firewall will generate frequent events. Use the following search to va index= sourcetype=pan:*| stats count by host ``` +# Vendor - Proofpoint + +## Product - Proofpoint Protection Server + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3080/ | +| Product Manual | https://proofpointcommunities.force.com/community/s/article/Remote-Syslog-Forwarding | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| pps_filter_log | | +| pps_mail_log | This sourcetype will conflict with sendmail itself, so will require that the PPS send syslog on a dedicated port or be uniquely identifiable with a hostname glob or CIDR block if this sourcetype is desired for PPS. | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| proofpoint_pps_filter | pps_filter_log | email | none | +| proofpoint_pps_sendmail | pps_mail_log | email | none | + + +### Filter type + +MSG Parse: This filter parses message content +* NOTE: This filter will simply parse the syslog message itself, and will _not_ perform the (required) re-assembly of related +messages to create meaningful final output. This will require follow-on processing in Splunk. + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Follow vendor configuration steps per referenced Product Manual + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT variable immediately below.| +| SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT variable immediately above. | + +### Verification + +One or two sourcetypes are included in Proofpoint PPS logs. The search below will surface both of them: + +``` +index= sourcetype=pps_*_log | stats count by host +``` + # Vendor - Symantec ## Product - ProxySG/ASG (Bluecoat) diff --git a/package/etc/conf.d/filters/proofpoint/pps.conf b/package/etc/conf.d/filters/proofpoint/pps.conf new file mode 100644 index 0000000..6fd213c --- /dev/null +++ b/package/etc/conf.d/filters/proofpoint/pps.conf @@ -0,0 +1,18 @@ +# Proofpoint + +filter f_proofpoint_pps_filter { + match("proofpoint_pps_filter", value("fields.sc4s_vendor_product") type(glob)) or + ( + ( + match('^(background|cvt|filter|pps)_instance\d+$' value("PROGRAM") type("pcre")) or + match('^\/opt\/proofpoint\/pps-\d\.\d\.\d\.\d+\/\S' value("PROGRAM") type("pcre")) or + match('^queued-(alert|default|reinject|released)$' value("PROGRAM") type("pcre")) + ) and + match('^rprt\s' value(MESSAGE) type("pcre")) + ); +}; + +filter f_proofpoint_pps_sendmail { + match('sendmail' value("PROGRAM") type("pcre")) and + match("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product") type(glob)); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl new file mode 100644 index 0000000..fa89c6c --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl @@ -0,0 +1,32 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_proofpoint_pps_filter); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_PROOFPOINT_PPS_FILTER); +{{- end }} + + rewrite { r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("proofpoint_pps_filter")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for PROOFPOINT_PPS_FILTER traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl new file mode 100644 index 0000000..db0baab --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl @@ -0,0 +1,32 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "PROOFPOINT_PPS_SENDMAIL" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_proofpoint_pps_sendmail); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_PROOFPOINT_PPS_SENDMAIL); +{{- end }} + + rewrite { r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("proofpoint_pps_sendmail")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_SENDMAIL traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for PROOFPOINT_PPS_SENDMAIL traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv index 1df4cc1..566c436 100644 --- a/package/etc/context-local/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -28,6 +28,8 @@ #pan_correlation,index,main #pan_userid,index,netauth #pan_unknown,index,netops -#sc4s_events,index,_internal +#proofpoint_pps_filter,index,email +#proofpoint_pps_sendmail,index,email +#sc4s_events,index,main #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/context-local/vendor_product_by_source.conf index 4da1866..bb1ede6 100644 --- a/package/etc/context-local/vendor_product_by_source.conf +++ b/package/etc/context-local/vendor_product_by_source.conf @@ -26,4 +26,12 @@ filter f_juniper_netscreen { filter f_cisco_nx_os { host("csconx-*" type(glob) ) or match("cisco_nx_os" value("fields.sc4s_presume")) +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.1.0/24) +}; +filter f_proofpoint_pps_filter { + host("pps-*" type(glob)) or + netmask(192.168.1.0/24) }; \ No newline at end of file diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/context-local/vendor_product_by_source.csv index 5193616..3f90603 100644 --- a/package/etc/context-local/vendor_product_by_source.csv +++ b/package/etc/context-local/vendor_product_by_source.csv @@ -3,4 +3,6 @@ f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" -f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" \ No newline at end of file +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index f8dbfb2..60d7cae 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -39,4 +39,9 @@ thawedPath = $SPLUNK_DB/netops/thaweddb [netproxy] homePath = $SPLUNK_DB/netproxy/db coldPath = $SPLUNK_DB/netproxy/colddb -thawedPath = $SPLUNK_DB/netproxy/thaweddb \ No newline at end of file +thawedPath = $SPLUNK_DB/netproxy/thaweddb + +[email] +homePath = $SPLUNK_DB/email/db +coldPath = $SPLUNK_DB/email/colddb +thawedPath = $SPLUNK_DB/email/thaweddb \ No newline at end of file diff --git a/tests/test_proofpoint.py b/tests/test_proofpoint.py new file mode 100644 index 0000000..4b04935 --- /dev/null +++ b/tests/test_proofpoint.py @@ -0,0 +1,56 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +# Apr 17 18:33:26 aplegw01 filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com +def test_proofpoint_pps_filter(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com\n") + message = mt.render(mark="<166>", host=host) + + sendsingle(message) + + st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"pps_filter_log\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 17 18:35:26 aplegw02 sendmail[56106]: w3HMZPVT056101: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok: Message 22675962 accepted) +def test_proofpoint_pps_mail(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} pps-{{ host }} sendmail[195529]: w3HMZPVT056101: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok: Message 22675962 accepted)\n") + message = mt.render(mark="<166>", host=host) + + sendsingle(message) + + st = env.from_string("search index=email host=\"pps-{{ host }}\" sourcetype=\"pps_mail_log\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +