From dc6f95ad338fbcd9b6cf8799b7b6429c335db3eb Mon Sep 17 00:00:00 2001 From: mbonsack Date: Sat, 28 Sep 2019 12:49:20 -0700 Subject: [PATCH] Fix/parser metadata (#100) * Parser/metdata update * Feature/proofpoint (#97) * Add Proofpoint PPS filter support * Fix parser/metdata/IOS filter --- docker-compose.yml | 2 +- docs/sources.md | 52 +++++++++++++++++ .../conf.d/conflib/_common/network.conf.tmpl | 0 .../{rfc_syslog.conf => syslog_format.conf} | 12 ++++ .../etc/conf.d/conflib/_common/utility.conf | 4 -- package/etc/conf.d/filters/cisco/ios.conf | 6 +- .../etc/conf.d/filters/proofpoint/pps.conf | 18 ++++++ .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_nx-os.conf.tmpl | 10 ++-- .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 32 +++++++++++ ..._rfc3164-proofpoint_pps_sendmail.conf.tmpl | 32 +++++++++++ package/etc/conf.d/sources/network.conf.tmpl | 4 +- package/etc/context-local/splunk_index.csv | 4 +- .../vendor_product_by_source.conf | 32 +++++++---- .../vendor_product_by_source.csv | 4 +- package/etc/templates/source_network.t | 6 +- .../apps/SA-syslog-ng/default/indexes.conf | 7 ++- tests/test_proofpoint.py | 56 +++++++++++++++++++ 18 files changed, 249 insertions(+), 34 deletions(-) delete mode 100644 package/etc/conf.d/conflib/_common/network.conf.tmpl rename package/etc/conf.d/conflib/_common/{rfc_syslog.conf => syslog_format.conf} (80%) delete mode 100644 package/etc/conf.d/conflib/_common/utility.conf create mode 100644 package/etc/conf.d/filters/proofpoint/pps.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl create mode 100644 tests/test_proofpoint.py diff --git a/docker-compose.yml b/docker-compose.yml index a1fc681..1765168 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - - SC4S_SOURCE_TLS_ENABLE=yes + - SC4S_SOURCE_TLS_ENABLE=no - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 volumes: diff --git a/docs/sources.md b/docs/sources.md index 363bbe5..19f26d1 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -493,6 +493,58 @@ An active firewall will generate frequent events. Use the following search to va index= sourcetype=pan:*| stats count by host ``` +# Vendor - Proofpoint + +## Product - Proofpoint Protection Server + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3080/ | +| Product Manual | https://proofpointcommunities.force.com/community/s/article/Remote-Syslog-Forwarding | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| pps_filter_log | | +| pps_mail_log | This sourcetype will conflict with sendmail itself, so will require that the PPS send syslog on a dedicated port or be uniquely identifiable with a hostname glob or CIDR block if this sourcetype is desired for PPS. | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| proofpoint_pps_filter | pps_filter_log | email | none | +| proofpoint_pps_sendmail | pps_mail_log | email | none | + + +### Filter type + +MSG Parse: This filter parses message content +* NOTE: This filter will simply parse the syslog message itself, and will _not_ perform the (required) re-assembly of related +messages to create meaningful final output. This will require follow-on processing in Splunk. + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Follow vendor configuration steps per referenced Product Manual + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT variable immediately below.| +| SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT variable immediately above. | + +### Verification + +One or two sourcetypes are included in Proofpoint PPS logs. The search below will surface both of them: + +``` +index= sourcetype=pps_*_log | stats count by host +``` + # Vendor - Symantec ## Product - ProxySG/ASG (Bluecoat) diff --git a/package/etc/conf.d/conflib/_common/network.conf.tmpl b/package/etc/conf.d/conflib/_common/network.conf.tmpl deleted file mode 100644 index e69de29..0000000 diff --git a/package/etc/conf.d/conflib/_common/rfc_syslog.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf similarity index 80% rename from package/etc/conf.d/conflib/_common/rfc_syslog.conf rename to package/etc/conf.d/conflib/_common/syslog_format.conf index 1809034..8c5794c 100644 --- a/package/etc/conf.d/conflib/_common/rfc_syslog.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -24,4 +24,16 @@ rewrite set_rfc3164{ }; filter f_is_rfc3164{ match("rfc3164" value("fields.sc4s_syslog_format")) +}; +rewrite set_cisco_ios{ + set("cisco_ios" value("fields.sc4s_syslog_format")); +}; +filter f_is_cisco_ios{ + match("cisco_ios" value("fields.sc4s_syslog_format")) +}; +rewrite set_no_parse{ + set("no_parse" value("fields.sc4s_syslog_format")); +}; +filter f_is_no_parse{ + match("no_parse" value("fields.sc4s_syslog_format")) }; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/utility.conf b/package/etc/conf.d/conflib/_common/utility.conf deleted file mode 100644 index 8b986c3..0000000 --- a/package/etc/conf.d/conflib/_common/utility.conf +++ /dev/null @@ -1,4 +0,0 @@ - -rewrite set_metadata_presume { - set("$(env SYSLOG_PRESUME_FILTER)" value("fields.sc4s_presume")); -}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco/ios.conf b/package/etc/conf.d/filters/cisco/ios.conf index 09a37ed..0e4c478 100644 --- a/package/etc/conf.d/filters/cisco/ios.conf +++ b/package/etc/conf.d/filters/cisco/ios.conf @@ -1,8 +1,6 @@ -rewrite set_metadata_vendor_product_cisco_ios{ - set("cisco_ios" value(".metadata.vendor_product")); -}; +# In general this will not be used; parser setting will override the need for this filter f_cisco_ios{ - match("cisco_ios", value(".metadata.vendor_product") type(glob)); + match("cisco_ios", value("fields.sc4s_vendor_product") type(glob)); }; diff --git a/package/etc/conf.d/filters/proofpoint/pps.conf b/package/etc/conf.d/filters/proofpoint/pps.conf new file mode 100644 index 0000000..6fd213c --- /dev/null +++ b/package/etc/conf.d/filters/proofpoint/pps.conf @@ -0,0 +1,18 @@ +# Proofpoint + +filter f_proofpoint_pps_filter { + match("proofpoint_pps_filter", value("fields.sc4s_vendor_product") type(glob)) or + ( + ( + match('^(background|cvt|filter|pps)_instance\d+$' value("PROGRAM") type("pcre")) or + match('^\/opt\/proofpoint\/pps-\d\.\d\.\d\.\d+\/\S' value("PROGRAM") type("pcre")) or + match('^queued-(alert|default|reinject|released)$' value("PROGRAM") type("pcre")) + ) and + match('^rprt\s' value(MESSAGE) type("pcre")) + ); +}; + +filter f_proofpoint_pps_sendmail { + match('sendmail' value("PROGRAM") type("pcre")) and + match("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product") type(glob)); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index 6518a0d..05b9e29 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -7,7 +7,7 @@ log { {{- if eq (.) "yes" }} source(s_default-ports); - filter(f_cisco_ios); + filter(f_is_cisco_ios); {{- end }} {{- if eq (.) "no" }} source (s_dedicated_port_CISCO_IOS); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl index 2caff20..03700cd 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl @@ -5,17 +5,17 @@ {{- end -}} {{ define "log_path" }} log { -{{- if eq (.) "yes"}} +{{- if eq (.) "yes" }} source(s_default-ports); filter(f_cisco_nx_os); -{{- end}} -{{- if eq (.) "no"}} +{{- end }} +{{- if eq (.) "no" }} source (s_dedicated_port_CISCO_NX_OS); -{{- end}} +{{- end }} rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))}; parser { - p_add_context_splunk(key("cisco_nx_os")); + p_add_context_splunk(key("cisco_nx_os")); }; destination(d_hec); #--HEC-- diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl new file mode 100644 index 0000000..fa89c6c --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl @@ -0,0 +1,32 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_proofpoint_pps_filter); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_PROOFPOINT_PPS_FILTER); +{{- end }} + + rewrite { r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("proofpoint_pps_filter")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for PROOFPOINT_PPS_FILTER traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl new file mode 100644 index 0000000..db0baab --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl @@ -0,0 +1,32 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "PROOFPOINT_PPS_SENDMAIL" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_proofpoint_pps_sendmail); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_PROOFPOINT_PPS_SENDMAIL); +{{- end }} + + rewrite { r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("proofpoint_pps_sendmail")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_SENDMAIL traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for PROOFPOINT_PPS_SENDMAIL traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index 42b4344..a9e8b78 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -81,7 +81,7 @@ source s_default-ports { rewrite(set_rfc5424_noversion); } elif { parser {cisco-parser()}; - rewrite(set_metadata_vendor_product_cisco_ios); + rewrite(set_cisco_ios); } else { parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); @@ -94,5 +94,5 @@ source s_default-ports { parser { vendor_product_by_source(); }; - }; + }; }; \ No newline at end of file diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv index 1df4cc1..566c436 100644 --- a/package/etc/context-local/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -28,6 +28,8 @@ #pan_correlation,index,main #pan_userid,index,netauth #pan_unknown,index,netops -#sc4s_events,index,_internal +#proofpoint_pps_filter,index,email +#proofpoint_pps_sendmail,index,email +#sc4s_events,index,main #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/context-local/vendor_product_by_source.conf index 4da1866..37e3412 100644 --- a/package/etc/context-local/vendor_product_by_source.conf +++ b/package/etc/context-local/vendor_product_by_source.conf @@ -4,26 +4,34 @@ #filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; filter f_test_test { - host("testvp-*" type(glob)) - or match("test_test" value("fields.sc4s_presume")) + host("testvp-*" type(glob)) or + netmask(192.168.100.1/24) }; filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) - or match("juniper_nsm" value("fields.sc4s_presume")) + host("jnpnsm-*" type(glob)) or + netmask(192.168.1.0/24) }; filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) - or match("juniper_nsm_idp" value("fields.sc4s_presume")) + host("jnpnsmidp-*" type(glob)) or + netmask(192.168.2.0/24) }; filter f_juniper_idp { - host("jnpidp-*" type(glob)) - or match("juniper_idp" value("fields.sc4s_presume")) + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) }; filter f_juniper_netscreen { - host("jnpns-*" type(glob) ) - or match("juniper_netscreen" value("fields.sc4s_presume")) + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) }; filter f_cisco_nx_os { - host("csconx-*" type(glob) ) - or match("cisco_nx_os" value("fields.sc4s_presume")) + host("csconx-*" type(glob)) or + netmask(192.168.5.0/24) +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; +filter f_proofpoint_pps_filter { + host("pps-*" type(glob)) or + netmask(192.168.7.0/24) }; \ No newline at end of file diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/context-local/vendor_product_by_source.csv index 5193616..3f90603 100644 --- a/package/etc/context-local/vendor_product_by_source.csv +++ b/package/etc/context-local/vendor_product_by_source.csv @@ -3,4 +3,6 @@ f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" -f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" \ No newline at end of file +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/templates/source_network.t b/package/etc/templates/source_network.t index 8651474..f804816 100644 --- a/package/etc/templates/source_network.t +++ b/package/etc/templates/source_network.t @@ -73,12 +73,14 @@ source s_dedicated_port_{{ .port_id}} { rewrite(set_rfc5424_noversion); {{- else if eq .parser "cisco_parser" }} parser {cisco-parser()}; - rewrite(set_metadata_vendor_product_cisco_ios); + rewrite(set_cisco_ios); {{- else if eq .parser "rfc3164" }} parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); }; rewrite(set_rfc3164); +{{- else if eq .parser "no_parse" }} + rewrite(set_no_parse); {{- else }} if {filter(f_rfc5424_strict); parser { @@ -93,7 +95,7 @@ source s_dedicated_port_{{ .port_id}} { rewrite(set_rfc5424_noversion); } elif { parser {cisco-parser()}; - rewrite(set_metadata_vendor_product_cisco_ios); + rewrite(set_cisco_ios); } else { parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index f8dbfb2..60d7cae 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -39,4 +39,9 @@ thawedPath = $SPLUNK_DB/netops/thaweddb [netproxy] homePath = $SPLUNK_DB/netproxy/db coldPath = $SPLUNK_DB/netproxy/colddb -thawedPath = $SPLUNK_DB/netproxy/thaweddb \ No newline at end of file +thawedPath = $SPLUNK_DB/netproxy/thaweddb + +[email] +homePath = $SPLUNK_DB/email/db +coldPath = $SPLUNK_DB/email/colddb +thawedPath = $SPLUNK_DB/email/thaweddb \ No newline at end of file diff --git a/tests/test_proofpoint.py b/tests/test_proofpoint.py new file mode 100644 index 0000000..4b04935 --- /dev/null +++ b/tests/test_proofpoint.py @@ -0,0 +1,56 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +# Apr 17 18:33:26 aplegw01 filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com +def test_proofpoint_pps_filter(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com\n") + message = mt.render(mark="<166>", host=host) + + sendsingle(message) + + st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"pps_filter_log\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 17 18:35:26 aplegw02 sendmail[56106]: w3HMZPVT056101: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok: Message 22675962 accepted) +def test_proofpoint_pps_mail(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} pps-{{ host }} sendmail[195529]: w3HMZPVT056101: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok: Message 22675962 accepted)\n") + message = mt.render(mark="<166>", host=host) + + sendsingle(message) + + st = env.from_string("search index=email host=\"pps-{{ host }}\" sourcetype=\"pps_mail_log\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +