Merge pull request #234 from splunk/feature/symantec-brightmail

Support Symantec Mail Gateway aka Brightmail
CSVD · Dec 13, 2019 · 37da425 · 37da425
2 parents da8ed77 + 5788298
commit 37da425
Show file tree

Hide file tree

Showing 5 changed files with 259 additions and 0 deletions.
diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md
@@ -49,3 +49,53 @@ An active proxy will generate frequent events. Use the following search to valid
 ```
 index=<asconfigured> sourcetype=bluecoat:proxysg:access:kv | stats count by host
 ```
+
+## Product - Mail Gateway (Brightmail)
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | TBD                                                            |
+| Product Manual | https://support.symantec.com/us/en/article.howto38250.html                                                       |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| symantec:smg        | Requires version TA 3.6                                                                                                    |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| symantec_brightmail      | symantec:smg     | email          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* No TA available
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL | no | Enable archive to disk for this specific source |
+| SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG | yes | Email processing events generated by the bmserver process will be grouped by host+program+pid+msg ID into a single event |
+### Verification
+
+An active mail server will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=symantec:smg | stats count by host
+```
diff --git a/package/etc/conf.d/filters/symantec/brightmail.conf b/package/etc/conf.d/filters/symantec/brightmail.conf
@@ -0,0 +1,6 @@
+filter f_symantec_brightmail {
+    message('\[Brightmail\]') or program("bmserver")
+};
+filter f_symantec_brightmail_details {
+    program("bmserver") and not message('\[Brightmail\]')
+};
diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl
@@ -0,0 +1,127 @@
+{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }}
+filter f_symantec_brightmail_complete{
+    match("yes", value("SMG.COMPLETE") type(glob));
+};
+
+parser symantec_brightmail_grouping {
+    csv-parser(
+        columns(PID, SMG.seq, MESSAGE)
+        delimiters(chars("|"))
+        flags(greedy)
+    );
+    grouping-by(
+        scope(program)
+        key("${SMG.seq}")
+        timeout(2)
+        aggregate(
+             value("MESSAGE" "${PID}|${SMG.seq}|$(implode ';' $(context-values ${MESSAGE}))")
+             value("SMG.COMPLETE" "yes")
+        )
+    );
+};
+{{- end }}
+
+{{ $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_DEFAULT);
+    filter(f_is_rfc3164);
+    filter(f_symantec_brightmail);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_SYMANTEC_BRIGHTMAIL);
+{{- end }}
+
+{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool)  }}
+    if {
+
+        filter(f_symantec_brightmail_details);
+        parser(symantec_brightmail_grouping);
+
+        if {
+            filter(f_symantec_brightmail_complete);
+
+            rewrite {
+                set("symantec_brightmail", value("fields.sc4s_vendor_product"));
+            };
+
+            rewrite {
+                r_set_splunk_dest_default(sourcetype("symantec:smg:mail"), index("email"), source("program:${.PROGRAM}") )
+
+            };
+
+            parser { p_add_context_splunk(key("symantec_brightmail")); };
+
+            parser (compliance_meta_by_source);
+
+            #We want to unset the fields we won't need, as this is copied into the
+            #disk queue for network destinations. This can be very disk expensive
+            #if we don't
+            rewrite {
+                set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+                unset(value("RAWMSG"));
+                unset(value("PROGRAM"));
+                unset(value("LEGACY_MSGHDR"));
+            };
+
+            {{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }}
+                destination(d_hec);
+            {{- end}}
+
+            {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }}
+                destination(d_archive);
+            {{- end}}
+
+        };
+    } else {
+{{- end }}
+
+        rewrite {
+            set("symantec_brightmail", value("fields.sc4s_vendor_product"));
+        };
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("symantec:smg"), index("email"), source("program:${.PROGRAM}") )
+
+        };
+
+        parser { p_add_context_splunk(key("symantec_brightmail")); };
+
+        parser (compliance_meta_by_source);
+
+        #We want to unset the fields we won't need, as this is copied into the
+        #disk queue for network destinations. This can be very disk expensive
+        #if we don't
+        rewrite {
+            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("LEGACY_MSGHDR"));
+        };
+
+    {{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }}
+        destination(d_hec);
+    {{- end}}
+
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }}
+        destination(d_archive);
+    {{- end}}
+{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }}
+    };
+{{- end}}
+
+
+    flags(flow-control,final);
+};
+{{- end}}
+
+{{- if or (or (getenv  (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT")) (getenv  (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT"))) (getenv  (print "SC4S_SYMANTEC_BRIGHTMAIL_NSS_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for SYMANTEC_BRIGHTMAIL traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for SYMANTEC_BRIGHTMAIL traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
@@ -4,6 +4,11 @@ homePath   = $SPLUNK_DB/syslogng_metrics/db
 coldPath   = $SPLUNK_DB/syslogng_metrics/colddb
 thawedPath = $SPLUNK_DB/syslogng_metrics/thaweddb
 
+[email]
+homePath   = $SPLUNK_DB/email/db
+coldPath   = $SPLUNK_DB/email/colddb
+thawedPath = $SPLUNK_DB/email/thaweddb
+
 [em_metrics]
 datatype=metric
 homePath   = $SPLUNK_DB/em_metrics/db

diff --git a/tests/test_symantec_brightmail.py b/tests/test_symantec_brightmail.py
@@ -0,0 +1,71 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import random
+import uuid
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+# <141>Oct 24 21:05:43 smg-1 conduit: [Brightmail] (NOTICE:7500.3119331456): [12066] 'BrightSig3 Newsletter Rules' were updated successfully.
+def test_symantec_brightmail(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} conduit: [Brightmail] (NOTICE:7500.3119331456): [12066] 'BrightSig3 Newsletter Rules' were updated successfully.")
+    message = mt.render(mark="<134>", host=host)
+    sendsingle(message)
+
+    st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"symantec:smg\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+def test_symantec_brightmail_msg(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    msgid = uuid.uuid4()
+
+    mt = env.from_string("""{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|VERDICT|someone@example.com|none|default|default\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|FIRED|someone@example.com|none\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|UNTESTED|someone@example.com|safe|opl|content_1574820902092|content_1574820956288|content_1574821059194|content_1574821017042|sys_deny_ip|sys_allow_ip|sys_deny_email|dns_allow|dns_deny|user_allow|user_deny|freq_va|freq_dha|freq_sa|connection_class_0|connection_class_1|connection_class_2|connection_class_3|connection_class_4|connection_class_5|connection_class_6|connection_class_7|connection_class_8|connection_class_9|blockedlang|knownlang\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|LOGICAL_IP|200.200.200.154\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|google-play_111-33.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|mac_appstore_136_33.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|6f0c8ad8-e0da-4bcc-aac9-8fa71ba43bb6.jpg\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|b340ec99-9c66-4a19-a2f4-ee467e0d63e1.jpg\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|product-logo.update.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|header-research.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|ms-logo-138.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|ATTACH|ms-logo-138.png|header-research.png|product-logo.update.png|b340ec99-9c66-4a19-a2f4-ee467e0d63e1.jpg|6f0c8ad8-e0da-4bcc-aac9-8fa71ba43bb6.jpg|mac_appstore_136_33.png|google-play_111-33.png\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|EHLO|mail6.bemta23.messagelabs.com\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|MSG_SIZE|94239\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|MSGID| <7jszytr60wmja@example.com>\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|SUBJECT|pulse: this is a subject\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|SOURCE|external\n
+{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195987|{{ MSGID }}|VERDICT|<none>|connection_class_1|default|static connection class 1\n""")
+    message = mt.render(mark="<1>", host=host, MSGID=msgid)
+    sendsingle(message)
+
+    st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"symantec:smg:mail\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#