Merge pull request #489 from splunk/feature/microsoft-mcas

Support Microsoft CAS

docs/sources/Microsoft/index.md

@@ -0,0 +1,57 @@
+# Vendor - Microsoft
+
+## Product - Cloud App Security (MCAS)
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/                                                              |
+| Splunk Add-on Source Specific | none |
+| Product Manual | https://docs.microsoft.com/en-us/cloud-app-security/siem                                                |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| cef        | Common sourcetype                                                                                                 |
+
+### Source
+
+| source    | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| microsoft:cas       | Common sourcetype                                                                                                 |
+
+### Index Configuration
+
+| key            | source     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| MCAS_SIEM_Agent      | microsoft:cas      | main          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Options
+
+Note listed for reference processing utilizes the Microsoft ArcSight log path as this format is a subtype of CEF
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_CEF_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CEF_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source |
+| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source |
+
+* NOTE:  Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how
+many ports are in use by this CEF source (or any others).  See the "Common Event Format" source
+documentation for more information.
+
+### Verification
+
+An active site will generate frequent events use the following search to check for new events
+
+Verify timestamp, and host values match as expected
+
+```
+index=<asconfigured> (sourcetype=cef source="microsoft:cas")
+```

package/etc/conf.d/context/common_event_format_source.csv

@@ -5,6 +5,8 @@ Cyber-Ark_Vault,sourcetype,cyberark:epv:cef
 Cyber-Ark_Vault,index,netauth
 CyberArk_PTA,sourcetype,cyberark:pta:cef
 CyberArk_PTA,index,main
+MCAS_SIEM_Agent,index,main
+MCAS_SIEM_Agent,source,microsoft:cas
 Microsoft_System or Application Event,source,CEFEventLog:System or Application Event
 Microsoft_System or Application Event,index,oswin
 Microsoft_Microsoft Windows,source,CEFEventLog:Microsoft Windows

tests/test_microsoft_mcas.py

@@ -0,0 +1,45 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import random
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+from .timeutils import *
+
+env = Environment()
+
+# 2020-05-15T13:25:05+00:00 HOSTNAME CEF:0|MCAS|SIEM_Agent|0.172.123|EVENT_CATEGORY_UPLOAD_DISCOVERY_FILE|Upload Cloud Discovery file|0|externalId=111005697_1589549105456_dc4b870227e1474f94cab2cb4d256d1c rt=1589549105456 start=1589549105456 end=1589549105456 msg=Upload Cloud Discovery file suser= destinationServiceName=Microsoft Cloud App Security dvc=111.222.18.21 requestClientApplication=Apache-HttpClient/4.5.10 (Java/1.8.0_222) cs1Label=portalURL cs1=https://companyname.portal.cloudappsecurity.com/#/audits?activity.id\=eq(111005697_1589549105456_dc4b870227e1474f94cab2cb4d256d1c,) cs2Label=uniqueServiceAppIds cs2=APPID_OFFICE,APPID_MCAS cs3Label=targetObjects cs3= cs4Label=policyIDs cs4= c6a1Label=“Device IPv6 Address” c6a1=
+def test_microsoft_mcas(record_property, setup_wordlist, setup_splunk, setup_sc4s):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    dt = datetime.datetime.now(datetime.timezone.utc)
+    iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
+
+    # Tune time functions
+    iso = dt.isoformat()[0:23]
+    epoch = epoch[:-3]
+
+    mt = env.from_string(
+        "{{ mark }} {{ iso }} {{ host }} "
+        + 'CEF:0|MCAS|SIEM_Agent|0.172.123|EVENT_CATEGORY_UPLOAD_DISCOVERY_FILE|Upload Cloud Discovery file|0|externalId=111005697_1589549105456_dc4b870227e1474f94cab2cb4d256d1c rt={{ epoch }} start={{ epoch }} end={{ epoch }} msg=Upload Cloud Discovery file suser= destinationServiceName=Microsoft Cloud App Security dvc=111.222.18.21 requestClientApplication=Apache-HttpClient/4.5.10 (Java/1.8.0_222) cs1Label=portalURL cs1=https://companyname.portal.cloudappsecurity.com/#/audits?activity.id\=eq(111005697_1589549105456_dc4b870227e1474f94cab2cb4d256d1c,) cs2Label=uniqueServiceAppIds cs2=APPID_OFFICE,APPID_MCAS cs3Label=targetObjects cs3= cs4Label=policyIDs cs4= c6a1Label="Device IPv6 Address" c6a1='
+        + "\n"
+    )
+    message = mt.render(mark="<134>", iso=iso, host=host, epoch=epoch)
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string(
+        'search _time={{ epoch }} index=main host="{{ host }}" source="microsoft:cas" sourcetype=cef'
+    )
+    search = st.render(epoch=epoch, host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1