From 31990cbbf1e76b31c009427ec0e21c428d7e6ded Mon Sep 17 00:00:00 2001
From: jstebbins-splunk <53191633+jstebbins-splunk@users.noreply.github.com>
Date: Fri, 23 Aug 2019 13:42:00 -0700
Subject: [PATCH] Filter dev (#57)
This change adds support for Symantec Bluecoat Proxy SG and ASG
---
.idea/encodings.xml | 4 +++
.idea/misc.xml | 2 +-
.idea/splunk-connect-for-syslog.iml | 2 +-
docs/sources.md | 36 +++++++++++++++++++
.../vendor_symantec/bluecoat_proxy.conf | 3 ++
.../log_paths/p_rfc3164-fortinet_fortios.conf | 3 +-
.../log_paths/p_rfc3164-paloalto-panos.conf | 3 +-
.../p_rfc_5424_noversion-symantec-proxy.conf | 16 +++++++++
package/etc/context-local/splunk_index.csv | 2 ++
tests/test_fortinet_ngfw.py | 6 ++--
tests/test_symantec_proxy.py | 34 ++++++++++++++++++
11 files changed, 104 insertions(+), 7 deletions(-)
create mode 100644 .idea/encodings.xml
create mode 100644 package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf
create mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf
create mode 100644 tests/test_symantec_proxy.py
diff --git a/.idea/encodings.xml b/.idea/encodings.xml
new file mode 100644
index 0000000..15a15b2
--- /dev/null
+++ b/.idea/encodings.xml
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
index 0c225df..10f4f51 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -7,4 +7,4 @@
-
\ No newline at end of file
+
diff --git a/.idea/splunk-connect-for-syslog.iml b/.idea/splunk-connect-for-syslog.iml
index a304766..3b1d69b 100644
--- a/.idea/splunk-connect-for-syslog.iml
+++ b/.idea/splunk-connect-for-syslog.iml
@@ -16,4 +16,4 @@
-
\ No newline at end of file
+
diff --git a/docs/sources.md b/docs/sources.md
index 3fd78da..d4167ef 100644
--- a/docs/sources.md
+++ b/docs/sources.md
@@ -167,4 +167,40 @@ An active firewall will generate frequent events use the following search to val
```
index= sourcetype=pan:*| stats count by host
+```
+
+# Vendor - Symantec
+
+## Product - ProxySG/ASG (Bluecoat)
+
+| Ref | Link |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on | https://splunkbase.splunk.com/app/2758/ |
+| Product Manual | https://support.symantec.com/us/en/article.tech242216.html |
+
+
+## Sourcetypes
+
+| sourcetype | notes |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| bluecoat:proxysg:access:kv | Requires version 3.6 |
+
+## Filter type
+
+MSG Parse: This filter parses message content
+
+## Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index as required.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+ * Select TCP or SSL transport option
+ * Ensure the format of the event is customized per Splunk documentation
+
+## Verification
+
+An active proxy will generate frequent events use the following search to validate events are present per source device
+
+```
+index= sourcetype=bluecoat:proxysg:access:kv | stats count by host
```
\ No newline at end of file
diff --git a/package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf b/package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf
new file mode 100644
index 0000000..afaff11
--- /dev/null
+++ b/package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf
@@ -0,0 +1,3 @@
+filter f_symantec_bluecoat_proxy {
+ program("bluecoat")
+};
\ No newline at end of file
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf
index 0707aaf..d81c565 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf
+++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf
@@ -27,4 +27,5 @@ log {
destination(d_hec); #--HEC--
flags(flow-control);
-};
\ No newline at end of file
+};
+
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf b/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf
index 84a60d2..956f747 100644
--- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf
+++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf
@@ -57,4 +57,5 @@ log {
destination(d_hec); #--HEC--
flags(flow-control);
-};
\ No newline at end of file
+};
+
diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf
new file mode 100644
index 0000000..4d2b8e4
--- /dev/null
+++ b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf
@@ -0,0 +1,16 @@
+log {
+ source(s_default-ports);
+ filter(f_is_rfc5424_noversion);
+ filter(f_symantec_bluecoat_proxy);
+
+ #set the source type based on program field and lookup index from the splunk context csv
+
+ parser {p_add_context_splunk(key("bluecoat:proxysg:access:kv")); };
+
+ #Using the 5424 parser the message content is all we need
+ rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
+
+ destination(d_hec); #--HEC--
+
+ flags(flow-control);
+};
\ No newline at end of file
diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv
index e495491..c1167c5 100644
--- a/package/etc/context-local/splunk_index.csv
+++ b/package/etc/context-local/splunk_index.csv
@@ -1,3 +1,5 @@
+bluecoat:proxysg:access:kv,index,main
+bluecoat:proxysg:access:kv,sourcetype,bluecoat:proxysg:access:kv
cisco:asa,index,main
cisco:asa,sourcetype,cisco:asa
cisco:ios,index,main
diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py
index df3aa36..3437f9e 100644
--- a/tests/test_fortinet_ngfw.py
+++ b/tests/test_fortinet_ngfw.py
@@ -17,7 +17,7 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
mt = env.from_string(
- "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n")
+ "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message)
@@ -38,7 +38,7 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
mt = env.from_string(
- "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n")
+ "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message)
@@ -58,7 +58,7 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
mt = env.from_string(
- "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n")
+ "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n")
message = mt.render(mark="<13>", host=host)
sendsingle(message)
diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py
new file mode 100644
index 0000000..8d84736
--- /dev/null
+++ b/tests/test_symantec_proxy.py
@@ -0,0 +1,34 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import random
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+# <134>1 2019-08-21T17:42:08.000z "sample_logs bluecoat[0]:SPLV5.1 c-ip=192.0.0.6 cs-bytes=6269 cs-categories="unavailable" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent="ocspd/1.0.3" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name="Explicit HTTP" service.group="Standard" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name="10.0.0.6-sample_logs" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url="http://randomserver:8000/en-US/app/examples/"
+def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk):
+ host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+ mt = env.from_string(
+ "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{host}} bluecoat[0]: SPLV5.1 c-ip=192.0.0.6 cs-bytes=6269 cs-categories=\"unavailable\" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent=\"ocspd/1.0.3\" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name=\"Explicit HTTP\" service.group=\"Standard\" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name=\"10.0.0.6-sample_logs\" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url=\"http://randomserver:8000/en-US/app/examples/\"")
+ message = mt.render(mark="<134>", host=host)
+ sendsingle(message)
+
+ st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"bluecoat:proxysg:access:kv\" | head 2")
+ search = st.render(host=host)
+
+ resultCount, eventCount = splunk_single(setup_splunk, search)
+
+ record_property("host", host)
+ record_property("resultCount", resultCount)
+ record_property("message", message)
+
+ assert resultCount == 1
+
+#