From 474ffa056b35910adc4dea9ae385d87f28b30b7a Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Thu, 7 May 2020 14:42:18 +0530 Subject: [PATCH 01/15] Route sep syslog to correct sub-sourcetype --- package/etc/conf.d/filters/symantec/ep.conf | 48 +++++++ .../conf.d/log_paths/lp-symantec_ep.conf.tmpl | 73 ++++++++++- .../splunk_index.csv.example | 2 +- tests/test_symantec_ep.py | 118 +++++++++++++++++- 4 files changed, 234 insertions(+), 7 deletions(-) diff --git a/package/etc/conf.d/filters/symantec/ep.conf b/package/etc/conf.d/filters/symantec/ep.conf index a9db248..3420415 100644 --- a/package/etc/conf.d/filters/symantec/ep.conf +++ b/package/etc/conf.d/filters/symantec/ep.conf @@ -1,3 +1,51 @@ filter f_symantec_ep { program("SymantecServer") +}; + +filter f_symantec_ep_proactive { + message(',Detection\stype:') +}; + +filter f_symantec_ep_risk { + message(',Risk\sname:') +}; + +filter f_symantec_ep_agt_system { + message(',Category:\s\d+,') +}; + +filter f_symantec_ep_packet { + message(',(?:Inbound|Outbound|Unknown),Application:') +}; + +filter f_symantec_ep_traffic { + message(',(?:Inbound|Outbound|Unknown),Begin(?:\sTime)?:') +}; + +filter f_symantec_ep_security { + message('CIDS\sSignature\sSubID:') +}; + +filter f_symantec_ep_scan { + message('Scan\sID:\s\d+') +}; + +filter f_symantec_ep_behavior { + message('Begin(?:\sTime)?:\s[^,]*,End(?:\sTime)?:') +}; + +filter f_symantec_ep_policy { + message('Admin:\s[^,]+,.*[Pp]olicy') +}; + +filter f_symantec_ep_admin { + message('Domain(?:\sName)?:\s[^,]{0,25},Admin:') +}; + +filter f_symantec_ep_agent { + message('(?:,The\smanagement\sserver|,The\sclient)') +}; + +filter f_symantec_ep_scm_system { + message('Site:\s[^,]+,Server(?:\sName)?:\s[^,]+,') }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-symantec_ep.conf.tmpl b/package/etc/conf.d/log_paths/lp-symantec_ep.conf.tmpl index 2f294ed..e093563 100644 --- a/package/etc/conf.d/log_paths/lp-symantec_ep.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-symantec_ep.conf.tmpl @@ -21,15 +21,78 @@ log { }; }; - + if { + filter(f_symantec_ep_proactive); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:proactive:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_risk); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:risk:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_agt_system); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:agt:system:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_packet); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:packet:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_traffic); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:traffic:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_security); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:security:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_scan); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:scan:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_behavior); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:behavior:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_policy); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:policy:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_admin); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:admin:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_agent); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:agent:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_scm_system); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:scm:system:syslog"), index("epav")) + }; + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:syslog"), index("epav")) + }; + }; rewrite { - set("symantec_ep_syslog", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("symantec:ep:syslog"), index("epav")) + set("Symantec Endpoint Protection", value("fields.sc4s_vendor_product")); }; - parser { p_add_context_splunk(key("symantec_ep_syslog")); }; + parser { p_add_context_splunk(key("symantec_ep")); }; parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_EP_HEC" "no")) }} destination(d_hec); diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 2fbd567..5652871 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -70,7 +70,7 @@ #sc4s_events,index,main #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics -#symanrtec_ep,index,epav +#symantec_ep,index,epav #vmware_nsx,index,main #zscaler_alerts,index,main #zscaler_dns,index,netdns diff --git a/tests/test_symantec_ep.py b/tests/test_symantec_ep.py index 0c0e75a..f91a735 100644 --- a/tests/test_symantec_ep.py +++ b/tests/test_symantec_ep.py @@ -30,7 +30,123 @@ def test_symantec_ep_agent(record_property, setup_wordlist, setup_splunk, setup_ sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:syslog"' + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:agent:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:41:51 xxxxx-xxxxx SymantecServer: yyyyyy,Category: 2,LiveUpdate Manager,Event Description: A LiveUpdate session ran successfully. No new updates were available.,Event time: 2020-04-14 10:41:33,Group Name: My Company\Default Group +def test_symantec_ep_agt_system(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: yyyyyy,Category: 2,LiveUpdate Manager,Event Description: A LiveUpdate session ran successfully. No new updates were available.,Event time: 2020-04-14 10:41:33,Group Name: My Company\Default Group" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:agt:system:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 09:07:42 xxxxx-xxxxx SymantecServer: Site: Site xxxxx-xxxxx,Server Name: xxxxx-xxxxx,Event Description: No updates found for Application Control Data 14.2 RU2. +def test_symantec_ep_scm_system(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + "SymantecServer: Site: Site xxxxx-xxxxx,Server Name: xxxxx-xxxxx,Event Description: No updates found for Application Control Data 14.2 RU2." + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:scm:system:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:03:23 xxxxx-xxxxx SymantecServer: Scan ID: 1581582179,Begin: 2020-04-14 10:01:04,End Time: 2020-04-14 10:02:14,Completed,Duration (seconds): 70,User1: Spiderman,User2: Spiderman,Scan started on selected drives and folders and all extensions.,Scan Complete: Risks: 0 Scanned: 1062 Files/Folders/Drives Omitted: 0 Trusted Files Skipped: 698,Command: Not a command scan (),Threats: 0,Infected: 0,Total files: 1062,Omitted: 0,Computer: yyyyyyy,IP Address: 1.1.1.1,Domain Name: Default,Group Name: My Company\Preprod Tuesday,Server Name: xxxxx-xxxxx +def test_symantec_ep_scan(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: Scan ID: 1581582179,Begin: 2020-04-14 10:01:04,End Time: 2020-04-14 10:02:14,Completed,Duration (seconds): 70,User1: Spiderman,User2: Spiderman,Scan started on selected drives and folders and all extensions.,Scan Complete: Risks: 0 Scanned: 1062 Files/Folders/Drives Omitted: 0 Trusted Files Skipped: 698,Command: Not a command scan (),Threats: 0,Infected: 0,Total files: 1062,Omitted: 0,Computer: yyyyyyy,IP Address: 1.1.1.1,Domain Name: Default,Group Name: My Company\Preprod Tuesday,Server Name: xxxxx-xxxxx" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:scan:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:42:32 xxxxx-xxxxx SymantecServer: yyyyyy,...,Blocked,C:\Program Files (x86)\Symantec\Symantec Endpoint Protection\14.2.3335.1000.105\Bin\ccSvcHst.exe,,Begin: 2020-04-14 10:36:40,End Time: 2020-04-14 10:36:40,Rule: ,3248,C:\PROGRAM FILES (X86)\BIGFIX ENTERPRISE\BES CLIENT\BESCLIENT.EXE,0,,C:\Program Files (x86)\Symantec\Symantec Endpoint Protection\14.2.3335.1000.105\Bin\ccSvcHst.exe,User Name: SYSTEM,Domain Name: ,Action Type: 55,File size (bytes): ,Device ID: +def test_symantec_ep_behavior(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: yyyyyy,...,Blocked,C:\Program Files (x86)\Symantec\Symantec Endpoint Protection\14.2.3335.1000.105\Bin\ccSvcHst.exe,,Begin: 2020-04-14 10:36:40,End Time: 2020-04-14 10:36:40,Rule: ,3248,C:\PROGRAM FILES (X86)\BIGFIX ENTERPRISE\BES CLIENT\BESCLIENT.EXE,0,,C:\Program Files (x86)\Symantec\Symantec Endpoint Protection\14.2.3335.1000.105\Bin\ccSvcHst.exe,User Name: SYSTEM,Domain Name: ,Action Type: 55,File size (bytes): ,Device ID: " + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:behavior:syslog"' ) search = st.render(epoch=epoch, host=host) From b56083eba10b4e858e040723d1c4b72bb0c8501b Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Thu, 7 May 2020 14:44:42 +0530 Subject: [PATCH 02/15] samples for remaining sourcetypes --- tests/test_symantec_ep.py | 203 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/tests/test_symantec_ep.py b/tests/test_symantec_ep.py index f91a735..de97c8c 100644 --- a/tests/test_symantec_ep.py +++ b/tests/test_symantec_ep.py @@ -157,3 +157,206 @@ def test_symantec_ep_behavior(record_property, setup_wordlist, setup_splunk, set record_property("message", message) assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: Site: Site_B,Server Name: Example Server B,Domain Name: Domain_B,Admin: Admin_B,Event Description: Administrator log on failed +def test_symantec_ep_admin(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: Site: Site_B,Server Name: Example Server B,Domain Name: Domain_B,Admin: Admin_B,Event Description: Administrator log on failed" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:admin:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: ccccc,Local Host IP: 10.0.8.1,Local Port: 50221,Remote Host IP: 10.0.1.2,Remote Host Name: qqqqq,Remote Port: 20362,Outbound,Application: C:/Windows/System32/example_y.exe,Action: Allowed +def test_symantec_ep_packet(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: ccccc,Local Host IP: 10.0.8.1,Local Port: 50221,Remote Host IP: 10.0.1.2,Remote Host Name: qqqqq,Remote Port: 20362,Outbound,Application: C:/Windows/System32/example_y.exe,Action: Allowed" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:packet:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: Site: Site_B,Server Name: Example Server B,Domain Name: Domain_B,Admin: Admin_B,"Event Description: Policy has been edited: Changed Console mode at [Default]",Client Policy +def test_symantec_ep_policy(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r'SymantecServer: Site: Site_B,Server Name: Example Server B,Domain Name: Domain_B,Admin: Admin_B,"Event Description: Policy has been edited: Changed Console mode at [Default]",Client Policy' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:policy:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: Potential risk found,Computer name: ooooo,IP Address: 10.0.0.2,Detection type: System Change HostFile,First Seen: Symantec has known about this file for more than 1 year.,Application name: Microsoft\xAE Windows\xAE Operating System,Application type: 127,Application version: 6.1.7600.16385,Hash type: SHA-256,Application hash: ded6fc40-4365-4ba0-8446-3fa77a30cb6e,Company name: KKK.,LLLL,MMMM,File size (bytes): 3507,Sensitivity: 2,Detection score: 3,COH Engine Version: ,Detection Submissions No,Permitted application reason: Not on the permitted application list,Disposition: Bad,Download site: http://attraction.example.org/,Web domain: tkhwesmptszdody.dm,Downloaded by: c:/users/administrator/desktop/tools/tools/xxxtools.exe,Prevalence: Unknown,Confidence: There is not enough information about this file to recommend it.,URL Tracking Status: on,Risk Level: High,Risk type: 3,Source: Heuristic Scan,Risk name: Trojan.Gen.2,Occurrences: 9,PolicyZZZ,Realtime deferred scanning,Actual action: Left alone,Requested action: Quarantined,Secondary action: Left alone,Event time: 2020-05-04 06:57:02,Inserted: 2020-05-04 06:57:02,End: 2020-05-04 06:57:02,Domain: Domain A,Group: My Company\Default Group,Server: Example Server C,User: user_b,Source computer: fffff,Source IP: 10.0.9.2,Intensive Protection Level: 0,Certificate issuer: Symantec,Certificate signer: Unizeto,Certificate thumbprint: e5:xx:74:3c:xx:01:c4:9b:xx:43:xx:bb:zz:e8:6a:81:10:9f:e4:xx,Signing timestamp: 0,Certificate serial number: 149843929435818692848040365716851702463 +def test_symantec_ep_proactive(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: Potential risk found,Computer name: ooooo,IP Address: 10.0.0.2,Detection type: System Change HostFile,First Seen: Symantec has known about this file for more than 1 year.,Application name: Microsoft\xAE Windows\xAE Operating System,Application type: 127,Application version: 6.1.7600.16385,Hash type: SHA-256,Application hash: ded6fc40-4365-4ba0-8446-3fa77a30cb6e,Company name: KKK.,LLLL,MMMM,File size (bytes): 3507,Sensitivity: 2,Detection score: 3,COH Engine Version: ,Detection Submissions No,Permitted application reason: Not on the permitted application list,Disposition: Bad,Download site: http://attraction.example.org/,Web domain: tkhwesmptszdody.dm,Downloaded by: c:/users/administrator/desktop/tools/tools/xxxtools.exe,Prevalence: Unknown,Confidence: There is not enough information about this file to recommend it.,URL Tracking Status: on,Risk Level: High,Risk type: 3,Source: Heuristic Scan,Risk name: Trojan.Gen.2,Occurrences: 9,PolicyZZZ,Realtime deferred scanning,Actual action: Left alone,Requested action: Quarantined,Secondary action: Left alone,Event time: 2020-05-04 06:57:02,Inserted: 2020-05-04 06:57:02,End: 2020-05-04 06:57:02,Domain: Domain A,Group: My Company\Default Group,Server: Example Server C,User: user_b,Source computer: fffff,Source IP: 10.0.9.2,Intensive Protection Level: 0,Certificate issuer: Symantec,Certificate signer: Unizeto,Certificate thumbprint: e5:xx:74:3c:xx:01:c4:9b:xx:43:xx:bb:zz:e8:6a:81:10:9f:e4:xx,Signing timestamp: 0,Certificate serial number: 149843929435818692848040365716851702463" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:proactive:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: qqqqq,Event Description: "Web Attack: Fake Scan Webpage 7",Local Host IP: 10.0.3.4,Local Host MAC: c1411f5F9502,Remote Host Name: eeeee,Remote Host IP: 10.0.3.6,Remote Host MAC: aD31CCFD3eFF,Inbound,TCP,Intrusion ID: 1,Begin: 2020-05-06 09:06:09,End Time: 2020-05-06 09:06:09,Occurrences: 3,Application: C:/Windows/System32/example_x.exe,Location: Internal,User Name: user_h,Domain Name: CompanyXX,Local Port: 1991,Remote Port: 46926,CIDS Signature ID: 25198,CIDS Signature string: Web Attack: Fake Scan Webpage 7,CIDS Signature SubID: 25378,Intrusion URL: https://www.example.org/,Intrusion Payload URL: http://www.example.com/,SHA-256: 6d2fe32dc4249ef7e7359c6d874fffbbf335e832e49a2681236e1b686af78794,MD-5: 70270ca63a3de2d8905a9181a0245e58 +def test_symantec_ep_security(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r'SymantecServer: qqqqq,Event Description: "Web Attack: Fake Scan Webpage 7",Local Host IP: 10.0.3.4,Local Host MAC: c1411f5F9502,Remote Host Name: eeeee,Remote Host IP: 10.0.3.6,Remote Host MAC: aD31CCFD3eFF,Inbound,TCP,Intrusion ID: 1,Begin: 2020-05-06 09:06:09,End Time: 2020-05-06 09:06:09,Occurrences: 3,Application: C:/Windows/System32/example_x.exe,Location: Internal,User Name: user_h,Domain Name: CompanyXX,Local Port: 1991,Remote Port: 46926,CIDS Signature ID: 25198,CIDS Signature string: Web Attack: Fake Scan Webpage 7,CIDS Signature SubID: 25378,Intrusion URL: https://www.example.org/,Intrusion Payload URL: http://www.example.com/,SHA-256: 6d2fe32dc4249ef7e7359c6d874fffbbf335e832e49a2681236e1b686af78794,MD-5: 70270ca63a3de2d8905a9181a0245e58' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:security:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: Security risk found,IP Address: 10.0.3.1,Computer name: qqqqq,Source: Definition downloader,Risk name: Backdoor.Joggver,Occurrences: 7,e:\resharper 9.1 + keygen\resharper.8.x.keygen.exe,"Still contains, 2 infected items",Actual action: Quarantined,Requested action: Process terminate pending restartLeft alone,Secondary action: Quarantined,Event time: 2020-05-06 08:29:27,Inserted: 2020-05-06 08:29:27,End: 2020-05-06 08:29:27,Last update time: 2020-05-06 08:29:27,Domain: SomeComp,Group: My Company\\Default Group,Server: Example Server C,User: user_h,Source computer: hhhhh,Source IP: 10.0.4.1,Disposition: Reputation was not used in this detection.,Download site: http://bbbb.example.com/,Web domain: gqtavlakkdkcryl.xn--pgbs0dh,Downloaded by: c:/program files (x86)/ggggg/cccc/application/cccc.exe,Prevalence: This file has been seen by fewer than 100 Symantec users.,Confidence: There is growing evidence that this file is trustworthy.,URL Tracking Status: off,First Seen: Reputation was not used in this detection.,Sensitivity: low,MDS,Application hash: 44d7fb7e-8c40-4a17-9aff-9c4aa0b96696,Hash type: SHA1,Company name: "Sample Inc. a wholly owned subsidiary of Dummy, Inc.",Application name: Setup Factory 7.0 Runtime,Application version: ,Application type: 127,File size (bytes): 1318,Category set: Security risk,Category type: UNKNOWN,Location: AZ - Office,Intensive Protection Level: 0,Certificate issuer: "Realtime deferred scanning",Certificate signer: Comodo,Certificate thumbprint: e5:xx:74:3c:xx:01:c4:9b:xx:43:xx:bb:zz:e8:6a:81:10:9f:e4:xx,Signing timestamp: 0,Certificate serial number: 903804111 +def test_symantec_ep_risk(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r'SymantecServer: Security risk found,IP Address: 10.0.3.1,Computer name: qqqqq,Source: Definition downloader,Risk name: Backdoor.Joggver,Occurrences: 7,e:\resharper 9.1 + keygen\resharper.8.x.keygen.exe,"Still contains, 2 infected items",Actual action: Quarantined,Requested action: Process terminate pending restartLeft alone,Secondary action: Quarantined,Event time: 2020-05-06 08:29:27,Inserted: 2020-05-06 08:29:27,End: 2020-05-06 08:29:27,Last update time: 2020-05-06 08:29:27,Domain: SomeComp,Group: My Company\\Default Group,Server: Example Server C,User: user_h,Source computer: hhhhh,Source IP: 10.0.4.1,Disposition: Reputation was not used in this detection.,Download site: http://bbbb.example.com/,Web domain: gqtavlakkdkcryl.xn--pgbs0dh,Downloaded by: c:/program files (x86)/ggggg/cccc/application/cccc.exe,Prevalence: This file has been seen by fewer than 100 Symantec users.,Confidence: There is growing evidence that this file is trustworthy.,URL Tracking Status: off,First Seen: Reputation was not used in this detection.,Sensitivity: low,MDS,Application hash: 44d7fb7e-8c40-4a17-9aff-9c4aa0b96696,Hash type: SHA1,Company name: "Sample Inc. a wholly owned subsidiary of Dummy, Inc.",Application name: Setup Factory 7.0 Runtime,Application version: ,Application type: 127,File size (bytes): 1318,Category set: Security risk,Category type: UNKNOWN,Location: AZ - Office,Intensive Protection Level: 0,Certificate issuer: "Realtime deferred scanning",Certificate signer: Comodo,Certificate thumbprint: e5:xx:74:3c:xx:01:c4:9b:xx:43:xx:bb:zz:e8:6a:81:10:9f:e4:xx,Signing timestamp: 0,Certificate serial number: 903804111' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:risk:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: nnnnn,Local Host IP: 10.0.0.2,Local Port: 10456,Local Host MAC: B9e90F5c3aC4,Remote Host IP: 10.0.9.2,Remote Host Name: lllll,Remote Port: 58999,Remote Host MAC: 7b6A329f7c1e,others,Inbound,Begin: 2020-05-06 09:18:32,End: 2020-05-06 09:18:32,Occurrences: 8,Application: C:/Windows/System32/example_y.EXE,Rule: Block all other IP traffic and log,Location: Public Network,User: user_f,Domain: XXXXDOMAIN,Action: Blocked,SHA-256: d1616b874a96df2515da372a90bddc00792cbff027f5e097cafa31d3aea8b310,MD-5: 82136b4240d6ce4ea7d03e51469a393b +def test_symantec_ep_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: nnnnn,Local Host IP: 10.0.0.2,Local Port: 10456,Local Host MAC: B9e90F5c3aC4,Remote Host IP: 10.0.9.2,Remote Host Name: lllll,Remote Port: 58999,Remote Host MAC: 7b6A329f7c1e,others,Inbound,Begin: 2020-05-06 09:18:32,End: 2020-05-06 09:18:32,Occurrences: 8,Application: C:/Windows/System32/example_y.EXE,Rule: Block all other IP traffic and log,Location: Public Network,User: user_f,Domain: XXXXDOMAIN,Action: Blocked,SHA-256: d1616b874a96df2515da372a90bddc00792cbff027f5e097cafa31d3aea8b310,MD-5: 82136b4240d6ce4ea7d03e51469a393b" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:traffic:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From 695de26a807cf5a29d8c7ab301a619dd34f8f383 Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Thu, 7 May 2020 14:45:36 +0530 Subject: [PATCH 03/15] Update Symantec EP document --- docs/sources/Symantec/index.md | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md index f2e7ab5..03e4f05 100644 --- a/docs/sources/Symantec/index.md +++ b/docs/sources/Symantec/index.md @@ -10,15 +10,27 @@ ### Sourcetypes -| sourcetype | notes | -|----------------|---------------------------------------------------------------------------------------------------------| -| symantec:ep:syslog | Warning the syslog method of accepting EP logs has been reported to show high data loss and is not Supported by Splunk | - -### Sourcetype and Index Configuration - -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| symantec_ep | symantec:ep:syslog | epav | none | +| sourcetype | notes | +|--------------------------------|---------------------------------------------------------------------------------------------------------| +| symantec:ep:syslog | Warning the syslog method of accepting EP logs has been reported to show high data loss and is not Supported by Splunk | +| symantec:ep:admin:syslog | none | +| symantec:ep:agent:syslog | none | +| symantec:ep:agt:system:syslog | none | +| symantec:ep:behavior:syslog | none | +| symantec:ep:packet:syslog | none | +| symantec:ep:policy:syslog | none | +| symantec:ep:proactive:syslog | none | +| symantec:ep:risk:syslog | none | +| symantec:ep:scan:syslog | none | +| symantec:ep:scm:system:syslog | none | +| symantec:ep:security:syslog | none | +| symantec:ep:traffic:syslog | none | + +### Index Configuration + +| key | index | notes | +|----------------|----------------|----------------| +| symantec_ep | epav | none | ### Filter type From 048e3b7426a7cea3a0092b4e84e810a9e4bd7fcf Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 20:24:14 -0400 Subject: [PATCH 04/15] TCP and TLS --- package/etc/go_templates/source_network.t | 8 ++++++-- tests/docker-compose.yml | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index fb59a33..adc09da 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -30,9 +30,10 @@ source s_{{ .port_id }} { {{- end}} {{- end}} {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} + {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} network ( - transport("tcp") - port({{ getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514" }}) + transport("tcp") + port({{ . }}) ip-protocol(4) max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) @@ -44,8 +45,10 @@ source s_{{ .port_id }} { chain-hostnames(off) flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); + {{- end }} {{- end}} {{- if (conv.ToBool (getenv "SC4S_SOURCE_TLS_ENABLE" "no")) }} + {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514") "," }} network( transport("tls") port({{ getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514" }}) @@ -66,6 +69,7 @@ source s_{{ .port_id }} { cipher-suite("{{- getenv "SC4S_SOURCE_TLS_CIPHER_SUITE" "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH" }}") ) ); + {{- end }} {{- end}} }; {{ if eq .parser "rfc3164" }} diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 0f070a0..c935baf 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -36,7 +36,8 @@ services: - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002 - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 - - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 + - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005,6005 + - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005,6005 - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 - SC4S_LISTEN_CISCO_ASA_UDP_PORT=5001 - SC4S_LISTEN_CISCO_IOS_UDP_PORT=5002 From f327c1157ecf12032092bd982cc6ad5c8af44cec Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 20:30:25 -0400 Subject: [PATCH 05/15] Support multiple ports per log path --- package/etc/go_templates/source_network.t | 14 +++++++++----- tests/docker-compose.yml | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index adc09da..59b9cf5 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -1,9 +1,12 @@ {{ define "UDP" }} +{{- $port_id := .port_id }} +{{- $port := .port }} + {{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_SOCKETS" "1"))}} syslog ( transport("udp") so-reuseport(1) - persist-name("{{ .port_id }}{{ .instance }}") - port({{ getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514" }}) + persist-name("{{ $port_id }}_{{ $port }}_{{ . }}") + port({{ $port }}) ip-protocol(4) so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "1703936"}}) keep-hostname(yes) @@ -13,6 +16,7 @@ chain-hostnames(off) flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); + {{- end}} {{- end}} {{ define "T1" }} @@ -24,13 +28,13 @@ source s_{{ .port_id }} { source { {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} {{- $port_id := .port_id }} -{{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_SOCKETS" "1"))}} -{{- $context := dict "instance" . "port_id" $port_id }} +{{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} +{{- $context := dict "port" . "port_id" $port_id }} {{- template "UDP" $context }} {{- end}} {{- end}} {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} - {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} + {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} network ( transport("tcp") port({{ . }}) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index c935baf..db3bad2 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -48,6 +48,7 @@ services: - SC4S_LISTEN_F5_BIGIP_UDP_PORT=5007 - SC4S_ARCHIVE_GLOBAL=no - SC4S_SOURCE_STORE_RAWMSG=yes + - SC4S_SOURCE_LISTEN_UDP_SOCKETS=2 splunk: build: context: ../splunk From fc931fa079fee91bc6d31a2539da555a2b7b37ff Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 20:33:17 -0400 Subject: [PATCH 06/15] Update configuration.md --- docs/configuration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 6fdf940..d9c5fc0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,10 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of kernel sockets per active UDP port, which configures multi-threading of the UDP input buffer in the kernel to prevent packet loss. Total UDP input buffer is the multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF | | SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | +### Multiple listening ports for the same source + +SC4S supports multiple network listening ports useful for a narrow set of compliance use cases. When configuring a source port variable use a comma ',' separated list with no spaces. + ## Syslog Source TLS Certificate Configuration * Create a folder ``/opt/sc4s/tls`` if not already done as part of the "getting started" process. From 7585981eb42372846b3990b3f41f6ca9b3a323aa Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 08:07:15 -0400 Subject: [PATCH 07/15] Fixes #273 make indexed fields optional The fields sc4s_syslog_product and sc4s_syslog_format are not made optional in this change as the current use is intergral to the logic tree a future change could change the internal field name and conditional include the field in the json payload. This change also adds new optional indexed fields for dest port when and container host. --- docs/configuration.md | 2 + docs/gettingstarted/docker-systemd-general.md | 1 + docs/gettingstarted/podman-systemd-general.md | 1 + .../conf.d/conflib/_splunk/splunkfields.conf | 23 ----------- .../conflib/_splunk/splunkfields.conf.tmpl | 41 +++++++++++++++++++ 5 files changed, 45 insertions(+), 23 deletions(-) delete mode 100644 package/etc/conf.d/conflib/_splunk/splunkfields.conf create mode 100644 package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl diff --git a/docs/configuration.md b/docs/configuration.md index 38bca4b..a30ec11 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,6 +10,7 @@ and variables needed to properly configure SC4S for your environment. | SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list | | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | | SC4S_GLOBAL_DNS_USE | yes or no(default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | +| SC4S_CONTAINER_HOST | string | variable passed to the container to identify the actual log host for container implementations | * NOTE: Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http destination does not support this feature. Moreover, HEC Ack would significantly degrade performance for streaming data such as @@ -26,6 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | +| SC4S_DEST_SPLUNK_INDEX_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is (container,log_host,dport,fromhostip,proto) ) ## Alternate Destination Configuration diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index f7fde28..5614607 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -59,6 +59,7 @@ ExecStartPre=/usr/bin/docker run \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \ + -e "SC4S_CONTAINER_HOST=$(hostname -s)" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index b0c026d..2b2269a 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -77,6 +77,7 @@ ExecStartPre=/usr/bin/podman run \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ + -e "SC4S_CONTAINER_HOST=$(hostname -s)" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf deleted file mode 100644 index bf911a6..0000000 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf +++ /dev/null @@ -1,23 +0,0 @@ -#Used to set indexed fields we will always use to global defaults -rewrite r_set_splunk_default { - set("SC4S:$SOURCE", value(".splunk.source")); - set($FACILITY, value("fields.sc4s_syslog_facility")); - set($LEVEL, value("fields.sc4s_syslog_severity")); - set($LOGHOST, value("fields.sc4s_log_host")); - set($SOURCEIP, value("fields.sc4s_fromhostip")); -}; -#used by each log-path to set index and sourcetype which may be -#overridden by user defined values -block rewrite r_set_splunk_dest_default( - index() - source("${.splunk.source}") - sourcetype() - template(`splunk-template`) - ) { - set("`index`", value(".splunk.index")); - set("`source`", value(".splunk.source")); - set("`sourcetype`", value(".splunk.sourcetype")); -}; - - - diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl new file mode 100644 index 0000000..9f43664 --- /dev/null +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -0,0 +1,41 @@ +#Used to set indexed fields we will always use to global defaults +rewrite r_set_splunk_default { + set("SC4S:$SOURCE", value(".splunk.source")); +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "facility" }} + set($FACILITY, value("fields.sc4s_syslog_facility")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "severity" }} + set($LEVEL, value("fields.sc4s_syslog_severity")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "log_host" }} +{{- if (getenv "SC4S_CONTAINER_HOST") }} + set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "container" }} + set($LOGHOST, value("fields.sc4s_container")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} + set($SOURCEIP, value("fields.sc4s_fromhostip")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "destport" }} + set($DESTPORT, value("fields.sc4s_destport")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "proto" }} + set($PROTO, value("fields.sc4s_proto")); +{{- end}} +}; +#used by each log-path to set index and sourcetype which may be +#overridden by user defined values +block rewrite r_set_splunk_dest_default( + index() + source("${.splunk.source}") + sourcetype() + template(`splunk-template`) + ) { + set("`index`", value(".splunk.index")); + set("`source`", value(".splunk.source")); + set("`sourcetype`", value(".splunk.sourcetype")); +}; + + + From a17cd4f3faa09d018946f465c154aaa862d171f5 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 11:12:20 -0400 Subject: [PATCH 08/15] Enhance Bluecoat Support a 5424 format and remove empty fields --- .../log_paths/lp-symantec_proxy.conf.tmpl | 5 +++- tests/test_symantec_proxy.py | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl index 8d5b475..1447711 100644 --- a/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl @@ -15,7 +15,6 @@ log { channel { # Listen on the default port (typically 514) for SYMANTEC_PROXY traffic source (s_DEFAULT); - filter(f_is_rfc5424_noversion); filter(f_symantec_bluecoat_proxy); flags(final); }; @@ -24,6 +23,10 @@ log { rewrite { set("bluecoat_proxy", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy")) + subst( + "([-_a-zA-Z\(\)]+=(\"-\"|-| ))", + "", value(MESSAGE) + ); }; parser {p_add_context_splunk(key("bluecoat_proxy")); }; diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py index ad90bbb..52221ea 100644 --- a/tests/test_symantec_proxy.py +++ b/tests/test_symantec_proxy.py @@ -41,3 +41,31 @@ def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk, setup assert resultCount == 1 # +# <111>1 2020-04-15T17:22:32.521Z animal-visual bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) +def test_bluecoatproxySG_kv_5424(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + iso = dt.isoformat()[0:23] + epoch = epoch[:-3] + + mt = env.from_string( + "{{ mark }}1 {{ iso }}Z {{host}} bluecoat - splunk_format - c-ip=192.0.0.6 cs-bytes=6269 cs-categories=\"unavailable\" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent=\"ocspd/1.0.3\" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name=\"Explicit HTTP\" service.group=\"Standard\" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name=\"10.0.0.6-sample_logs\" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url=\"http://randomserver:8000/en-US/app/examples/\"") + message = mt.render(mark="<134>", iso=iso, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netproxy host=\"{{ host }}\" sourcetype=\"bluecoat:proxysg:access:kv\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# \ No newline at end of file From e873be4b9d55a7f15873a42aa6f5d7e09c21abf5 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 12:54:57 -0400 Subject: [PATCH 09/15] INDEXED not INDEX and fix missing end --- docs/configuration.md | 2 +- .../conf.d/conflib/_splunk/splunkfields.conf.tmpl | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a30ec11..eeffb1e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | -| SC4S_DEST_SPLUNK_INDEX_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is (container,log_host,dport,fromhostip,proto) ) +| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is (container,log_host,dport,fromhostip,proto) ) ## Alternate Destination Configuration diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 9f43664..1d838f9 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -1,26 +1,27 @@ #Used to set indexed fields we will always use to global defaults rewrite r_set_splunk_default { set("SC4S:$SOURCE", value(".splunk.source")); -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "facility" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "facility" }} set($FACILITY, value("fields.sc4s_syslog_facility")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "severity" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "severity" }} set($LEVEL, value("fields.sc4s_syslog_severity")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "log_host" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "log_host" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "container" }} +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "container" }} set($LOGHOST, value("fields.sc4s_container")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} set($SOURCEIP, value("fields.sc4s_fromhostip")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "destport" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "destport" }} set($DESTPORT, value("fields.sc4s_destport")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "proto" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "proto" }} set($PROTO, value("fields.sc4s_proto")); {{- end}} }; From 832da7a1d23d31aa451ae6aac83b2cbac7495bda Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 12:59:41 -0400 Subject: [PATCH 10/15] include all fields in default --- docs/configuration.md | 2 +- .../conf.d/conflib/_splunk/splunkfields.conf.tmpl | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index eeffb1e..7d77fd9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | -| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is (container,log_host,dport,fromhostip,proto) ) +| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is all fields ) ## Alternate Destination Configuration diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 1d838f9..62bfca7 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -1,27 +1,27 @@ #Used to set indexed fields we will always use to global defaults rewrite r_set_splunk_default { set("SC4S:$SOURCE", value(".splunk.source")); -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "facility" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "facility" }} set($FACILITY, value("fields.sc4s_syslog_facility")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "severity" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "severity" }} set($LEVEL, value("fields.sc4s_syslog_severity")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "log_host" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "log_host" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); {{- end}} {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "container" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "container" }} set($LOGHOST, value("fields.sc4s_container")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} set($SOURCEIP, value("fields.sc4s_fromhostip")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "destport" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,destport,fromhostip,proto") ",") has "destport" }} set($DESTPORT, value("fields.sc4s_destport")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "proto" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,destport,fromhostip,proto") ",") has "proto" }} set($PROTO, value("fields.sc4s_proto")); {{- end}} }; From 661a4e4258902c5efb37c0ca81a8fca6457fcc46 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 8 May 2020 10:15:10 -0700 Subject: [PATCH 11/15] Refine mulitple unique port docs * Refine mulitple unique ports docs; change section it appears in --- docs/configuration.md | 4 ---- docs/sources/index.md | 10 ++++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6f707fd..38bca4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,10 +113,6 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of kernel sockets per active UDP port, which configures multi-threading of the UDP input buffer in the kernel to prevent packet loss. Total UDP input buffer is the multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF | | SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | -### Multiple listening ports for the same source - -SC4S supports multiple network listening ports useful for a narrow set of compliance use cases. When configuring a source port variable use a comma ',' separated list with no spaces. - ## Syslog Source TLS Certificate Configuration * Create a folder ``/opt/sc4s/tls`` if not already done as part of the "getting started" process. diff --git a/docs/sources/index.md b/docs/sources/index.md index 8a416bf..c709a62 100644 --- a/docs/sources/index.md +++ b/docs/sources/index.md @@ -19,3 +19,13 @@ A key aspect of SC4S is to properly set Splunk metadata prior to the data arrivi It is understood that default values will need to be changed in many installations. Each source documented in this section has a table entitled "Sourcetype and Index Configuration", which highlights the default index and sourcetype for each source. See the section "SC4S metadata configuration" in the "Configuration" page for more information on how to override the default values in this table. +## Unique listening ports + +SC4S supports unique listening ports for each source technology/log path (e.g. Cisco ASA), which is useful when the device is +sending data on a port different from the typical default syslog port (UDP port 514). In some cases, when the source device emits data that +is not able to be distinguished from other device types, a unique port is sometimes required. The specific environment variables used for +setting "unique ports" are outlined in each source document in this section. + +In most cases only one "unique port" is needed for each source. However, SC4S also supports multiple network listening ports per source, +which can be useful for a narrow set of compliance use cases. When configuring a source port variable to enable multiple ports, use a +comma-separated list with no spaces (e.g. `SC4S_LISTEN_CISCO_ASA_UDP_PORT=5005,6005`). From 01fbe9fec602bf18cd1486267ddf4ee08fcd8eb8 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Fri, 8 May 2020 11:04:54 -0700 Subject: [PATCH 12/15] Update splunkfields.conf.tmpl --- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 62bfca7..f340bb3 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -9,7 +9,7 @@ rewrite r_set_splunk_default { {{- end}} {{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "log_host" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} - set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); + set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost")); {{- end}} {{- end}} {{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "container" }} From 0e9e17c30438d0cef5f69f772654643c4761acdf Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 14:05:53 -0400 Subject: [PATCH 13/15] change sc4s_log_host to sc4s_loghost --- docs/troubleshooting.md | 2 +- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 03c4d48..fdf331c 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -75,7 +75,7 @@ don't expect, check to see that the index is created in Splunk, or that a `lastC cause for almost _all_ `400` errors. * If you continue to the individual log entries in these directories, you will see entries of the form ```bash -curl -k -u "sc4s HEC debug:a778f63a-5dff-4e3c-a72c-a03183659e94" "https://splunk.smg.aws:8088/services/collector/event" -d '{"time":"1584556114.271","sourcetype":"sc4s:events","source":"SC4S:s_internal","index":"main","host":"e3563b0ea5d8","fields":{"sc4s_syslog_severity":"notice","sc4s_syslog_facility":"syslog","sc4s_log_host":"e3563b0ea5d8","sc4s_fromhostip":"127.0.0.1"},"event":"syslog-ng starting up; version='3.26.1'"}' +curl -k -u "sc4s HEC debug:a778f63a-5dff-4e3c-a72c-a03183659e94" "https://splunk.smg.aws:8088/services/collector/event" -d '{"time":"1584556114.271","sourcetype":"sc4s:events","source":"SC4S:s_internal","index":"main","host":"e3563b0ea5d8","fields":{"sc4s_syslog_severity":"notice","sc4s_syslog_facility":"syslog","sc4s_loghost":"e3563b0ea5d8","sc4s_fromhostip":"127.0.0.1"},"event":"syslog-ng starting up; version='3.26.1'"}' ``` * These commands, with minimal modifications (e.g. multiple URLs specified or elements that needs shell escapes) can be run directly on the command line to determine what, exactly, the HEC endpoint is returning. This can be used to refine th index or other parameter to correct the diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 62bfca7..f340bb3 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -9,7 +9,7 @@ rewrite r_set_splunk_default { {{- end}} {{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "log_host" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} - set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); + set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost")); {{- end}} {{- end}} {{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "container" }} From 0f6081415eea1e4cb53ec3a5bfb82d014ac76de4 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 14:11:34 -0400 Subject: [PATCH 14/15] loghost --- docs/configuration.md | 2 +- .../conf.d/conflib/_splunk/splunkfields.conf.tmpl | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7d77fd9..f8f3920 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | -| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is all fields ) +| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,loghost,dport,fromhostip,proto | list of sc4s indexed fields default list is all fields ) ## Alternate Destination Configuration diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index f340bb3..320acf8 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -1,27 +1,27 @@ #Used to set indexed fields we will always use to global defaults rewrite r_set_splunk_default { set("SC4S:$SOURCE", value(".splunk.source")); -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "facility" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "facility" }} set($FACILITY, value("fields.sc4s_syslog_facility")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "severity" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "severity" }} set($LEVEL, value("fields.sc4s_syslog_severity")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "log_host" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "loghost" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost")); {{- end}} {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "container" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "container" }} set($LOGHOST, value("fields.sc4s_container")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "fromhostip" }} set($SOURCEIP, value("fields.sc4s_fromhostip")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,destport,fromhostip,proto") ",") has "destport" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,destport,fromhostip,proto") ",") has "destport" }} set($DESTPORT, value("fields.sc4s_destport")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,destport,fromhostip,proto") ",") has "proto" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,destport,fromhostip,proto") ",") has "proto" }} set($PROTO, value("fields.sc4s_proto")); {{- end}} }; From bd615a7524d9f31f801435329aa720349ee3767b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 14:25:25 -0400 Subject: [PATCH 15/15] update docs --- docs/sources/Symantec/index.md | 7 ++++++- tests/test_symantec_proxy.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md index f2e7ab5..f0285f1 100644 --- a/docs/sources/Symantec/index.md +++ b/docs/sources/Symantec/index.md @@ -31,7 +31,12 @@ MSG Parse: This filter parses message content * Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. * Refer to the Splunk TA documentation for the specific customer format required for proxy configuration * Select TCP or SSL transport option - * Ensure the format of the event is customized per Splunk documentation + * Ensure the format of the event is customized as follows + +``` +<111>1 $(date)T$(x-bluecoat-hour-utc):$(x-bluecoat-minute-utc):$(x-bluecoat-second-utc).000z $(x-bluecoat-appliance-name) bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) + +``` ### Options diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py index 52221ea..a65b55f 100644 --- a/tests/test_symantec_proxy.py +++ b/tests/test_symantec_proxy.py @@ -41,7 +41,8 @@ def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk, setup assert resultCount == 1 # -# <111>1 2020-04-15T17:22:32.521Z animal-visual bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) +# <111>1 $(date)T$(x-bluecoat-hour-utc):$(x-bluecoat-minute-utc):$(x-bluecoat-second-utc).000z $(x-bluecoat-appliance-name) bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) +# def test_bluecoatproxySG_kv_5424(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))