From b4af6ab1f3d3e14cd66baaa869bcba115757979e Mon Sep 17 00:00:00 2001 From: "Mahir Chavda (C)" Date: Mon, 20 Apr 2020 17:15:56 +0530 Subject: [PATCH 01/71] Add f5 bigip irule and ASM filters --- package/etc/conf.d/filters/f5/bigip.conf.tmpl | 3 +- .../conf.d/log_paths/lp-f5_bigip.conf.tmpl | 47 +++++- .../splunk_index.csv.example | 1 + tests/test_f5_bigip.py | 143 ++++++++++++++++++ 4 files changed, 191 insertions(+), 3 deletions(-) diff --git a/package/etc/conf.d/filters/f5/bigip.conf.tmpl b/package/etc/conf.d/filters/f5/bigip.conf.tmpl index a0138bb..a4a4f62 100644 --- a/package/etc/conf.d/filters/f5/bigip.conf.tmpl +++ b/package/etc/conf.d/filters/f5/bigip.conf.tmpl @@ -5,7 +5,8 @@ filter f_f5_bigip { or program("mcpd") or program("apmd") or program("tmm\d?") - or program('^f5_irule='); + or program('^f5_irule=') + or message('^f5_asm=Splunk-F5-ASM'); }; filter f_f5_bigip_irule { diff --git a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl index 934f71c..ee9e332 100644 --- a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl @@ -51,15 +51,58 @@ log { # rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; } elif { filter { - program('f5_irule=') + program('^f5_irule=') + }; + if { + filter { + program('^f5_irule=Splunk-iRule-HTTP') + }; + rewrite { + r_set_splunk_dest_default(sourcetype("f5:bigip:ltm:http:irule"), index("netops")) + }; + } elif { + filter { + program('^f5_irule=Splunk-iRule-DNS_REQUEST') + }; + rewrite { + r_set_splunk_dest_default(sourcetype("f5:bigip:gtm:dns:request:irule"), index("netops")) + }; + } elif { + filter { + program('^f5_irule=Splunk-iRule-DNS_RESPONSE') + }; + rewrite { + r_set_splunk_dest_default(sourcetype("f5:bigip:gtm:dns:response:irule"), index("netops")) + }; + } elif { + filter { + program('^f5_irule=Splunk-iRule-LB_FAILED') + }; + rewrite { + r_set_splunk_dest_default(sourcetype("f5:bigip:ltm:failed:irule"), index("netops")) + }; + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("f5:bigip:irule"), index("netops")) + }; }; rewrite { set("f5_bigip_irule", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("f5:bigip:irule"), index("netops")) }; parser { p_add_context_splunk(key("f5_bigip_irule")); }; parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + } elif { + filter { + message('^f5_asm=Splunk-F5-ASM') + }; + rewrite { + set("f5_bigip_asm", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("f5:bigip:asm:syslog"), index("netops")) + }; + parser { p_add_context_splunk(key("f5_bigip_asm")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; } elif { filter(f_f5_bigip); rewrite { diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 8a77f3d..361ed9d 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -27,6 +27,7 @@ #forcepoint_webprotect,index,netproxy #f5_bigip,index,netops #f5_bigip_irule,index,netops +#f5_bigip_asm,index,netops #f5_bigip_nix,index,netops #fortinet_fortios_event,index,netops #fortinet_fortios_log,index,netops diff --git a/tests/test_f5_bigip.py b/tests/test_f5_bigip.py index 3b17512..1d99448 100644 --- a/tests/test_f5_bigip.py +++ b/tests/test_f5_bigip.py @@ -24,6 +24,7 @@ #Jan 17 04:03:37 SV5-F5-5600-2 warning tmm1[23068]: 01260009:4: Connection error: ssl_passthru:5234: not SSL (40) #Jan 17 04:42:37 SV5-F5-5600-2.splunk.com notice mcpd[10653]: 01070638:5: Pool /Common/infra-docs-pool member /Common/go_web3:4000 monitor status down. [ /Common/tcp_half_open: down; last error: ] [ was up for 837hrs:31mins:36sec ] #Jan 17 04:42:37 SV5-F5-5600-2 notice apmd[11023]: 01490248:5: /Common/Network_Access_02:Common:8c6be305: Received client info - Hostname: Type: IE Version: 8 Platform: Win7 CPU: WOW64 UI Mode: Full Javascript Support: 1 ActiveX Support: 1 Plugin Support: 0 +#Apr 07 11:39:53 192.168.128.217 notice mcpd[6760]: 01070417:5: AUDIT - client Unknown, user admin - transaction #29194914-3 - object 0 - modify { gtm_rule { gtm_rule_name "/Common/Splunk_DNS_REQUEST" gtm_rule_definition "when DNS_REQUEST { set client_addr [IP::client_addr] set dns_server_addr [IP::local_addr] set question_name [DNS::question name] set question_class [DNS::question class] set question_type [DNS::question type] set data_center [whereami] set geo_information [join [whereis $client_addr] ;] set gtm_server [whoami] set wideip [wideip name] set dns_len [DNS::len] set hsl [HSL::open -proto UDP -pool Pool-syslog] HSL::send $hsl \"<190>,f5_irule=Splunk-iRule-DNS_REQUEST,src_ip=10.0.0.1,dns_server_ip=10.0.0.2,src_geo_info=dummy_geo_information,question_name=test.dummy_url1.com,question_class=IN,question_type=AB,data_center=/Common/Dummy-data-center-01,gtm_server=/Common/GTM-01,wideip=/Common/home.url.com,dns_len=34 } } [Status=Command OK] #2019-12-12T15:54:12.972208-08:00 10.160.21.242,f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.32.30.21,vip=10.156.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015 testdata_nix = [ @@ -40,6 +41,7 @@ '{{ mark }}{{ bsd }} {{ host }} warning tmm1[23068]: 01260009:4: Connection error: ssl_passthru:5234: not SSL (40)', '{{ mark }}{{ bsd }} {{ host }} notice mcpd[10653]: 01070638:5: Pool /Common/infra-docs-pool member /Common/go_web3:4000 monitor status down. [ /Common/tcp_half_open: down; last error: ] [ was up for 837hrs:31mins:36sec ]', '{{ mark }}{{ bsd }} {{ host }} notice apmd[11023]: 01490248:5: /Common/Network_Access_02:Common:8c6be305: Received client info - Hostname: Type: IE Version: 8 Platform: Win7 CPU: WOW64 UI Mode: Full Javascript Support: 1 ActiveX Support: 1 Plugin Support: 0', +'{{ mark }}{{ bsd }} {{ host }} notice mcpd[6760]: 01070417:5: AUDIT - client Unknown, user admin - transaction #29194914-3 - object 0 - modify { gtm_rule { gtm_rule_name "/Common/Splunk_DNS_REQUEST" gtm_rule_definition "when DNS_REQUEST { set client_addr [IP::client_addr] set dns_server_addr [IP::local_addr] set question_name [DNS::question name] set question_class [DNS::question class] set question_type [DNS::question type] set data_center [whereami] set geo_information [join [whereis $client_addr] ;] set gtm_server [whoami] set wideip [wideip name] set dns_len [DNS::len] set hsl [HSL::open -proto UDP -pool Pool-syslog] HSL::send $hsl \"<190>,f5_irule=Splunk-iRule-DNS_REQUEST,src_ip=10.0.0.1,dns_server_ip=10.0.0.2,src_geo_info=dummy_geo_information,question_name=test.dummy_url1.com,question_class=IN,question_type=AB,data_center=/Common/Dummy-data-center-01,gtm_server=/Common/GTM-01,wideip=/Common/home.url.com,dns_len=34 } } [Status=Command OK]' ] testdata_irule = [ '{{ mark }}{{ iso }} {{ host }},f5_irule=Splunk-HSL-iRule-HTTP,src_ip=10.111.30.21,vip=10.1111.1.160,http_method=GET,http_host=confluence.splunk.com: 443,http_uri=/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_url=confluence.splunk.com:443/download/attachments/185799227/Dynamic%20Lookups%20in%20RZ%20-%20architecture.png?version=1&modificationDate=1574471645759&api=v2,http_version=1.1,http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",http_content_type=,http_referrer="https://confluence.splunk.com/display/SEC/Dynamic+Lookups+in+RZ",req_start_time=2019/12/12 15:54:12,cookie="optimizelyBuckets _ga __ktt _gid optimizelyEndUserId __lc.visitor_id.3988321 _cs_c SPLUNK_SUB_LOGIN confluence.list.pages.cookie __kti __ktv _gcl_au crowd.token_key __utmv SPLUNK_USER_LOGIN_STATUS OptanonConsent trackAffiliate lc_sso3988321 _fbp _fbc confluence.browse.space.cookie _biz_pendingA ELOQUA __utmz ajs_group_id SPLUNK_SUB_SIGNUP _biz_nA _cs_id _hjid __utma mywork.tab.tasks optimizelySegments __utmc SPLUNK_AFFILIATE_CODE JSESSIONID Apache _biz_uid distance ajs_anonymous_id _biz_flagsA _st _gaexp __kts",user=,virtual_server="/Common/confluence-pool 10.156.18.12 8090",bytes_in=0,res_start_time=2019/12/12 15:54:12,node=10.156.18.12,node_port=8090,http_status=200,req_elapsed_time=21,bytes_out=75366#015' @@ -208,3 +210,144 @@ def test_f5_bigip_app_structured(record_property, setup_wordlist, get_host_key, record_property("message", message) assert resultCount == 1 + +# Apr 07 11:39:47 192.168.128.217,f5_irule=Splunk-iRule-HTTP,src_ip=192.168.128.62,vip=192.168.131.188,http_method=GET,http_host=test.url.com:80,http_uri=/test.html,http_url=test.url.com:80/test.html,http_method=GET,http_version=1.1,http_user_agent="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36",http_content_type=,http_referrer="",req_start_time=2020/04/07 11:39:47,cookie="",user=admin,virtual_server="/Common/Pool-02 0",bytes_in=0,res_start_time=2020/04/07 11:39:47,node=192.168.1.13,node_port=80,http_status=301,req_elapsed_time=2,bytes_out=145 +def test_f5_bigip_irule_http(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ bsd }} {{ host }},f5_irule=Splunk-iRule-HTTP,src_ip=192.168.128.62,vip=192.168.131.188,http_method=GET,http_host=test.url.com:80,http_uri=/test.html,http_url=test.url.com:80/test.html,http_method=GET,http_version=1.1,http_user_agent="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36",http_content_type=,http_referrer="",req_start_time=2020/04/07 11:39:47,cookie="",user=admin,virtual_server="/Common/Pool-02 0",bytes_in=0,res_start_time=2020/04/07 11:39:47,node=192.168.1.13,node_port=80,http_status=301,req_elapsed_time=2,bytes_out=145' + "\n") + message = mt.render(mark="<166>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:ltm:http:irule\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 07 11:38:50 192.168.128.63,f5_irule=Splunk-iRule-DNS_REQUEST,src_ip=192.168.128.62,dns_server_ip=192.168.128.63,src_geo_info=,question_name=test.url.com,question_class=IN,question_type=A,data_center=/Common/Data-Center-02,gtm_server=/Common/GTM-02,wideip=/Common/test.url.com,dns_len=34 +def test_f5_bigip_irule_dns_request(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ bsd }} {{ host }},f5_irule=Splunk-iRule-DNS_REQUEST,src_ip=192.168.128.62,dns_server_ip=192.168.128.63,src_geo_info=,question_name=test.url.com,question_class=IN,question_type=A,data_center=/Common/Data-Center-02,gtm_server=/Common/GTM-02,wideip=/Common/test.url.com,dns_len=34' + "\n") + message = mt.render(mark="<166>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:gtm:dns:request:irule\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 07 11:40:20 192.168.128.63,f5_irule=Splunk-iRule-DNS_RESPONSE,src_ip=192.168.128.62,dns_server_ip=192.168.128.217,question_name=dr.sg.baidu.com,is_wideip=0,answer="test.url.com 30 IN A 192.168.131.189" +def test_f5_bigip_irule_dns_response(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ bsd }} {{ host }},f5_irule=Splunk-iRule-DNS_RESPONSE,src_ip=192.168.128.62,dns_server_ip=192.168.128.217,question_name=dr.sg.baidu.com,is_wideip=0,answer="test.url.com 30 IN A 192.168.131.189' + "\n") + message = mt.render(mark="<166>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:gtm:dns:response:irule\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 07 11:39:24 192.168.128.217,f5_irule=Splunk-iRule-LB_FAILED,src_ip=192.168.128.62,vip=192.168.131.189,http_method=GET,http_host=test.url.com:80,http_uri=/index.html,http_url=test.url.com:80/index.html,http_method=GET,http_version=1.1,http_user_agent="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)",http_content_type=,http_referrer="",req_start_time=2020/04/07 11:39:24,cookie="",user=,virtual_server="/Common/Pool-01 0",bytes_in=0 +def test_f5_bigip_irule_lb_failed(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ bsd }} {{ host }},f5_irule=Splunk-iRule-LB_FAILED,src_ip=192.168.128.62,vip=192.168.131.189,http_method=GET,http_host=test.url.com:80,http_uri=/index.html,http_url=test.url.com:80/index.html,http_method=GET,http_version=1.1,http_user_agent="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)",http_content_type=,http_referrer="",req_start_time=2020/04/07 11:39:24,cookie="",user=,virtual_server="/Common/Pool-01 0",bytes_in=0' + "\n") + message = mt.render(mark="<166>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:ltm:failed:irule\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <131>Apr 07 11:40:26 bigip-2.test_domain.com ASM:f5_asm=Splunk-F5-ASM,attack_type="SQL-Injection",date_time="2020-04-07 11:40:26",dest_ip=192.168.131.2,dest_port=80,geo_info="N/A",headers="Host: 192.168.131.2\\r\\nConnection: keep-alive\\r\\nCache-Control: max-age=0\\r\\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\\r\\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36\\r\\nAccept-Encoding: gzip, deflate, sdch\\r\\nAccept-Language: zh-CN,zh;q=0.8\\r\\nCookie: TS01aac4be=01953d3060e3cf18e66518dbb5e1d643669c9ff7afa0583160b6c34a3ead57baf615f8ec45\\r\\nIf-None-Match: ""864bfa9-50-507180d6d3b5a""\\r\\nIf-Modified-Since: Wed, 05 Nov 2014 08:06:09 GMT\\r\\n\\r\\n",http_class="/Common/ASM_Test",ip_addr_intelli="N/A",ip_client=72.6.2.84,ip_route_domain="72.6.2.84%0",is_trunct=,manage_ip_addr=192.168.1.2,method="GET",policy_apply_date="2015-02-06 11:07:22",policy_name="/Common/ASM_Test",protocol="HTTP",query_str="",req="Host: 192.168.131.2\\r\\nConnection: keep-alive\\r\\nCache-Control: max-age=0\\r\\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\\r\\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36\\r\\nAccept-Encoding: gzip, deflate, sdch\\r\\nAccept-Language: zh-CN,zh;q=0.8\\r\\nCookie: TS01aac4be=01953d3060e3cf18e66518dbb5e1d643669c9ff7afa0583160b6c34a3ead57baf615f8ec45\\r\\nIf-None-Match: ""864bfa9-50-507180d6d3b5a""\\r\\nIf-Modified-Since: Wed, 05 Nov 2014 08:06:09 GMT\\r\\n\\r\\n",req_status="passed",resp="HTTP/1.1 200 OK Content-type: text/html Content-Length: 7 ",resp_code="200",route_domain="0",session_id="d4f876aaf07d1c0d",severity="Informational",sig_ids="",sig_names="",src_port="39861",sub_violates="HTTP protocol compliance failed:Unparsable request content",support_id="12921611355731185944",unit_host="bigip-2.test_domain.com",uri="/some-path/secret.php",username="N/A",violate_details="14VIOL_HTTP_PROTOCOL6553665536SFRUUCB2ZXJzaW9uIG5vdCBmb3VuZA==",violate_rate="5",violations="",virus_name="Melissa",x_fwd_hdr_val="N/A" +def test_f5_bigip_asm_syslog(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): + host = get_host_key + host = "bigip-2.test_domain.com" + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }}{{ bsd }} {{ host }} ASM:f5_asm=Splunk-F5-ASM,attack_type="SQL-Injection",date_time="2020-04-07 11:40:26",dest_ip=192.168.131.2,dest_port=80,geo_info="N/A",headers="Host: 192.168.131.2\\r\\nConnection: keep-alive\\r\\nCache-Control: max-age=0\\r\\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\\r\\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36\\r\\nAccept-Encoding: gzip, deflate, sdch\\r\\nAccept-Language: zh-CN,zh;q=0.8\\r\\nCookie: TS01aac4be=01953d3060e3cf18e66518dbb5e1d643669c9ff7afa0583160b6c34a3ead57baf615f8ec45\\r\\nIf-None-Match: ""864bfa9-50-507180d6d3b5a""\\r\\nIf-Modified-Since: Wed, 05 Nov 2014 08:06:09 GMT\\r\\n\\r\\n",http_class="/Common/ASM_Test",ip_addr_intelli="N/A",ip_client=72.6.2.84,ip_route_domain="72.6.2.84%0",is_trunct=,manage_ip_addr=192.168.1.2,method="GET",policy_apply_date="2015-02-06 11:07:22",policy_name="/Common/ASM_Test",protocol="HTTP",query_str="",req="Host: 192.168.131.2\\r\\nConnection: keep-alive\\r\\nCache-Control: max-age=0\\r\\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\\r\\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36\\r\\nAccept-Encoding: gzip, deflate, sdch\\r\\nAccept-Language: zh-CN,zh;q=0.8\\r\\nCookie: TS01aac4be=01953d3060e3cf18e66518dbb5e1d643669c9ff7afa0583160b6c34a3ead57baf615f8ec45\\r\\nIf-None-Match: ""864bfa9-50-507180d6d3b5a""\\r\\nIf-Modified-Since: Wed, 05 Nov 2014 08:06:09 GMT\\r\\n\\r\\n",req_status="passed",resp="HTTP/1.1 200 OK Content-type: text/html Content-Length: 7 ",resp_code="200",route_domain="0",session_id="d4f876aaf07d1c0d",severity="Informational",sig_ids="",sig_names="",src_port="39861",sub_violates="HTTP protocol compliance failed:Unparsable request content",support_id="12921611355731185944",unit_host="bigip-2.test_domain.com",uri="/some-path/secret.php",username="N/A",violate_details="14VIOL_HTTP_PROTOCOL6553665536SFRUUCB2ZXJzaW9uIG5vdCBmb3VuZA==",violate_rate="5",violations="",virus_name="Melissa",x_fwd_hdr_val="N/A"' + "\n") + message = mt.render(mark="<166>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:asm:syslog\" host=\"{{ host }}\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From 7cae90965dfa80870a2b4fe93f8aea464df3d26b Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 4 May 2020 11:14:19 -0700 Subject: [PATCH 02/71] Document netfilter bug * Document RHEL/podman netfilter table update bug * Remove `SC4S_DEST_SPLUNK_HEC_WORKERS=6` from documentation (default in code is 10) --- docs/gettingstarted/byoe-rhel7.md | 1 - docs/gettingstarted/docker-swarm-general.md | 1 - docs/gettingstarted/docker-swarm-rhel7.md | 1 - docs/gettingstarted/docker-systemd-general.md | 1 - docs/gettingstarted/podman-systemd-general.md | 37 +++++++++++++++++-- 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 1aeea8d..47f6493 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -159,7 +159,6 @@ sudo bash /opt/sc4s/bin/preconfig.sh SYSLOGNG_OPTS=-f /etc/syslog-ng/syslog-ng.conf SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index e216dc3..dea3f12 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -107,7 +107,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index d1c795d..8c1d6f1 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -115,7 +115,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 4cc1a49..f7fde28 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -113,7 +113,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 3ebdade..7bb6ee6 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -1,8 +1,37 @@ -# WARNING: Do _not_ use Podman with RHEL/CentOS 7.x or earlier! +# UPDATE: Podman/RHEL UDP data block issue: netfilter connection table -There have been cases where UDP packet loss is noted when Podman is used with RHEL/CentOS 7.x versions. Stay tuned; the cause is -currently unkown. +We have determined the root cause for the issue with UDP data blocking and Podman/RHEL. The crux of the issue is that the netfilter +connection tables are _not_ udpdated when a new container starts _and_ there is a constant stream of UDP traffic from a given IP destined +for a given port. The table is _only_ updated if the trafffic pauses for the length of the connection table timeout (30 seconds by default). + +Therefore, if you attempt to start up sc4s on a server to which, for example, a firewall is sending a steady stream of UDP events, the kernel +will mistakenly keep trying to route the packets to the server itself rather than through the virtual network created by the new container. +Until the firewall pauses its output stream (unlikely) _or_ the workaround provided below is applied, traffic from that particular firewall +will never been seen by the container (and hence sc4s). + +## WORKAROUND + +There is a utility called `conntrack` that allows you to view/manipulate the netfilter connection tables in real time. Follow the steps below +to install and run it each time sc4s starts. It should be available in all RHEL 7/8 subscriptions. + +``` + install conntrack +``` + +After this is done, add the following entry to the unit file (and/or use the command when starting sc4s manually): + +``` +ExecStartPost=sleep 2; conntrack -D -p udp +``` + +This command will delete the old (stale) UDP entries two seconds after the container starts and allow the system to build a new table that +will properly route to the container when it sees UDP traffic. Note that this command resets the table for _all_ UDP +ports; for a purpose-built sc4s server this should not cause issues. If for any reason more granular control over _which_ UPD ports are +reset is desired, there are additional arguments to `conntrack` that can be used to select the specific UDP ports that are deleted in the +table. See the man page for `conntrack` for more information. + +The unit file entry above has been added to the example below for completeness. # Install podman @@ -53,6 +82,7 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ "$SC4S_TLS_DIR" \ --name SC4S \ --rm $SC4S_IMAGE +ExecStartPost=sleep 2; conntrack -D -p udp ``` * Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication @@ -100,7 +130,6 @@ SC4S is almost entirely controlled through environment variables, which are read ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` From c91f57faa120eecbfcb48341eb692b6aff88b4ad Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Wed, 6 May 2020 16:06:18 +0530 Subject: [PATCH 03/71] Update index from netops to netwaf for the f5_bigip_asm events --- package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl | 2 +- package/etc/context_templates/splunk_index.csv.example | 2 +- tests/test_f5_bigip.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl index ee9e332..fa32d23 100644 --- a/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl @@ -98,7 +98,7 @@ log { }; rewrite { set("f5_bigip_asm", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("f5:bigip:asm:syslog"), index("netops")) + r_set_splunk_dest_default(sourcetype("f5:bigip:asm:syslog"), index("netwaf")) }; parser { p_add_context_splunk(key("f5_bigip_asm")); }; parser (compliance_meta_by_source); diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 694dec6..2fbd567 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -27,7 +27,7 @@ #forcepoint_webprotect,index,netproxy #f5_bigip,index,netops #f5_bigip_irule,index,netops -#f5_bigip_asm,index,netops +#f5_bigip_asm,index,netwaf #f5_bigip_nix,index,netops #fortinet_fortios_event,index,netops #fortinet_fortios_log,index,netops diff --git a/tests/test_f5_bigip.py b/tests/test_f5_bigip.py index 1d99448..ca31ca6 100644 --- a/tests/test_f5_bigip.py +++ b/tests/test_f5_bigip.py @@ -341,7 +341,7 @@ def test_f5_bigip_asm_syslog(record_property, setup_wordlist, get_host_key, setu sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops _time={{ epoch }} sourcetype=\"f5:bigip:asm:syslog\" host=\"{{ host }}\"") + "search index=netwaf _time={{ epoch }} sourcetype=\"f5:bigip:asm:syslog\" host=\"{{ host }}\"") search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) From 723869288ee12ed3a43f805b7fea26270b6fd8e6 Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Wed, 6 May 2020 17:04:33 +0530 Subject: [PATCH 04/71] Update F5 document --- docs/gettingstarted/index.md | 1 + docs/sources/F5/index.md | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md index 9a33256..4514c7f 100644 --- a/docs/gettingstarted/index.md +++ b/docs/gettingstarted/index.md @@ -40,6 +40,7 @@ using the SC4S defaults. SC4S can be easily customized to use different indexes * netfw * netids * netops +* netwaf * netproxy * netipam * oswinsec diff --git a/docs/sources/F5/index.md b/docs/sources/F5/index.md index 7ec2c86..8a3f1ce 100644 --- a/docs/sources/F5/index.md +++ b/docs/sources/F5/index.md @@ -15,15 +15,17 @@ |----------------|---------------------------------------------------------------------------------------------------------| | f5:bigip:syslog | None | | f5:bigip:irule | None | +| f5:bigip:asm:syslog | None | | nix:syslog | None | -### Sourcetype and Index Configuration +### Index Configuration -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| f5_bigip | f5:bigip:syslog | netops | none | -| f5_bigip_irule | f5:bigip:syslog | netops | none | -| f5_bigip_nix | nix:syslog | netops | if `f_f5_bigip` is not set the index osnix will be used | +| key | index | notes | +|----------------|----------------|----------------| +| f5_bigip | netops | none | +| f5_bigip_irule | netops | none | +| f5_bigip_asm | netwaf | none | +| f5_bigip_nix | netops | if `f_f5_bigip` is not set the index osnix will be used | ### Filter type From f7ab9914f27904ccf5767adc8b5a014782044ffc Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Wed, 6 May 2020 17:06:08 +0530 Subject: [PATCH 05/71] Add new irule sourcetype in the F5 document --- docs/sources/F5/index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/sources/F5/index.md b/docs/sources/F5/index.md index 8a3f1ce..952c8ab 100644 --- a/docs/sources/F5/index.md +++ b/docs/sources/F5/index.md @@ -15,6 +15,10 @@ |----------------|---------------------------------------------------------------------------------------------------------| | f5:bigip:syslog | None | | f5:bigip:irule | None | +| f5:bigip:ltm:http:irule | None | +| f5:bigip:gtm:dns:request:irule | None | +| f5:bigip:gtm:dns:response:irule | None | +| f5:bigip:ltm:failed:irule | None | | f5:bigip:asm:syslog | None | | nix:syslog | None | From cef810ff65c0ff06b9c5fa24f42815a76ab0aacd Mon Sep 17 00:00:00 2001 From: Jay Shah Date: Wed, 6 May 2020 22:23:35 +0530 Subject: [PATCH 06/71] Changed event format to KV from JSON for Juniper --- .../etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl index 7b743cb..48187f0 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl @@ -54,7 +54,7 @@ log { #We want to unset the fields we won't need, as this is copied into the #disk queue for network destinations. This can be very disk expensive #if we don't - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG")); }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_JUNOS_STRUCTURED_HEC" "no")) }} destination(d_hec); From 30c20f5c651f4c3420f58c34b6757422d6b35e29 Mon Sep 17 00:00:00 2001 From: Jay Shah Date: Thu, 7 May 2020 12:59:41 +0530 Subject: [PATCH 07/71] Added support of secintel and aamw sourcetypes --- docs/sources/Juniper/index.md | 6 +- .../lp-juniper_junos_structured.conf.tmpl | 6 ++ .../splunk_index.csv.example | 2 + tests/test_juniper_junos_rfc5124.py | 56 +++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/sources/Juniper/index.md b/docs/sources/Juniper/index.md index e497d0c..f8fccff 100644 --- a/docs/sources/Juniper/index.md +++ b/docs/sources/Juniper/index.md @@ -12,7 +12,11 @@ | sourcetype | notes | |--------------------------|------------------------------------------------------------------| | juniper:junos:firewall | None | -| juniper:junos:idp | None | +| juniper:junos:firewall:structured | None | +| juniper:junos:idp | None | +| juniper:junos:idp:structured | None | +| juniper:junos:aamw:structured | None | +| juniper:junos:secintel:structured | None | ### Sourcetype and Index Configuration diff --git a/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl index 48187f0..0927d87 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl @@ -36,6 +36,12 @@ log { } elif (program('RT_UTM')) { rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw")) }; parser {p_add_context_splunk(key("juniper_junos_utm_structured")); }; + } elif (program('RT_AAMW')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:aamw:structured"), index("netfw")) }; + parser {p_add_context_splunk(key("juniper_junos_aamw_structured")); }; + } elif (program('RT_SECINTEL')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:secintel:structured"), index("netfw")) }; + parser {p_add_context_splunk(key("juniper_junos_secintel_structured")); }; } # Legacy Netscreen IDP is handled in the "p_rfc3164-juniper-idp.conf" log path # diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 8a77f3d..afffde3 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -44,6 +44,8 @@ #juniper_junos_fw_structured,index,netfw #juniper_junos_ids_structured,index,netids #juniper_junos_utm_structured,index,netfw +#juniper_junos_aamw_structured,index,netfw +#juniper_junos_secintel_structured,index,netfw #juniper_junos_fw,index,netfw #juniper_junos_ids,index,netids #juniper_junos_utm,index,netfw diff --git a/tests/test_juniper_junos_rfc5124.py b/tests/test_juniper_junos_rfc5124.py index 458536c..c7ff8e9 100644 --- a/tests/test_juniper_junos_rfc5124.py +++ b/tests/test_juniper_junos_rfc5124.py @@ -99,4 +99,60 @@ def test_juniper_junos_fw_structured(record_property, setup_wordlist, get_host_k assert resultCount == 1 +# <165>1 2007-02-15T09:17:15.719Z aamw1 RT_AAMW - AAMW_ACTION_LOG [junos@2636.1.1.1.2.129 hostname="dummy_host" file-category="executable" verdict-number="10" malware-info="Testfile" action="PERMIT" list-hit="N/A" file-hash-lookup="FALSE" source-address="1.1.1.1" source-port="60148" destination-address="10.0.0.1" destination-port="80" protocol-id="6" application="HTTP" nested-application="N/A" policy-name="test-policy" username="N/A" roles="N/A" session-id-32="502156" source-zone-name="Inside" destination-zone-name="Outside" sample-sha256="e038b5168d9209267058112d845341cae83d92b1d1af0a10b66830acb7529494" file-name="dummy_file" url="dummy_url"] +# @pytest.mark.xfail +def test_juniper_junos_aamw_structured(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): + host = get_host_key + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + iso = dt.isoformat()[0:23] + epoch = epoch[:-3] + + mt = env.from_string( + "{{ mark }} {{ iso }}Z {{ host }} RT_AAMW - AAMW_ACTION_LOG [junos@2636.1.1.1.2.129 hostname=\"dummy_host\" file-category=\"executable\" verdict-number=\"10\" malware-info=\"Testfile\" action=\"PERMIT\" list-hit=\"N/A\" file-hash-lookup=\"FALSE\" source-address=\"1.1.1.1\" source-port=\"60148\" destination-address=\"10.0.0.1\" destination-port=\"80\" protocol-id=\"6\" application=\"HTTP\" nested-application=\"N/A\" policy-name=\"test-policy\" username=\"N/A\" roles=\"N/A\" session-id-32=\"502156\" source-zone-name=\"Inside\" destination-zone-name=\"Outside\" sample-sha256=\"e038b5168d9209267058112d845341cae83d92b1d1af0a10b66830acb7529494\" file-name=\"dummy_file\" url=\"dummy_url\"]") + message = mt.render(mark="<165>1", iso=iso, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:aamw:structured\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +# <165>1 2007-02-15T09:17:15.719Z secintel1 RT_SECINTEL - SECINTEL_ACTION_LOG [junos@2636.1.1.1.2.129 category="secintel" sub-category="CC" action="BLOCK" action-detail="CLOSE REDIRECT MSG" http-host="dummy_host" threat-severity="10" source-address="1.1.1.1" source-port="36612" destination-address="10.0.0.1" destination-port="80" protocol-id="6" application="HTTP" nested-application="N/A" feed-name="cc_url_data" policy-name="test" profile-name="test-profile" username="N/A" roles="N/A" session-id-32="502362" source-zone-name="Inside" destination-zone-name="Outside" occur-count="0"] +# @pytest.mark.xfail +def test_juniper_junos_secintel_structured(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): + host = get_host_key + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + iso = dt.isoformat()[0:23] + epoch = epoch[:-3] + + mt = env.from_string( + "{{ mark }} {{ iso }}Z {{ host }} RT_SECINTEL - SECINTEL_ACTION_LOG [junos@2636.1.1.1.2.129 category=\"secintel\" sub-category=\"CC\" action=\"BLOCK\" action-detail=\"CLOSE REDIRECT MSG\" http-host=\"dummy_host\" threat-severity=\"10\" source-address=\"1.1.1.1\" source-port=\"36612\" destination-address=\"10.0.0.1\" destination-port=\"80\" protocol-id=\"6\" application=\"HTTP\" nested-application=\"N/A\" feed-name=\"cc_url_data\" policy-name=\"test\" profile-name=\"test-profile\" username=\"N/A\" roles=\"N/A\" session-id-32=\"502362\" source-zone-name=\"Inside\" destination-zone-name=\"Outside\" occur-count=\"0\"]") + message = mt.render(mark="<23>1", iso=iso, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:secintel:structured\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From 474ffa056b35910adc4dea9ae385d87f28b30b7a Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Thu, 7 May 2020 14:42:18 +0530 Subject: [PATCH 08/71] Route sep syslog to correct sub-sourcetype --- package/etc/conf.d/filters/symantec/ep.conf | 48 +++++++ .../conf.d/log_paths/lp-symantec_ep.conf.tmpl | 73 ++++++++++- .../splunk_index.csv.example | 2 +- tests/test_symantec_ep.py | 118 +++++++++++++++++- 4 files changed, 234 insertions(+), 7 deletions(-) diff --git a/package/etc/conf.d/filters/symantec/ep.conf b/package/etc/conf.d/filters/symantec/ep.conf index a9db248..3420415 100644 --- a/package/etc/conf.d/filters/symantec/ep.conf +++ b/package/etc/conf.d/filters/symantec/ep.conf @@ -1,3 +1,51 @@ filter f_symantec_ep { program("SymantecServer") +}; + +filter f_symantec_ep_proactive { + message(',Detection\stype:') +}; + +filter f_symantec_ep_risk { + message(',Risk\sname:') +}; + +filter f_symantec_ep_agt_system { + message(',Category:\s\d+,') +}; + +filter f_symantec_ep_packet { + message(',(?:Inbound|Outbound|Unknown),Application:') +}; + +filter f_symantec_ep_traffic { + message(',(?:Inbound|Outbound|Unknown),Begin(?:\sTime)?:') +}; + +filter f_symantec_ep_security { + message('CIDS\sSignature\sSubID:') +}; + +filter f_symantec_ep_scan { + message('Scan\sID:\s\d+') +}; + +filter f_symantec_ep_behavior { + message('Begin(?:\sTime)?:\s[^,]*,End(?:\sTime)?:') +}; + +filter f_symantec_ep_policy { + message('Admin:\s[^,]+,.*[Pp]olicy') +}; + +filter f_symantec_ep_admin { + message('Domain(?:\sName)?:\s[^,]{0,25},Admin:') +}; + +filter f_symantec_ep_agent { + message('(?:,The\smanagement\sserver|,The\sclient)') +}; + +filter f_symantec_ep_scm_system { + message('Site:\s[^,]+,Server(?:\sName)?:\s[^,]+,') }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-symantec_ep.conf.tmpl b/package/etc/conf.d/log_paths/lp-symantec_ep.conf.tmpl index 2f294ed..e093563 100644 --- a/package/etc/conf.d/log_paths/lp-symantec_ep.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-symantec_ep.conf.tmpl @@ -21,15 +21,78 @@ log { }; }; - + if { + filter(f_symantec_ep_proactive); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:proactive:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_risk); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:risk:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_agt_system); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:agt:system:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_packet); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:packet:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_traffic); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:traffic:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_security); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:security:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_scan); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:scan:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_behavior); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:behavior:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_policy); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:policy:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_admin); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:admin:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_agent); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:agent:syslog"), index("epav")) + }; + } elif { + filter(f_symantec_ep_scm_system); + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:scm:system:syslog"), index("epav")) + }; + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:ep:syslog"), index("epav")) + }; + }; rewrite { - set("symantec_ep_syslog", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("symantec:ep:syslog"), index("epav")) + set("Symantec Endpoint Protection", value("fields.sc4s_vendor_product")); }; - parser { p_add_context_splunk(key("symantec_ep_syslog")); }; + parser { p_add_context_splunk(key("symantec_ep")); }; parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_EP_HEC" "no")) }} destination(d_hec); diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 2fbd567..5652871 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -70,7 +70,7 @@ #sc4s_events,index,main #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics -#symanrtec_ep,index,epav +#symantec_ep,index,epav #vmware_nsx,index,main #zscaler_alerts,index,main #zscaler_dns,index,netdns diff --git a/tests/test_symantec_ep.py b/tests/test_symantec_ep.py index 0c0e75a..f91a735 100644 --- a/tests/test_symantec_ep.py +++ b/tests/test_symantec_ep.py @@ -30,7 +30,123 @@ def test_symantec_ep_agent(record_property, setup_wordlist, setup_splunk, setup_ sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:syslog"' + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:agent:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:41:51 xxxxx-xxxxx SymantecServer: yyyyyy,Category: 2,LiveUpdate Manager,Event Description: A LiveUpdate session ran successfully. No new updates were available.,Event time: 2020-04-14 10:41:33,Group Name: My Company\Default Group +def test_symantec_ep_agt_system(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: yyyyyy,Category: 2,LiveUpdate Manager,Event Description: A LiveUpdate session ran successfully. No new updates were available.,Event time: 2020-04-14 10:41:33,Group Name: My Company\Default Group" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:agt:system:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 09:07:42 xxxxx-xxxxx SymantecServer: Site: Site xxxxx-xxxxx,Server Name: xxxxx-xxxxx,Event Description: No updates found for Application Control Data 14.2 RU2. +def test_symantec_ep_scm_system(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + "SymantecServer: Site: Site xxxxx-xxxxx,Server Name: xxxxx-xxxxx,Event Description: No updates found for Application Control Data 14.2 RU2." + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:scm:system:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:03:23 xxxxx-xxxxx SymantecServer: Scan ID: 1581582179,Begin: 2020-04-14 10:01:04,End Time: 2020-04-14 10:02:14,Completed,Duration (seconds): 70,User1: Spiderman,User2: Spiderman,Scan started on selected drives and folders and all extensions.,Scan Complete: Risks: 0 Scanned: 1062 Files/Folders/Drives Omitted: 0 Trusted Files Skipped: 698,Command: Not a command scan (),Threats: 0,Infected: 0,Total files: 1062,Omitted: 0,Computer: yyyyyyy,IP Address: 1.1.1.1,Domain Name: Default,Group Name: My Company\Preprod Tuesday,Server Name: xxxxx-xxxxx +def test_symantec_ep_scan(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: Scan ID: 1581582179,Begin: 2020-04-14 10:01:04,End Time: 2020-04-14 10:02:14,Completed,Duration (seconds): 70,User1: Spiderman,User2: Spiderman,Scan started on selected drives and folders and all extensions.,Scan Complete: Risks: 0 Scanned: 1062 Files/Folders/Drives Omitted: 0 Trusted Files Skipped: 698,Command: Not a command scan (),Threats: 0,Infected: 0,Total files: 1062,Omitted: 0,Computer: yyyyyyy,IP Address: 1.1.1.1,Domain Name: Default,Group Name: My Company\Preprod Tuesday,Server Name: xxxxx-xxxxx" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:scan:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:42:32 xxxxx-xxxxx SymantecServer: yyyyyy,...,Blocked,C:\Program Files (x86)\Symantec\Symantec Endpoint Protection\14.2.3335.1000.105\Bin\ccSvcHst.exe,,Begin: 2020-04-14 10:36:40,End Time: 2020-04-14 10:36:40,Rule: ,3248,C:\PROGRAM FILES (X86)\BIGFIX ENTERPRISE\BES CLIENT\BESCLIENT.EXE,0,,C:\Program Files (x86)\Symantec\Symantec Endpoint Protection\14.2.3335.1000.105\Bin\ccSvcHst.exe,User Name: SYSTEM,Domain Name: ,Action Type: 55,File size (bytes): ,Device ID: +def test_symantec_ep_behavior(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: yyyyyy,...,Blocked,C:\Program Files (x86)\Symantec\Symantec Endpoint Protection\14.2.3335.1000.105\Bin\ccSvcHst.exe,,Begin: 2020-04-14 10:36:40,End Time: 2020-04-14 10:36:40,Rule: ,3248,C:\PROGRAM FILES (X86)\BIGFIX ENTERPRISE\BES CLIENT\BESCLIENT.EXE,0,,C:\Program Files (x86)\Symantec\Symantec Endpoint Protection\14.2.3335.1000.105\Bin\ccSvcHst.exe,User Name: SYSTEM,Domain Name: ,Action Type: 55,File size (bytes): ,Device ID: " + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:behavior:syslog"' ) search = st.render(epoch=epoch, host=host) From b56083eba10b4e858e040723d1c4b72bb0c8501b Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Thu, 7 May 2020 14:44:42 +0530 Subject: [PATCH 09/71] samples for remaining sourcetypes --- tests/test_symantec_ep.py | 203 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/tests/test_symantec_ep.py b/tests/test_symantec_ep.py index f91a735..de97c8c 100644 --- a/tests/test_symantec_ep.py +++ b/tests/test_symantec_ep.py @@ -157,3 +157,206 @@ def test_symantec_ep_behavior(record_property, setup_wordlist, setup_splunk, set record_property("message", message) assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: Site: Site_B,Server Name: Example Server B,Domain Name: Domain_B,Admin: Admin_B,Event Description: Administrator log on failed +def test_symantec_ep_admin(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: Site: Site_B,Server Name: Example Server B,Domain Name: Domain_B,Admin: Admin_B,Event Description: Administrator log on failed" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:admin:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: ccccc,Local Host IP: 10.0.8.1,Local Port: 50221,Remote Host IP: 10.0.1.2,Remote Host Name: qqqqq,Remote Port: 20362,Outbound,Application: C:/Windows/System32/example_y.exe,Action: Allowed +def test_symantec_ep_packet(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: ccccc,Local Host IP: 10.0.8.1,Local Port: 50221,Remote Host IP: 10.0.1.2,Remote Host Name: qqqqq,Remote Port: 20362,Outbound,Application: C:/Windows/System32/example_y.exe,Action: Allowed" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:packet:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: Site: Site_B,Server Name: Example Server B,Domain Name: Domain_B,Admin: Admin_B,"Event Description: Policy has been edited: Changed Console mode at [Default]",Client Policy +def test_symantec_ep_policy(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r'SymantecServer: Site: Site_B,Server Name: Example Server B,Domain Name: Domain_B,Admin: Admin_B,"Event Description: Policy has been edited: Changed Console mode at [Default]",Client Policy' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:policy:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: Potential risk found,Computer name: ooooo,IP Address: 10.0.0.2,Detection type: System Change HostFile,First Seen: Symantec has known about this file for more than 1 year.,Application name: Microsoft\xAE Windows\xAE Operating System,Application type: 127,Application version: 6.1.7600.16385,Hash type: SHA-256,Application hash: ded6fc40-4365-4ba0-8446-3fa77a30cb6e,Company name: KKK.,LLLL,MMMM,File size (bytes): 3507,Sensitivity: 2,Detection score: 3,COH Engine Version: ,Detection Submissions No,Permitted application reason: Not on the permitted application list,Disposition: Bad,Download site: http://attraction.example.org/,Web domain: tkhwesmptszdody.dm,Downloaded by: c:/users/administrator/desktop/tools/tools/xxxtools.exe,Prevalence: Unknown,Confidence: There is not enough information about this file to recommend it.,URL Tracking Status: on,Risk Level: High,Risk type: 3,Source: Heuristic Scan,Risk name: Trojan.Gen.2,Occurrences: 9,PolicyZZZ,Realtime deferred scanning,Actual action: Left alone,Requested action: Quarantined,Secondary action: Left alone,Event time: 2020-05-04 06:57:02,Inserted: 2020-05-04 06:57:02,End: 2020-05-04 06:57:02,Domain: Domain A,Group: My Company\Default Group,Server: Example Server C,User: user_b,Source computer: fffff,Source IP: 10.0.9.2,Intensive Protection Level: 0,Certificate issuer: Symantec,Certificate signer: Unizeto,Certificate thumbprint: e5:xx:74:3c:xx:01:c4:9b:xx:43:xx:bb:zz:e8:6a:81:10:9f:e4:xx,Signing timestamp: 0,Certificate serial number: 149843929435818692848040365716851702463 +def test_symantec_ep_proactive(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: Potential risk found,Computer name: ooooo,IP Address: 10.0.0.2,Detection type: System Change HostFile,First Seen: Symantec has known about this file for more than 1 year.,Application name: Microsoft\xAE Windows\xAE Operating System,Application type: 127,Application version: 6.1.7600.16385,Hash type: SHA-256,Application hash: ded6fc40-4365-4ba0-8446-3fa77a30cb6e,Company name: KKK.,LLLL,MMMM,File size (bytes): 3507,Sensitivity: 2,Detection score: 3,COH Engine Version: ,Detection Submissions No,Permitted application reason: Not on the permitted application list,Disposition: Bad,Download site: http://attraction.example.org/,Web domain: tkhwesmptszdody.dm,Downloaded by: c:/users/administrator/desktop/tools/tools/xxxtools.exe,Prevalence: Unknown,Confidence: There is not enough information about this file to recommend it.,URL Tracking Status: on,Risk Level: High,Risk type: 3,Source: Heuristic Scan,Risk name: Trojan.Gen.2,Occurrences: 9,PolicyZZZ,Realtime deferred scanning,Actual action: Left alone,Requested action: Quarantined,Secondary action: Left alone,Event time: 2020-05-04 06:57:02,Inserted: 2020-05-04 06:57:02,End: 2020-05-04 06:57:02,Domain: Domain A,Group: My Company\Default Group,Server: Example Server C,User: user_b,Source computer: fffff,Source IP: 10.0.9.2,Intensive Protection Level: 0,Certificate issuer: Symantec,Certificate signer: Unizeto,Certificate thumbprint: e5:xx:74:3c:xx:01:c4:9b:xx:43:xx:bb:zz:e8:6a:81:10:9f:e4:xx,Signing timestamp: 0,Certificate serial number: 149843929435818692848040365716851702463" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:proactive:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: qqqqq,Event Description: "Web Attack: Fake Scan Webpage 7",Local Host IP: 10.0.3.4,Local Host MAC: c1411f5F9502,Remote Host Name: eeeee,Remote Host IP: 10.0.3.6,Remote Host MAC: aD31CCFD3eFF,Inbound,TCP,Intrusion ID: 1,Begin: 2020-05-06 09:06:09,End Time: 2020-05-06 09:06:09,Occurrences: 3,Application: C:/Windows/System32/example_x.exe,Location: Internal,User Name: user_h,Domain Name: CompanyXX,Local Port: 1991,Remote Port: 46926,CIDS Signature ID: 25198,CIDS Signature string: Web Attack: Fake Scan Webpage 7,CIDS Signature SubID: 25378,Intrusion URL: https://www.example.org/,Intrusion Payload URL: http://www.example.com/,SHA-256: 6d2fe32dc4249ef7e7359c6d874fffbbf335e832e49a2681236e1b686af78794,MD-5: 70270ca63a3de2d8905a9181a0245e58 +def test_symantec_ep_security(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r'SymantecServer: qqqqq,Event Description: "Web Attack: Fake Scan Webpage 7",Local Host IP: 10.0.3.4,Local Host MAC: c1411f5F9502,Remote Host Name: eeeee,Remote Host IP: 10.0.3.6,Remote Host MAC: aD31CCFD3eFF,Inbound,TCP,Intrusion ID: 1,Begin: 2020-05-06 09:06:09,End Time: 2020-05-06 09:06:09,Occurrences: 3,Application: C:/Windows/System32/example_x.exe,Location: Internal,User Name: user_h,Domain Name: CompanyXX,Local Port: 1991,Remote Port: 46926,CIDS Signature ID: 25198,CIDS Signature string: Web Attack: Fake Scan Webpage 7,CIDS Signature SubID: 25378,Intrusion URL: https://www.example.org/,Intrusion Payload URL: http://www.example.com/,SHA-256: 6d2fe32dc4249ef7e7359c6d874fffbbf335e832e49a2681236e1b686af78794,MD-5: 70270ca63a3de2d8905a9181a0245e58' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:security:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: Security risk found,IP Address: 10.0.3.1,Computer name: qqqqq,Source: Definition downloader,Risk name: Backdoor.Joggver,Occurrences: 7,e:\resharper 9.1 + keygen\resharper.8.x.keygen.exe,"Still contains, 2 infected items",Actual action: Quarantined,Requested action: Process terminate pending restartLeft alone,Secondary action: Quarantined,Event time: 2020-05-06 08:29:27,Inserted: 2020-05-06 08:29:27,End: 2020-05-06 08:29:27,Last update time: 2020-05-06 08:29:27,Domain: SomeComp,Group: My Company\\Default Group,Server: Example Server C,User: user_h,Source computer: hhhhh,Source IP: 10.0.4.1,Disposition: Reputation was not used in this detection.,Download site: http://bbbb.example.com/,Web domain: gqtavlakkdkcryl.xn--pgbs0dh,Downloaded by: c:/program files (x86)/ggggg/cccc/application/cccc.exe,Prevalence: This file has been seen by fewer than 100 Symantec users.,Confidence: There is growing evidence that this file is trustworthy.,URL Tracking Status: off,First Seen: Reputation was not used in this detection.,Sensitivity: low,MDS,Application hash: 44d7fb7e-8c40-4a17-9aff-9c4aa0b96696,Hash type: SHA1,Company name: "Sample Inc. a wholly owned subsidiary of Dummy, Inc.",Application name: Setup Factory 7.0 Runtime,Application version: ,Application type: 127,File size (bytes): 1318,Category set: Security risk,Category type: UNKNOWN,Location: AZ - Office,Intensive Protection Level: 0,Certificate issuer: "Realtime deferred scanning",Certificate signer: Comodo,Certificate thumbprint: e5:xx:74:3c:xx:01:c4:9b:xx:43:xx:bb:zz:e8:6a:81:10:9f:e4:xx,Signing timestamp: 0,Certificate serial number: 903804111 +def test_symantec_ep_risk(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r'SymantecServer: Security risk found,IP Address: 10.0.3.1,Computer name: qqqqq,Source: Definition downloader,Risk name: Backdoor.Joggver,Occurrences: 7,e:\resharper 9.1 + keygen\resharper.8.x.keygen.exe,"Still contains, 2 infected items",Actual action: Quarantined,Requested action: Process terminate pending restartLeft alone,Secondary action: Quarantined,Event time: 2020-05-06 08:29:27,Inserted: 2020-05-06 08:29:27,End: 2020-05-06 08:29:27,Last update time: 2020-05-06 08:29:27,Domain: SomeComp,Group: My Company\\Default Group,Server: Example Server C,User: user_h,Source computer: hhhhh,Source IP: 10.0.4.1,Disposition: Reputation was not used in this detection.,Download site: http://bbbb.example.com/,Web domain: gqtavlakkdkcryl.xn--pgbs0dh,Downloaded by: c:/program files (x86)/ggggg/cccc/application/cccc.exe,Prevalence: This file has been seen by fewer than 100 Symantec users.,Confidence: There is growing evidence that this file is trustworthy.,URL Tracking Status: off,First Seen: Reputation was not used in this detection.,Sensitivity: low,MDS,Application hash: 44d7fb7e-8c40-4a17-9aff-9c4aa0b96696,Hash type: SHA1,Company name: "Sample Inc. a wholly owned subsidiary of Dummy, Inc.",Application name: Setup Factory 7.0 Runtime,Application version: ,Application type: 127,File size (bytes): 1318,Category set: Security risk,Category type: UNKNOWN,Location: AZ - Office,Intensive Protection Level: 0,Certificate issuer: "Realtime deferred scanning",Certificate signer: Comodo,Certificate thumbprint: e5:xx:74:3c:xx:01:c4:9b:xx:43:xx:bb:zz:e8:6a:81:10:9f:e4:xx,Signing timestamp: 0,Certificate serial number: 903804111' + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:risk:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 14 10:10:10 dummyhost SymantecServer: nnnnn,Local Host IP: 10.0.0.2,Local Port: 10456,Local Host MAC: B9e90F5c3aC4,Remote Host IP: 10.0.9.2,Remote Host Name: lllll,Remote Port: 58999,Remote Host MAC: 7b6A329f7c1e,others,Inbound,Begin: 2020-05-06 09:18:32,End: 2020-05-06 09:18:32,Occurrences: 8,Application: C:/Windows/System32/example_y.EXE,Rule: Block all other IP traffic and log,Location: Public Network,User: user_f,Domain: XXXXDOMAIN,Action: Blocked,SHA-256: d1616b874a96df2515da372a90bddc00792cbff027f5e097cafa31d3aea8b310,MD-5: 82136b4240d6ce4ea7d03e51469a393b +def test_symantec_ep_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}{{ bsd }} {{host}} " + r"SymantecServer: nnnnn,Local Host IP: 10.0.0.2,Local Port: 10456,Local Host MAC: B9e90F5c3aC4,Remote Host IP: 10.0.9.2,Remote Host Name: lllll,Remote Port: 58999,Remote Host MAC: 7b6A329f7c1e,others,Inbound,Begin: 2020-05-06 09:18:32,End: 2020-05-06 09:18:32,Occurrences: 8,Application: C:/Windows/System32/example_y.EXE,Rule: Block all other IP traffic and log,Location: Public Network,User: user_f,Domain: XXXXDOMAIN,Action: Blocked,SHA-256: d1616b874a96df2515da372a90bddc00792cbff027f5e097cafa31d3aea8b310,MD-5: 82136b4240d6ce4ea7d03e51469a393b" + ) + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=epav host="{{ host }}" sourcetype="symantec:ep:traffic:syslog"' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From 695de26a807cf5a29d8c7ab301a619dd34f8f383 Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Thu, 7 May 2020 14:45:36 +0530 Subject: [PATCH 10/71] Update Symantec EP document --- docs/sources/Symantec/index.md | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md index f2e7ab5..03e4f05 100644 --- a/docs/sources/Symantec/index.md +++ b/docs/sources/Symantec/index.md @@ -10,15 +10,27 @@ ### Sourcetypes -| sourcetype | notes | -|----------------|---------------------------------------------------------------------------------------------------------| -| symantec:ep:syslog | Warning the syslog method of accepting EP logs has been reported to show high data loss and is not Supported by Splunk | - -### Sourcetype and Index Configuration - -| key | sourcetype | index | notes | -|----------------|----------------|----------------|----------------| -| symantec_ep | symantec:ep:syslog | epav | none | +| sourcetype | notes | +|--------------------------------|---------------------------------------------------------------------------------------------------------| +| symantec:ep:syslog | Warning the syslog method of accepting EP logs has been reported to show high data loss and is not Supported by Splunk | +| symantec:ep:admin:syslog | none | +| symantec:ep:agent:syslog | none | +| symantec:ep:agt:system:syslog | none | +| symantec:ep:behavior:syslog | none | +| symantec:ep:packet:syslog | none | +| symantec:ep:policy:syslog | none | +| symantec:ep:proactive:syslog | none | +| symantec:ep:risk:syslog | none | +| symantec:ep:scan:syslog | none | +| symantec:ep:scm:system:syslog | none | +| symantec:ep:security:syslog | none | +| symantec:ep:traffic:syslog | none | + +### Index Configuration + +| key | index | notes | +|----------------|----------------|----------------| +| symantec_ep | epav | none | ### Filter type From 084d9364471a44681b836fd2c769d4b16f1e6c8b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 14:45:07 -0400 Subject: [PATCH 11/71] Support reverse DNS --- docs/configuration.md | 1 + package/Dockerfile | 2 +- package/etc/{syslog-ng.conf => syslog-ng.conf.tmpl} | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) rename package/etc/{syslog-ng.conf => syslog-ng.conf.tmpl} (94%) diff --git a/docs/configuration.md b/docs/configuration.md index 6fdf940..4105d17 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,6 +9,7 @@ and variables needed to properly configure SC4S for your environment. |----------|---------------|-------------| | SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list | | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | +| SC4S_GLOBAL_DNS_USE | yes/no (default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | * NOTE: Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http destination does not support this feature. Moreover, HEC Ack would significantly degrade performance for streaming data such as diff --git a/package/Dockerfile b/package/Dockerfile index bd5e477..b461c2b 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -13,7 +13,7 @@ COPY --from=hairyhenderson/gomplate:v3.5.0 /gomplate /usr/local/bin/gomplate COPY goss.yaml goss.yaml -COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf +COPY etc/syslog-ng.conf.tmpl /opt/syslog-ng/etc/syslog-ng.conf.tmpl COPY etc/conf.d /opt/syslog-ng/etc/conf.d COPY etc/go_templates /opt/syslog-ng/etc/go_templates COPY etc/context_templates /opt/syslog-ng/etc/context_templates diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf.tmpl similarity index 94% rename from package/etc/syslog-ng.conf rename to package/etc/syslog-ng.conf.tmpl index 03ba65c..8730725 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf.tmpl @@ -18,9 +18,9 @@ options { time_reopen (10); log_fifo_size (10000); chain_hostnames (off); - use_dns (no); + use_dns ({{getenv "SC4S_GLOBAL_DNS_USE" "no"}}); use_fqdn (no); - dns-cache(no); + dns-cache({{getenv "SC4S_GLOBAL_DNS_CACHE" "yes"}}); create_dirs (no); keep-hostname (yes); create_dirs(yes); From 28b41d99ce3c0206a72e14b4f093132cf7354a41 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 14:56:56 -0400 Subject: [PATCH 12/71] Update syslog-ng.conf.tmpl --- package/etc/syslog-ng.conf.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/syslog-ng.conf.tmpl b/package/etc/syslog-ng.conf.tmpl index 8730725..98db6a2 100644 --- a/package/etc/syslog-ng.conf.tmpl +++ b/package/etc/syslog-ng.conf.tmpl @@ -17,12 +17,12 @@ options { flush_lines (100); time_reopen (10); log_fifo_size (10000); - chain_hostnames (off); + chain_hostnames (yes); use_dns ({{getenv "SC4S_GLOBAL_DNS_USE" "no"}}); use_fqdn (no); dns-cache({{getenv "SC4S_GLOBAL_DNS_CACHE" "yes"}}); create_dirs (no); - keep-hostname (yes); + keep-hostname (no); create_dirs(yes); dir_perm(0750); stats-freq(30); From eeacb62c240d6e0cf85d84044ea0dbd5a020b513 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 15:20:01 -0400 Subject: [PATCH 13/71] Update config.yml --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a8f8ce..92332a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,15 +215,18 @@ jobs: name: Build Docker command: | if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="$(./semtag getcurrent)"; fi + VERSION_DOCKER_MMP=$(echo $VERSION | sed -n 's/v\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p') echo ${VERSION} >package/VERSION echo ${CIRCLE_SHA1}=${VERSION} docker build --build-arg BRANCH=${SYSLOG} package -t splunk/scs:${CIRCLE_SHA1} docker tag splunk/scs:${CIRCLE_SHA1} splunk/scs:${VERSION} + docker tag splunk/scs:${CIRCLE_SHA1} splunk/scs:${VERSION_DOCKER_MMP} docker tag splunk/scs:${CIRCLE_SHA1} splunk/scs:${CIRCLE_SHA1:0:7} docker push splunk/scs:${CIRCLE_SHA1} docker push splunk/scs:${VERSION} + docker push splunk/scs:${VERSION_DOCKER_MMP} docker push splunk/scs:${CIRCLE_SHA1:0:7} - docker save splunk/scs:${VERSION} | gzip -c > /tmp/workspace/oci_container.tar.gz + docker save splunk/scs:${VERSION_DOCKER_MMP} | gzip -c > /tmp/workspace/oci_container.tar.gz - go/install - run: name: "Publish on GitHub" From 6f1eb2b68ba023a8929c7e15e05d60b76c056a86 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 15:29:53 -0400 Subject: [PATCH 14/71] Update base image to 3.27.1 --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 92332a4..6ad2412 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,7 +90,7 @@ jobs: docker: - image: circleci/python:3.7 environment: - SYSLOG: "syslog-ng-3.26.1" + SYSLOG: "syslog-ng-3.27.1" SPLUNK_VERSION: "8.0.2" <<: *test test-sc4s-next-splunk-8-0: @@ -104,14 +104,14 @@ jobs: docker: - image: circleci/python:3.7 environment: - SYSLOG: "syslog-ng-3.26.1" + SYSLOG: "syslog-ng-3.27.1" SPLUNK_VERSION: "7.3.4" <<: *test test-sc4s-current-splunk-7-2: docker: - image: circleci/python:3.7 environment: - SYSLOG: "syslog-ng-3.26.1" + SYSLOG: "syslog-ng-3.27.1" SPLUNK_VERSION: "7.2.9" <<: *test From 9eaebb30bf16d5991cde9cf592a236ce0e679aba Mon Sep 17 00:00:00 2001 From: mbonsack Date: Thu, 7 May 2020 13:01:24 -0700 Subject: [PATCH 15/71] Update configuration.md --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4105d17..38bca4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,7 +9,7 @@ and variables needed to properly configure SC4S for your environment. |----------|---------------|-------------| | SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list | | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | -| SC4S_GLOBAL_DNS_USE | yes/no (default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | +| SC4S_GLOBAL_DNS_USE | yes or no(default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | * NOTE: Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http destination does not support this feature. Moreover, HEC Ack would significantly degrade performance for streaming data such as From dc15a9f1f2866d40818e800154ce213d24097316 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 7 May 2020 13:55:46 -0700 Subject: [PATCH 16/71] Update 5424 templates to include RAWMSG * Update 5424 output templates to include RAWMSG. Too confusing for most users without them, and there are some small parts of the message that are excluded (such as colons) that are needed to reassemble the raw message with the constituent macros. --- package/etc/conf.d/conflib/_common/templates.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index fc0c7a0..ce18953 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -80,6 +80,7 @@ template t_JSON_5424 { template('$(format-json --scope rfc5424 --pair PRI="<$PRI>" --key ISODATE + --key RAWMSG --exclude DATE --exclude FACILITY --exclude PRIORITY @@ -95,6 +96,7 @@ template t_JSON_5424_SDATA { template('$(format-json --scope rfc5424 --pair PRI="<$PRI>" --key ISODATE + --key RAWMSG --exclude DATE --exclude HOST --exclude FACILITY From c97d86af21f551de00c69ff13b3a7a9f94385a52 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 7 May 2020 14:56:10 -0700 Subject: [PATCH 17/71] Update conntrack unit file entry with required space * `ExecStartPost` command requires a space around the semicolon separating the two individual commands. --- docs/gettingstarted/podman-systemd-general.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 7bb6ee6..b0c026d 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -19,10 +19,12 @@ to install and run it each time sc4s starts. It should be available in all RHEL install conntrack ``` -After this is done, add the following entry to the unit file (and/or use the command when starting sc4s manually): +After this is done, add the following entry to the unit file (and/or use the command when starting sc4s manually). +Note that the space on either side of the semicolon in the `ExecStartPost` entry is _required_ and systemd +will error out if it is missing. ``` -ExecStartPost=sleep 2; conntrack -D -p udp +ExecStartPost=sleep 2 ; conntrack -D -p udp ``` This command will delete the old (stale) UDP entries two seconds after the container starts and allow the system to build a new table that @@ -82,7 +84,7 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ "$SC4S_TLS_DIR" \ --name SC4S \ --rm $SC4S_IMAGE -ExecStartPost=sleep 2; conntrack -D -p udp +ExecStartPost=sleep 2 ; conntrack -D -p udp ``` * Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication From 048e3b7426a7cea3a0092b4e84e810a9e4bd7fcf Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 20:24:14 -0400 Subject: [PATCH 18/71] TCP and TLS --- package/etc/go_templates/source_network.t | 8 ++++++-- tests/docker-compose.yml | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index fb59a33..adc09da 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -30,9 +30,10 @@ source s_{{ .port_id }} { {{- end}} {{- end}} {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} + {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} network ( - transport("tcp") - port({{ getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514" }}) + transport("tcp") + port({{ . }}) ip-protocol(4) max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) @@ -44,8 +45,10 @@ source s_{{ .port_id }} { chain-hostnames(off) flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); + {{- end }} {{- end}} {{- if (conv.ToBool (getenv "SC4S_SOURCE_TLS_ENABLE" "no")) }} + {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514") "," }} network( transport("tls") port({{ getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514" }}) @@ -66,6 +69,7 @@ source s_{{ .port_id }} { cipher-suite("{{- getenv "SC4S_SOURCE_TLS_CIPHER_SUITE" "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH" }}") ) ); + {{- end }} {{- end}} }; {{ if eq .parser "rfc3164" }} diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 0f070a0..c935baf 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -36,7 +36,8 @@ services: - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002 - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 - - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 + - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005,6005 + - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005,6005 - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 - SC4S_LISTEN_CISCO_ASA_UDP_PORT=5001 - SC4S_LISTEN_CISCO_IOS_UDP_PORT=5002 From f327c1157ecf12032092bd982cc6ad5c8af44cec Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 20:30:25 -0400 Subject: [PATCH 19/71] Support multiple ports per log path --- package/etc/go_templates/source_network.t | 14 +++++++++----- tests/docker-compose.yml | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index adc09da..59b9cf5 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -1,9 +1,12 @@ {{ define "UDP" }} +{{- $port_id := .port_id }} +{{- $port := .port }} + {{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_SOCKETS" "1"))}} syslog ( transport("udp") so-reuseport(1) - persist-name("{{ .port_id }}{{ .instance }}") - port({{ getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514" }}) + persist-name("{{ $port_id }}_{{ $port }}_{{ . }}") + port({{ $port }}) ip-protocol(4) so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "1703936"}}) keep-hostname(yes) @@ -13,6 +16,7 @@ chain-hostnames(off) flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}) ); + {{- end}} {{- end}} {{ define "T1" }} @@ -24,13 +28,13 @@ source s_{{ .port_id }} { source { {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} {{- $port_id := .port_id }} -{{- range (math.Seq (getenv "SC4S_SOURCE_LISTEN_UDP_SOCKETS" "1"))}} -{{- $context := dict "instance" . "port_id" $port_id }} +{{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} +{{- $context := dict "port" . "port_id" $port_id }} {{- template "UDP" $context }} {{- end}} {{- end}} {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} - {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} + {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} network ( transport("tcp") port({{ . }}) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index c935baf..db3bad2 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -48,6 +48,7 @@ services: - SC4S_LISTEN_F5_BIGIP_UDP_PORT=5007 - SC4S_ARCHIVE_GLOBAL=no - SC4S_SOURCE_STORE_RAWMSG=yes + - SC4S_SOURCE_LISTEN_UDP_SOCKETS=2 splunk: build: context: ../splunk From fc931fa079fee91bc6d31a2539da555a2b7b37ff Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 20:33:17 -0400 Subject: [PATCH 20/71] Update configuration.md --- docs/configuration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 6fdf940..d9c5fc0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,10 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of kernel sockets per active UDP port, which configures multi-threading of the UDP input buffer in the kernel to prevent packet loss. Total UDP input buffer is the multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF | | SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | +### Multiple listening ports for the same source + +SC4S supports multiple network listening ports useful for a narrow set of compliance use cases. When configuring a source port variable use a comma ',' separated list with no spaces. + ## Syslog Source TLS Certificate Configuration * Create a folder ``/opt/sc4s/tls`` if not already done as part of the "getting started" process. From 17f94a13c71f4f41392464c2ba08cd252e833db8 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 7 May 2020 22:25:03 -0400 Subject: [PATCH 21/71] Reduce noise from checkpoint by loguid --- docs/sources/Checkpoint/index.md | 1 + .../log_paths/lp-checkpoint_splunk.conf.tmpl | 169 +++++++++++------- tests/docker-compose.yml | 1 + tests/test_checkpoint.py | 31 ++++ 4 files changed, 140 insertions(+), 62 deletions(-) diff --git a/docs/sources/Checkpoint/index.md b/docs/sources/Checkpoint/index.md index ee2c590..5c1186b 100644 --- a/docs/sources/Checkpoint/index.md +++ b/docs/sources/Checkpoint/index.md @@ -51,6 +51,7 @@ MSG Parse: This filter parses message content | SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the port number defined | | SC4S_ARCHIVE_CHECKPOINT_SPLUNK | no | Enable archive to disk for this specific source | | SC4S_DEST_CHECKPOINT_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL | no | Suppress any duplicate product+loguid pairs processed with 2 seconds | ### Verification diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index 673b11c..cd227fc 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -1,3 +1,17 @@ + +parser checkpoint_grouping { + grouping-by( + scope(host) + key("${.kv.product}${.kv.loguid}") + aggregate( + value("MESSAGE" "${MESSAGE}") + value(".cpfinal" "1" ) + inherit-mode(context) + ) + timeout(1) + ); +}; + # Checkpoint {{- /* The following provides a unique port source configuration if env var(s) are set */}} {{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "rfc3164" }} @@ -32,55 +46,85 @@ log { r_set_splunk_dest_default(sourcetype("cp_log"), index("netops")) }; - parser {p_add_context_splunk(key("checkpoint_splunk")); }; - - if { - filter(f_checkpoint_splunk_NetworkTraffic); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; - } elif { - filter(f_checkpoint_splunk_Web); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; - } elif { - filter(f_checkpoint_splunk_NetworkSessions); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; - } elif { - filter(f_checkpoint_splunk_IDS_Malware); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; - } elif { - filter(f_checkpoint_splunk_IDS); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; - } elif { - filter(f_checkpoint_splunk_email); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; - } elif { - filter(f_checkpoint_splunk_DLP); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; - } elif { - filter(f_checkpoint_splunk_syslog); - if { - parser { - syslog-parser(template("${.kv.default_device_message}") flags(guess-timezone, no-hostname)); - date-parser-nofilter(format("%s") template("${.kv.time}")); - }; + + if { + parser {p_add_context_splunk(key("checkpoint_splunk")); }; + + {{- if (conv.ToBool (getenv "SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL" "no")) }} + parser(checkpoint_grouping); + filter { + match("1" value(".cpfinal")) }; - - rewrite { - set("${.kv.hostname}", value("HOST")); - set("checkpoint_splunk", value("fields.sc4s_vendor_product")); - subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); - set("${PROGRAM}", value(".PROGRAM")); - subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + {{- end }} + + if { + filter(f_checkpoint_splunk_NetworkTraffic); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; + } elif { + filter(f_checkpoint_splunk_Web); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; + } elif { + filter(f_checkpoint_splunk_NetworkSessions); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; + } elif { + filter(f_checkpoint_splunk_IDS_Malware); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; + } elif { + filter(f_checkpoint_splunk_IDS); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; + } elif { + filter(f_checkpoint_splunk_email); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; + } elif { + filter(f_checkpoint_splunk_DLP); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; + } elif { + filter(f_checkpoint_splunk_syslog); + if { + parser { + syslog-parser(template("${.kv.default_device_message}") flags(guess-timezone, no-hostname)); + date-parser-nofilter(format("%s") template("${.kv.time}")); + }; }; - rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) }; - parser { p_add_context_splunk(key("checkpoint_os")); }; - + + rewrite { + set("${.kv.hostname}", value("HOST")); + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) }; + parser { p_add_context_splunk(key("checkpoint_os")); }; + + }; + + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + + {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no")) }} + destination(d_hec); + {{- end}} + + {{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK" "no")) }} + destination(d_archive); + {{- end}} + + {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} + + {{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} + }; } else { filter(f_nix_syslog); @@ -93,25 +137,26 @@ log { rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}")) }; parser { p_add_context_splunk(key("checkpoint_os")); }; - }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; -{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no")) }} - destination(d_hec); -{{- end}} + {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CHECKPOINT_SPLUNK_HEC" "no")) }} + destination(d_hec); + {{- end}} -{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK" "no")) }} - destination(d_archive); -{{- end}} + {{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK" "no")) }} + destination(d_archive); + {{- end}} -{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} - {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); -{{- end }} + {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} -{{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }} - {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); -{{- end }} + {{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} + + }; flags(flow-control,final); }; diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 0f070a0..04944c4 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -47,6 +47,7 @@ services: - SC4S_LISTEN_F5_BIGIP_UDP_PORT=5007 - SC4S_ARCHIVE_GLOBAL=no - SC4S_SOURCE_STORE_RAWMSG=yes + - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes splunk: build: context: ../splunk diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index d6ac8aa..06714f6 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -68,6 +68,37 @@ def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splun assert resultCount == 1 + +def test_checkpoint_splunk_firewall_noise(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions for Checkpoint + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") + message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"") + search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + def test_checkpoint_splunk_firewall2(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) From d1ccba9f584e8fbaa633fcfe58f1a68671bd8607 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 07:19:20 -0400 Subject: [PATCH 22/71] new test and docs fix --- docs/sources/Checkpoint/index.md | 2 +- .../log_paths/lp-checkpoint_splunk.conf.tmpl | 2 +- tests/test_checkpoint.py | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/sources/Checkpoint/index.md b/docs/sources/Checkpoint/index.md index 5c1186b..7bcde6e 100644 --- a/docs/sources/Checkpoint/index.md +++ b/docs/sources/Checkpoint/index.md @@ -51,7 +51,7 @@ MSG Parse: This filter parses message content | SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the port number defined | | SC4S_ARCHIVE_CHECKPOINT_SPLUNK | no | Enable archive to disk for this specific source | | SC4S_DEST_CHECKPOINT_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | -| SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL | no | Suppress any duplicate product+loguid pairs processed with 2 seconds | +| SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL | no | Suppress any duplicate product+loguid pairs processed within 2 seconds of the last matching event | ### Verification diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index cd227fc..c5a884a 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -8,7 +8,7 @@ parser checkpoint_grouping { value(".cpfinal" "1" ) inherit-mode(context) ) - timeout(1) + timeout(2) ); }; diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 06714f6..7baf2e8 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -98,6 +98,35 @@ def test_checkpoint_splunk_firewall_noise(record_property, setup_wordlist, setup assert resultCount == 1 +def test_checkpoint_splunk_firewall_noise2(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions for Checkpoint + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") + message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sleep(5) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"") + search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 2 def test_checkpoint_splunk_firewall2(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) From 7585981eb42372846b3990b3f41f6ca9b3a323aa Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 08:07:15 -0400 Subject: [PATCH 23/71] Fixes #273 make indexed fields optional The fields sc4s_syslog_product and sc4s_syslog_format are not made optional in this change as the current use is intergral to the logic tree a future change could change the internal field name and conditional include the field in the json payload. This change also adds new optional indexed fields for dest port when and container host. --- docs/configuration.md | 2 + docs/gettingstarted/docker-systemd-general.md | 1 + docs/gettingstarted/podman-systemd-general.md | 1 + .../conf.d/conflib/_splunk/splunkfields.conf | 23 ----------- .../conflib/_splunk/splunkfields.conf.tmpl | 41 +++++++++++++++++++ 5 files changed, 45 insertions(+), 23 deletions(-) delete mode 100644 package/etc/conf.d/conflib/_splunk/splunkfields.conf create mode 100644 package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl diff --git a/docs/configuration.md b/docs/configuration.md index 38bca4b..a30ec11 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,6 +10,7 @@ and variables needed to properly configure SC4S for your environment. | SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list | | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | | SC4S_GLOBAL_DNS_USE | yes or no(default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | +| SC4S_CONTAINER_HOST | string | variable passed to the container to identify the actual log host for container implementations | * NOTE: Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http destination does not support this feature. Moreover, HEC Ack would significantly degrade performance for streaming data such as @@ -26,6 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | +| SC4S_DEST_SPLUNK_INDEX_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is (container,log_host,dport,fromhostip,proto) ) ## Alternate Destination Configuration diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index f7fde28..5614607 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -59,6 +59,7 @@ ExecStartPre=/usr/bin/docker run \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \ + -e "SC4S_CONTAINER_HOST=$(hostname -s)" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index b0c026d..2b2269a 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -77,6 +77,7 @@ ExecStartPre=/usr/bin/podman run \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ + -e "SC4S_CONTAINER_HOST=$(hostname -s)" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf deleted file mode 100644 index bf911a6..0000000 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf +++ /dev/null @@ -1,23 +0,0 @@ -#Used to set indexed fields we will always use to global defaults -rewrite r_set_splunk_default { - set("SC4S:$SOURCE", value(".splunk.source")); - set($FACILITY, value("fields.sc4s_syslog_facility")); - set($LEVEL, value("fields.sc4s_syslog_severity")); - set($LOGHOST, value("fields.sc4s_log_host")); - set($SOURCEIP, value("fields.sc4s_fromhostip")); -}; -#used by each log-path to set index and sourcetype which may be -#overridden by user defined values -block rewrite r_set_splunk_dest_default( - index() - source("${.splunk.source}") - sourcetype() - template(`splunk-template`) - ) { - set("`index`", value(".splunk.index")); - set("`source`", value(".splunk.source")); - set("`sourcetype`", value(".splunk.sourcetype")); -}; - - - diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl new file mode 100644 index 0000000..9f43664 --- /dev/null +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -0,0 +1,41 @@ +#Used to set indexed fields we will always use to global defaults +rewrite r_set_splunk_default { + set("SC4S:$SOURCE", value(".splunk.source")); +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "facility" }} + set($FACILITY, value("fields.sc4s_syslog_facility")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "severity" }} + set($LEVEL, value("fields.sc4s_syslog_severity")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "log_host" }} +{{- if (getenv "SC4S_CONTAINER_HOST") }} + set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "container" }} + set($LOGHOST, value("fields.sc4s_container")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} + set($SOURCEIP, value("fields.sc4s_fromhostip")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "destport" }} + set($DESTPORT, value("fields.sc4s_destport")); +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "proto" }} + set($PROTO, value("fields.sc4s_proto")); +{{- end}} +}; +#used by each log-path to set index and sourcetype which may be +#overridden by user defined values +block rewrite r_set_splunk_dest_default( + index() + source("${.splunk.source}") + sourcetype() + template(`splunk-template`) + ) { + set("`index`", value(".splunk.index")); + set("`source`", value(".splunk.source")); + set("`sourcetype`", value(".splunk.sourcetype")); +}; + + + From a17cd4f3faa09d018946f465c154aaa862d171f5 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 11:12:20 -0400 Subject: [PATCH 24/71] Enhance Bluecoat Support a 5424 format and remove empty fields --- .../log_paths/lp-symantec_proxy.conf.tmpl | 5 +++- tests/test_symantec_proxy.py | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl index 8d5b475..1447711 100644 --- a/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl @@ -15,7 +15,6 @@ log { channel { # Listen on the default port (typically 514) for SYMANTEC_PROXY traffic source (s_DEFAULT); - filter(f_is_rfc5424_noversion); filter(f_symantec_bluecoat_proxy); flags(final); }; @@ -24,6 +23,10 @@ log { rewrite { set("bluecoat_proxy", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy")) + subst( + "([-_a-zA-Z\(\)]+=(\"-\"|-| ))", + "", value(MESSAGE) + ); }; parser {p_add_context_splunk(key("bluecoat_proxy")); }; diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py index ad90bbb..52221ea 100644 --- a/tests/test_symantec_proxy.py +++ b/tests/test_symantec_proxy.py @@ -41,3 +41,31 @@ def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk, setup assert resultCount == 1 # +# <111>1 2020-04-15T17:22:32.521Z animal-visual bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) +def test_bluecoatproxySG_kv_5424(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + iso = dt.isoformat()[0:23] + epoch = epoch[:-3] + + mt = env.from_string( + "{{ mark }}1 {{ iso }}Z {{host}} bluecoat - splunk_format - c-ip=192.0.0.6 cs-bytes=6269 cs-categories=\"unavailable\" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent=\"ocspd/1.0.3\" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name=\"Explicit HTTP\" service.group=\"Standard\" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name=\"10.0.0.6-sample_logs\" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url=\"http://randomserver:8000/en-US/app/examples/\"") + message = mt.render(mark="<134>", iso=iso, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netproxy host=\"{{ host }}\" sourcetype=\"bluecoat:proxysg:access:kv\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# \ No newline at end of file From e873be4b9d55a7f15873a42aa6f5d7e09c21abf5 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 12:54:57 -0400 Subject: [PATCH 25/71] INDEXED not INDEX and fix missing end --- docs/configuration.md | 2 +- .../conf.d/conflib/_splunk/splunkfields.conf.tmpl | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a30ec11..eeffb1e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | -| SC4S_DEST_SPLUNK_INDEX_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is (container,log_host,dport,fromhostip,proto) ) +| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is (container,log_host,dport,fromhostip,proto) ) ## Alternate Destination Configuration diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 9f43664..1d838f9 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -1,26 +1,27 @@ #Used to set indexed fields we will always use to global defaults rewrite r_set_splunk_default { set("SC4S:$SOURCE", value(".splunk.source")); -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "facility" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "facility" }} set($FACILITY, value("fields.sc4s_syslog_facility")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "severity" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "severity" }} set($LEVEL, value("fields.sc4s_syslog_severity")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "log_host" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "log_host" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "container" }} +{{- end}} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "container" }} set($LOGHOST, value("fields.sc4s_container")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} set($SOURCEIP, value("fields.sc4s_fromhostip")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "destport" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "destport" }} set($DESTPORT, value("fields.sc4s_destport")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEX_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "proto" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "proto" }} set($PROTO, value("fields.sc4s_proto")); {{- end}} }; From 832da7a1d23d31aa451ae6aac83b2cbac7495bda Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 12:59:41 -0400 Subject: [PATCH 26/71] include all fields in default --- docs/configuration.md | 2 +- .../conf.d/conflib/_splunk/splunkfields.conf.tmpl | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index eeffb1e..7d77fd9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | -| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is (container,log_host,dport,fromhostip,proto) ) +| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is all fields ) ## Alternate Destination Configuration diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 1d838f9..62bfca7 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -1,27 +1,27 @@ #Used to set indexed fields we will always use to global defaults rewrite r_set_splunk_default { set("SC4S:$SOURCE", value(".splunk.source")); -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "facility" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "facility" }} set($FACILITY, value("fields.sc4s_syslog_facility")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "severity" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "severity" }} set($LEVEL, value("fields.sc4s_syslog_severity")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "log_host" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "log_host" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); {{- end}} {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "container" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "container" }} set($LOGHOST, value("fields.sc4s_container")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} set($SOURCEIP, value("fields.sc4s_fromhostip")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "destport" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,destport,fromhostip,proto") ",") has "destport" }} set($DESTPORT, value("fields.sc4s_destport")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container,log_host,destport,fromhostip,proto") ",") has "proto" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,destport,fromhostip,proto") ",") has "proto" }} set($PROTO, value("fields.sc4s_proto")); {{- end}} }; From 661a4e4258902c5efb37c0ca81a8fca6457fcc46 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 8 May 2020 10:15:10 -0700 Subject: [PATCH 27/71] Refine mulitple unique port docs * Refine mulitple unique ports docs; change section it appears in --- docs/configuration.md | 4 ---- docs/sources/index.md | 10 ++++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6f707fd..38bca4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,10 +113,6 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_LISTEN_UDP_SOCKETS | 5 | Number of kernel sockets per active UDP port, which configures multi-threading of the UDP input buffer in the kernel to prevent packet loss. Total UDP input buffer is the multiple of SC4S_SOURCE_LISTEN_UDP_SOCKETS * SC4S_SOURCE_UDP_SO_RCVBUFF | | SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | -### Multiple listening ports for the same source - -SC4S supports multiple network listening ports useful for a narrow set of compliance use cases. When configuring a source port variable use a comma ',' separated list with no spaces. - ## Syslog Source TLS Certificate Configuration * Create a folder ``/opt/sc4s/tls`` if not already done as part of the "getting started" process. diff --git a/docs/sources/index.md b/docs/sources/index.md index 8a416bf..c709a62 100644 --- a/docs/sources/index.md +++ b/docs/sources/index.md @@ -19,3 +19,13 @@ A key aspect of SC4S is to properly set Splunk metadata prior to the data arrivi It is understood that default values will need to be changed in many installations. Each source documented in this section has a table entitled "Sourcetype and Index Configuration", which highlights the default index and sourcetype for each source. See the section "SC4S metadata configuration" in the "Configuration" page for more information on how to override the default values in this table. +## Unique listening ports + +SC4S supports unique listening ports for each source technology/log path (e.g. Cisco ASA), which is useful when the device is +sending data on a port different from the typical default syslog port (UDP port 514). In some cases, when the source device emits data that +is not able to be distinguished from other device types, a unique port is sometimes required. The specific environment variables used for +setting "unique ports" are outlined in each source document in this section. + +In most cases only one "unique port" is needed for each source. However, SC4S also supports multiple network listening ports per source, +which can be useful for a narrow set of compliance use cases. When configuring a source port variable to enable multiple ports, use a +comma-separated list with no spaces (e.g. `SC4S_LISTEN_CISCO_ASA_UDP_PORT=5005,6005`). From 01fbe9fec602bf18cd1486267ddf4ee08fcd8eb8 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Fri, 8 May 2020 11:04:54 -0700 Subject: [PATCH 28/71] Update splunkfields.conf.tmpl --- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 62bfca7..f340bb3 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -9,7 +9,7 @@ rewrite r_set_splunk_default { {{- end}} {{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "log_host" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} - set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); + set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost")); {{- end}} {{- end}} {{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "container" }} From 0e9e17c30438d0cef5f69f772654643c4761acdf Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 14:05:53 -0400 Subject: [PATCH 29/71] change sc4s_log_host to sc4s_loghost --- docs/troubleshooting.md | 2 +- package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 03c4d48..fdf331c 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -75,7 +75,7 @@ don't expect, check to see that the index is created in Splunk, or that a `lastC cause for almost _all_ `400` errors. * If you continue to the individual log entries in these directories, you will see entries of the form ```bash -curl -k -u "sc4s HEC debug:a778f63a-5dff-4e3c-a72c-a03183659e94" "https://splunk.smg.aws:8088/services/collector/event" -d '{"time":"1584556114.271","sourcetype":"sc4s:events","source":"SC4S:s_internal","index":"main","host":"e3563b0ea5d8","fields":{"sc4s_syslog_severity":"notice","sc4s_syslog_facility":"syslog","sc4s_log_host":"e3563b0ea5d8","sc4s_fromhostip":"127.0.0.1"},"event":"syslog-ng starting up; version='3.26.1'"}' +curl -k -u "sc4s HEC debug:a778f63a-5dff-4e3c-a72c-a03183659e94" "https://splunk.smg.aws:8088/services/collector/event" -d '{"time":"1584556114.271","sourcetype":"sc4s:events","source":"SC4S:s_internal","index":"main","host":"e3563b0ea5d8","fields":{"sc4s_syslog_severity":"notice","sc4s_syslog_facility":"syslog","sc4s_loghost":"e3563b0ea5d8","sc4s_fromhostip":"127.0.0.1"},"event":"syslog-ng starting up; version='3.26.1'"}' ``` * These commands, with minimal modifications (e.g. multiple URLs specified or elements that needs shell escapes) can be run directly on the command line to determine what, exactly, the HEC endpoint is returning. This can be used to refine th index or other parameter to correct the diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 62bfca7..f340bb3 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -9,7 +9,7 @@ rewrite r_set_splunk_default { {{- end}} {{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "log_host" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} - set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_log_host")); + set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost")); {{- end}} {{- end}} {{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "container" }} From 0f6081415eea1e4cb53ec3a5bfb82d014ac76de4 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 14:11:34 -0400 Subject: [PATCH 30/71] loghost --- docs/configuration.md | 2 +- .../conf.d/conflib/_splunk/splunkfields.conf.tmpl | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7d77fd9..f8f3920 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | -| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,log_host,dport,fromhostip,proto | list of sc4s indexed fields default list is all fields ) +| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,loghost,dport,fromhostip,proto | list of sc4s indexed fields default list is all fields ) ## Alternate Destination Configuration diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index f340bb3..320acf8 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -1,27 +1,27 @@ #Used to set indexed fields we will always use to global defaults rewrite r_set_splunk_default { set("SC4S:$SOURCE", value(".splunk.source")); -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "facility" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "facility" }} set($FACILITY, value("fields.sc4s_syslog_facility")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "severity" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "severity" }} set($LEVEL, value("fields.sc4s_syslog_severity")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "log_host" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "loghost" }} {{- if (getenv "SC4S_CONTAINER_HOST") }} set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost")); {{- end}} {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "container" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "container" }} set($LOGHOST, value("fields.sc4s_container")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,dport,fromhostip,proto") ",") has "fromhostip" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "fromhostip" }} set($SOURCEIP, value("fields.sc4s_fromhostip")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,destport,fromhostip,proto") ",") has "destport" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,destport,fromhostip,proto") ",") has "destport" }} set($DESTPORT, value("fields.sc4s_destport")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,log_host,destport,fromhostip,proto") ",") has "proto" }} +{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,destport,fromhostip,proto") ",") has "proto" }} set($PROTO, value("fields.sc4s_proto")); {{- end}} }; From bd615a7524d9f31f801435329aa720349ee3767b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 14:25:25 -0400 Subject: [PATCH 31/71] update docs --- docs/sources/Symantec/index.md | 7 ++++++- tests/test_symantec_proxy.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md index f2e7ab5..f0285f1 100644 --- a/docs/sources/Symantec/index.md +++ b/docs/sources/Symantec/index.md @@ -31,7 +31,12 @@ MSG Parse: This filter parses message content * Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. * Refer to the Splunk TA documentation for the specific customer format required for proxy configuration * Select TCP or SSL transport option - * Ensure the format of the event is customized per Splunk documentation + * Ensure the format of the event is customized as follows + +``` +<111>1 $(date)T$(x-bluecoat-hour-utc):$(x-bluecoat-minute-utc):$(x-bluecoat-second-utc).000z $(x-bluecoat-appliance-name) bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) + +``` ### Options diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py index 52221ea..a65b55f 100644 --- a/tests/test_symantec_proxy.py +++ b/tests/test_symantec_proxy.py @@ -41,7 +41,8 @@ def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk, setup assert resultCount == 1 # -# <111>1 2020-04-15T17:22:32.521Z animal-visual bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) +# <111>1 $(date)T$(x-bluecoat-hour-utc):$(x-bluecoat-minute-utc):$(x-bluecoat-second-utc).000z $(x-bluecoat-appliance-name) bluecoat - splunk_format - c-ip=$(c-ip) Content-Type=$(quot)$(rs(Content-Type))$(quot) cs-auth-group=$(cs-auth-group) cs-bytes=$(cs-bytes) cs-categories=$(quot)$(cs-categories)$(quot) cs-host=$(cs-host) cs-ip=$(cs-ip) cs-method=$(cs-method) cs-uri-extension=$(cs-uri-extension) cs-uri-path=$(cs-uri-path) cs-uri-port=$(cs-uri-port) cs-uri-query=$(quot)$(cs-uri-query)$(quot) cs-uri-scheme=$(cs-uri-scheme) cs-User-Agent=$(quot)$(cs(User-Agent))$(quot) cs-username=$(cs-username) dnslookup-time=$(dnslookup-time) duration=$(duration) rs-status=$(rs-status) rs-version=$(rs-version) rs_Content_Type=$(rs-Content-Type) s-action=$(s-action) s-ip=$(s-ip) service.name=$(service.name) service.group=$(service.group) s-supplier-ip=$(s-supplier-ip) s-supplier-name=$(s-supplier-name) sc-bytes=$(sc-bytes) sc-filter-result=$(sc-filter-result) sc-filter-result=$(sc-filter-result) sc-status=$(sc-status) time-taken=$(time-taken) x-bluecoat-appliance-name=$(x-bluecoat-appliance-name) x-bluecoat-appliance-primary-address=$(x-bluecoat-appliance-primary-address) x-bluecoat-application-name=$(x-bluecoat-application-name) x-bluecoat-application-operation=$(x-bluecoat-application-operation) x-bluecoat-proxy-primary-address=$(x-bluecoat-proxy-primary-address) x-bluecoat-transaction-uuid=$(x-bluecoat-transaction-uuid) x-exception-id=$(x-exception-id) x-virus-id=$(x-virus-id) c-url=$(quot)$(url)$(quot) cs-Referer=$(quot)$(cs(Referer))$(quot) c-cpu=$(c-cpu) c-uri-pathquery=$(c-uri-pathquery) connect-time=$(connect-time) cs-auth-groups=$(cs-auth-groups) cs-headerlength=$(cs-headerlength) cs-threat-risk=$(cs-threat-risk) r-ip=$(r-ip) r-supplier-ip=$(r-supplier-ip) rs-time-taken=$(rs-time-taken) rs-server=$(rs(server)) s-connect-type=$(s-connect-type) s-icap-status=$(s-icap-status) s-sitename=$(s-sitename) s-source-port=$(s-source-port) s-supplier-country=$(s-supplier-country) sc-Content-Encoding=$(sc(Content-Encoding)) sr-Accept-Encoding=$(sr(Accept-Encoding)) x-auth-credential-type=$(x-auth-credential-type) x-cookie-date=$(x-cookie-date) x-cs-certificate-subject=$(x-cs-certificate-subject) x-cs-connection-negotiated-cipher=$(x-cs-connection-negotiated-cipher) x-cs-connection-negotiated-cipher-size=$(x-cs-connection-negotiated-cipher-size) x-cs-connection-negotiated-ssl-version=$(x-cs-connection-negotiated-ssl-version) x-cs-ocsp-error=$(x-cs-ocsp-error) x-cs-Referer-uri=$(x-cs(Referer)-uri) x-cs-Referer-uri-address=$(x-cs(Referer)-uri-address) x-cs-Referer-uri-extension=$(x-cs(Referer)-uri-extension) x-cs-Referer-uri-host=$(x-cs(Referer)-uri-host) x-cs-Referer-uri-hostname=$(x-cs(Referer)-uri-hostname) x-cs-Referer-uri-path=$(x-cs(Referer)-uri-path) x-cs-Referer-uri-pathquery=$(x-cs(Referer)-uri-pathquery) x-cs-Referer-uri-port=$(x-cs(Referer)-uri-port) x-cs-Referer-uri-query=$(x-cs(Referer)-uri-query) x-cs-Referer-uri-scheme=$(x-cs(Referer)-uri-scheme) x-cs-Referer-uri-stem=$(x-cs(Referer)-uri-stem) x-exception-category=$(x-exception-category) x-exception-category-review-message=$(x-exception-category-review-message) x-exception-company-name=$(x-exception-company-name) x-exception-contact=$(x-exception-contact) x-exception-details=$(x-exception-details) x-exception-header=$(x-exception-header) x-exception-help=$(x-exception-help) x-exception-last-error=$(x-exception-last-error) x-exception-reason=$(x-exception-reason) x-exception-sourcefile=$(x-exception-sourcefile) x-exception-sourceline=$(x-exception-sourceline) x-exception-summary=$(x-exception-summary) x-icap-error-code=$(x-icap-error-code) x-rs-certificate-hostname=$(x-rs-certificate-hostname) x-rs-certificate-hostname-category=$(x-rs-certificate-hostname-category) x-rs-certificate-observed-errors=$(x-rs-certificate-observed-errors) x-rs-certificate-subject=$(x-rs-certificate-subject) x-rs-certificate-validate-status=$(x-rs-certificate-validate-status) x-rs-connection-negotiated-cipher=$(x-rs-connection-negotiated-cipher) x-rs-connection-negotiated-cipher-size=$(x-rs-connection-negotiated-cipher-size) x-rs-connection-negotiated-ssl-version=$(x-rs-connection-negotiated-ssl-version) x-rs-ocsp-error=$(x-rs-ocsp-error) +# def test_bluecoatproxySG_kv_5424(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) From caaf5dc2d37d9cb362e1c727ad981e37f17cf08b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 8 May 2020 14:33:42 -0400 Subject: [PATCH 32/71] Update docker-compose-ci.yml --- tests/docker-compose-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/docker-compose-ci.yml b/tests/docker-compose-ci.yml index e45bda0..6221947 100644 --- a/tests/docker-compose-ci.yml +++ b/tests/docker-compose-ci.yml @@ -55,6 +55,8 @@ services: - SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT=5005 - SC4S_LISTEN_PFSENSE_UDP_PORT=5006 - SC4S_ARCHIVE_GLOBAL=no + - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + splunk: build: context: ../splunk From dd50bd88af6dbde767adfe91148956dc460af185 Mon Sep 17 00:00:00 2001 From: nkaleiya Date: Sat, 9 May 2020 12:15:43 +0530 Subject: [PATCH 33/71] Adding the support for Cisco WSA --- docs/sources/Cisco/index.md | 55 +++++++- package/etc/conf.d/filters/cisco/wsa.conf | 7 ++ .../conf.d/log_paths/lp-cisco_wsa.conf.tmpl | 72 +++++++++++ .../vendor_product_by_source.conf.example | 6 + .../vendor_product_by_source.csv.example | 4 +- tests/test_cisco_wsa.py | 118 ++++++++++++++++++ 6 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 package/etc/conf.d/filters/cisco/wsa.conf create mode 100644 package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl create mode 100644 tests/test_cisco_wsa.py diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index 9467fa2..9fccf8d 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -375,4 +375,57 @@ Use the following search to validate events are present index= sourcetype=cisco:ucm ``` -Verify timestamp, and host values match as expected \ No newline at end of file +Verify timestamp, and host values match as expected + +## Product - WSA + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/1747/ | +| Product Manual | https://www.cisco.com/c/en/us/td/docs/security/wsa/wsa11-7/user_guide/b_WSA_UserGuide_11_7.html | + +* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf `` update the host or ip mask for ``f_cisco_wsa`` to identiy the wsa events prior to WSA v11.7 and ``f_cisco_wsa11_7`` to identify the events since WSA v11.7. + + +### Sourcetypes + +| cisco:wsa:l4tm | The L4TM logs of Cisco IronPort WSA record sites added to the L4TM block and allow lists. | +| cisco:wsa:squid | The access logs of Cisco IronPort WSA version prior to 11.7 record Web Proxy client history in squid. | +| cisco:wsa:squid:new | The access logs of Cisco IronPort WSA version since 11.7 record Web Proxy client history in squid. | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_wsa_l4tm | cisco:wsa:l4tm | netops | None | +| cisco_wsa_squid | cisco:wsa:squid | netops | None | +| cisco_wsa_squid_new | cisco:wsa:squid:new | netops | None | + +### Filter type + +IP, Netmask or Host + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* WSA Follow vendor configuration steps per Product Manual. +* Ensure host and timestamp are included. + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_WSA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CISCO_WSA_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CISCO_WSA | no | Enable archive to disk for this specific source | +| SC4S_DEST_CISCO_WSA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +Use the following search to validate events are present + +``` +index=netops sourcetype=cisco:wsa:* +``` + +Verify timestamp, and host values match as expected \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco/wsa.conf b/package/etc/conf.d/filters/cisco/wsa.conf new file mode 100644 index 0000000..6f86606 --- /dev/null +++ b/package/etc/conf.d/filters/cisco/wsa.conf @@ -0,0 +1,7 @@ +filter f_cisco_wsa11_7{ + match("cisco_wsa11_7", value("fields.sc4s_vendor_product") type(glob)); +}; + +filter f_cisco_wsa{ + match('cisco_wsa*', value("fields.sc4s_vendor_product") type(glob)); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl new file mode 100644 index 0000000..7722dd4 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl @@ -0,0 +1,72 @@ +# Cisco WSA +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_WSA" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log{ + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_WSA_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_WSA_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_WSA_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for CISCO_WSA traffic + source (s_CISCO_WSA); + flags (final); + }; +{{- end}} + channel { + # Listen on the default port (typically 514) for CISCO_WSA traffic + source (s_DEFAULT); + filter(f_cisco_wsa); + flags(final); + }; + }; + + if{ + filter{ + message('(\d*|\s+|\w+|\(|\)|\+|\:|\-)*(Info|Warning|Alert|Debug):.*') + }; + rewrite { + set("cisco_wsa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:wsa:l4tm"), index("netops")) + }; + parser { p_add_context_splunk(key("cisco_wsa")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + + } elif { + filter(f_cisco_wsa11_7); + rewrite { + set("cisco_wsa11_7", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:wsa:squid:new"), index("netops"),source("wsa_11.7")) + }; + parser { p_add_context_splunk(key("cisco_wsa")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + + } else { + rewrite { + set("cisco_wsa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:wsa:squid"), index("netops")) + }; + parser { p_add_context_splunk(key("cisco_wsa")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + }; + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_WSA_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CISCO_WSA" "no")) }} + destination(d_archive); +{{- end}} + +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_WSA_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_WSA_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + + flags(flow-control,final); +}; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.conf.example b/package/etc/context_templates/vendor_product_by_source.conf.example index 1eb20df..da7d7aa 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf.example +++ b/package/etc/context_templates/vendor_product_by_source.conf.example @@ -80,3 +80,9 @@ filter f_tzfixny { host("tzfny-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) }; +filter f_cisco_wsa{ + host("cisco_wsa" type(glob)) +}; +filter f_cisco_wsa11_7{ + host("cisco_wsa11_7" type(glob)) +}; diff --git a/package/etc/context_templates/vendor_product_by_source.csv.example b/package/etc/context_templates/vendor_product_by_source.csv.example index 86ece07..8a69fea 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv.example +++ b/package/etc/context_templates/vendor_product_by_source.csv.example @@ -17,4 +17,6 @@ f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" f_schneider_apc,sc4s_vendor_product,"schneider_apc" f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" f_tzfixhst,sc4s_time_zone,"Pacific/Honolulu" -f_tzfixny,sc4s_time_zone,"America/New_York" \ No newline at end of file +f_tzfixny,sc4s_time_zone,"America/New_York" +f_cisco_wsa,sc4s_vendor_product,"cisco_wsa" +f_cisco_wsa11_7,sc4s_vendor_product,"cisco_wsa11_7" \ No newline at end of file diff --git a/tests/test_cisco_wsa.py b/tests/test_cisco_wsa.py new file mode 100644 index 0000000..f396ffd --- /dev/null +++ b/tests/test_cisco_wsa.py @@ -0,0 +1,118 @@ +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * +from .timeutils import * +import pytest +env = Environment() + + +testdata_squid_11_7 = [ +'{{ mark }}{{ bsd }} {{ host }} 1588851315.000 382 10.0.0.13 TCP_CLIENT_REFRESH_MISS_SSL/201 4646 GET http://test_web.com/page2/b.txt Conner_Fitzerald DEFAULT_PARENT/www.xxxxxxx14.com application/x-javascript OTHER_382-NONE-CyberRange_Inside_NoAuth-OMSPolicy-random_policy-random_policy-DIRECT "Anonymous_Suspect_Vendor" "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52" - -', +'{{ mark }}{{ bsd }} {{ host }} 1588850982.000 331 10.0.0.12 TCP_DENIED/403 3197 POST http://test_web.net/contents/content5.jpg Tom_Lawrence DIRECT/www.xxxxxxx7.com application/x-javascript DEFAULT_CASE_331-Auth-APJC_Cisco_Corporate-OMSPolicy-DefaultGroup-NONE-DefaultRouting "Anonymous_Suspect_Vendor" "Mozilla/5.0 (X11; U; Linux arm7tdmi; rv:1.8.1.11) Gecko/20071130 Minimo/0.025" - -', +'{{ mark }}{{ bsd }} {{ host }} 1588851529.000 252 10.0.0.2 NONE/504 3040 GET http://test_web.net/users/user5.jpg Tom_Lawrence DIRECT/www.xxxxxxx5.com application/pkix-crl PASSTHRU_ADMIN_252-Decrypt_VFS-WebxOnly-RFS_Transparent_Proxy_Test-random_policy-NONE-DefaultRouting "abcd" "Mozilla/5.0 (X11; U; Linux arm7tdmi; rv:1.8.1.11) Gecko/20071130 Minimo/0.025" - -' +] + +testdata_l4tm = [ +'{{ mark }}{{ bsd }} {{ host }} Mon May 04 12:59:59 2020 Info: Firewall noted TCP data from 10.0.0.15 to 61.79.37.205(www.xxxxxxx7.com):1283.', +'{{ mark }}{{ bsd }} {{ host }} 04 May 2020 12:59:57 (GMT-1:00) Info: Address 143.164.34.50 discovered for www.xxxxxxx4.com (www.xxxxxxx4.com) added to firewall greylist.', +'{{ mark }}{{ bsd }} {{ host }} Mon May 04 12:59:54 2020 Info: Begin Logfile', +'{{ mark }}{{ bsd }} {{ host }} Mon May 04 12:59:49 2020 Info: Version: 9.0.0-485 SN: 848F69E6010F-JYFZWQ1', +'{{ mark }}{{ bsd }} {{ host }} 04 May 2020 12:59:59 (GMT+5:00) Info: Firewall blocked TCP data from 10.0.0.3:1148 to 96.246.56.182.', +'{{ mark }}{{ bsd }} {{ host }} Mon May 04 12:59:58 2020 Info: Time offset from UTC: 113 seconds', +'{{ mark }}{{ bsd }} {{ host }} Mon May 04 12:59:59 2020 Info: Firewall noted TCP data from 10.0.0.15 to 61.79.37.205(www.xxxxxxx7.com):1283.' +] +testdata_squid = [ +'{{ mark }}{{ bsd }} {{ host }} 1588851279.000 184 10.0.0.6 TCP_CLIENT_REFRESH_MISS/404 461 POST http://test_web.net/users/user2.jpg - DEFAULT_PARENT/www.xxxxxxx15.com application/javascript DEFAULT_CASE_184-NONE-CyberRange_DC_NoAuth-RFS_Transparent_Proxy_Test-random_policy-DefaultGroup-RoutingPolicy "abcd" 486', +'{{ mark }}{{ bsd }} {{ host }} 1588851133.000 258 10.0.0.12 TCP_MISS/200 4687 GET http://test_web.net/users/user2.jpg Tom_Lawrence DIRECT/www.xxxxxxx15.com image/gif BLOCK_AMW_RESP_URL_258-Allow_All_iDevices-APJC_Cisco_Corporate-RFS_Transparent_Proxy_Test-NONE-random_policy-random_policy "random_name"', +'{{ mark }}{{ bsd }} {{ host }} 1588851234.000 17 10.0.0.5 TCP_CLIENT_REFRESH_MISS_SSL/200 1939 HEAD http://test_web.net/contents/content4.jpg - NONE/www.xxxxxxx15.com application/javascript ALLOW_WBRS_17-AccessPolicy-CyberRange_Inside_NoAuth-RFS_Transparent_Proxy_Test-DefaultGroup-random_policy-RoutingPolicy - 486', +'{{ mark }}{{ bsd }} {{ host }} 1588850961.000 245 2001:b8f9:c5c2:f730::2 TCP_DENIED/403 0 GET http://test_web.net/users/user1.jpg Alexei_Romanov NONE/www.xxxxxxx6.com application/x-javascript BLOCK_WEBCAT_245-Allow_All_iDevices-CyberRange_Inside_NoAuth-OMSPolicy-DataSecurityPolicy-DefaultGroup-DIRECT -', +'{{ mark }}{{ bsd }} {{ host }} 1588850940.000 26 2001:44c4:cf35:1b78::6 TCP_MISS/204 4525 POST http://test_web.com/page1/a.txt Andy_Lloyd DIRECT/www.xxxxxxx3.com image/jpeg DEFAULT_CASE_26-NONE-CyberRange_Inside_NoAuth-OMSPolicy-DataSecurityPolicy-ExternalDLPolicy-RoutingPolicy "Anonymous_Suspect_Vendor" 100', +'{{ mark }}{{ bsd }} {{ host }} 1588851442.000 6 10.0.0.7 TCP_CLIENT_REFRESH_MISS/404 1932 GET http://test_web.com/page2/b.txt - DEFAULT_PARENT/www.xxxxxxx8.com - DEFAULT_CASE_6-AP_Subnet_2-NONE-RFS_Transparent_Proxy_Test-NONE-ExternalDLPolicy-RoutingPolicy <nc,5.0,-,"-",-,-,-,-,"-",-,-,-,"-",-,-,"-","-",-,-,nc,-,"-","-","Unknown","Unknown","-","-",0.63,0,-,"-","-",-,"-",-,-,"-","-"> - "03/Jan/2015:07:09:50 +1100" NONE -', +'{{ mark }}{{ bsd }} {{ host }} 1588850950.000 262 10.0.0.7 TCP_MISS_SSL/204 953 POST http://test_web.net/contents/content3.jpg Alexei_Romanov NONE/www.xxxxxxx10.com application/x-javascript DEFAULT_CASE_262-Internet_Access_with_Streaming-ID.ACMETECHISE-NONE-DefaultGroup-random_policy-RoutingPolicy "Anonymous_Suspect_Vendor" 123 "07/052020:11:29:10 +1332" NONE "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/125.4 (KHTML, like Gecko, Safari) OmniWeb/v563.15"' + +] + +@pytest.mark.parametrize("event", testdata_squid_11_7) +def test_cisco_wsa_squid_11_7(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): + host = "cisco_wsa11_7" + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<13>", bsd=bsd ,host=host ) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops _time={{ epoch }} sourcetype=\"cisco:wsa:squid:new\" _raw=\"{{ message }}\"") + message1 = mt.render(mark="", bsd="", host="") + search = st.render(epoch=epoch ,host=host, message=message1.lstrip().replace('"','\\"')) + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +@pytest.mark.parametrize("event", testdata_squid) +def test_cisco_wsa_squid(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): + host = "cisco_wsa" + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<13>", bsd=bsd ,host=host ) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops _time={{ epoch }} sourcetype=\"cisco:wsa:squid\" _raw=\"{{ message }}\"") + message1 = mt.render(mark="", bsd="", host="") + search = st.render(epoch=epoch ,host=host, message=message1.lstrip().replace('"','\\"')) + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +@pytest.mark.parametrize("event", testdata_l4tm) +def test_cisco_wsa_l4tm(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): + host = "cisco_wsa" + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<13>", bsd=bsd, host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=netops _time={{ epoch }} sourcetype=\"cisco:wsa:l4tm\" _raw=\"{{ message }}\"") + + message1 = mt.render(mark="", bsd="", host="") + search = st.render(epoch=epoch, host=host, message=message1.lstrip()) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file From 80296a7f695ad736f0dc53fe86f8df4011f87b59 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 11 May 2020 11:18:18 -0400 Subject: [PATCH 34/71] Update snmptrapd.conf --- package/snmp/snmptrapd.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/package/snmp/snmptrapd.conf b/package/snmp/snmptrapd.conf index c14e9a6..64d88a5 100644 --- a/package/snmp/snmptrapd.conf +++ b/package/snmp/snmptrapd.conf @@ -1,3 +1,4 @@ authCommunity log,execute,net public +format1 %.4y-%.2m-%.2l %.2h:%.2j:%.2k %B [%b]:\n%v\n format2 %.4y-%.2m-%.2l %.2h:%.2j:%.2k %B [%b]:\n%v\n outputOption s \ No newline at end of file From 3375ab1744dc48d8252550ced169e23aeab9c899 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 11 May 2020 12:37:17 -0400 Subject: [PATCH 35/71] Correct Palo HIPMATCH events --- docs/sources/PaloaltoNetworks/index.md | 4 +-- .../etc/conf.d/filters/paloalto/panos.conf | 2 +- .../log_paths/lp-paloalto_panos.conf.tmpl | 6 ++-- .../splunk_index.csv.example | 2 +- tests/test_palo_alto.py | 33 +++++++++++++++++++ 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/docs/sources/PaloaltoNetworks/index.md b/docs/sources/PaloaltoNetworks/index.md index bc27602..2fae016 100644 --- a/docs/sources/PaloaltoNetworks/index.md +++ b/docs/sources/PaloaltoNetworks/index.md @@ -17,7 +17,7 @@ | pan:threat | None | | pan:system | None | | pan:config | None | -| pan:hipwatch | None | +| pan:hipmatch | None | | pan:correlation | None | ### Sourcetype and Index Configuration @@ -29,7 +29,7 @@ | pan_threat | pan:threat | netproxy | none | | pan_system | pan:system | netops | none | | pan_config | pan:config | netops | none | -| pan_hipwatch | pan:hipwatch | netops | none | +| hipmatch | pan:hipmatch | netops | none | | pan_correlation | pan:correlation | netops | none | ### Filter type diff --git a/package/etc/conf.d/filters/paloalto/panos.conf b/package/etc/conf.d/filters/paloalto/panos.conf index 9c5109b..efe8304 100644 --- a/package/etc/conf.d/filters/paloalto/panos.conf +++ b/package/etc/conf.d/filters/paloalto/panos.conf @@ -5,5 +5,5 @@ filter f_is_palalto_format{ }; filter f_paloalto_panos { - message(',[0-9A-F]+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),'); + message(',[0-9A-F]+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPMATCH|CORRELATION|USERID),'); }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl index d5d7774..f07df1c 100644 --- a/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl @@ -90,7 +90,7 @@ log { }; rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"))}; parser {p_add_context_splunk(key("pan_config")); }; - } elif (match('HIPWATCH', value('.pan.type'))) { + } elif (match('HIPMATCH', value('.pan.type'))) { parser { csv-parser( columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_user","vsys","host_name","os","src_ip","hip_name","hip_count","hip_type","future_use3","future_use4","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") @@ -98,8 +98,8 @@ log { delimiters(',') ); }; - rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"))}; - parser {p_add_context_splunk(key("pan_hipwatch")); }; + rewrite { r_set_splunk_dest_default(sourcetype("pan:hipmatch"), index("main"))}; + parser {p_add_context_splunk(key("pan_hipmatch")); }; } elif (match('CORRELATION', value('.pan.type'))) { parser { csv-parser( diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index b52f7ea..f447021 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -61,7 +61,7 @@ #pan_threat,index,netproxy #pan_system,index,netops #pan_config,index,netops -#pan_hipwatch,index,main +#pan_hipmatch,index,main #pan_correlation,index,main #pan_userid,index,netauth #pan_unknown,index,netops diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index 25dbecc..9633b32 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -167,3 +167,36 @@ def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk, record_property("message", message) assert resultCount == 1 + + +#<14>May 11 10:13:22 xxxxxx 1,2020/05/11 10:13:22,015451000001111,HIPMATCH,0,2049,2020/05/11 10:13:22,xx.xx,vsys1,xx-xxxxx-MB,Mac,10.252.31.187,GP-HIP,1,profile,0,0,1052623,0x0,17,11,12,0,,xxxxx,1,0.0.0.0, +def test_palo_alto_hipmatch(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + time = dt.strftime("%Y/%m/%d %H:%M:%S.%f")[:-3] + tzoffset = tzoffset[0:3] + ":" + tzoffset[3:] + epoch = epoch[:-3] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} 1,{{ time }},015451000001111,HIPMATCH,0,2049,{{ time }},xxxx.xxx,vsys1,xx-xxxxxx-MB,Mac,10.252.31.187,GP-HIP,1,profile,0,0,1052623,0x0,17,11,12,0,,{{ host }},1,0.0.0.0,\n") + message = mt.render(mark="<111>", bsd=bsd, host=host, time=time) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search _time={{ epoch }} index=main host=\"{{ host }}\" sourcetype=\"pan:hipmatch\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + From ee28b952cfc53a3ca9ba522d56462352d1dc9225 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Tue, 12 May 2020 22:19:18 -0700 Subject: [PATCH 36/71] Fix indexed field gomplate template * Fix indexed field gomplate template * Add "none" as an option for indexed fields * Update and expand docs --- docs/configuration.md | 2 +- docs/gettingstarted/docker-systemd-general.md | 2 +- docs/gettingstarted/podman-systemd-general.md | 2 +- .../conflib/_splunk/splunkfields.conf.tmpl | 38 +++++++++++-------- .../conf.d/destinations/splunk_hec.conf.tmpl | 5 ++- 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index f8f3920..60b9800 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | -| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,loghost,dport,fromhostip,proto | list of sc4s indexed fields default list is all fields ) +| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,loghost,destport,fromhostip,proto, or none | List of sc4s indexed fields (default is the entire list except "none"). If this veriable is not set, the default indexed fields `sc4s_vendor_product` and `sc4d_syslog_format` _will_ appear. If no indexed fields are desired (including the two defaults mentioned), set the value to the single value of "none". This list maps to the following indexed fields that will appear in all Splunk events:
facility: sc4s_syslog_facility
severity: sc4s_syslog_severity
container: sc4s_container
loghost: sc4s_loghost
dport: sc4s_destport
fromhostip: sc4s_fromhostip
proto: sc4s_proto ## Alternate Destination Configuration diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 5614607..f18cb08 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -59,7 +59,7 @@ ExecStartPre=/usr/bin/docker run \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \ - -e "SC4S_CONTAINER_HOST=$(hostname -s)" \ + -e "SC4S_CONTAINER_HOST=$(`hostname -s`)" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 2b2269a..afbcefd 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -77,7 +77,7 @@ ExecStartPre=/usr/bin/podman run \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ - -e "SC4S_CONTAINER_HOST=$(hostname -s)" \ + -e "SC4S_CONTAINER_HOST=$(`hostname -s`)" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index 320acf8..fea713b 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -1,28 +1,36 @@ #Used to set indexed fields we will always use to global defaults rewrite r_set_splunk_default { - set("SC4S:$SOURCE", value(".splunk.source")); -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "facility" }} - set($FACILITY, value("fields.sc4s_syslog_facility")); + set("SC4S:$SOURCE", value(".splunk.source")); +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "facility") }} + set($FACILITY, value("fields.sc4s_syslog_facility")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "severity" }} - set($LEVEL, value("fields.sc4s_syslog_severity")); +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "severity") }} + set($LEVEL, value("fields.sc4s_syslog_severity")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "loghost" }} +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "loghost") }} {{- if (getenv "SC4S_CONTAINER_HOST") }} - set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost")); + set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost")); {{- end}} {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "container" }} - set($LOGHOST, value("fields.sc4s_container")); +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "container") }} + set($LOGHOST, value("fields.sc4s_container")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") has "fromhostip" }} - set($SOURCEIP, value("fields.sc4s_fromhostip")); +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "fromhostip") }} + set($SOURCEIP, value("fields.sc4s_fromhostip")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,destport,fromhostip,proto") ",") has "destport" }} - set($DESTPORT, value("fields.sc4s_destport")); +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "destport") }} + set($DESTPORT, value("fields.sc4s_destport")); {{- end}} -{{- if (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,destport,fromhostip,proto") ",") has "proto" }} - set($PROTO, value("fields.sc4s_proto")); +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "proto") }} + channel { + if (match("6" value("PROTO"))) { + rewrite { set("TCP", value("fields.sc4s_proto")); }; + } elif (match("17" value("PROTO"))) { + rewrite { set("UDP", value("fields.sc4s_proto")); }; + } else { + rewrite { set($PROTO, value("fields.sc4s_proto")); }; + }; + }; {{- end}} }; #used by each log-path to set index and sourcetype which may be diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 0b2b349..d0a3341 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -47,6 +47,9 @@ destination d_hec { sourcetype=${.splunk.sourcetype} index=${.splunk.index} event="$MSG" - fields.*)') + {{- if ne (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS") "none" }} + fields.* + {{- end }} + )') ); }; From d29bd31a5e7d49c1fd47ffada6e21ab75d94ef65 Mon Sep 17 00:00:00 2001 From: nkaleiya Date: Wed, 13 May 2020 16:43:02 +0530 Subject: [PATCH 37/71] Updated timestamp extraction and Updated example conf as per alphabatical order --- .../conf.d/log_paths/lp-cisco_wsa.conf.tmpl | 26 +++++++++++++++++++ .../vendor_product_by_source.conf.example | 12 ++++----- .../vendor_product_by_source.csv.example | 6 ++--- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl index 7722dd4..e9c3704 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl @@ -34,6 +34,19 @@ log{ } elif { filter(f_cisco_wsa11_7); + parser { + filter { + program( + '(?:(?\d{10})(?:.(?\d{1,9}))?)' + flags(store-matches) + ); + }; + parser { + date-parser-nofilter(format('%s.%f') + template("${EPOCH}.${TIMESECFRAC}") + ); + }; + }; rewrite { set("cisco_wsa11_7", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("cisco:wsa:squid:new"), index("netops"),source("wsa_11.7")) @@ -43,6 +56,19 @@ log{ rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; } else { + parser { + filter { + program( + '(?:(?\d{10})(?:.(?\d{1,9}))?)' + flags(store-matches) + ); + }; + parser { + date-parser-nofilter(format('%s.%f') + template("${EPOCH}.${TIMESECFRAC}") + ); + }; + }; rewrite { set("cisco_wsa", value("fields.sc4s_vendor_product")); r_set_splunk_dest_default(sourcetype("cisco:wsa:squid"), index("netops")) diff --git a/package/etc/context_templates/vendor_product_by_source.conf.example b/package/etc/context_templates/vendor_product_by_source.conf.example index da7d7aa..8e27762 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf.example +++ b/package/etc/context_templates/vendor_product_by_source.conf.example @@ -39,6 +39,12 @@ filter f_cisco_meraki { host("testcm-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) }; +filter f_cisco_wsa{ + host("cisco_wsa" type(glob)) +}; +filter f_cisco_wsa11_7{ + host("cisco_wsa11_7" type(glob)) +}; filter f_cisco_nx_os { host("csconx-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) @@ -80,9 +86,3 @@ filter f_tzfixny { host("tzfny-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) }; -filter f_cisco_wsa{ - host("cisco_wsa" type(glob)) -}; -filter f_cisco_wsa11_7{ - host("cisco_wsa11_7" type(glob)) -}; diff --git a/package/etc/context_templates/vendor_product_by_source.csv.example b/package/etc/context_templates/vendor_product_by_source.csv.example index 8a69fea..d1e29bd 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv.example +++ b/package/etc/context_templates/vendor_product_by_source.csv.example @@ -2,6 +2,8 @@ f_test_test,sc4s_vendor_product,"test_test" f_brocade_syslog,sc4s_vendor_product,"brocade_syslog" f_null_queue,sc4s_vendor_product,"null_queue" f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" +f_cisco_wsa,sc4s_vendor_product,"cisco_wsa" +f_cisco_wsa11_7,sc4s_vendor_product,"cisco_wsa11_7" f_citrix_netscaler,sc4s_vendor_product,"citrix_netscaler" f_dell_rsa_secureid,sc4s_vendor_product,"dell_rsa_secureid" f_f5_bigip,sc4s_vendor_product,"f5_bigip" @@ -17,6 +19,4 @@ f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" f_schneider_apc,sc4s_vendor_product,"schneider_apc" f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" f_tzfixhst,sc4s_time_zone,"Pacific/Honolulu" -f_tzfixny,sc4s_time_zone,"America/New_York" -f_cisco_wsa,sc4s_vendor_product,"cisco_wsa" -f_cisco_wsa11_7,sc4s_vendor_product,"cisco_wsa11_7" \ No newline at end of file +f_tzfixny,sc4s_time_zone,"America/New_York" \ No newline at end of file From 50a1dfb5dcc6beb1e3fe91c6f4d3ce68015e8c9f Mon Sep 17 00:00:00 2001 From: nkaleiya Date: Wed, 13 May 2020 19:40:20 +0530 Subject: [PATCH 38/71] Updated test cases and log path file --- package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl | 4 ++++ tests/test_cisco_wsa.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl index e9c3704..9403f7d 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_wsa.conf.tmpl @@ -35,6 +35,7 @@ log{ } elif { filter(f_cisco_wsa11_7); parser { + channel { filter { program( '(?:(?\d{10})(?:.(?\d{1,9}))?)' @@ -46,6 +47,7 @@ log{ template("${EPOCH}.${TIMESECFRAC}") ); }; + }; }; rewrite { set("cisco_wsa11_7", value("fields.sc4s_vendor_product")); @@ -57,6 +59,7 @@ log{ } else { parser { + channel { filter { program( '(?:(?\d{10})(?:.(?\d{1,9}))?)' @@ -68,6 +71,7 @@ log{ template("${EPOCH}.${TIMESECFRAC}") ); }; + }; }; rewrite { set("cisco_wsa", value("fields.sc4s_vendor_product")); diff --git a/tests/test_cisco_wsa.py b/tests/test_cisco_wsa.py index f396ffd..7443348 100644 --- a/tests/test_cisco_wsa.py +++ b/tests/test_cisco_wsa.py @@ -51,7 +51,7 @@ def test_cisco_wsa_squid_11_7(record_property, setup_wordlist, get_host_key, set sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops _time={{ epoch }} sourcetype=\"cisco:wsa:squid:new\" _raw=\"{{ message }}\"") + "search index=netops sourcetype=\"cisco:wsa:squid:new\" _raw=\"{{ message }}\"") message1 = mt.render(mark="", bsd="", host="") search = st.render(epoch=epoch ,host=host, message=message1.lstrip().replace('"','\\"')) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -78,7 +78,7 @@ def test_cisco_wsa_squid(record_property, setup_wordlist, get_host_key, setup_sp sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops _time={{ epoch }} sourcetype=\"cisco:wsa:squid\" _raw=\"{{ message }}\"") + "search index=netops sourcetype=\"cisco:wsa:squid\" _raw=\"{{ message }}\"") message1 = mt.render(mark="", bsd="", host="") search = st.render(epoch=epoch ,host=host, message=message1.lstrip().replace('"','\\"')) resultCount, eventCount = splunk_single(setup_splunk, search) From eb9bf427b7e546e4438f3a2e5a4f8ed381bd4f91 Mon Sep 17 00:00:00 2001 From: nkaleiya Date: Wed, 13 May 2020 19:47:04 +0530 Subject: [PATCH 39/71] Updated test cases --- tests/test_cisco_wsa.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cisco_wsa.py b/tests/test_cisco_wsa.py index 7443348..a69273f 100644 --- a/tests/test_cisco_wsa.py +++ b/tests/test_cisco_wsa.py @@ -53,7 +53,7 @@ def test_cisco_wsa_squid_11_7(record_property, setup_wordlist, get_host_key, set st = env.from_string( "search index=netops sourcetype=\"cisco:wsa:squid:new\" _raw=\"{{ message }}\"") message1 = mt.render(mark="", bsd="", host="") - search = st.render(epoch=epoch ,host=host, message=message1.lstrip().replace('"','\\"')) + search = st.render(host=host, message=message1.lstrip().replace('"','\\"')) resultCount, eventCount = splunk_single(setup_splunk, search) record_property("host", host) @@ -80,7 +80,7 @@ def test_cisco_wsa_squid(record_property, setup_wordlist, get_host_key, setup_sp st = env.from_string( "search index=netops sourcetype=\"cisco:wsa:squid\" _raw=\"{{ message }}\"") message1 = mt.render(mark="", bsd="", host="") - search = st.render(epoch=epoch ,host=host, message=message1.lstrip().replace('"','\\"')) + search = st.render(host=host, message=message1.lstrip().replace('"','\\"')) resultCount, eventCount = splunk_single(setup_splunk, search) record_property("host", host) From f07ff23f93d1108d48ff0f1eee94a44631d8f4ee Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Wed, 13 May 2020 14:54:35 -0400 Subject: [PATCH 40/71] Update source_network.t --- package/etc/go_templates/source_network.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 59b9cf5..a8e1887 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -28,7 +28,7 @@ source s_{{ .port_id }} { source { {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} {{- $port_id := .port_id }} -{{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514") "," }} +{{- range split (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514") "," }} {{- $context := dict "port" . "port_id" $port_id }} {{- template "UDP" $context }} {{- end}} From bb92e3b63e6ff6bf5346de80f5194ca74cec4fdf Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Wed, 13 May 2020 15:47:15 -0700 Subject: [PATCH 41/71] Reformat indexed fields list for proper rendering * Reformat indexed fields env var list so that single line renders properly as multiple lines --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 60b9800..79fbb94 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | -| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,severity,container,loghost,destport,fromhostip,proto, or none | List of sc4s indexed fields (default is the entire list except "none"). If this veriable is not set, the default indexed fields `sc4s_vendor_product` and `sc4d_syslog_format` _will_ appear. If no indexed fields are desired (including the two defaults mentioned), set the value to the single value of "none". This list maps to the following indexed fields that will appear in all Splunk events:
facility: sc4s_syslog_facility
severity: sc4s_syslog_severity
container: sc4s_container
loghost: sc4s_loghost
dport: sc4s_destport
fromhostip: sc4s_fromhostip
proto: sc4s_proto +| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,
severity,
container,
loghost,
destport,
fromhostip,
proto

none | List of sc4s indexed fields that will be included with each event in Splunk (default is the entire list except "none"). Two other indexed fields, `sc4s_vendor_product` and `sc4s_syslog_format`, will also appear along with the fields selected via the list and cannot be turned on or off individually. If no indexed fields are desired (including the two internal ones), set the value to the single value of "none". When setting this variable, separate multiple entries with commas and do not include extra spaces.

This list maps to the following indexed fields that will appear in all Splunk events:
facility: sc4s_syslog_facility
severity: sc4s_syslog_severity
container: sc4s_container
loghost: sc4s_loghost
dport: sc4s_destport
fromhostip: sc4s_fromhostip
proto: sc4s_proto ## Alternate Destination Configuration From 8b79d9a68696921231aaee19e675f77e27dbd6fd Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Wed, 13 May 2020 21:02:23 -0700 Subject: [PATCH 42/71] Update splunkfields gomplate template logic * Update `splunkfields.conf.tmpl` gomplate logic to fix typo in desthost field --- docs/gettingstarted/docker-systemd-general.md | 3 ++- docs/gettingstarted/podman-systemd-general.md | 3 ++- .../conf.d/conflib/_splunk/splunkfields.conf.tmpl | 14 +++++++------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index f18cb08..f23926c 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -53,13 +53,14 @@ TimeoutStartSec=0 Restart=always ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE +ExecStartPre=/usr/bin/bash -c "/usr/bin/systemctl set-environment SC4SHOST=$(hostname -s)" ExecStartPre=/usr/bin/docker run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \ - -e "SC4S_CONTAINER_HOST=$(`hostname -s`)" \ + -e "SC4S_CONTAINER_HOST=${SC4SHOST}" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index afbcefd..f033343 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -71,13 +71,14 @@ TimeoutStartSec=0 Restart=always ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE +ExecStartPre=/usr/bin/bash -c "/usr/bin/systemctl set-environment SC4SHOST=$(hostname -s)" ExecStartPre=/usr/bin/podman run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ - -e "SC4S_CONTAINER_HOST=$(`hostname -s`)" \ + -e "SC4S_CONTAINER_HOST=${SC4SHOST}" \ --env-file=/opt/sc4s/env_file \ "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl index fea713b..989e4bf 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf.tmpl @@ -1,27 +1,27 @@ #Used to set indexed fields we will always use to global defaults rewrite r_set_splunk_default { set("SC4S:$SOURCE", value(".splunk.source")); -{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "facility") }} +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility") ",") "facility") }} set($FACILITY, value("fields.sc4s_syslog_facility")); {{- end}} -{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "severity") }} +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "severity") ",") "severity") }} set($LEVEL, value("fields.sc4s_syslog_severity")); {{- end}} -{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "loghost") }} +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "loghost") ",") "loghost") }} {{- if (getenv "SC4S_CONTAINER_HOST") }} set("{{ getenv "SC4S_CONTAINER_HOST" }}", value("fields.sc4s_loghost")); {{- end}} {{- end}} -{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "container") }} +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "container") ",") "container") }} set($LOGHOST, value("fields.sc4s_container")); {{- end}} -{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "fromhostip") }} +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "fromhostip") ",") "fromhostip") }} set($SOURCEIP, value("fields.sc4s_fromhostip")); {{- end}} -{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "destport") }} +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "destport") ",") "destport") }} set($DESTPORT, value("fields.sc4s_destport")); {{- end}} -{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "facility,severity,container,loghost,dport,fromhostip,proto") ",") "proto") }} +{{- if (has (split (getenv "SC4S_DEST_SPLUNK_INDEXED_FIELDS" "proto") ",") "proto") }} channel { if (match("6" value("PROTO"))) { rewrite { set("TCP", value("fields.sc4s_proto")); }; From 5b23e7a03e71ce4fed7422784e77c7a398dcfcf4 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 14 May 2020 07:31:52 -0400 Subject: [PATCH 43/71] Update test_checkpoint.py --- tests/test_checkpoint.py | 171 ++++++++++++++++++++++++++++----------- 1 file changed, 123 insertions(+), 48 deletions(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 7baf2e8..22fc362 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -14,8 +14,10 @@ env = Environment() # -#Oct 8 15:00:25 DEVICENAME time=1570561225|hostname=devicename|severity=Informational|confidence_level=Unknown|product=IPS|action=Drop|ifdir=inbound|ifname=bond2|loguid={0x5d9cdcc9,0x8d159f,0x5f19f392,0x1897a828}|origin=1.1.1.1|time=1570561225|version=1|attack=Streaming Engine: TCP Segment Limit Enforcement|attack_info=TCP segment out of maximum allowed sequence. Packet dropped.|chassis_bladed_system=[ 1_3 ]|dst=10.10.10.10|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|performance_impact=0|protection_id=tcp_segment_limit|protection_name=TCP Segment Limit Enforcement|protection_type=settings_tcp|proto=6|rule=393|rule_name=10.384_..|rule_uid={9F77F944-8DD5-4ADF-803A-785D03B3A2E8}|s_port=46455|service=443|smartdefense_profile=Recommended_Protection_ded9e8d8ee89d|src=1.1.1.2| -def test_checkpoint_splunk_ips(record_property, setup_wordlist, setup_splunk, setup_sc4s): +# Oct 8 15:00:25 DEVICENAME time=1570561225|hostname=devicename|severity=Informational|confidence_level=Unknown|product=IPS|action=Drop|ifdir=inbound|ifname=bond2|loguid={0x5d9cdcc9,0x8d159f,0x5f19f392,0x1897a828}|origin=1.1.1.1|time=1570561225|version=1|attack=Streaming Engine: TCP Segment Limit Enforcement|attack_info=TCP segment out of maximum allowed sequence. Packet dropped.|chassis_bladed_system=[ 1_3 ]|dst=10.10.10.10|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|performance_impact=0|protection_id=tcp_segment_limit|protection_name=TCP Segment Limit Enforcement|protection_type=settings_tcp|proto=6|rule=393|rule_name=10.384_..|rule_uid={9F77F944-8DD5-4ADF-803A-785D03B3A2E8}|s_port=46455|service=443|smartdefense_profile=Recommended_Protection_ded9e8d8ee89d|src=1.1.1.2| +def test_checkpoint_splunk_ips( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -25,13 +27,18 @@ def test_checkpoint_splunk_ips(record_property, setup_wordlist, setup_splunk, se epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|severity=Informational|confidence_level=Unknown|product=IPS|action=Drop|ifdir=inbound|ifname=bond2|loguid={0x5d9cdcc9,0x8d159f,0x5f19f392,0x1897a828}|origin=1.1.1.1|time={{ epoch }}|version=1|attack=Streaming Engine: TCP Segment Limit Enforcement|attack_info=TCP segment out of maximum allowed sequence. Packet dropped.|chassis_bladed_system=[ 1_3 ]|dst=10.10.10.10|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|performance_impact=0|protection_id=tcp_segment_limit|protection_name=TCP Segment Limit Enforcement|protection_type=settings_tcp|proto=6|rule=393|rule_name=10.384_..|rule_uid={9F77F944-8DD5-4ADF-803A-785D03B3A2E8}|s_port=46455|service=443|smartdefense_profile=Recommended_Protection_ded9e8d8ee89d|src=1.1.1.2|\n") + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|severity=Informational|confidence_level=Unknown|product=IPS|action=Drop|ifdir=inbound|ifname=bond2|loguid={0x5d9cdcc9,0x8d159f,0x5f19f392,0x1897a828}|origin=1.1.1.1|time={{ epoch }}|version=1|attack=Streaming Engine: TCP Segment Limit Enforcement|attack_info=TCP segment out of maximum allowed sequence. Packet dropped.|chassis_bladed_system=[ 1_3 ]|dst=10.10.10.10|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|performance_impact=0|protection_id=tcp_segment_limit|protection_name=TCP Segment Limit Enforcement|protection_type=settings_tcp|proto=6|rule=393|rule_name=10.384_..|rule_uid={9F77F944-8DD5-4ADF-803A-785D03B3A2E8}|s_port=46455|service=443|smartdefense_profile=Recommended_Protection_ded9e8d8ee89d|src=1.1.1.2|\n" + ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netids host=\"{{ host }}\" sourcetype=\"cp_log\"") - search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + st = env.from_string( + 'search _time={{ epoch }} index=netids host="{{ host }}" sourcetype="cp_log"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -41,8 +48,11 @@ def test_checkpoint_splunk_ips(record_property, setup_wordlist, setup_splunk, se assert resultCount == 1 -#$Oct 8 15:48:31 DEVICENAME time=1570564111|hostname=devicename|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time=1570564111|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11_..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2| -def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splunk, setup_sc4s): + +# $Oct 8 15:48:31 DEVICENAME time=1570564111|hostname=devicename|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time=1570564111|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11_..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2| +def test_checkpoint_splunk_firewall( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -52,13 +62,18 @@ def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splun epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n" + ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"") - search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + st = env.from_string( + 'search _time={{ epoch }} index=netfw host="{{ host }}" sourcetype="cp_log"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -69,7 +84,9 @@ def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splun assert resultCount == 1 -def test_checkpoint_splunk_firewall_noise(record_property, setup_wordlist, setup_splunk, setup_sc4s): +def test_checkpoint_splunk_firewall_noise( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -79,7 +96,8 @@ def test_checkpoint_splunk_firewall_noise(record_property, setup_wordlist, setup epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n" + ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) @@ -87,8 +105,12 @@ def test_checkpoint_splunk_firewall_noise(record_property, setup_wordlist, setup sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"") - search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + st = env.from_string( + 'search _time={{ epoch }} index=netfw host="{{ host }}" sourcetype="cp_log"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -98,7 +120,10 @@ def test_checkpoint_splunk_firewall_noise(record_property, setup_wordlist, setup assert resultCount == 1 -def test_checkpoint_splunk_firewall_noise2(record_property, setup_wordlist, setup_splunk, setup_sc4s): + +def test_checkpoint_splunk_firewall_noise2( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -108,17 +133,21 @@ def test_checkpoint_splunk_firewall_noise2(record_property, setup_wordlist, setu epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n" + ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - sleep(5) - sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sleep(10) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"") - search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + st = env.from_string( + 'search _time={{ epoch }} index=netfw host="{{ host }}" sourcetype="cp_log"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -128,7 +157,10 @@ def test_checkpoint_splunk_firewall_noise2(record_property, setup_wordlist, setu assert resultCount == 2 -def test_checkpoint_splunk_firewall2(record_property, setup_wordlist, setup_splunk, setup_sc4s): + +def test_checkpoint_splunk_firewall2( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -138,13 +170,18 @@ def test_checkpoint_splunk_firewall2(record_property, setup_wordlist, setup_splu epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|severity=Medium|product=Firewall|action=Drop|ifdir=inbound|ifname=eth1|loguid={0x0,0x0,0x0,0x0}|origin=111.89.111.53|originsicname=CN\=xx-xx.xx.net,O\=cma-xx.xx.net.xx|sequencenum=64|time={{epoch}}|version=5|dst=10.11.11.11|inspection_category=anomaly|foo=bar: bat mark||\n") + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|severity=Medium|product=Firewall|action=Drop|ifdir=inbound|ifname=eth1|loguid={0x0,0x0,0x0,0x0}|origin=111.89.111.53|originsicname=CN\=xx-xx.xx.net,O\=cma-xx.xx.net.xx|sequencenum=64|time={{epoch}}|version=5|dst=10.11.11.11|inspection_category=anomaly|foo=bar: bat mark||\n" + ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\"") - search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + st = env.from_string( + 'search _time={{ epoch }} index=netfw host="{{ host }}" sourcetype="cp_log"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -154,8 +191,11 @@ def test_checkpoint_splunk_firewall2(record_property, setup_wordlist, setup_splu assert resultCount == 1 -#Oct 9 12:01:16 DEVICENAME |hostname=DEVICENAME|product=mds-query-tool|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time=1570641309|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login| -def test_checkpoint_splunk_mds(record_property, setup_wordlist, setup_splunk, setup_sc4s): + +# Oct 9 12:01:16 DEVICENAME |hostname=DEVICENAME|product=mds-query-tool|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time=1570641309|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login| +def test_checkpoint_splunk_mds( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -165,13 +205,18 @@ def test_checkpoint_splunk_mds(record_property, setup_wordlist, setup_splunk, se epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} |hostname={{ host }}|product=mds-query-tool|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time={{ epoch }}|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login|\n") + "{{ mark }} {{ bsd }} {{ host }} |hostname={{ host }}|product=mds-query-tool|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time={{ epoch }}|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login|\n" + ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"cp_log\"") - search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + st = env.from_string( + 'search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="cp_log"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -181,8 +226,11 @@ def test_checkpoint_splunk_mds(record_property, setup_wordlist, setup_splunk, se assert resultCount == 1 -#Oct 9 12:01:16 DEVICENAME|hostname=DEVICENAME|product=CPMI Client|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time=1570641173|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login -def test_checkpoint_splunk_cpmi(record_property, setup_wordlist, setup_splunk, setup_sc4s): + +# Oct 9 12:01:16 DEVICENAME|hostname=DEVICENAME|product=CPMI Client|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time=1570641173|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login +def test_checkpoint_splunk_cpmi( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -192,13 +240,18 @@ def test_checkpoint_splunk_cpmi(record_property, setup_wordlist, setup_splunk, s epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} |hostname={{ host }}|product=CPMI Client|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time={{ epoch }}|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login\n") + "{{ mark }} {{ bsd }} {{ host }} |hostname={{ host }}|product=CPMI Client|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time={{ epoch }}|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login\n" + ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"cp_log\"") - search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + st = env.from_string( + 'search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="cp_log"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -208,8 +261,11 @@ def test_checkpoint_splunk_cpmi(record_property, setup_wordlist, setup_splunk, s assert resultCount == 1 -#Oct 9 12:01:16 DEVICENAME |hostname=DEVICENAME|product=WEB_API|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time=1570640578|version=5|administrator=tufinapi|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login -def test_checkpoint_splunk_web_api(record_property, setup_wordlist, setup_splunk, setup_sc4s): + +# Oct 9 12:01:16 DEVICENAME |hostname=DEVICENAME|product=WEB_API|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time=1570640578|version=5|administrator=tufinapi|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login +def test_checkpoint_splunk_web_api( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -219,13 +275,18 @@ def test_checkpoint_splunk_web_api(record_property, setup_wordlist, setup_splunk epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} |hostname={{ host }}|product=WEB_API|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time={{ epoch }}|version=5|administrator=tufinapi|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login\n") + "{{ mark }} {{ bsd }} {{ host }} |hostname={{ host }}|product=WEB_API|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time={{ epoch }}|version=5|administrator=tufinapi|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login\n" + ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"cp_log\"") - search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + st = env.from_string( + 'search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="cp_log"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -235,8 +296,11 @@ def test_checkpoint_splunk_web_api(record_property, setup_wordlist, setup_splunk assert resultCount == 1 -#Oct 9 11:05:15 DEVICENAME time=1570633513|hostname=DEVICENAME|product=SmartConsole|action=Accept|ifdir=outbound|origin=1.1.1.1|4.4.4.4|sequencenum=1|time=1570633513|version=5|additional_info=Authentication method: Password based application token|administrator=psanadhya|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log In|operation_number=10|subject=Administrator Login| -def test_checkpoint_splunk_smartconsole(record_property, setup_wordlist, setup_splunk, setup_sc4s): + +# Oct 9 11:05:15 DEVICENAME time=1570633513|hostname=DEVICENAME|product=SmartConsole|action=Accept|ifdir=outbound|origin=1.1.1.1|4.4.4.4|sequencenum=1|time=1570633513|version=5|additional_info=Authentication method: Password based application token|administrator=psanadhya|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log In|operation_number=10|subject=Administrator Login| +def test_checkpoint_splunk_smartconsole( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -246,13 +310,18 @@ def test_checkpoint_splunk_smartconsole(record_property, setup_wordlist, setup_s epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=SmartConsole|action=Accept|ifdir=outbound|origin=1.1.1.1|4.4.4.4|sequencenum=1|time={{ epoch }}|version=5|additional_info=Authentication method: Password based application token|administrator=psanadhya|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log In|operation_number=10|subject=Administrator Login|\n") + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=SmartConsole|action=Accept|ifdir=outbound|origin=1.1.1.1|4.4.4.4|sequencenum=1|time={{ epoch }}|version=5|additional_info=Authentication method: Password based application token|administrator=psanadhya|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log In|operation_number=10|subject=Administrator Login|\n" + ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"cp_log\"") - search = st.render(epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + st = env.from_string( + 'search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="cp_log"' + ) + search = st.render( + epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -262,18 +331,24 @@ def test_checkpoint_splunk_smartconsole(record_property, setup_wordlist, setup_s assert resultCount == 1 -#<6>kernel: sd 2:0:0:0: SCSI error: return code = 0x00040000 -def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk, setup_sc4s): + +# <6>kernel: sd 2:0:0:0: SCSI error: return code = 0x00040000 +def test_checkpoint_splunk_os( + record_property, setup_wordlist, setup_splunk, setup_sc4s +): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string( - "{{ mark }}kernel: sd 2:0:0:0: SCSI error: return code = 0x{{pid}}\n") + "{{ mark }}kernel: sd 2:0:0:0: SCSI error: return code = 0x{{pid}}\n" + ) message = mt.render(mark="<6>", pid=pid) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search earliest=-1m@m latest=+1m@m index=osnix \"0x{{ pid }}\" sourcetype=\"nix:syslog\"") + st = env.from_string( + 'search earliest=-1m@m latest=+1m@m index=osnix "0x{{ pid }}" sourcetype="nix:syslog"' + ) search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) From 8cabdbe3d259cf3426d6ad33492fd1b5b9701ee2 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 14 May 2020 07:48:11 -0400 Subject: [PATCH 44/71] Update test_checkpoint.py --- tests/test_checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 22fc362..762c53b 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -137,10 +137,10 @@ def test_checkpoint_splunk_firewall_noise2( ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) - sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) sleep(10) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( 'search _time={{ epoch }} index=netfw host="{{ host }}" sourcetype="cp_log"' From 4b81261dc11d484c879cca3f75ea295297966eb2 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 14 May 2020 07:57:06 -0400 Subject: [PATCH 45/71] Update test_checkpoint.py --- tests/test_checkpoint.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 762c53b..998748d 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -27,7 +27,7 @@ def test_checkpoint_splunk_ips( epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|severity=Informational|confidence_level=Unknown|product=IPS|action=Drop|ifdir=inbound|ifname=bond2|loguid={0x5d9cdcc9,0x8d159f,0x5f19f392,0x1897a828}|origin=1.1.1.1|time={{ epoch }}|version=1|attack=Streaming Engine: TCP Segment Limit Enforcement|attack_info=TCP segment out of maximum allowed sequence. Packet dropped.|chassis_bladed_system=[ 1_3 ]|dst=10.10.10.10|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|performance_impact=0|protection_id=tcp_segment_limit|protection_name=TCP Segment Limit Enforcement|protection_type=settings_tcp|proto=6|rule=393|rule_name=10.384_..|rule_uid={9F77F944-8DD5-4ADF-803A-785D03B3A2E8}|s_port=46455|service=443|smartdefense_profile=Recommended_Protection_ded9e8d8ee89d|src=1.1.1.2|\n" + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|severity=Informational|confidence_level=Unknown|product=IPS|action=Drop|ifdir=inbound|ifname=bond2|loguid={{ host }}{0x5d9cdcc9,0x8d159f,0x5f19f392,0x1897a828}|origin=1.1.1.1|time={{ epoch }}|version=1|attack=Streaming Engine: TCP Segment Limit Enforcement|attack_info=TCP segment out of maximum allowed sequence. Packet dropped.|chassis_bladed_system=[ 1_3 ]|dst=10.10.10.10|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|performance_impact=0|protection_id=tcp_segment_limit|protection_name=TCP Segment Limit Enforcement|protection_type=settings_tcp|proto=6|rule=393|rule_name=10.384_..|rule_uid={9F77F944-8DD5-4ADF-803A-785D03B3A2E8}|s_port=46455|service=443|smartdefense_profile=Recommended_Protection_ded9e8d8ee89d|src=1.1.1.2|\n" ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) @@ -62,7 +62,7 @@ def test_checkpoint_splunk_firewall( epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n" + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={{ host }}{0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n" ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) @@ -133,7 +133,7 @@ def test_checkpoint_splunk_firewall_noise2( epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n" + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={{ host }}{0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n" ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) @@ -170,7 +170,7 @@ def test_checkpoint_splunk_firewall2( epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|severity=Medium|product=Firewall|action=Drop|ifdir=inbound|ifname=eth1|loguid={0x0,0x0,0x0,0x0}|origin=111.89.111.53|originsicname=CN\=xx-xx.xx.net,O\=cma-xx.xx.net.xx|sequencenum=64|time={{epoch}}|version=5|dst=10.11.11.11|inspection_category=anomaly|foo=bar: bat mark||\n" + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|severity=Medium|product=Firewall|action=Drop|ifdir=inbound|ifname=eth1|loguid={{ host }}{0x0,0x0,0x0,0x0}|origin=111.89.111.53|originsicname=CN\=xx-xx.xx.net,O\=cma-xx.xx.net.xx|sequencenum=64|time={{epoch}}|version=5|dst=10.11.11.11|inspection_category=anomaly|foo=bar: bat mark||\n" ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) @@ -373,7 +373,7 @@ def test_checkpoint_splunk_os_nested( epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Syslog|ifdir=inbound|loguid={0x0,0x0,0x0,0x0}|origin=10.0.0.0|sequencenum=3|time={{ epoch }}|version=5|default_device_message=<134>ctasd[5665]: Save SenderId lists finished |facility=local use 0|\n" + "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Syslog|ifdir=inbound|loguid={{ host }}{0x0,0x0,0x0,0x0}|origin=10.0.0.0|sequencenum=3|time={{ epoch }}|version=5|default_device_message=<134>ctasd[5665]: Save SenderId lists finished |facility=local use 0|\n" ) message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) From c205f3979c2734dee9ab67c59010ecaf3c4650ca Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 14 May 2020 12:27:25 -0400 Subject: [PATCH 46/71] Add soup for nxos --- .../etc/conf.d/conflib/_common/templates.conf | 4 ++ package/etc/conf.d/filters/cisco/nx-os.conf | 38 ++++++++++++++++++- .../conf.d/log_paths/lp-cisco_nxos.conf.tmpl | 2 +- .../conf.d/log_paths/lp-cisco_z_ios.conf.tmpl | 8 +++- tests/test_cisco_nx_os.py | 26 +++++++++++++ 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index ce18953..dc5f58f 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -42,6 +42,10 @@ template t_hdr_msg { template("${MSGHDR}${MESSAGE}"); }; +template t_program_nopid_msg { + template("${PROGRAM}: ${MESSAGE}"); + }; + template t_legacy_hdr_msg { template("${LEGACY_MSGHDR}${MESSAGE}"); }; diff --git a/package/etc/conf.d/filters/cisco/nx-os.conf b/package/etc/conf.d/filters/cisco/nx-os.conf index 2f9cacc..33bfef7 100644 --- a/package/etc/conf.d/filters/cisco/nx-os.conf +++ b/package/etc/conf.d/filters/cisco/nx-os.conf @@ -1,3 +1,39 @@ filter f_cisco_nx_os { - match("cisco_nx_os", value("fields.sc4s_vendor_product") type(glob) ); + match("cisco_nx_os", value("fields.sc4s_vendor_product") type(glob) ) + or program('^%ACLMGR-\d*-') + or program('^%ACLQOS-SLOT\d*-\d*-') + or program('^%AFM-\d*-') + or program('^%ARP-\d*-') + or program('^%AUTHPRIV-\d*-') + or program('^%COPP-\d*-') + or program('^%DAEMON-\d*-') + or program('^%EEM_ACTION-\d*-') + or program('^%EIGRP-\d*-') + or program('^%ETHPORT-\d*-') + or program('^%ETH_PORT_CHANNEL-\d*-') + or program('^%IGMP-\d*-') + or program('^%IM-\d*-') + or program('^%INTERFACE_VLAN-\d*-') + or program('^%KERN-\d*-') + or program('^%L2FM-\d*-') + or program('^%LIBIFMGR-\d*-') + or program('^%LICMGR-\d*-') + or program('^%LOCAL\d*-\d*-') + or program('^%LOG_-\d*-') + or program('^%LOG_LOCAL\d*-\d*-') + or program('^%M2FIB-SLOT\d*-\d*-') + or program('^%METROPOLIS_USD-SLOT') + or program('^%MODULE-\d*-') + or program('^%NOHMS-\d*-') + or program('^%NTP-\d*-') + or program('^%PFMA-\d*-') + or program('^%PLATFORM-\d*-') + or program('^%SYSMGR-STANDBY-\d*-') + or program('^%SYSMGR-\d*-') + or program('^%TACACS-\d*-') + or program('^%USBHSD-\d*-') + or program('^%USER-\d*-') + or program('^%VIM-\d*-') + or program('^%VPC-\d*-') + or program('^%VSHD-\d*-') ; }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl index ed387d2..b490903 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl @@ -28,7 +28,7 @@ log { parser { p_add_context_splunk(key("cisco_nx_os")); }; parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_nopid_msg))" value("MSG")); }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_NXOS_HEC" "no")) }} destination(d_hec); diff --git a/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl index b4c6eea..93a2c54 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl @@ -27,7 +27,13 @@ log { }; parser { p_add_context_splunk(key("cisco_ios")); }; parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + if { + filter{ program('^%')}; + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_nopid_msg))" value("MSG")); }; + } else { + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + }; + {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_IOS_HEC" "no")) }} destination(d_hec); diff --git a/tests/test_cisco_nx_os.py b/tests/test_cisco_nx_os.py index fe7cfa0..3f386f5 100644 --- a/tests/test_cisco_nx_os.py +++ b/tests/test_cisco_nx_os.py @@ -39,6 +39,32 @@ def test_cisco_nx_os(record_property, setup_wordlist, get_host_key, setup_splunk assert resultCount == 1 +def test_cisco_nx_os_soup(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} %MODULE-5-MOD_OK: Module 1 is online") + message = mt.render(mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"cisco:ios\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + # Nov 1 14:07:58 excal-113 %MODULE-5-MOD_OK: Module 1 is online # @pytest.mark.xfail #def test_cisco_nx_os_singleport(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): From 2842b88046e81c3c308619b71da09c244bfc9267 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 14 May 2020 18:15:49 -0400 Subject: [PATCH 47/71] Remove extra test --- tests/test_checkpoint.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 998748d..873c050 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -121,43 +121,6 @@ def test_checkpoint_splunk_firewall_noise( assert resultCount == 1 -def test_checkpoint_splunk_firewall_noise2( - record_property, setup_wordlist, setup_splunk, setup_sc4s -): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - - dt = datetime.datetime.now() - iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) - - # Tune time functions for Checkpoint - epoch = epoch[:-7] - - mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} time={{ epoch }}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={{ host }}{0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={{ epoch }}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11:..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n" - ) - message = mt.render(mark="<111>", host=host, bsd=bsd, epoch=epoch) - - sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - sleep(10) - sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - - st = env.from_string( - 'search _time={{ epoch }} index=netfw host="{{ host }}" sourcetype="cp_log"' - ) - search = st.render( - epoch=epoch, bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset - ) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 2 - - def test_checkpoint_splunk_firewall2( record_property, setup_wordlist, setup_splunk, setup_sc4s ): From fdf38f04197d19ad33895d010fe5e3617b275d36 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 14 May 2020 18:21:17 -0400 Subject: [PATCH 48/71] Correct missed reference to syslog-ng version --- .circleci/config.yml | 2 +- package/etc/syslog-ng.conf.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6ad2412..c93f78c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -195,7 +195,7 @@ jobs: docker: - image: circleci/python:3.7 environment: - SYSLOG: "syslog-ng-3.26.1" + SYSLOG: "syslog-ng-3.27.1" steps: - setup_remote_docker: docker_layer_caching: true diff --git a/package/etc/syslog-ng.conf.tmpl b/package/etc/syslog-ng.conf.tmpl index 98db6a2..1a06e5f 100644 --- a/package/etc/syslog-ng.conf.tmpl +++ b/package/etc/syslog-ng.conf.tmpl @@ -1,4 +1,4 @@ -@version:3.26 +@version:3.27 # syslog-ng configuration file. From 545587907937ef84c3120b6cbbd95ec9be2941d4 Mon Sep 17 00:00:00 2001 From: Nandini Vij Date: Thu, 14 May 2020 15:54:03 -0700 Subject: [PATCH 49/71] added @nandinivij to reviewers --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ef960bb..f9e23aa 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ * @rfaircloth-splunk -package/etc/ @rfaircloth @mbonsack \ No newline at end of file +package/etc/ @rfaircloth @mbonsack @nandinivij From 8eaea05bc30a3b537f4bdee836f9c7c2a7266e7e Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 15 May 2020 15:50:00 -0700 Subject: [PATCH 50/71] Fix regex for new statistic in syslog-ng 3.27 * Update `sc4s_internal.conf.tmpl` to update regex to handle new connection statistic in syslog-ng version 3.27 --- package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl index 1ce291f..d2a9c0c 100644 --- a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl @@ -9,7 +9,7 @@ log { rewrite { subst('Log statistics; ', '', value("MESSAGE"), flags("utf8" "global")); subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global")); - subst('(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27,? ?', + subst('(?[^= ]+)=\x27(?[^\(]+)\((?\S+(?=\)=))(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27,? ?', '{"time": "$S_UNIXTIME","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}} ', value("MESSAGE") flags("utf8" "global") From a84f57d8fa8a29b160d487a4a5244511ee4876a7 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 16 May 2020 09:47:23 -0400 Subject: [PATCH 51/71] Fixed test case for corrected new metric name and for vs-code execution with -det args --- package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl | 4 ++-- tests/test_common.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl index d2a9c0c..0bfba04 100644 --- a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl @@ -1,13 +1,13 @@ log { source(s_internal); - if (match("^Log statistics; " value("MESSAGE"))) { + if (match("Log statistics; " value("MESSAGE"))) { rewrite { r_set_splunk_dest_default(sourcetype("sc4s:metrics"), index("em_metrics")) }; parser {p_add_context_splunk(key("sc4s_metrics")); }; rewrite { - subst('Log statistics; ', '', value("MESSAGE"), flags("utf8" "global")); + subst('.*Log statistics; ', '', value("MESSAGE"), flags("utf8" "global")); subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global")); subst('(?[^= ]+)=\x27(?[^\(]+)\((?\S+(?=\)=))(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27,? ?', '{"time": "$S_UNIXTIME","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}} diff --git a/tests/test_common.py b/tests/test_common.py index d826ade..d2b54f2 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -109,14 +109,14 @@ def test_fallback(record_property, setup_wordlist, setup_splunk, setup_sc4s): def test_metrics(record_property, setup_wordlist, setup_splunk, setup_sc4s): st = env.from_string( - 'mcatalog values(metric_name) WHERE metric_name="syslogng.d_*#0" AND ("index"="*" OR "index"="_*") BY index | fields index') + 'mcatalog values(metric_name) WHERE metric_name="syslogng.*" AND ("index"="*" OR "index"="_*") BY metric_name | fields metric_name') search = st.render() resultCount, eventCount = splunk_single(setup_splunk, search) record_property("resultCount", resultCount) - assert resultCount == 1 + assert resultCount != 0 def test_tz_guess(record_property, setup_wordlist, setup_splunk, setup_sc4s): From 0f7c2d09472727e61fb8513c9fe0d995ccb142db Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 16 May 2020 12:38:20 -0400 Subject: [PATCH 52/71] Cisco IOS XR Improvements --- .../conf.d/filters/cisco/cisco_syslog.conf | 42 +++++++++---------- tests/test_cisco_ios.py | 39 +++++++++++------ 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 3938476..82cd4db 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -11,38 +11,38 @@ filter f_is_cisco_syslog{ parser cisco-parser-ex{ channel { filter { - message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); - }; - - parser { date-parser-nofilter(format( - '%b %d %H:%M:%S.%f', - '%b %d %H:%M:%S', - '%b %d %I:%M:%S %p.%f', - '%b %d %I:%M:%S %p', - '%b %d %Y %H:%M:%S.%f', - '%b %d %Y %H:%M:%S') - template("$8")); + message( + '^<\d*> ?(?:\d+\: )?(?RP\/\d*\/RSP\d*\/CPU\d*:)?(?:(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])): ?)?(?:\d+: )?(?:(?:\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?(?(?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?(?: [AP]M)?)(?: [A-Z]{3,3})?)? ?(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: (?(?:(?[^\[]{1,30})\[(?\d*)\]: ?)?(?\%[^\: ]+)\:? ?.*)' + flags(store-matches) + ); }; rewrite { set( - "${4}", + "${H1}", value("HOST") - condition(not match('^\d+$', value('4')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('4'))) + condition(not match('^\d+$', value('H1')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H1'))) ); set( - "${13}", + "${H2}", value("HOST") - condition(not match('^\d+$', value('13')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) + condition(not match('^\d+$', value('H2')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H2'))) ); set( - "${15}", - value("PROGRAM") - ); - set( - "${14}", + "${CISCOMESSAGE}", value("MESSAGE") - ); + ); + }; + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("${CISCOTS}")); + }; + }; }; diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index 108bec2..ab24128 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -11,6 +11,7 @@ from .timeutils import * import pytest + env = Environment() @@ -27,7 +28,7 @@ # foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # 101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # *Mar 1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) - +# <189>357492: RP/0/RSP0/CPU0:May 14 16:44:40.145 : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576 testdata = [ "{{ mark }}{{ seq }}: {{ host }}: 6340004: *{{ bsd }}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet", "{{ mark }}{{ seq }}: {{ host }}: *{{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated {{ bsd }}.{{ millisec }}", @@ -37,21 +38,24 @@ "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ millisec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ host }}: {{ bsd }}.{{ millisec }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure. {{ bsd }}.{{ millisec }}", - "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}" + "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}", + "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from {{ host }} has reached 792340, max 1048576", + "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ host }}:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576", ] - testdata_uptime = [ "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}{{ seq }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ seq }}: {{ host }}: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.", - "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}" + "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}", ] @pytest.mark.parametrize("event", testdata) -def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): +def test_cisco_ios( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): host = get_host_key dt = datetime.datetime.now() @@ -64,15 +68,23 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, - millisec=millisec, microsec=microsec, tzname=tzname, host=host) + message = mt.render( + mark="<166>", + seq=20, + bsd=bsd, + time=time, + millisec=millisec, + microsec=microsec, + tzname=tzname, + host=host, + ) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") - search = st.render(epoch=epoch, millisec=millisec, - microsec=microsec, host=host) + 'search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + ) + search = st.render(epoch=epoch, millisec=millisec, microsec=microsec, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -84,7 +96,9 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, @pytest.mark.parametrize("event", testdata_uptime) -def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): +def test_cisco_ios_uptime( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): host = get_host_key mt = env.from_string(event + "\n") @@ -93,7 +107,8 @@ def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_s sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops earliest=-1m@m latest=+1m@m sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") + 'search index=netops earliest=-1m@m latest=+1m@m sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + ) search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) From 070a15007e9be58736de95c3cdc105199f61512b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 16 May 2020 14:55:18 -0400 Subject: [PATCH 53/71] Cisco IOS XR Enhancement --- .../conf.d/filters/cisco/cisco_syslog.conf | 44 +++++++++---------- tests/test_cisco_ios.py | 41 +++++++++++------ 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 3938476..6766d1a 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -11,38 +11,38 @@ filter f_is_cisco_syslog{ parser cisco-parser-ex{ channel { filter { - message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); - }; - - parser { date-parser-nofilter(format( - '%b %d %H:%M:%S.%f', - '%b %d %H:%M:%S', - '%b %d %I:%M:%S %p.%f', - '%b %d %I:%M:%S %p', - '%b %d %Y %H:%M:%S.%f', - '%b %d %Y %H:%M:%S') - template("$8")); + message( + '^<\d*> ?(?:\d+\: )?(?RP\/\d*\/RSP\d*\/CPU\d*:)?(?:(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])): ?)?(?:\d+: )?(?:(?:\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?(?(?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?(?: [AP]M)?)(?: [A-Z]{3,3})?)? ?(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: (?(?:(?[^\[]{1,30})\[(?\d*)\]: ?)?(?\%[^\: ]+)\:? ?.*)' + flags(store-matches) + ); }; rewrite { set( - "${4}", + "${H1}", value("HOST") - condition(not match('^\d+$', value('4')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('4'))) + condition(not match('^\d+$', value('H1')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H1'))) ); set( - "${13}", + "${H2}", value("HOST") - condition(not match('^\d+$', value('13')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) + condition(not match('^\d+$', value('H2')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H2'))) ); set( - "${15}", - value("PROGRAM") - ); - set( - "${14}", + "${CISCOMESSAGE}", value("MESSAGE") - ); + ); + }; + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("${CISCOTS}")); + }; + }; -}; +}; \ No newline at end of file diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index 108bec2..30ea68d 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -11,6 +11,7 @@ from .timeutils import * import pytest + env = Environment() @@ -27,7 +28,7 @@ # foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # 101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # *Mar 1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) - +# <189>357492: RP/0/RSP0/CPU0:May 14 16:44:40.145 : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576 testdata = [ "{{ mark }}{{ seq }}: {{ host }}: 6340004: *{{ bsd }}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet", "{{ mark }}{{ seq }}: {{ host }}: *{{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated {{ bsd }}.{{ millisec }}", @@ -37,21 +38,24 @@ "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ millisec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ host }}: {{ bsd }}.{{ millisec }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure. {{ bsd }}.{{ millisec }}", - "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}" + "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}", + "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from {{ host }} has reached 792340, max 1048576", + "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ host }}:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576", ] - testdata_uptime = [ "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}{{ seq }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ seq }}: {{ host }}: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.", - "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}" + "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}", ] @pytest.mark.parametrize("event", testdata) -def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): +def test_cisco_ios( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): host = get_host_key dt = datetime.datetime.now() @@ -64,15 +68,23 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, - millisec=millisec, microsec=microsec, tzname=tzname, host=host) + message = mt.render( + mark="<166>", + seq=20, + bsd=bsd, + time=time, + millisec=millisec, + microsec=microsec, + tzname=tzname, + host=host, + ) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") - search = st.render(epoch=epoch, millisec=millisec, - microsec=microsec, host=host) + 'search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + ) + search = st.render(epoch=epoch, millisec=millisec, microsec=microsec, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -84,7 +96,9 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, @pytest.mark.parametrize("event", testdata_uptime) -def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): +def test_cisco_ios_uptime( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): host = get_host_key mt = env.from_string(event + "\n") @@ -93,7 +107,8 @@ def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_s sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - "search index=netops earliest=-1m@m latest=+1m@m sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") + 'search index=netops earliest=-1m@m latest=+1m@m sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' + ) search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -102,4 +117,4 @@ def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_s record_property("resultCount", resultCount) record_property("message", message) - assert resultCount == 1 + assert resultCount == 1 \ No newline at end of file From 44397431b1e2be8e28ee455bd76f929cdc055f41 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 08:46:15 -0400 Subject: [PATCH 54/71] Update for cisco IOS-XR --- .../conf.d/filters/cisco/cisco_syslog.conf | 93 ++++++++++++------- .../conf.d/log_paths/lp-cisco_z_ios.conf.tmpl | 2 +- 2 files changed, 62 insertions(+), 33 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 6766d1a..e9acf58 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -10,39 +10,68 @@ filter f_is_cisco_syslog{ parser cisco-parser-ex{ channel { - filter { - message( - '^<\d*> ?(?:\d+\: )?(?RP\/\d*\/RSP\d*\/CPU\d*:)?(?:(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])): ?)?(?:\d+: )?(?:(?:\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?(?(?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?(?: [AP]M)?)(?: [A-Z]{3,3})?)? ?(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: (?(?:(?[^\[]{1,30})\[(?\d*)\]: ?)?(?\%[^\: ]+)\:? ?.*)' - flags(store-matches) - ); - }; + if { + #Cisco IOS-XR devices with node-id format + filter { + message('^<\d*>(?:(\d+)\: )?(RP\/\d*\/RSP\d*\/CPU\d*:)?(?:([^\: ]+):)(?:(\*)?(\w\w\w {1,2}\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}\.\d+))? : ?([^\[]{1,30}\[\d*\]: ?\%[^\: ]+\:? ?.*)' flags(store-matches)); + }; - rewrite { - set( - "${H1}", - value("HOST") - condition(not match('^\d+$', value('H1')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H1'))) - ); - set( - "${H2}", - value("HOST") - condition(not match('^\d+$', value('H2')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H2'))) - ); - set( - "${CISCOMESSAGE}", - value("MESSAGE") - ); - - }; - parser { date-parser-nofilter(format( - '%b %d %H:%M:%S.%f', - '%b %d %H:%M:%S', - '%b %d %I:%M:%S %p.%f', - '%b %d %I:%M:%S %p', - '%b %d %Y %H:%M:%S.%f', - '%b %d %Y %H:%M:%S') - template("${CISCOTS}")); - }; + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("$5")); + }; + rewrite { + set( + "${3}", + value("HOST") + condition(not match('^\d+$', value('3')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('3'))) + ); + set( + "${6}", + value("MESSAGE") + ); + }; + } else { + # All other cisco syslog + filter { + message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); + }; + + rewrite { + set( + "${4}", + value("HOST") + condition(not match('^\d+$', value('4')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('4'))) + ); + set( + "${13}", + value("HOST") + condition(not match('^\d+$', value('13')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) + ); + set( + "${15}", + value("PROGRAM") + ); + set( + "${14}", + value("MESSAGE") + ); + }; + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("$8")); + }; + }; }; }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl index b4c6eea..f3f06c1 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_z_ios.conf.tmpl @@ -46,4 +46,4 @@ log { {{- end }} flags(flow-control,final); -}; +}; \ No newline at end of file From 547f313a2c6bcb0c586ed260de4fc250e42786c2 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 09:37:20 -0400 Subject: [PATCH 55/71] Additional type of ACL log from APIC --- package/etc/conf.d/filters/cisco/apic.conf | 4 ++-- tests/test_cisco_apic.py | 27 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/apic.conf b/package/etc/conf.d/filters/cisco/apic.conf index ea6660d..22478a6 100644 --- a/package/etc/conf.d/filters/cisco/apic.conf +++ b/package/etc/conf.d/filters/cisco/apic.conf @@ -1,6 +1,6 @@ filter f_cisco_apic { program('^%LOG_LOCAL\d-\d-'); - or - program('^%ACLLOG-\d-ACLLOG_PKTLOG'); + or program('^%LOG_-\d-'); + or program('^%ACLLOG-\d-ACLLOG_PKTLOG'); }; \ No newline at end of file diff --git a/tests/test_cisco_apic.py b/tests/test_cisco_apic.py index 5f59fea..0de42d7 100644 --- a/tests/test_cisco_apic.py +++ b/tests/test_cisco_apic.py @@ -14,7 +14,7 @@ env = Environment() #<11>July 22 22:45:28 apic1 %LOG_LOCAL0-2-SYSTEM_MSG [F0110][soaking][node-failed][critical][topology/pod-1/node-102/fault-F0110] Node 102 not reachable. unknown -def test_cisco_aci(record_property, setup_wordlist, setup_splunk, setup_sc4s): +def test_cisco_aci_loglocal(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) dt = datetime.datetime.now() @@ -39,6 +39,31 @@ def test_cisco_aci(record_property, setup_wordlist, setup_splunk, setup_sc4s): assert resultCount == 1 +def test_cisco_aci_log(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions for Cisco APIC + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} %LOG_-2-SYSTEM_MSG [F0110][soaking][node-failed][critical][topology/pod-1/node-102/fault-F0110]\n") + message = mt.render(mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"cisco:apic:events\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + #%ACLLOG-5-ACLLOG_PKTLOG def test_cisco_aci_acl(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) From e205558668f9b80b234e0bbafcfc9c648c5d0a1e Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 09:44:04 -0400 Subject: [PATCH 56/71] Update nx-os.conf --- package/etc/conf.d/filters/cisco/nx-os.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/nx-os.conf b/package/etc/conf.d/filters/cisco/nx-os.conf index 33bfef7..80b94c1 100644 --- a/package/etc/conf.d/filters/cisco/nx-os.conf +++ b/package/etc/conf.d/filters/cisco/nx-os.conf @@ -19,8 +19,6 @@ filter f_cisco_nx_os { or program('^%LIBIFMGR-\d*-') or program('^%LICMGR-\d*-') or program('^%LOCAL\d*-\d*-') - or program('^%LOG_-\d*-') - or program('^%LOG_LOCAL\d*-\d*-') or program('^%M2FIB-SLOT\d*-\d*-') or program('^%METROPOLIS_USD-SLOT') or program('^%MODULE-\d*-') From 563e9e70adf1d5ffea23c7b88f5a2fb390f3cd04 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 09:53:25 -0400 Subject: [PATCH 57/71] Revert "Cisco IOS XR Improvements" This reverts commit 0f7c2d09472727e61fb8513c9fe0d995ccb142db. --- .../conf.d/filters/cisco/cisco_syslog.conf | 42 +++++++++---------- tests/test_cisco_ios.py | 39 ++++++----------- 2 files changed, 33 insertions(+), 48 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 82cd4db..3938476 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -11,38 +11,38 @@ filter f_is_cisco_syslog{ parser cisco-parser-ex{ channel { filter { - message( - '^<\d*> ?(?:\d+\: )?(?RP\/\d*\/RSP\d*\/CPU\d*:)?(?:(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])): ?)?(?:\d+: )?(?:(?:\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?(?(?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?(?: [AP]M)?)(?: [A-Z]{3,3})?)? ?(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: (?(?:(?[^\[]{1,30})\[(?\d*)\]: ?)?(?\%[^\: ]+)\:? ?.*)' - flags(store-matches) - ); + message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); + }; + + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("$8")); }; rewrite { set( - "${H1}", + "${4}", value("HOST") - condition(not match('^\d+$', value('H1')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H1'))) + condition(not match('^\d+$', value('4')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('4'))) ); set( - "${H2}", + "${13}", value("HOST") - condition(not match('^\d+$', value('H2')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H2'))) + condition(not match('^\d+$', value('13')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) ); set( - "${CISCOMESSAGE}", + "${15}", + value("PROGRAM") + ); + set( + "${14}", value("MESSAGE") - ); - + ); }; - parser { date-parser-nofilter(format( - '%b %d %H:%M:%S.%f', - '%b %d %H:%M:%S', - '%b %d %I:%M:%S %p.%f', - '%b %d %I:%M:%S %p', - '%b %d %Y %H:%M:%S.%f', - '%b %d %Y %H:%M:%S') - template("${CISCOTS}")); - }; - }; }; diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index ab24128..108bec2 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -11,7 +11,6 @@ from .timeutils import * import pytest - env = Environment() @@ -28,7 +27,7 @@ # foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # 101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # *Mar 1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) -# <189>357492: RP/0/RSP0/CPU0:May 14 16:44:40.145 : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576 + testdata = [ "{{ mark }}{{ seq }}: {{ host }}: 6340004: *{{ bsd }}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet", "{{ mark }}{{ seq }}: {{ host }}: *{{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated {{ bsd }}.{{ millisec }}", @@ -38,24 +37,21 @@ "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ millisec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ host }}: {{ bsd }}.{{ millisec }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure. {{ bsd }}.{{ millisec }}", - "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}", - "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from {{ host }} has reached 792340, max 1048576", - "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ host }}:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576", + "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}" ] + testdata_uptime = [ "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}{{ seq }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ seq }}: {{ host }}: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.", - "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}", + "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}" ] @pytest.mark.parametrize("event", testdata) -def test_cisco_ios( - record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event -): +def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): host = get_host_key dt = datetime.datetime.now() @@ -68,23 +64,15 @@ def test_cisco_ios( microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render( - mark="<166>", - seq=20, - bsd=bsd, - time=time, - millisec=millisec, - microsec=microsec, - tzname=tzname, - host=host, - ) + message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, + millisec=millisec, microsec=microsec, tzname=tzname, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' - ) - search = st.render(epoch=epoch, millisec=millisec, microsec=microsec, host=host) + "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") + search = st.render(epoch=epoch, millisec=millisec, + microsec=microsec, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -96,9 +84,7 @@ def test_cisco_ios( @pytest.mark.parametrize("event", testdata_uptime) -def test_cisco_ios_uptime( - record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event -): +def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): host = get_host_key mt = env.from_string(event + "\n") @@ -107,8 +93,7 @@ def test_cisco_ios_uptime( sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=netops earliest=-1m@m latest=+1m@m sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' - ) + "search index=netops earliest=-1m@m latest=+1m@m sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) From 5a3d6e156424ff4fce74738efcd33cf6963e43ba Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 09:54:01 -0400 Subject: [PATCH 58/71] Revert "Cisco IOS XR Improvements" This reverts commit 0f7c2d09472727e61fb8513c9fe0d995ccb142db. --- .../conf.d/filters/cisco/cisco_syslog.conf | 42 +++++++++---------- tests/test_cisco_ios.py | 39 ++++++----------- 2 files changed, 33 insertions(+), 48 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 82cd4db..3938476 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -11,38 +11,38 @@ filter f_is_cisco_syslog{ parser cisco-parser-ex{ channel { filter { - message( - '^<\d*> ?(?:\d+\: )?(?RP\/\d*\/RSP\d*\/CPU\d*:)?(?:(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])): ?)?(?:\d+: )?(?:(?:\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?(?(?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?(?: [AP]M)?)(?: [A-Z]{3,3})?)? ?(?

(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: (?(?:(?[^\[]{1,30})\[(?\d*)\]: ?)?(?\%[^\: ]+)\:? ?.*)' - flags(store-matches) - ); + message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); + }; + + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S') + template("$8")); }; rewrite { set( - "${H1}", + "${4}", value("HOST") - condition(not match('^\d+$', value('H1')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H1'))) + condition(not match('^\d+$', value('4')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('4'))) ); set( - "${H2}", + "${13}", value("HOST") - condition(not match('^\d+$', value('H2')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('H2'))) + condition(not match('^\d+$', value('13')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) ); set( - "${CISCOMESSAGE}", + "${15}", + value("PROGRAM") + ); + set( + "${14}", value("MESSAGE") - ); - + ); }; - parser { date-parser-nofilter(format( - '%b %d %H:%M:%S.%f', - '%b %d %H:%M:%S', - '%b %d %I:%M:%S %p.%f', - '%b %d %I:%M:%S %p', - '%b %d %Y %H:%M:%S.%f', - '%b %d %Y %H:%M:%S') - template("${CISCOTS}")); - }; - }; }; diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index ab24128..108bec2 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -11,7 +11,6 @@ from .timeutils import * import pytest - env = Environment() @@ -28,7 +27,7 @@ # foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # 101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. # *Mar 1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) -# <189>357492: RP/0/RSP0/CPU0:May 14 16:44:40.145 : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576 + testdata = [ "{{ mark }}{{ seq }}: {{ host }}: 6340004: *{{ bsd }}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet", "{{ mark }}{{ seq }}: {{ host }}: *{{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated {{ bsd }}.{{ millisec }}", @@ -38,24 +37,21 @@ "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ millisec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ host }}: {{ bsd }}.{{ millisec }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure. {{ bsd }}.{{ millisec }}", - "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}", - "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from {{ host }} has reached 792340, max 1048576", - "{{ mark }}{{ seq }}: RP/0/RSP0/CPU0:{{ host }}:{{ bsd }}.{{ millisec }} : bgp[1051]: %ROUTING-BGP-5-MAXPFX : No. of IPv4 Unicast prefixes received from xx.xx.xx.xx has reached 792340, max 1048576", + "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}" ] + testdata_uptime = [ "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ", "{{ mark }}{{ seq }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}", "{{ mark }}{{ seq }}: {{ host }}: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.", - "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}", + "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}" ] @pytest.mark.parametrize("event", testdata) -def test_cisco_ios( - record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event -): +def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): host = get_host_key dt = datetime.datetime.now() @@ -68,23 +64,15 @@ def test_cisco_ios( microsec = iso[20:26] mt = env.from_string(event + "\n") - message = mt.render( - mark="<166>", - seq=20, - bsd=bsd, - time=time, - millisec=millisec, - microsec=microsec, - tzname=tzname, - host=host, - ) + message = mt.render(mark="<166>", seq=20, bsd=bsd, time=time, + millisec=millisec, microsec=microsec, tzname=tzname, host=host) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' - ) - search = st.render(epoch=epoch, millisec=millisec, microsec=microsec, host=host) + "search index=netops (_time={{ epoch }} OR _time={{ epoch }}.{{ millisec }} OR _time={{ epoch }}.{{ microsec }}) sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") + search = st.render(epoch=epoch, millisec=millisec, + microsec=microsec, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -96,9 +84,7 @@ def test_cisco_ios( @pytest.mark.parametrize("event", testdata_uptime) -def test_cisco_ios_uptime( - record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event -): +def test_cisco_ios_uptime(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): host = get_host_key mt = env.from_string(event + "\n") @@ -107,8 +93,7 @@ def test_cisco_ios_uptime( sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=netops earliest=-1m@m latest=+1m@m sourcetype="cisco:ios" (host="{{ host }}" OR "{{ host }}")' - ) + "search index=netops earliest=-1m@m latest=+1m@m sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\")") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) From 45f96190f4e0da7ddb57ed8963f6220a10113bf1 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 18 May 2020 10:09:36 -0700 Subject: [PATCH 59/71] Update lookahead for metrics regex * Update lookahead in `lp-sc4s_internal.conf.tmpl` to include _either_ a `=` or `,` character in the lookahead for SourceID --- package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl index 0bfba04..f015298 100644 --- a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl @@ -9,7 +9,7 @@ log { rewrite { subst('.*Log statistics; ', '', value("MESSAGE"), flags("utf8" "global")); subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global")); - subst('(?[^= ]+)=\x27(?[^\(]+)\((?\S+(?=\)=))(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27,? ?', + subst('(?[^= ]+)=\x27(?[^\(]+)\((?\S+(?=\)[=,]))(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27,? ?', '{"time": "$S_UNIXTIME","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}} ', value("MESSAGE") flags("utf8" "global") From b441cd2b2219e329642f119adda1443c12a5ad93 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 14:20:38 -0400 Subject: [PATCH 60/71] Fix TZ issue when lower case --- package/etc/conf.d/filters/cisco/cisco_syslog.conf | 2 +- tests/test_cisco_ios.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 3938476..88ca544 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -11,7 +11,7 @@ filter f_is_cisco_syslog{ parser cisco-parser-ex{ channel { filter { - message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); + message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Za-z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); }; parser { date-parser-nofilter(format( diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index 108bec2..91bf27c 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -37,7 +37,8 @@ "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ microsec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ seq }}: {{ host }}: {{ bsd }}.{{ millisec }}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated", "{{ mark }}{{ host }}: {{ bsd }}.{{ millisec }}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure. {{ bsd }}.{{ millisec }}", - "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}" + "{{ mark }}*{{ bsd }}.{{ millisec }} {{ tzname }}: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}", + "{{ mark }}84027: {{ bsd }}.{{ millisec }} dst: %SYS-5-CONFIG_I: Configured from console by username on vty0 ({{ host }})" ] testdata_uptime = [ From f066cd1367866297c91506151bda1a59c397c40e Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 17:15:26 -0400 Subject: [PATCH 61/71] Support cisco firepower unified logs --- docs/sources/Cisco/index.md | 9 ++++-- .../conf.d/log_paths/lp-cisco_asa.conf.tmpl | 28 +++++++++++++---- tests/test_cisco_asa.py | 30 +++++++++++++++++++ 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index 9fccf8d..fd58a34 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -95,13 +95,14 @@ Use the following search to validate events are present index= sourcetype=cisco:apic:* ``` -Verify timestamp, and host values match as expected +Verify timestamp, and host values match as expected ## Product - ASA AND FTD (Firepower) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | https://splunkbase.splunk.com/app/1620/ | +| Splunk Add-on for ASA | https://splunkbase.splunk.com/app/1620/ | +| Cisco eStreamer for Splunk | https://splunkbase.splunk.com/app/1629/ | | Product Manual | https://www.cisco.com/c/en/us/td/docs/security/asa/asa82/configuration/guide/config/monitor_syslog.html | @@ -109,13 +110,15 @@ Verify timestamp, and host values match as expected | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| cisco:asa | cisco FTD Firepower will also use this source type | +| cisco:asa | cisco FTD Firepower will also use this source type except those noted below | +| cisco:firepower:syslog | FTD Unified events see https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide.pdf | ### Sourcetype and Index Configuration | key | sourcetype | index | notes | |----------------|----------------|----------------|----------------| | cisco_asa | cisco:asa | netfw | none | +| cisco_ftd | cisco:firepower:syslog | netfw | none | ### Filter type diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl index 76c8a8b..b60f1d6 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl @@ -21,13 +21,29 @@ log { }; }; - rewrite { - set("cisco_asa", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw")) + if { + filter { + message('^%FTD-\d+-43000\d: ') or + match('^%FTD-\d+-43000\d:', value("LEGACY_MSGHDR")); + }; + rewrite { + set("cisco_ftd", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:firepower:syslog"), index("netfw")) + }; + parser {p_add_context_splunk(key("cisco_ftd")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + + } else { + rewrite { + set("cisco_asa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw")) + }; + parser {p_add_context_splunk(key("cisco_asa")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + }; - parser {p_add_context_splunk(key("cisco_asa")); }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_ASA_HEC" "no")) }} destination(d_hec); diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index 77a934f..6657876 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -100,3 +100,33 @@ def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk, setup_ record_property("message", message) assert resultCount == 1 + +#<118>2020-02-04T11:00:54Z %FTD-6-430003: DeviceUUID: 90e14378-2081-11e8-a7fa-d34972ba379f, AccessControlRuleAction: Allow, SrcIP: 75.150.94.75, DstIP: 172.30.0.2, SrcPort: 59698, DstPort: 8027, Protocol: tcp, IngressInterface: Outside2, EgressInterface: DMZ, IngressZone: Outside, EgressZone: DMZ, ACPolicy: Rapid7 5525X, AccessControlRuleName: Allow MDM - Out to DMZ, Prefilter Policy: Default Prefilter Policy, User: No Authentication Required, ConnectionDuration: 600, InitiatorPackets: 0, ResponderPackets: 0, InitiatorBytes: 31, ResponderBytes: 0, NAPPolicy: Balanced Security and Connectivity +def test_cisco_ftd(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + +# Get UTC-based 'dt' time structure + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + # iso from included timeutils is from local timezone; need to keep iso as UTC + iso = dt.isoformat()[0:19] + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ iso }}Z {{ host }} : %FTD-6-430003: DeviceUUID: 90e14378-2081-11e8-a7fa-d34972ba379f, AccessControlRuleAction: Allow, SrcIP: 75.150.94.75, DstIP: 172.30.0.2, SrcPort: 59698, DstPort: 8027, Protocol: tcp, IngressInterface: Outside2, EgressInterface: DMZ, IngressZone: Outside, EgressZone: DMZ, ACPolicy: Rapid7 5525X, AccessControlRuleName: Allow MDM - Out to DMZ, Prefilter Policy: Default Prefilter Policy, User: No Authentication Required, ConnectionDuration: 600, InitiatorPackets: 0, ResponderPackets: 0, InitiatorBytes: 31, ResponderBytes: 0, NAPPolicy: Balanced Security and Connectivity\n") + message = mt.render(mark="<166>", iso=iso, epoch=epoch, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cisco:firepower:syslog\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file From fa80547d16a8703e2edb4f2c411c9615f1e8a711 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 18:01:14 -0400 Subject: [PATCH 62/71] Update cisco_syslog.conf --- package/etc/conf.d/filters/cisco/cisco_syslog.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index a4d977b..dad2c7a 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -12,7 +12,7 @@ parser cisco-parser-ex{ channel { if { #Cisco IOS-XR devices with node-id format - filter { + filter { message('^<\d*>(?:(\d+)\: )?(RP\/\d*\/RSP\d*\/CPU\d*:)?(?:([^\: ]+):)(?:(\*)?(\w\w\w {1,2}\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}\.\d+))? : ?([^\[]{1,30}\[\d*\]: ?\%[^\: ]+\:? ?.*)' flags(store-matches)); }; @@ -40,7 +40,8 @@ parser cisco-parser-ex{ } else { # All other cisco syslog filter { - message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Za-z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); + not message('%(ASA|FTD-)') + and message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Za-z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); }; rewrite { From 7e380ebb27946f2fb189e756e86d6f1e2af3869c Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 18 May 2020 18:07:34 -0400 Subject: [PATCH 63/71] Update cisco_syslog.conf --- package/etc/conf.d/filters/cisco/cisco_syslog.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index dad2c7a..e750fc7 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -40,8 +40,7 @@ parser cisco-parser-ex{ } else { # All other cisco syslog filter { - not message('%(ASA|FTD-)') - and message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Za-z]{3,3})?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); + message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Za-z]{3,3} )?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); }; rewrite { From fc4a1973f053d9f423965ee914cee8f62cacd7b5 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Tue, 19 May 2020 10:06:45 -0700 Subject: [PATCH 64/71] TLS multi-port template fix * Update `source_network.t` gomplate construct to properly parse tls multi-port --- package/etc/go_templates/source_network.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index a8e1887..9aeb850 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -55,7 +55,7 @@ source s_{{ .port_id }} { {{- range split (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514") "," }} network( transport("tls") - port({{ getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514" }}) + port({{ . }}) ip-protocol(4) max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) From bc6319e829a1f362bac9a380cb05255d195f0309 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 19 May 2020 13:41:11 -0400 Subject: [PATCH 65/71] Fix --- .../conf.d/filters/cisco/cisco_syslog.conf | 4 +- tests/test_cisco_nx_os.py | 70 ++++++++++++++++--- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index e750fc7..9dbcd43 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -52,7 +52,9 @@ parser cisco-parser-ex{ set( "${13}", value("HOST") - condition(not match('^\d+$', value('13')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) + condition(not match('^\d+$', value('13')) + and not match('^[A-Za-z]{1,3}' value('13')) + and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) ); set( "${15}", diff --git a/tests/test_cisco_nx_os.py b/tests/test_cisco_nx_os.py index 3f386f5..97888c2 100644 --- a/tests/test_cisco_nx_os.py +++ b/tests/test_cisco_nx_os.py @@ -13,7 +13,9 @@ env = Environment() # Nov 1 14:07:58 excal-113 %MODULE-5-MOD_OK: Module 1 is online -def test_cisco_nx_os(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): +def test_cisco_nx_os( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s +): host = get_host_key dt = datetime.datetime.now() @@ -23,12 +25,17 @@ def test_cisco_nx_os(record_property, setup_wordlist, get_host_key, setup_splunk epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} csconx-{{ host }} %MODULE-5-MOD_OK: Module 1 is online") - message = mt.render(mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + "{{ mark }} {{ bsd }} csconx-{{ host }} %MODULE-5-MOD_OK: Module 1 is online" + ) + message = mt.render( + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netops host=\"csconx-{{ host }}\" sourcetype=\"cisco:ios\"") + st = env.from_string( + 'search _time={{ epoch }} index=netops host="csconx-{{ host }}" sourcetype="cisco:ios"' + ) search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -39,7 +46,10 @@ def test_cisco_nx_os(record_property, setup_wordlist, get_host_key, setup_splunk assert resultCount == 1 -def test_cisco_nx_os_soup(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): + +def test_cisco_nx_os_soup( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s +): host = get_host_key dt = datetime.datetime.now() @@ -49,12 +59,17 @@ def test_cisco_nx_os_soup(record_property, setup_wordlist, get_host_key, setup_s epoch = epoch[:-7] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} %MODULE-5-MOD_OK: Module 1 is online") - message = mt.render(mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset) + "{{ mark }} {{ bsd }} {{ host }} %MODULE-5-MOD_OK: Module 1 is online" + ) + message = mt.render( + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netops host=\"{{ host }}\" sourcetype=\"cisco:ios\"") + st = env.from_string( + 'search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="cisco:ios"' + ) search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -65,9 +80,46 @@ def test_cisco_nx_os_soup(record_property, setup_wordlist, get_host_key, setup_s assert resultCount == 1 + +# <187>364241: May 19 16:58:44.814 GMT: %ADJ-3-RESOLVE_REQ: Adj resolve request: Failed to resolve 1.1.1.1 Vlan1 +def test_cisco_nx_os_soup2( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s +): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }}364241: {{ bsd }} GMT: %ADJ-3-RESOLVE_REQ: Adj resolve request: Failed to resolve {{ host }} Vlan1\n" + ) + message = mt.render( + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} host!=GMT index=netops sourcetype="cisco:ios" {{ host }}' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +#%ADJ-3-RESOLVE_REQ # Nov 1 14:07:58 excal-113 %MODULE-5-MOD_OK: Module 1 is online # @pytest.mark.xfail -#def test_cisco_nx_os_singleport(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): +# def test_cisco_nx_os_singleport(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): # host = get_host_key # # dt = datetime.datetime.now() From e483a896e28dc552db164f9f918834aee024c98f Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 19 May 2020 14:11:52 -0400 Subject: [PATCH 66/71] Update cisco_syslog.conf --- package/etc/conf.d/filters/cisco/cisco_syslog.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index e750fc7..f4194b1 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -36,6 +36,11 @@ parser cisco-parser-ex{ "${6}", value("MESSAGE") ); + set( + "${HOST_FROM}" + value("HOST") + condition(not host(".")) + ); }; } else { # All other cisco syslog From ee938f70122c3641b304feb276eb187a64784033 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 19 May 2020 16:03:34 -0400 Subject: [PATCH 67/71] Update cisco_syslog.conf --- .../etc/conf.d/filters/cisco/cisco_syslog.conf | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 9dbcd43..a714288 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -40,7 +40,8 @@ parser cisco-parser-ex{ } else { # All other cisco syslog filter { - message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Za-z]{3,3} )?)? ?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' flags(store-matches)); + message('^<\d*> ?(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}))?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( [A-Za-z]{3,3} )?)? ?((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))? ?: ((\%[^\: ]+)\:? ?.*)' + flags(store-matches)); }; rewrite { @@ -50,18 +51,15 @@ parser cisco-parser-ex{ condition(not match('^\d+$', value('4')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('4'))) ); set( - "${13}", + "${11}", value("HOST") - condition(not match('^\d+$', value('13')) - and not match('^[A-Za-z]{1,3}' value('13')) - and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('13'))) - ); - set( - "${15}", - value("PROGRAM") + condition( + not match('^\d+$', value('11')) + and match('^[A-Za-z]{1,3}.+' value('11')) + and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('11'))) ); set( - "${14}", + "${12}", value("MESSAGE") ); }; From 273e96f06aaeb924daaaafc5f41ac00ff4c1a2fd Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 19 May 2020 16:27:57 -0400 Subject: [PATCH 68/71] Update test_cisco_nx_os.py --- tests/test_cisco_nx_os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cisco_nx_os.py b/tests/test_cisco_nx_os.py index 97888c2..bb08dcb 100644 --- a/tests/test_cisco_nx_os.py +++ b/tests/test_cisco_nx_os.py @@ -97,7 +97,7 @@ def test_cisco_nx_os_soup2( "{{ mark }}364241: {{ bsd }} GMT: %ADJ-3-RESOLVE_REQ: Adj resolve request: Failed to resolve {{ host }} Vlan1\n" ) message = mt.render( - mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + mark="<111>", bsd=bsd, date=date, time=time, tzoffset=tzoffset ) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) From 235f2a182f981c4828de9097d7b5bc4eabb3ed5e Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 19 May 2020 16:53:58 -0400 Subject: [PATCH 69/71] fix test --- package/etc/conf.d/filters/cisco/cisco_syslog.conf | 2 +- tests/test_cisco_nx_os.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index 90ecbbb..d1a77ae 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -60,7 +60,7 @@ parser cisco-parser-ex{ value("HOST") condition( not match('^\d+$', value('11')) - and match('^[A-Za-z]{1,3}.+' value('11')) + and match('^[A-Za-z0-9]{4,}$' value('11')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('11'))) ); set( diff --git a/tests/test_cisco_nx_os.py b/tests/test_cisco_nx_os.py index bb08dcb..97888c2 100644 --- a/tests/test_cisco_nx_os.py +++ b/tests/test_cisco_nx_os.py @@ -97,7 +97,7 @@ def test_cisco_nx_os_soup2( "{{ mark }}364241: {{ bsd }} GMT: %ADJ-3-RESOLVE_REQ: Adj resolve request: Failed to resolve {{ host }} Vlan1\n" ) message = mt.render( - mark="<111>", bsd=bsd, date=date, time=time, tzoffset=tzoffset + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset ) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) From 5915fe012f97223f16ec5a1886d6d53d338ae1a8 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 19 May 2020 17:22:27 -0400 Subject: [PATCH 70/71] Update cisco_syslog.conf --- package/etc/conf.d/filters/cisco/cisco_syslog.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index d1a77ae..a4f0381 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -60,7 +60,7 @@ parser cisco-parser-ex{ value("HOST") condition( not match('^\d+$', value('11')) - and match('^[A-Za-z0-9]{4,}$' value('11')) + and match('^.{4,}$' value('11')) and match('^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' value('11'))) ); set( From 8744a48593f14995c1037bd06ff7af76296e0440 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 19 May 2020 18:29:37 -0400 Subject: [PATCH 71/71] Update cisco_syslog.conf --- package/etc/conf.d/filters/cisco/cisco_syslog.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package/etc/conf.d/filters/cisco/cisco_syslog.conf b/package/etc/conf.d/filters/cisco/cisco_syslog.conf index a4f0381..aae8f84 100644 --- a/package/etc/conf.d/filters/cisco/cisco_syslog.conf +++ b/package/etc/conf.d/filters/cisco/cisco_syslog.conf @@ -67,6 +67,11 @@ parser cisco-parser-ex{ "${12}", value("MESSAGE") ); + set( + "${HOST_FROM}" + value("HOST") + condition(not host(".")) + ); }; parser { date-parser-nofilter(format( '%b %d %H:%M:%S.%f',