From 18f3bd2447962047c245a2a588aa2667ec354534 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Mon, 26 Aug 2019 19:16:39 -0400 Subject: [PATCH] Feature/testmultiindex (#63) * Update CI to support index specific tests --- .circleci/config.yml | 3 ++ docker-compose-ci.yml | 4 +- package/etc/context-local/splunk_index.csv | 46 +++++++++---------- .../apps}/SA-syslog-ng/default/indexes.conf | 21 +++++++++ test-with-compose.sh | 5 +- tests/test_cisco_asa.py | 4 +- tests/test_cisco_ios.py | 2 +- tests/test_fortinet_ngfw.py | 6 +-- tests/test_juniper.py | 8 ++-- tests/test_juniper_junos_rfc3164.py | 8 ++-- tests/test_juniper_junos_rfc5124.py | 6 +-- tests/test_palo_alto.py | 4 +- tests/test_symantec_proxy.py | 2 +- 13 files changed, 74 insertions(+), 45 deletions(-) rename splunk/{ => etc/apps}/SA-syslog-ng/default/indexes.conf (53%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 69f02a8..89d1e1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,9 +120,12 @@ jobs: docker container create --name dummy \ -v sc4s-tests:/work/tests \ -v sc4s-results:/work/test-results \ + -v splunk-etc:/work/splunk-etc \ registry.access.redhat.com/ubi7/ubi docker cp tests/ dummy:/work/tests/ + docker cp ./splunk/etc/* dummy:/work/splunk-etc/ docker rm dummy + - run: name: Docker Compose build command: | diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index fec924c..26eced6 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -79,7 +79,7 @@ services: - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng + - splunk-etc:/opt/splunk/etc # logging: # driver: splunk # options: @@ -93,3 +93,5 @@ volumes: external: true sc4s-results: external: true + splunk-etc: + external: true \ No newline at end of file diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv index 58a2010..4f98d23 100644 --- a/package/etc/context-local/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -1,50 +1,50 @@ -bluecoat:proxysg:access:kv,index,main +bluecoat:proxysg:access:kv,index,netproxy bluecoat:proxysg:access:kv,sourcetype,bluecoat:proxysg:access:kv -cisco:asa,index,main +cisco:asa,index,netfw cisco:asa,sourcetype,cisco:asa -cisco:ios,index,main +cisco:ios,index,netops cisco:ios,sourcetype,cisco:ios cisco:nx-os,index,main cisco:nx-os,sourcetype,cisco:nx-os fgt_event,sourcetype,fgt_event -fgt_event,index,main +fgt_event,index,netops fgt_log,sourcetype,fgt_log -fgt_log,index,main +fgt_log,index,netops fgt_traffic,sourcetype,fgt_traffic -fgt_traffic,index,main +fgt_traffic,index,netfw fgt_utm,sourcetype,fgt_utm -fgt_utm,index,main +fgt_utm,index,netids juniper:structured,index,main juniper:structured,sourcetype,juniper:structured -juniper:junos:firewall:structured,index,main +juniper:junos:firewall:structured,index,netfw juniper:junos:firewall:structured,sourcetype,juniper:junos:firewall:structured -juniper:junos:idp:structured,index,main +juniper:junos:idp:structured,index,netids juniper:junos:idp:structured,sourcetype,juniper:junos:idp:structured -juniper:junos:firewall,index,main +juniper:junos:firewall,index,netfw juniper:junos:firewall,sourcetype,juniper:junos:firewall -juniper:junos:idp,index,main +juniper:junos:idp,index,netids juniper:junos:idp,sourcetype,juniper:junos:idp -juniper:nsm:idp,index,main +juniper:nsm:idp,index,netids juniper:nsm:idp,sourcetype,juniper:nsm:idp -juniper:sslvpn,index,main +juniper:sslvpn,index,netfw juniper:sslvpn,sourcetype,juniper:sslvpn -juniper:netscreen,index,main +juniper:netscreen,index,netfw juniper:netscreen,sourcetype,netscreen:firewall -juniper:idp,index,main +juniper:idp,index,netids juniper:idp,sourcetype,juniper:idp -juniper:idp:structured,index,main +juniper:idp:structured,index,netids juniper:idp:structured,sourcetype,juniper:idp:structured -juniper:nsm,index,main +juniper:nsm,index,netids juniper:nsm,sourcetype,juniper:nsm -juniper:legacy,index,main +juniper:legacy,index,netfw juniper:legacy,sourcetype,juniper:legacy -pan:traffic,index,main -pan:threat,index,main -pan:system,index,main -pan:config,index,main +pan:traffic,index,netfw +pan:threat,index,netproxy +pan:system,index,netops +pan:config,index,netops pan:hipwatch,index,main pan:correlation,index,main -pan:userid,index,main +pan:userid,index,netops pan:unknown,index,main pan:traffic,sourcetype,pan:traffic pan:threat,sourcetype,pan:threat diff --git a/splunk/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf similarity index 53% rename from splunk/SA-syslog-ng/default/indexes.conf rename to splunk/etc/apps/SA-syslog-ng/default/indexes.conf index e795078..f8dbfb2 100644 --- a/splunk/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -19,3 +19,24 @@ thawedPath = $SPLUNK_DB/test/thaweddb homePath = $SPLUNK_DB/test2/db coldPath = $SPLUNK_DB/test2/colddb thawedPath = $SPLUNK_DB/test2/thaweddb + + +[netfw] +homePath = $SPLUNK_DB/netfw/db +coldPath = $SPLUNK_DB/netfw/colddb +thawedPath = $SPLUNK_DB/netfw/thaweddb + +[netids] +homePath = $SPLUNK_DB/netids/db +coldPath = $SPLUNK_DB/netids/colddb +thawedPath = $SPLUNK_DB/netids/thaweddb + +[netops] +homePath = $SPLUNK_DB/netops/db +coldPath = $SPLUNK_DB/netops/colddb +thawedPath = $SPLUNK_DB/netops/thaweddb + +[netproxy] +homePath = $SPLUNK_DB/netproxy/db +coldPath = $SPLUNK_DB/netproxy/colddb +thawedPath = $SPLUNK_DB/netproxy/thaweddb \ No newline at end of file diff --git a/test-with-compose.sh b/test-with-compose.sh index 3f08cbf..b569b7f 100755 --- a/test-with-compose.sh +++ b/test-with-compose.sh @@ -8,16 +8,19 @@ mkdir test-results docker volume create sc4s-tests docker volume create sc4s-results +docker volume create splunk-etc docker container create --name dummy \ -v sc4s-tests:/work/tests \ -v sc4s-results:/work/test-results \ + -v splunk-etc:/work/splunk-etc \ registry.access.redhat.com/ubi7/ubi docker cp tests/ dummy:/work/tests/ +docker cp ./splunk/etc/* dummy:/work/splunk-etc/ docker rm dummy docker-compose build -docker-compose up --abort-on-container-exit --exit-code-from test +docker-compose up --abort-on-container-exit --exit-code-from test docker container create --name dummy \ -v sc4s-tests:/work/tests \ diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index 65d122e..7203f32 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -25,7 +25,7 @@ def test_cisco_asa_tradditional(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -47,7 +47,7 @@ def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-005424\"| head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-005424\"| head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index ca60605..13ec3f5 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -22,7 +22,7 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") + st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index 3437f9e..f68f13a 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -22,7 +22,7 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") + st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -42,7 +42,7 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk): message = mt.render(mark="<13>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -62,7 +62,7 @@ def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk): message = mt.render(mark="<13>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_juniper.py b/tests/test_juniper.py index 102dca3..c4bdb81 100644 --- a/tests/test_juniper.py +++ b/tests/test_juniper.py @@ -22,7 +22,7 @@ def test_juniper_nsm_standard(record_property, setup_wordlist, get_host_key, set sendsingle(message) - st = env.from_string("search index=main host=\"jnpnsm-{{ host }}\" sourcetype=\"juniper:nsm\" | head 2") + st = env.from_string("search index=netids host=\"jnpnsm-{{ host }}\" sourcetype=\"juniper:nsm\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -44,7 +44,7 @@ def test_juniper_nsm_idp_standard(record_property, setup_wordlist, get_host_key, sendsingle(message) - st = env.from_string("search index=main host=\"jnpnsmidp-{{ host }}\" sourcetype=\"juniper:nsm:idp\" | head 2") + st = env.from_string("search index=netids host=\"jnpnsmidp-{{ host }}\" sourcetype=\"juniper:nsm:idp\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -66,7 +66,7 @@ def test_juniper_netscreen(record_property, setup_wordlist, get_host_key, setup_ sendsingle(message) - st = env.from_string("search index=main host=\"jnpns-{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") + st = env.from_string("search index=netfw host=\"jnpns-{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -88,7 +88,7 @@ def test_juniper_netscreen_singleport(record_property, setup_wordlist, get_host_ sendsingle(message, host="sc4s-juniper") - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_juniper_junos_rfc3164.py b/tests/test_juniper_junos_rfc3164.py index 5af94b7..3ec13ff 100644 --- a/tests/test_juniper_junos_rfc3164.py +++ b/tests/test_juniper_junos_rfc3164.py @@ -20,7 +20,7 @@ def test_juniper_utm_standard(record_property, setup_wordlist, get_host_key, set sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -41,7 +41,7 @@ def test_juniper_firewall_standard(record_property, setup_wordlist, get_host_key sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -63,7 +63,7 @@ def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:sslvpn\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:sslvpn\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -85,7 +85,7 @@ def test_juniper_idp_standard(record_property, setup_wordlist, get_host_key, set sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:idp\" | head 2") + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:idp\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_juniper_junos_rfc5124.py b/tests/test_juniper_junos_rfc5124.py index 9ad8b22..b9ff921 100644 --- a/tests/test_juniper_junos_rfc5124.py +++ b/tests/test_juniper_junos_rfc5124.py @@ -44,7 +44,7 @@ def test_juniper_junos_idp_structured(record_property, setup_wordlist, get_host_ sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:idp:structured\" | head 2") + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:idp:structured\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -69,7 +69,7 @@ def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, s sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:idp:structured\" | head 2") + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:idp:structured\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -91,7 +91,7 @@ def test_juniper_junos_fw_structured(record_property, setup_wordlist, get_host_k sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall:structured\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall:structured\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index 62470b7..2d40c8f 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -22,7 +22,7 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -44,7 +44,7 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:threat\" | head 2") + st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"pan:threat\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py index 8d84736..0dddfac 100644 --- a/tests/test_symantec_proxy.py +++ b/tests/test_symantec_proxy.py @@ -20,7 +20,7 @@ def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk): message = mt.render(mark="<134>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"bluecoat:proxysg:access:kv\" | head 2") + st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"bluecoat:proxysg:access:kv\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search)