From cb8ebded433cff243aef4f1cde7bfb9aea9b7120 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Tue, 27 Aug 2019 15:09:11 -0400 Subject: [PATCH] Release/0.4.0 (#67) v0.4.0 - Add source support for Cisco NX - Add source support for Symantec Proxy SG and ASG (Formerly bluecoat) - Add support for mapping network source IP OR parsed host to specific vendor product where MSG parsing is impossible - Code cleanup and simplification - Begin using SEMVER for releases --- .circleci/config.yml | 49 ++++- .gitignore | 8 +- .idea/misc.xml | 10 - .idea/modules.xml | 8 - .idea/sc4s.iml | 13 -- .idea/splunk-connect-for-syslog.iml | 19 -- .idea/vcs.xml | 10 - CHANGELOG | 6 + docker-compose-ci.yml | 21 +- docker-compose-debug.yml | 56 +++++ docker-compose.yml | 31 ++- docs/gettingstarted.md | 58 ++++- docs/sources.md | 119 +++++++++-- package/Dockerfile | 2 +- package/etc/conf.d/blocks/b_parsers.conf | 69 ------ .../conf.d/conflib/_common/rfc_syslog.conf | 27 +++ .../_common/templates.conf} | 6 +- .../etc/conf.d/conflib/_common/utility.conf | 4 + .../vendor_product_by_source_context.conf | 9 + .../_splunk}/splunk_context.conf | 2 +- .../conf.d/conflib/_splunk/splunkfields.conf | 23 ++ .../etc/conf.d/conflib/vendor_cisco/asa.conf | 3 + .../etc/conf.d/conflib/vendor_cisco/ios.conf | 8 + .../conf.d/conflib/vendor_cisco/nx-os.conf | 3 + .../conflib/vendor_fortinet/fortios.conf | 3 + .../conf.d/conflib/vendor_juniper/junos.conf | 8 + .../conf.d/conflib/vendor_juniper/legacy.conf | 19 ++ .../etc/conf.d/conflib/vendor_pan/panos.conf | 3 + .../vendor_symantec/bluecoat_proxy.conf | 3 + .../{d_destinations.conf => splunk_hec.conf} | 8 +- .../filters/cisco_asa_tradditional.conf | 24 --- package/etc/conf.d/filters/juniper_junos.conf | 65 ------ .../log_paths/P_rfc3164-juniper_nsm_idp.conf | 15 ++ package/etc/conf.d/log_paths/internal.conf | 35 +++ .../log_paths/p_rfc3164-cisco-nx-os.conf | 14 ++ .../conf.d/log_paths/p_rfc3164-cisco_asa.conf | 21 ++ .../p_rfc3164-cisco_ios.conf} | 13 +- .../p_rfc3164-fortinet_fortios.conf} | 12 +- .../log_paths/p_rfc3164-juniper-idp.conf | 15 ++ .../log_paths/p_rfc3164-juniper-junos.conf | 24 +++ .../p_rfc3164-juniper_netscreen.conf | 14 ++ .../log_paths/p_rfc3164-juniper_nsm.conf | 19 ++ .../p_rfc3164-paloalto-panos.conf} | 36 ++-- .../p_rfc5424_noversion-cisco_asa.conf | 16 ++ .../p_rfc_5424_noversion-symantec-proxy.conf | 16 ++ .../p_rfc_5424_strict-juniper-junos.conf | 27 +++ package/etc/conf.d/log_paths/zfallback.conf | 11 + package/etc/conf.d/sources/network.conf | 104 +++++---- .../splunk_index.csv | 48 +++-- .../vendor_product_by_source.conf | 29 +++ .../vendor_product_by_source.csv | 6 + package/etc/syslog-ng.conf | 61 +----- .../apps}/SA-syslog-ng/default/indexes.conf | 21 ++ test-with-compose.sh | 5 +- tests/Dockerfile | 4 +- tests/entrypoint.sh | 23 ++ tests/requirements.txt | 1 - tests/sendmessage.py | 4 +- tests/splunkutils.py | 2 +- tests/test_cisco_asa.py | 11 +- tests/test_cisco_ios.py | 4 +- tests/test_cisco_nx_os.py | 55 +++++ tests/test_common.py | 51 +++++ tests/test_fortinet_ngfw.py | 16 +- tests/test_juniper.py | 200 ++---------------- tests/test_juniper_junos_rfc3164.py | 97 +++++++++ tests/test_juniper_junos_rfc5124.py | 105 +++++++++ tests/test_palo_alto.py | 8 +- tests/test_poc.py | 35 --- tests/test_symantec_proxy.py | 34 +++ tests/wait-for | 79 +++++++ 71 files changed, 1323 insertions(+), 665 deletions(-) delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/sc4s.iml delete mode 100644 .idea/splunk-connect-for-syslog.iml delete mode 100644 .idea/vcs.xml create mode 100644 CHANGELOG create mode 100644 docker-compose-debug.yml delete mode 100644 package/etc/conf.d/blocks/b_parsers.conf create mode 100644 package/etc/conf.d/conflib/_common/rfc_syslog.conf rename package/etc/conf.d/{templates/t_templates.conf => conflib/_common/templates.conf} (92%) create mode 100644 package/etc/conf.d/conflib/_common/utility.conf create mode 100644 package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf rename package/etc/conf.d/{blocks => conflib/_splunk}/splunk_context.conf (74%) create mode 100644 package/etc/conf.d/conflib/_splunk/splunkfields.conf create mode 100644 package/etc/conf.d/conflib/vendor_cisco/asa.conf create mode 100644 package/etc/conf.d/conflib/vendor_cisco/ios.conf create mode 100644 package/etc/conf.d/conflib/vendor_cisco/nx-os.conf create mode 100644 package/etc/conf.d/conflib/vendor_fortinet/fortios.conf create mode 100644 package/etc/conf.d/conflib/vendor_juniper/junos.conf create mode 100644 package/etc/conf.d/conflib/vendor_juniper/legacy.conf create mode 100644 package/etc/conf.d/conflib/vendor_pan/panos.conf create mode 100644 package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf rename package/etc/conf.d/destinations/{d_destinations.conf => splunk_hec.conf} (92%) delete mode 100644 package/etc/conf.d/filters/cisco_asa_tradditional.conf delete mode 100644 package/etc/conf.d/filters/juniper_junos.conf create mode 100644 package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf create mode 100644 package/etc/conf.d/log_paths/internal.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf rename package/etc/conf.d/{filters/cisco_ios.conf => log_paths/p_rfc3164-cisco_ios.conf} (60%) rename package/etc/conf.d/{filters/fortinet_fgt.conf => log_paths/p_rfc3164-fortinet_fortios.conf} (80%) create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf rename package/etc/conf.d/{filters/paloalto_networks.conf => log_paths/p_rfc3164-paloalto-panos.conf} (69%) create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf create mode 100644 package/etc/conf.d/log_paths/zfallback.conf rename package/etc/{context => context-local}/splunk_index.csv (64%) create mode 100644 package/etc/context-local/vendor_product_by_source.conf create mode 100644 package/etc/context-local/vendor_product_by_source.csv rename splunk/{ => etc/apps}/SA-syslog-ng/default/indexes.conf (53%) create mode 100755 tests/entrypoint.sh create mode 100644 tests/test_cisco_nx_os.py create mode 100644 tests/test_common.py create mode 100644 tests/test_juniper_junos_rfc3164.py create mode 100644 tests/test_juniper_junos_rfc5124.py delete mode 100644 tests/test_poc.py create mode 100644 tests/test_symantec_proxy.py create mode 100755 tests/wait-for diff --git a/.circleci/config.yml b/.circleci/config.yml index aa9293b..4a82547 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,16 +120,21 @@ jobs: docker container create --name dummy \ -v sc4s-tests:/work/tests \ -v sc4s-results:/work/test-results \ + -v splunk-etc:/work/splunk-etc \ registry.access.redhat.com/ubi7/ubi docker cp tests/ dummy:/work/tests/ + docker cp ./splunk/etc/* dummy:/work/splunk-etc/ docker rm dummy -# - run: -# name: Docker Compose build -# command: | -# docker-compose build + - run: + name: Docker Compose build + command: | + docker-compose build test - run: name: Docker Compose up - command: docker-compose -f ./docker-compose-ci.yml up --abort-on-container-exit + command: | + docker-compose -f ./docker-compose-ci.yml up -d splunk + sleep 70 + docker-compose -f ./docker-compose-ci.yml up --abort-on-container-exit - run: name: Collect test results from Docker Compose command: | @@ -204,6 +209,31 @@ jobs: name: Docker push tag command: docker push $IMAGE_NAME:edge + publish-version: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false + environment: + IMAGE_NAME: rfaircloth/scs + steps: + - checkout + + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker tag image + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker tag $IMAGE_NAME:$CIRCLE_SHA1 splunk/scs:$SEMVER + - run: + name: Docker push tag + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker push splunk/scs:$SEMVER publish-latest: environment: IMAGE_NAME: rfaircloth/scs @@ -256,6 +286,15 @@ workflows: branches: only: - develop + - publish-version: + requires: + - dgoss + - test-unit + filters: + branches: + only: + - master + - /^release\/.*/ - publish-latest: requires: - dgoss diff --git a/.gitignore b/.gitignore index 68dbac4..59343f4 100644 --- a/.gitignore +++ b/.gitignore @@ -378,11 +378,5 @@ fabric.properties !/package/scripts/splunkmetrics.sh !/package/scripts/ .ecs -======= /test-results/ -/.idea/workspace.xml -/.idea/tasks.xml -!/package/scripts/switch_transport.sh -!/package/scripts/splunkmetrics.sh -!/package/scripts/ ->>>>>>> master +/.idea/ \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 7e25d8b..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 4e16e75..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/sc4s.iml b/.idea/sc4s.iml deleted file mode 100644 index bda388e..0000000 --- a/.idea/sc4s.iml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/splunk-connect-for-syslog.iml b/.idea/splunk-connect-for-syslog.iml deleted file mode 100644 index 14d7789..0000000 --- a/.idea/splunk-connect-for-syslog.iml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 6310177..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..4b84880 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,6 @@ +0.4.0 + - Add source support for Cisco NX + - Add source support for Symantec Proxy SG and ASG (Formerly bluecoat) + - Add support for mapping network source IP OR parsed host to specific vendor product where MSG parsing is impossible + - Code cleanup and simplification + - Begin using SEMVER for releases \ No newline at end of file diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index b7dbd53..5201e5b 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -39,6 +39,23 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + sc4s-juniper: + image: rfaircloth/scs:${CIRCLE_SHA1} + hostname: sc4s-juniper + ports: + - "514" + stdin_open: true + tty: true + links: + - splunk + environment: + - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} + - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} + - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + - SYSLOG_PRESUME_FILTER=juniper_netscreen # logging: # driver: splunk # options: @@ -62,7 +79,7 @@ services: - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng + - splunk-etc:/opt/splunk/etc # logging: # driver: splunk # options: @@ -76,3 +93,5 @@ volumes: external: true sc4s-results: external: true + splunk-etc: + external: true diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml new file mode 100644 index 0000000..55cb296 --- /dev/null +++ b/docker-compose-debug.yml @@ -0,0 +1,56 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +version: "3.2" +services: + + sc4s: + image: splunk/scs:latest + build: + context: ./package + args: + RH_ORG: ${RH_ORG} + RH_ACTIVATION: ${RH_ACTIVATION} + entrypoint: + - "tail" + - "-f" + - "/dev/null" + hostname: sc4s + ports: + - "514" + - "601" + - "514/udp" + - "5514" + - "5514/udp" + stdin_open: true + tty: true + links: + - splunk + environment: + - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} + - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} + - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + splunk: + image: splunk/splunk:latest + hostname: splunk + ports: + - "8000:8000" + - "8088:8088" + - "8089:8089" + environment: + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} + - SPLUNK_START_ARGS=${SPLUNK_START_ARGS} + - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} + - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} + - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} + volumes: + - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng diff --git a/docker-compose.yml b/docker-compose.yml index fd562ee..b0135dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ services: links: - splunk - sc4s + - sc4s-juniper volumes: - sc4s-tests:/work/tests - sc4s-results:/work/test-results @@ -44,6 +45,32 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + sc4s-juniper: + image: splunk/scs:latest + build: + context: ./package +# cache_from: +# - registry.access.redhat.com/ubi7/ubi +# - registry.access.redhat.com/rhel7/rhel +# - rfaircloth/scs:edge + args: + RH_ORG: ${RH_ORG} + RH_ACTIVATION: ${RH_ACTIVATION} + hostname: sc4s-juniper + ports: + - "514" + stdin_open: true + tty: true + links: + - splunk + environment: + - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} + - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} + - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + - SYSLOG_PRESUME_FILTER=juniper_netscreen # logging: # driver: splunk # options: @@ -67,7 +94,7 @@ services: - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng + - splunk-etc:/opt/splunk/etc # logging: # driver: splunk # options: @@ -81,3 +108,5 @@ volumes: external: true sc4s-results: external: true + splunk-etc: + external: true \ No newline at end of file diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index d1e71cc..8abfe3b 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -22,11 +22,11 @@ services: image: splunk/scs:latest hostname: sc4s ports: - - "514" - - "601" - - "514/udp" - - "5514" - - "5514/udp" + - "514:514" + - "601:601" + - "514:514/udp" + - "5514:5514" + - "5514:5514/udp" stdin_open: true tty: true environment: @@ -36,16 +36,25 @@ services: - SPLUNK_DEFAULT_INDEX= - SPLUNK_METRICS_INDEX=em_metrics volumes: - - splunk_index.csv:/opt/syslog-ng/etc/context/splunk_index.csv + - ./sc4s/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv + - ./sc4s/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv ``` -* Download the latest context.csv file to the current directory +## Configure index destinations for Splunk +* Download the latest context.csv file to a subdirectory sc4s below the docker-compose.yml file created above ```bash -wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context/splunk_index.csv +wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` - * Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. +## Configure sources by source IP or host name +* This step is required even if not used +* Download the latest vendor_product_by_source.conf file to a subdirectory sc4s below the docker-compose.yml file created above +```bash +wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf +``` +* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. + * Start SC4S ```bash @@ -56,3 +65,34 @@ docker stack deploy --compose-file docker-compose.yml sc4s ## Scale out Additional hosts can be deployed for syslog collection from additional network zones and locations + + +## Single Source Technology instance - Alpha + +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. In the following configration example a dedicated port is opened (6514) for legacy juniper netscreen devices + +This approach is "alpha" and subject to change + +```yaml +version: "3" +services: + sc4s-juniper-netscreen: + image: splunk/scs:latest + hostname: sc4s-juniper-netscreen + ports: + - "6514:514" + - "6514:514/udp" + stdin_open: true + tty: true + environment: + - SPLUNK_HEC_URL=https://foo:8088/services/collector/event + - SPLUNK_HEC_TOKEN= + - SPLUNK_CONNECT_METHOD=hec + - SPLUNK_DEFAULT_INDEX= + - SPLUNK_METRICS_INDEX=em_metrics + - SYSLOG_PRESUME_FILTER=f_juniper_netscreen + volumes: + - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +``` \ No newline at end of file diff --git a/docs/sources.md b/docs/sources.md index 3fd78da..24e9d72 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -8,7 +8,7 @@ | Product Manual | https://www.cisco.com/c/en/us/td/docs/security/asa/asa82/configuration/guide/config/monitor_syslog.html | -## Sourcetypes +### Sourcetypes | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| @@ -16,11 +16,11 @@ | cisco:pix | Not supported | | cisco:fwsm | Not supported | -## Filter type +### Filter type MSG Parse: This filter parses message content -## Setup and Configuration +### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index as required. @@ -31,7 +31,7 @@ MSG Parse: This filter parses message content * device-id is hostname and included * timestamp is included -## Verification +### Verification Use the following search to validate events are present @@ -39,6 +39,63 @@ Use the following search to validate events are present index= sourcetype=cisco:asa ``` +Verify timestamp, and host values match as expected + +## Product - IOS and NX-OS based equipment + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/1467/ | +| IOS Manual | https://www.cisco.com/c/en/us/td/docs/switches/lan/catalyst2960/software/release/12-2_55_se/configuration/guide/scg_2960/swlog.html | +| NX-OS Manual | https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/6-x/system_management/configuration/guide/b_Cisco_Nexus_9000_Series_NX-OS_System_Management_Configuration_Guide/sm_5syslog.html| +| Cisco ACI | https://community.cisco.com/legacyfs/online/attachments/document/technote-aci-syslog_external-v1.pdf | +| Cisco WLC & AP | https://www.cisco.com/c/en/us/support/docs/wireless/4100-series-wireless-lan-controllers/107252-WLC-Syslog-Server.html#anc8 | +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cisco:ios | This source type is also used for NX-OS, ACI and WLC product lines | + +### Filter type + +* Cisco IOS products can be identified by message parsing alone +* Cisco NX OS, WLC, and ACI products must be identified by host or ip assignment update the filter `f_cisco_nx_os` as required + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index as required. +* IOS Follow vendor configuration steps per Product Manual above ensure: + * Ensure a reliable NTP server is set and synced + * Log Level is 6 "Informational" + * Protocol is TCP/IP + * permit-hostdown is on + * device-id is hostname and included + * timestamp is included +* NX-OS Follow vendor configuration steps per Product Manual above ensure: + * Ensure a reliable NTP server is set and synced + * Log Level is 6 "Informational" user may select alternate levels by module based on use cases + * Protocol is TCP/IP + * device-id is hostname and included + * timestamp is included and milisecond accuracy selected +* ACI Logging configuration of the ACI product often varies by use case. + * Ensure NTP sync is configured and active + * Ensure proper host names are configured +* WLC + * Ensure NTP sync is configured and active + * Ensure proper host names are configured + * For security use cases per AP logging is required + + +### Verification + +Use the following search to validate events are present, for NX-OS, WLC and ACI products ensure each host filter condition is verified + +``` +index= sourcetype=cisco:ios | stats count by host +``` + + Verify timestamp, and host values match as expected # Vendor - Fortinet @@ -51,7 +108,7 @@ Verify timestamp, and host values match as expected | Product Manual | https://docs.fortinet.com/product/fortigate/6.2 | -## Sourcetypes +### Sourcetypes | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| @@ -60,11 +117,11 @@ Verify timestamp, and host values match as expected | fgt_utm | None | | fgt_event | None -## Filter type +### Filter type MSG Parse: This filter parses message content -## Setup and Configuration +### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index as required. @@ -103,7 +160,7 @@ end ``` -## Verification +### Verification An active firewall will generate frequent events, in addition fortigate has the ability to test logging functionality using a build in command @@ -117,7 +174,7 @@ Verify timestamp, and host values match as expected index= (sourcetype=fgt_log OR sourcetype=fgt_traffic OR sourcetype=fgt_utm) ``` -###UTM Message type +### UTM Message type ![FortiGate UTM message](FortiGate_utm.png) @@ -140,7 +197,7 @@ Verify timestamp, and host values match as expected | Product Manual | https://docs.paloaltonetworks.com/pan-os/9-0/pan-os-admin/monitoring/use-syslog-for-monitoring/configure-syslog-monitoring.html | -## Sourcetypes +### Sourcetypes | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| @@ -148,11 +205,11 @@ Verify timestamp, and host values match as expected | pan:traffic | None | | pan:threat | None | -## Filter type +### Filter type MSG Parse: This filter parses message content -## Setup and Configuration +### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index as required. @@ -161,10 +218,46 @@ MSG Parse: This filter parses message content * Select IETF Format * Ensure the format of the event is not customized -## Verification +### Verification An active firewall will generate frequent events use the following search to validate events are present per source device ``` index= sourcetype=pan:*| stats count by host +``` + +# Vendor - Symantec + +## Product - ProxySG/ASG (Bluecoat) + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2758/ | +| Product Manual | https://support.symantec.com/us/en/article.tech242216.html | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| bluecoat:proxysg:access:kv | Requires version 3.6 | + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index as required. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Verification + +An active proxy will generate frequent events use the following search to validate events are present per source device + +``` +index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` \ No newline at end of file diff --git a/package/Dockerfile b/package/Dockerfile index 2f8698e..4e3b4c4 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -83,7 +83,7 @@ COPY goss.yaml /etc/goss.yaml COPY --from=0 /opt/syslog-ng /opt/syslog-ng COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d -COPY etc/context /opt/syslog-ng/etc/context +COPY etc/context-local /opt/syslog-ng/etc/context-local COPY sbin/entrypoint.sh /sbin/entrypoint.sh RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer diff --git a/package/etc/conf.d/blocks/b_parsers.conf b/package/etc/conf.d/blocks/b_parsers.conf deleted file mode 100644 index 8aae8e9..0000000 --- a/package/etc/conf.d/blocks/b_parsers.conf +++ /dev/null @@ -1,69 +0,0 @@ -#this parser can be used for "almost compliant" 5424 messages -block parser p_RFC5424-lite(prefix(".rfc5424-lite.")) { - channel { - rewrite { - # Remove Version Identifier - subst('(<\d+>)\d+' "$1" value("MSG")); - - }; - parser { - # by this time this message will parse with the BSD 3164 parser - syslog-parser(time-zone(`default-timezone`)); - # syslog-parser(); - - # extract name-value pairs. - # kv-parser(prefix("`prefix`")); - }; - }; -}; - -block parser p_test-parser(prefix(".testparse.")) { - channel { - rewrite { - # Put your rewrite rule here;use "MSG" as no-parse() puts entire payload there - # subst('(>.{12})\s' "${1}T" value("MSG")); - }; - parser { - # Don't initially parse so full output can be seen - # syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); - # syslog-parser(); - # syslog-parser(time-zone(`default-timezone`)); - # date-parser(format("%Y-%m-%d %H:%M:%S") template("$(substr ${MESSAGE} 0 19)") time-zone(`default-timezone`)); - }; - }; -}; - -#Used by fallback to log everything we can -block rewrite r_set_splunk(sourcetype(`splunk-sourcetype`) - index(`splunk-index`) - template(`splunk-default-template`) - vendor("unassigned") - category("") - filename("messages") - ) { - set("`index`", value(".splunk.index")); - set("`sourcetype`", value(".splunk.sourcetype")); - set("`template`", value(".splunk.template")); - set("`vendor`", value(".splunk.vendor")); - set("`category`", value(".splunk.category")); - set("`filename`", value(".splunk.filename")); - set($FACILITY, value("fields.syslog_facility")); - set($LEVEL, value("fields.syslog_severity")); - set($LOGHOST, value("fields.syslog_server")); - }; - -#Used when for normal filters index and sourcetype must be set in a prior re-write or context -block rewrite r_set_splunk_basic( - template(`splunk-default-template`) - vendor("unassigned") - category("") - filename("messages") - ) { - set("`template`", value(".splunk.template")); - set("`vendor`", value(".splunk.vendor")); - set("`category`", value(".splunk.category")); - set("`filename`", value(".splunk.filename")); - set($FACILITY, value("fields.syslog_facility")); - set($LEVEL, value("fields.syslog_severity")); - set($LOGHOST, value("fields.syslog_server")); - }; diff --git a/package/etc/conf.d/conflib/_common/rfc_syslog.conf b/package/etc/conf.d/conflib/_common/rfc_syslog.conf new file mode 100644 index 0000000..1809034 --- /dev/null +++ b/package/etc/conf.d/conflib/_common/rfc_syslog.conf @@ -0,0 +1,27 @@ +filter f_rfc5424_strict{ + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); +}; +filter f_rfc5424_noversion{ + message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); +}; +rewrite set_rfcnonconformant{ + set("rfc5424_nonconform" value("fields.sc4s_syslog_format")); +}; +rewrite set_rfc5424_strict{ + set("rfc5424_strict" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_strict{ + match("rfc5424_strict" value("fields.sc4s_syslog_format")) +}; +rewrite set_rfc5424_noversion{ + set("rfc5424_noversion" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_noversion{ + match("rfc5424_noversion" value("fields.sc4s_syslog_format")) +}; +rewrite set_rfc3164{ + set("rfc3164" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc3164{ + match("rfc3164" value("fields.sc4s_syslog_format")) +}; \ No newline at end of file diff --git a/package/etc/conf.d/templates/t_templates.conf b/package/etc/conf.d/conflib/_common/templates.conf similarity index 92% rename from package/etc/conf.d/templates/t_templates.conf rename to package/etc/conf.d/conflib/_common/templates.conf index 0190f1f..b07a8a5 100644 --- a/package/etc/conf.d/templates/t_templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -48,5 +48,9 @@ template t_hdr_sdata_msg { # =============================================================================================== template t_JSON { - template("$(format-json --scope all-nv-pairs)"); + template("$(format-json --scope all-nv-pairs + --exclude fields.* + --exclude .splunk.* + --exclude HOST --exclude HOSTFROMgit m + )"); }; diff --git a/package/etc/conf.d/conflib/_common/utility.conf b/package/etc/conf.d/conflib/_common/utility.conf new file mode 100644 index 0000000..8b986c3 --- /dev/null +++ b/package/etc/conf.d/conflib/_common/utility.conf @@ -0,0 +1,4 @@ + +rewrite set_metadata_presume { + set("$(env SYSLOG_PRESUME_FILTER)" value("fields.sc4s_presume")); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf new file mode 100644 index 0000000..cb238de --- /dev/null +++ b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf @@ -0,0 +1,9 @@ +block parser vendor_product_by_source() { + add-contextual-data( + selector(filters("/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf")), + database("context-local/vendor_product_by_source.csv") + ignore-case(yes) + prefix("fields.") + ); +}; + diff --git a/package/etc/conf.d/blocks/splunk_context.conf b/package/etc/conf.d/conflib/_splunk/splunk_context.conf similarity index 74% rename from package/etc/conf.d/blocks/splunk_context.conf rename to package/etc/conf.d/conflib/_splunk/splunk_context.conf index e2e0af9..e2d1fbb 100644 --- a/package/etc/conf.d/blocks/splunk_context.conf +++ b/package/etc/conf.d/conflib/_splunk/splunk_context.conf @@ -1,7 +1,7 @@ block parser p_add_context_splunk(key("syslogng-fallback")) { add-contextual-data( selector("`key`"), - database("context/splunk_index.csv"), + database("context-local/splunk_index.csv"), prefix(".splunk.") ); }; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf new file mode 100644 index 0000000..7d96599 --- /dev/null +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf @@ -0,0 +1,23 @@ +#Used to set index fields we will always use +rewrite r_set_splunk_basic { + set("`index`", value(".splunk.index")); + + set("`sourcetype`", value(".splunk.sourcetype")); + set("`source`", value(".splunk.source")); + set($FACILITY, value("fields.sc4s_syslog_facility")); + set($LEVEL, value("fields.sc4s_syslog_severity")); + set($LOGHOST, value("fields.sc4s_syslog_server")); + set($SOURCEIP, value("fields.sc4s_fromhostip")); +}; + +#rewrite + +#Used by fallback to log everything we can +block rewrite r_set_splunk( + template(`splunk-default-template`) + ) { + set("`template`", value("fields.sc4s_template")); + }; + + + diff --git a/package/etc/conf.d/conflib/vendor_cisco/asa.conf b/package/etc/conf.d/conflib/vendor_cisco/asa.conf new file mode 100644 index 0000000..bbd3157 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_cisco/asa.conf @@ -0,0 +1,3 @@ +filter f_cisco_asa { + message('^%ASA-\d+-\d{1,10}: '); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/vendor_cisco/ios.conf b/package/etc/conf.d/conflib/vendor_cisco/ios.conf new file mode 100644 index 0000000..09a37ed --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_cisco/ios.conf @@ -0,0 +1,8 @@ +rewrite set_metadata_vendor_product_cisco_ios{ + set("cisco_ios" value(".metadata.vendor_product")); +}; + +filter f_cisco_ios{ + match("cisco_ios", value(".metadata.vendor_product") type(glob)); +}; + diff --git a/package/etc/conf.d/conflib/vendor_cisco/nx-os.conf b/package/etc/conf.d/conflib/vendor_cisco/nx-os.conf new file mode 100644 index 0000000..2f9cacc --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_cisco/nx-os.conf @@ -0,0 +1,3 @@ +filter f_cisco_nx_os { + match("cisco_nx_os", value("fields.sc4s_vendor_product") type(glob) ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/vendor_fortinet/fortios.conf b/package/etc/conf.d/conflib/vendor_fortinet/fortios.conf new file mode 100644 index 0000000..6c08a87 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_fortinet/fortios.conf @@ -0,0 +1,3 @@ +filter f_fortinet_fortios { + message('devid=\"?F[G|W|6K].+type=\"?(traffic|utm|event)'); +}; diff --git a/package/etc/conf.d/conflib/vendor_juniper/junos.conf b/package/etc/conf.d/conflib/vendor_juniper/junos.conf new file mode 100644 index 0000000..49ae1de --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_juniper/junos.conf @@ -0,0 +1,8 @@ +filter f_juniper_junos { + match('^\[junos@2636' value("SDATA")) + or match('syslog@juniper.net' value("SDATA")) +}; + +filter f_juniper_junos_legacy { + program("RT_IDP|RT_FLOW|RT_IDS|RT_UTM|Juniper"); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/vendor_juniper/legacy.conf b/package/etc/conf.d/conflib/vendor_juniper/legacy.conf new file mode 100644 index 0000000..8ec73a3 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_juniper/legacy.conf @@ -0,0 +1,19 @@ + + +filter f_juniper_nsm { + match("^juniper_nsm$", value("fields.sc4s_vendor_product")); + +}; +filter f_juniper_nsm_idp { + match("juniper_nsm_idp", value("fields.sc4s_vendor_product") type(glob) ); + +}; + +filter f_juniper_netscreen { + match("juniper_netscreen", value("fields.sc4s_vendor_product") type(glob) ); + +}; + +filter f_juniper_idp { + match("juniper_idp", value("fields.sc4s_vendor_product") type(glob) ); +}; diff --git a/package/etc/conf.d/conflib/vendor_pan/panos.conf b/package/etc/conf.d/conflib/vendor_pan/panos.conf new file mode 100644 index 0000000..da0cb04 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_pan/panos.conf @@ -0,0 +1,3 @@ +filter f_paloalto_panos { + message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf b/package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf new file mode 100644 index 0000000..afaff11 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf @@ -0,0 +1,3 @@ +filter f_symantec_bluecoat_proxy { + program("bluecoat") +}; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/d_destinations.conf b/package/etc/conf.d/destinations/splunk_hec.conf similarity index 92% rename from package/etc/conf.d/destinations/d_destinations.conf rename to package/etc/conf.d/destinations/splunk_hec.conf index 5c59aee..81f3187 100644 --- a/package/etc/conf.d/destinations/d_destinations.conf +++ b/package/etc/conf.d/destinations/splunk_hec.conf @@ -27,7 +27,7 @@ destination d_hec { password("`SPLUNK_HEC_TOKEN`") persist-name("splunk") disk-buffer(mem-buf-length(15000) - disk-buf-size(20000) + disk-buf-size(200000) reliable(no) dir("/opt/syslog-ng/var/data/disk-buffer/")) tls(peer-verify(no) @@ -41,11 +41,11 @@ destination d_hec { ) body('$(format-json time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} + host=${HOST} + source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$(template ${.splunk.template} $(template `splunk-default-template`)) + event=$(template ${fields.sc4s_template} $(template `splunk-default-template`)) fields.*)') ); }; diff --git a/package/etc/conf.d/filters/cisco_asa_tradditional.conf b/package/etc/conf.d/filters/cisco_asa_tradditional.conf deleted file mode 100644 index 57d25ce..0000000 --- a/package/etc/conf.d/filters/cisco_asa_tradditional.conf +++ /dev/null @@ -1,24 +0,0 @@ -log { - source(s_default-ports); - - filter {tags("cisco:asa") - or message('^.{10,60} : %ASA-\d+-\d{1,10}: ') - ;}; - - #Cisco ASA can send 3264 or 5424 either way we are good with basic parsing - parser { - #basic parsing - syslog-parser(time-zone(`default-timezone`)); - }; - - #set the source type based on program field and lookup index from the splunk context csv - parser {p_add_context_splunk(key("cisco:asa")); }; - - #rewrite the final message for splunk json - #sourcetype and index are set in the filter defaults won't be used - rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- - - destination(d_hec); #--HEC-- - - flags(flow-control,final); -}; diff --git a/package/etc/conf.d/filters/juniper_junos.conf b/package/etc/conf.d/filters/juniper_junos.conf deleted file mode 100644 index 3a205e4..0000000 --- a/package/etc/conf.d/filters/juniper_junos.conf +++ /dev/null @@ -1,65 +0,0 @@ -# =============================================================================================== -# Juniper Standard and Structured logging -# =============================================================================================== - -log { - - source(s_default-ports); - -# For newer (structured) format; RFC 5424 compliant. The existing TA will not work for this format; these filters -# are included for future TA development. The exception is NSM IDP, for which we're awaiting a data -# sample to confirm that it follows the RFC 5424 format. -# If it is determined that NSM IDP ("syslog@juniper.net") messages are indeed 3164 format, remove the check for -# the "syslog@juniper.net" message below. It will then be caught by the standard section following. - -junction { - channel { - filter { tags("juniper_structured") or message('junos@2636') or message('syslog@juniper.net'); }; - parser { syslog-parser(flags(syslog-protocol)); }; - if (tags("juniper_JunOS_IDP") or program('RT_IDP')) { - parser {p_add_context_splunk(key("juniper:junos:idp:structured")); }; - } elif (tags("juniper_JunOS_Firewall") or program('RT_FLOW|RT_IDS|RT_UTM')) { - parser {p_add_context_splunk(key("juniper:junos:firewall:structured")); }; - } - elif (tags("juniper_idp") or program('Jnpr')) { - parser {p_add_context_splunk(key("juniper:idp:structured")); }; - } - else { - parser {p_add_context_splunk(key("juniper:structured")); }; - }; - rewrite { r_set_splunk_basic(template("t_hdr_sdata_msg"))}; - destination(d_hec); #--HEC-- - flags(final); - }; - -# For legacy "standard" format. The existing Juniper TA will work for these data sources. -# Remove the message filter below when tag support is implemented. - - channel { - filter { tags("juniper_standard") or message('RT_IDP:|RT_FLOW:|RT_IDS:|RT_UTM:|Juniper:|traffic\,\s+traffic\s+log|syslog@juniper.net|predefined\,\s+|NetScreen\s+device_id\=') }; -# filter { tags("juniper_standard") }; - parser { syslog-parser(time-zone(`default-timezone`)); }; - if (program('RT_IDP')) { - parser {p_add_context_splunk(key("juniper:junos:idp")); }; - } elif (program('RT_FLOW|RT_IDS|RT_UTM')) { - parser {p_add_context_splunk(key("juniper:junos:firewall")); }; - } elif (program('Juniper')) { - parser {p_add_context_splunk(key("juniper:sslvpn")); }; - } elif (message('traffic\,\s+traffic\s+log')) { - parser {p_add_context_splunk(key("juniper:nsm")); }; - } elif (message('syslog@juniper.net')) { - parser {p_add_context_splunk(key("juniper:idp")); }; - } elif (message('predefined\,\s+')) { - parser {p_add_context_splunk(key("juniper:nsm:idp")); }; - } elif (message('NetScreen\s+device_id\=')) { - parser {p_add_context_splunk(key("juniper:netscreen")); }; - } - else { - parser {p_add_context_splunk(key("juniper:legacy")); }; - }; - rewrite { r_set_splunk_basic(template("t_standard")) }; - destination(d_hec); #--HEC-- - flags(final); - }; - }; -}; diff --git a/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf b/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf new file mode 100644 index 0000000..377820c --- /dev/null +++ b/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf @@ -0,0 +1,15 @@ +log { + + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_nsm_idp); + + parser { + p_add_context_splunk(key("juniper:nsm:idp")); + }; + rewrite { + r_set_splunk(template("t_standard")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; diff --git a/package/etc/conf.d/log_paths/internal.conf b/package/etc/conf.d/log_paths/internal.conf new file mode 100644 index 0000000..879925f --- /dev/null +++ b/package/etc/conf.d/log_paths/internal.conf @@ -0,0 +1,35 @@ +log { + source(s_internal); + + if (match("^Log statistics; " value("MESSAGE"))) { + + parser {p_add_context_splunk(key("syslog-ng:metrics")); }; + rewrite { + subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', + ' + {"time": "$S_UNIXTIME.$S_MSEC", + "event": "metric", + "host": "$HOST", + "index": "${.splunk.index}", + "source": "internal", + "sourcetype": "${.splunk.sourcetype}", + "fields": { + "source_name": "${SourceName}", + "source_instance": "${SourceInstance}", + "state": "${State}", + "type": "${Type}", + "_value": ${Number}, + "metric_name": "syslogng.${SourceId}" + } + } + ', + value("MESSAGE") flags("utf8" "global") + ); + }; + destination(d_hecmetrics); #--HEC-- + } else { + parser {p_add_context_splunk(key("syslog-ng:events")); }; + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- + destination(d_hec); #--HEC-- + }; +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf new file mode 100644 index 0000000..e9f018c --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf @@ -0,0 +1,14 @@ +log { + + source(s_default-ports); + filter(f_cisco_nx_os); + + parser { + p_add_context_splunk(key("cisco:nx-os")); + }; + rewrite { + r_set_splunk(template("t_hdr_msg")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf new file mode 100644 index 0000000..428cb0f --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf @@ -0,0 +1,21 @@ + +# =============================================================================================== +# Cisco ASA (Firewall) +# =============================================================================================== + +log { + source(s_default-ports); + + filter(f_is_rfc3164); + filter(f_cisco_asa); + #set the source type based on program field and lookup index from the splunk context csv + + parser {p_add_context_splunk(key("cisco:asa")); }; + + #Using the 5424 parser the message content is all we need + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco_ios.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf similarity index 60% rename from package/etc/conf.d/filters/cisco_ios.conf rename to package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf index cac0bb7..780130e 100644 --- a/package/etc/conf.d/filters/cisco_ios.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf @@ -1,3 +1,4 @@ + # =============================================================================================== # Cisco IOS (Route/Switch) # =============================================================================================== @@ -5,12 +6,16 @@ log { source(s_default-ports); + filter(f_cisco_ios); - parser { cisco-parser(); }; - parser {p_add_context_splunk(key("cisco:ios")); }; - rewrite { r_set_splunk_basic(template("t_msg_only"))}; #--HEC-- + parser { + p_add_context_splunk(key("cisco:ios")); + }; + rewrite { + r_set_splunk(template("t_msg_only")) #--HEC-- + }; destination(d_hec); #--HEC-- - flags(flow-control,final); + flags(flow-control); }; diff --git a/package/etc/conf.d/filters/fortinet_fgt.conf b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf similarity index 80% rename from package/etc/conf.d/filters/fortinet_fgt.conf rename to package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf index d6205b5..d81c565 100644 --- a/package/etc/conf.d/filters/fortinet_fgt.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf @@ -1,12 +1,9 @@ log { source(s_default-ports); - - filter {tags("fgt_log") - or message('devid=\"?F[G|W|6K].+type=\"?(traffic|utm|event)'); - }; + filter(f_is_rfc3164); + filter(f_fortinet_fortios); parser { - syslog-parser(); kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone(`default-timezone`)); }; @@ -26,8 +23,9 @@ log { #rewrite the final message for splunk json #sourcetype and index are set in the filter defaults won't be used - rewrite {r_set_splunk_basic(template("t_standard")) }; #--HEC-- + rewrite {r_set_splunk(template("t_standard")) }; #--HEC-- destination(d_hec); #--HEC-- - flags(flow-control,final); + flags(flow-control); }; + diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf new file mode 100644 index 0000000..4b84403 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf @@ -0,0 +1,15 @@ +log { + + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_idp); + + parser { + p_add_context_splunk(key("juniper:idp")); + }; + rewrite { + r_set_splunk(template("t_standard")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf new file mode 100644 index 0000000..e689cf6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf @@ -0,0 +1,24 @@ +# =============================================================================================== +# Juniper Standard and Structured logging +# =============================================================================================== + +log { + + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_junos_legacy); + + if (program('RT_IDP')) { + parser {p_add_context_splunk(key("juniper:junos:idp")); }; + } elif (program('RT_FLOW|RT_IDS|RT_UTM')) { + parser {p_add_context_splunk(key("juniper:junos:firewall")); }; + } elif (program('Juniper')) { + parser {p_add_context_splunk(key("juniper:sslvpn")); }; + } else { + parser {p_add_context_splunk(key("juniper:legacy")); }; + }; + + rewrite { r_set_splunk(template("t_standard")) }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf new file mode 100644 index 0000000..efd046c --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf @@ -0,0 +1,14 @@ +log { + + source(s_default-ports); + filter(f_juniper_netscreen); + + parser { + p_add_context_splunk(key("juniper:netscreen")); + }; + rewrite { + r_set_splunk(template("t_standard")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf new file mode 100644 index 0000000..04810cc --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf @@ -0,0 +1,19 @@ +# =============================================================================================== +# Juniper NSM +# =============================================================================================== + +log { + + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_nsm); + + parser { + p_add_context_splunk(key("juniper:nsm")); + }; + rewrite { + r_set_splunk(template("t_standard")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; diff --git a/package/etc/conf.d/filters/paloalto_networks.conf b/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf similarity index 69% rename from package/etc/conf.d/filters/paloalto_networks.conf rename to package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf index a3e1016..956f747 100644 --- a/package/etc/conf.d/filters/paloalto_networks.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf @@ -1,22 +1,21 @@ - - log { source(s_default-ports); + filter(f_is_rfc3164); + filter(f_paloalto_panos); - filter {tags("pan:log") - or message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),') - ;}; - - # the palo message does not include a program value in the header unfortunatly + # the palo message does not include a program value in the header unfortunatly # the use of colon in the message tricks the RFC3164 parser we will re-write the message # so the parser will not incorrectly break it apart. # while we are at it we will save the mesage type into the program field so parser can find it rewrite { - subst('^([^ ]+ \w+ \d{2} \d{2}:\d{2}:\d{2} [^ ]+)( \d+,[^,]+,[^,]+,)([^,]+)(.*)', '$1 $3:$2$3$4', value("MESSAGE") ); + set("${LEGACY_MSGHDR}${MESSAGE}" value("MESSAGE")); + unset(value("LEGACY_MSGHDR")); + unset(value("PROGRAM")); + }; parser { #basic parsing - syslog-parser(time-zone(`default-timezone`)); + #we need to actual even time from the field GeneratedTime use csv parser to get it out csv-parser( columns('FUTURE_USE', 'ReceiveTime', 'SerialNumber', 'Type', 'Subtype', 'FUTURE_USE2', 'GeneratedTime') @@ -33,19 +32,19 @@ log { }; #set the source type based on program field and lookup index from the splunk context csv - if (program('THREAT')) { + if (message(',\d+,THREAT')) { parser {p_add_context_splunk(key("pan:threat")); }; - } elif (program('TRAFFIC')) { + } elif (message(',\d+,TRAFFIC')) { parser {p_add_context_splunk(key("pan:traffic")); }; - } elif (program('SYSTEM')) { + } elif (message(',\d+,SYSTEM')) { parser {p_add_context_splunk(key("pan:system")); }; - } elif (program('CONFIG')) { + } elif (message(',\d+,CONFIG')) { parser {p_add_context_splunk(key("pan:config")); }; - } elif (program('HIPWATCH')) { + } elif (message(',\d+,HIPWATCH')) { parser {p_add_context_splunk(key("pan:hipwatch")); }; - } elif (program('CORRELATION')) { + } elif (message(',\d+,CORRELATION')) { parser {p_add_context_splunk(key("pan:correlation")); }; - } elif (program('USERID')) { + } elif (message(',\d+,USERID')) { parser {p_add_context_splunk(key("pan:userid")); }; } else { parser {p_add_context_splunk(key("pan:log")); }; @@ -53,9 +52,10 @@ log { #rewrite the final message for splunk json #sourcetype and index are set in the filter defaults won't be used - rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- destination(d_hec); #--HEC-- - flags(flow-control,final); + flags(flow-control); }; + diff --git a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf new file mode 100644 index 0000000..295669a --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf @@ -0,0 +1,16 @@ +log { + source(s_default-ports); + filter(f_is_rfc5424_noversion); + filter(f_cisco_asa); + + #set the source type based on program field and lookup index from the splunk context csv + + parser {p_add_context_splunk(key("cisco:asa")); }; + + #Using the 5424 parser the message content is all we need + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf new file mode 100644 index 0000000..4d2b8e4 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf @@ -0,0 +1,16 @@ +log { + source(s_default-ports); + filter(f_is_rfc5424_noversion); + filter(f_symantec_bluecoat_proxy); + + #set the source type based on program field and lookup index from the splunk context csv + + parser {p_add_context_splunk(key("bluecoat:proxysg:access:kv")); }; + + #Using the 5424 parser the message content is all we need + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf new file mode 100644 index 0000000..c49854b --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf @@ -0,0 +1,27 @@ +# =============================================================================================== +# Juniper Structured logging +# =============================================================================================== + +log { + + source(s_default-ports); + filter(f_is_rfc5424_strict); + filter(f_juniper_junos); + + if (program('RT_IDP')) { + parser {p_add_context_splunk(key("juniper:junos:idp:structured")); }; + } elif (program('RT_FLOW|RT_IDS|RT_UTM')) { + parser {p_add_context_splunk(key("juniper:junos:firewall:structured")); }; + } + elif (program('Jnpr')) { + parser {p_add_context_splunk(key("juniper:idp:structured")); }; + } + else { + parser {p_add_context_splunk(key("juniper:structured")); }; + }; + + rewrite { r_set_splunk(template("t_hdr_sdata_msg"))}; + destination(d_hec); #--HEC-- + + +}; diff --git a/package/etc/conf.d/log_paths/zfallback.conf b/package/etc/conf.d/log_paths/zfallback.conf new file mode 100644 index 0000000..4b1b5d5 --- /dev/null +++ b/package/etc/conf.d/log_paths/zfallback.conf @@ -0,0 +1,11 @@ +log { + source(s_default-ports); + + parser { + p_add_context_splunk(key("syslog-ng:fallback")); + }; + + rewrite { r_set_splunk(template("t_JSON"))}; #--HEC-- + destination(d_hec); #--HEC-- + flags(flow-control,fallback); +}; diff --git a/package/etc/conf.d/sources/network.conf b/package/etc/conf.d/sources/network.conf index e3ed6a9..70a5f7b 100644 --- a/package/etc/conf.d/sources/network.conf +++ b/package/etc/conf.d/sources/network.conf @@ -11,51 +11,65 @@ # source s_network-defaults { default-network-drivers() }; source s_default-ports { - `splunk-udp-driver` ( - transport("udp") - port(`udp-listening-port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=`udp-listening-port`/udp") -# teardown("firewall-cmd --permanent --remove-port=`udp-listening-port`/udp") -# ) - so-rcvbuf(`splunk-rcvbuf`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); + channel { + source { + `splunk-udp-driver` ( + transport("udp") + port(`udp-listening-port`) + ip-protocol(`ip-version`) + so-rcvbuf(`splunk-rcvbuf`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); - `splunk-tcp-driver` ( - transport("tcp") - port(`tcp-listening-port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=`tcp-listening-port`/tcp") -# teardown("firewall-cmd --permanent --remove-port=`tcp-listening-port`/tcp") -# ) - max-connections(`splunk-max-connections`) - log-iw-size(`splunk-window-size`) - log-fetch-limit(`splunk-fetch-limit`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - }; + `splunk-tcp-driver` ( + transport("tcp") + port(`tcp-listening-port`) + ip-protocol(`ip-version`) + max-connections(`splunk-max-connections`) + log-iw-size(`splunk-window-size`) + log-fetch-limit(`splunk-fetch-limit`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); + }; + #TODO: #60 Remove this function with enhancement + rewrite(set_metadata_presume); + rewrite(set_rfcnonconformant); + rewrite(r_set_splunk_basic); -source s_all-ports { - `splunk-udp-driver` ( transport("udp") port(514) flags(no-parse) persist-name("example-persist-name1")); - `splunk-udp-driver` ( transport("udp") port(919) flags(no-parse)); - `splunk-udp-driver` ( transport("udp") port(920) flags(no-parse)); - `splunk-udp-driver` ( transport("udp") port(921) flags(no-parse)); - - `splunk-tcp-driver` ( transport("tcp") port(919) flags(no-parse)); - `splunk-tcp-driver` ( transport("tcp") port(920) flags(no-parse)); - `splunk-tcp-driver` ( transport("tcp") port(921) flags(no-parse)); - }; + if { + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_strict); + } elif { + filter(f_rfc5424_noversion); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_noversion); + } elif { + parser {cisco-parser()}; + rewrite(set_metadata_vendor_product_cisco_ios); + } else { + parser { + syslog-parser(time-zone(`default-timezone`) flags(store-raw-message)); + }; + rewrite(set_rfc3164); + }; + parser { + vendor_product_by_source(); + }; + }; +}; diff --git a/package/etc/context/splunk_index.csv b/package/etc/context-local/splunk_index.csv similarity index 64% rename from package/etc/context/splunk_index.csv rename to package/etc/context-local/splunk_index.csv index e495491..e3d7f9d 100644 --- a/package/etc/context/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -1,46 +1,50 @@ -cisco:asa,index,main +bluecoat:proxysg:access:kv,index,netproxy +bluecoat:proxysg:access:kv,sourcetype,bluecoat:proxysg:access:kv +cisco:asa,index,netfw cisco:asa,sourcetype,cisco:asa -cisco:ios,index,main +cisco:ios,index,netops cisco:ios,sourcetype,cisco:ios +cisco:nx-os,index,netops +cisco:nx-os,sourcetype,cisco:ios fgt_event,sourcetype,fgt_event -fgt_event,index,main +fgt_event,index,netops fgt_log,sourcetype,fgt_log -fgt_log,index,main +fgt_log,index,netops fgt_traffic,sourcetype,fgt_traffic -fgt_traffic,index,main +fgt_traffic,index,netfw fgt_utm,sourcetype,fgt_utm -fgt_utm,index,main +fgt_utm,index,netids juniper:structured,index,main juniper:structured,sourcetype,juniper:structured -juniper:junos:firewall:structured,index,main +juniper:junos:firewall:structured,index,netfw juniper:junos:firewall:structured,sourcetype,juniper:junos:firewall:structured -juniper:junos:idp:structured,index,main +juniper:junos:idp:structured,index,netids juniper:junos:idp:structured,sourcetype,juniper:junos:idp:structured -juniper:junos:firewall,index,main +juniper:junos:firewall,index,netfw juniper:junos:firewall,sourcetype,juniper:junos:firewall -juniper:junos:idp,index,main +juniper:junos:idp,index,netids juniper:junos:idp,sourcetype,juniper:junos:idp -juniper:nsm:idp,index,main +juniper:nsm:idp,index,netids juniper:nsm:idp,sourcetype,juniper:nsm:idp -juniper:sslvpn,index,main +juniper:sslvpn,index,netfw juniper:sslvpn,sourcetype,juniper:sslvpn -juniper:netscreen,index,main +juniper:netscreen,index,netfw juniper:netscreen,sourcetype,netscreen:firewall -juniper:idp,index,main +juniper:idp,index,netids juniper:idp,sourcetype,juniper:idp -juniper:idp:structured,index,main +juniper:idp:structured,index,netids juniper:idp:structured,sourcetype,juniper:idp:structured -juniper:nsm,index,main +juniper:nsm,index,netids juniper:nsm,sourcetype,juniper:nsm -juniper:legacy,index,main +juniper:legacy,index,netfw juniper:legacy,sourcetype,juniper:legacy -pan:traffic,index,main -pan:threat,index,main -pan:system,index,main -pan:config,index,main +pan:traffic,index,netfw +pan:threat,index,netproxy +pan:system,index,netops +pan:config,index,netops pan:hipwatch,index,main pan:correlation,index,main -pan:userid,index,main +pan:userid,index,netops pan:unknown,index,main pan:traffic,sourcetype,pan:traffic pan:threat,sourcetype,pan:threat diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/context-local/vendor_product_by_source.conf new file mode 100644 index 0000000..4da1866 --- /dev/null +++ b/package/etc/context-local/vendor_product_by_source.conf @@ -0,0 +1,29 @@ +@version: 3.22 +#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a +#bug/limitation of selector files. The better syntax should be as follows +#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; + +filter f_test_test { + host("testvp-*" type(glob)) + or match("test_test" value("fields.sc4s_presume")) +}; +filter f_juniper_nsm { + host("jnpnsm-*" type(glob)) + or match("juniper_nsm" value("fields.sc4s_presume")) +}; +filter f_juniper_nsm_idp { + host("jnpnsmidp-*" type(glob)) + or match("juniper_nsm_idp" value("fields.sc4s_presume")) +}; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) + or match("juniper_idp" value("fields.sc4s_presume")) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob) ) + or match("juniper_netscreen" value("fields.sc4s_presume")) +}; +filter f_cisco_nx_os { + host("csconx-*" type(glob) ) + or match("cisco_nx_os" value("fields.sc4s_presume")) +}; \ No newline at end of file diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/context-local/vendor_product_by_source.csv new file mode 100644 index 0000000..5193616 --- /dev/null +++ b/package/etc/context-local/vendor_product_by_source.csv @@ -0,0 +1,6 @@ +f_test_test,sc4s_vendor_product,"test_test" +f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" +f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" +f_juniper_idp,sc4s_vendor_product,"juniper_idp" +f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" \ No newline at end of file diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 565b79e..27e1a6e 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -48,7 +48,7 @@ options { # =============================================================================================== # Splunk metadata (HEC/Kafka transport only) # =============================================================================================== -@define splunk-sourcetype "syslog:fallback" +@define splunk-sourcetype "syslog-ng:fallback" @define splunk-index "main" # =============================================================================================== @@ -74,61 +74,16 @@ options { # =============================================================================================== @include "scl.conf" @include "conf.d/templates/*.conf" +@include "conf.d/conflib/*.conf" +@include "conf.d/conflib/*/*.conf" +@include "conf.d/conflib/blocks/*.conf" +@include "conf.d/conflib/blocks/*/*.conf" + @include "conf.d/sources/*.conf" -@include "conf.d/blocks/*.conf" @include "conf.d/destinations/*.conf" -log { - source(s_internal); - - if (match("^Log statistics; " value("MESSAGE"))) { - - parser {p_add_context_splunk(key("syslog-ng:metrics")); }; - rewrite { - subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', - ' - {"time": "$S_UNIXTIME.$S_MSEC", - "event": "metric", - "host": "$HOST", - "index": "${.splunk.index}", - "source": "internal", - "sourcetype": "${.splunk.sourcetype}", - "fields": { - "source_name": "${SourceName}", - "source_instance": "${SourceInstance}", - "state": "${State}", - "type": "${Type}", - "_value": ${Number}, - "metric_name": "syslogng.${SourceId}" - } - } - ', - value("MESSAGE") flags("utf8" "global") - ); - }; - destination(d_hecmetrics); #--HEC-- - } else { - parser {p_add_context_splunk(key("syslog-ng:events")); }; - rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- - destination(d_hec); #--HEC-- - }; - flags(final); -}; - - -@include "conf.d/prefilters/*.conf" -@include "conf.d/filters/*.conf" +@include "conf.d/log_paths/*.conf" +@include "conf.d/plugin/*/log_paths/*.conf" -@include "conf.d/plugin/*/blocks/*.conf" -@include "conf.d/plugin/*/prefilters/*.conf" -@include "conf.d/plugin/*/filters/*.conf" - -log { - source(s_default-ports); - parser {p_add_context_splunk(key("syslog-ng:fallback")); }; - rewrite { r_set_splunk(template("t_JSON"))}; #--HEC-- - destination(d_hec); #--HEC-- - flags(flow-control,fallback); -}; diff --git a/splunk/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf similarity index 53% rename from splunk/SA-syslog-ng/default/indexes.conf rename to splunk/etc/apps/SA-syslog-ng/default/indexes.conf index e795078..f8dbfb2 100644 --- a/splunk/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -19,3 +19,24 @@ thawedPath = $SPLUNK_DB/test/thaweddb homePath = $SPLUNK_DB/test2/db coldPath = $SPLUNK_DB/test2/colddb thawedPath = $SPLUNK_DB/test2/thaweddb + + +[netfw] +homePath = $SPLUNK_DB/netfw/db +coldPath = $SPLUNK_DB/netfw/colddb +thawedPath = $SPLUNK_DB/netfw/thaweddb + +[netids] +homePath = $SPLUNK_DB/netids/db +coldPath = $SPLUNK_DB/netids/colddb +thawedPath = $SPLUNK_DB/netids/thaweddb + +[netops] +homePath = $SPLUNK_DB/netops/db +coldPath = $SPLUNK_DB/netops/colddb +thawedPath = $SPLUNK_DB/netops/thaweddb + +[netproxy] +homePath = $SPLUNK_DB/netproxy/db +coldPath = $SPLUNK_DB/netproxy/colddb +thawedPath = $SPLUNK_DB/netproxy/thaweddb \ No newline at end of file diff --git a/test-with-compose.sh b/test-with-compose.sh index 3f08cbf..b569b7f 100755 --- a/test-with-compose.sh +++ b/test-with-compose.sh @@ -8,16 +8,19 @@ mkdir test-results docker volume create sc4s-tests docker volume create sc4s-results +docker volume create splunk-etc docker container create --name dummy \ -v sc4s-tests:/work/tests \ -v sc4s-results:/work/test-results \ + -v splunk-etc:/work/splunk-etc \ registry.access.redhat.com/ubi7/ubi docker cp tests/ dummy:/work/tests/ +docker cp ./splunk/etc/* dummy:/work/splunk-etc/ docker rm dummy docker-compose build -docker-compose up --abort-on-container-exit --exit-code-from test +docker-compose up --abort-on-container-exit --exit-code-from test docker container create --name dummy \ -v sc4s-tests:/work/tests \ diff --git a/tests/Dockerfile b/tests/Dockerfile index cb630f3..ddb8c55 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -13,5 +13,7 @@ COPY requirements.txt / RUN pip3 install -r /requirements.txt RUN mkdir -p /work/tests RUN mkdir -p /work/test-results/functional +COPY entrypoint.sh / +COPY wait-for /bin/ #WORKDIR /work -CMD cd /work;python -m pytest --junitxml=/work/test-results/functional/functional.xml +CMD /entrypoint.sh \ No newline at end of file diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh new file mode 100755 index 0000000..646dd56 --- /dev/null +++ b/tests/entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/sh +echo Check for sc4s +wait-for sc4s:514 -t 0 -- echo scs is up +echo check for splunk web +wait-for splunk:8000 -t 0 -- echo splunkweb is up +echo check for splunk mgmt +wait-for splunk:8089 -t 0 -- echo splunkmgmt is up +echo check for splunk hec +wait-for splunk:8088 -t 0 -- echo splunkhec is up + +sleep 30 + +echo Check for sc4s +wait-for sc4s:514 -t 0 -- echo scs is up +echo check for splunk web +wait-for splunk:8000 -t 0 -- echo splunkweb is up +echo check for splunk mgmt +wait-for splunk:8089 -t 0 -- echo splunkmgmt is up +echo check for splunk hec +wait-for splunk:8088 -t 0 -- echo splunkhec is up + + +cd /work;python -m pytest --junitxml=/work/test-results/functional/functional.xml diff --git a/tests/requirements.txt b/tests/requirements.txt index 149b7a8..cc706aa 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,4 +11,3 @@ jinja2 jinja2-time http://dev.splunk.com/goto/sdk-python flake8 -flaky diff --git a/tests/sendmessage.py b/tests/sendmessage.py index 6388c78..6b591ad 100644 --- a/tests/sendmessage.py +++ b/tests/sendmessage.py @@ -8,9 +8,9 @@ from time import sleep -def sendsingle(message): +def sendsingle(message, host="sc4s", port=514): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_address = ('sc4s', 514) + server_address = (host, port) tried = 0 while True: diff --git a/tests/splunkutils.py b/tests/splunkutils.py index 7227ae9..bb77a58 100644 --- a/tests/splunkutils.py +++ b/tests/splunkutils.py @@ -33,5 +33,5 @@ def splunk_single(service, search): break else: tried += 1 - sleep(1) + sleep(5) return resultCount, eventCount diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index 308356a..7203f32 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause import random -from flaky import flaky from jinja2 import Environment from .sendmessage import * @@ -17,17 +16,16 @@ # Apr 15 2017 00:21:14 192.168.12.1 : %ASA-5-111010: User 'john', running 'CLI' from IP 0.0.0.0, executed 'dir disk0:/dap.xml' # Apr 15 2017 00:22:27 192.168.12.1 : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:81.24.28.226 dst inside:72.142.17.10 (type 3, code 0) on outside interface. Original IP payload: udp src 72.142.17.10/40998 dst 194.153.237.66/53. # Apr 15 2017 00:22:42 192.168.12.1 : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 -@flaky(max_runs=3, min_passes=2) def test_cisco_asa_tradditional(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") message = mt.render(mark="<111>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:asa\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -40,17 +38,16 @@ def test_cisco_asa_tradditional(record_property, setup_wordlist, setup_splunk): # <166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 -@flaky(max_runs=3, min_passes=2) def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/5424 to outside:72.142.18.38/23\n") + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} : %ASA-3-005424: TCP access denied by ACL from 179.236.133.160/5424 to outside:72.142.18.38/23\n") message = mt.render(mark="<166>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:asa\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-005424\"| head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index f729df9..13ec3f5 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause from jinja2 import Environment -from flaky import flaky from .sendmessage import * from .splunkutils import * @@ -14,7 +13,6 @@ # <190>30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated -@flaky(max_runs=3, min_passes=2) def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key @@ -24,7 +22,7 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") + st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_cisco_nx_os.py b/tests/test_cisco_nx_os.py new file mode 100644 index 0000000..27b1155 --- /dev/null +++ b/tests/test_cisco_nx_os.py @@ -0,0 +1,55 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +# Nov 1 14:07:58 excal-113 %MODULE-5-MOD_OK: Module 1 is online +def test_cisco_nx_os(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} csconx-{{ host }} %MODULE-5-MOD_OK: Module 1 is online") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netops host=\"csconx-{{ host }}\" sourcetype=\"cisco:ios\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Nov 1 14:07:58 excal-113 %MODULE-5-MOD_OK: Module 1 is online +# @pytest.mark.xfail +#def test_cisco_nx_os_singleport(record_property, setup_wordlist, get_host_key, setup_splunk): +# host = get_host_key +# +# mt = env.from_string( +# "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} %MODULE-5-MOD_OK: Module 1 is online") +# message = mt.render(mark="<23>", host=host) +# +# sendsingle(message, host="sc4s-nx-os") +# +# st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") +# search = st.render(host=host) +# +# resultCount, eventCount = splunk_single(setup_splunk, search) +# +# record_property("host", host) +# record_property("resultCount", resultCount) +# record_property("message", message) +# +# assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_common.py b/tests/test_common.py new file mode 100644 index 0000000..24cb44a --- /dev/null +++ b/tests/test_common.py @@ -0,0 +1,51 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +def test_defaultroute(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4sdefault[0]: test\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"syslog-ng:fallback\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_tag(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} sc4sdefault[0]: test\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"testvp-{{ host }}\" sourcetype=\"syslog-ng:fallback\" sc4s_vendor_product=test_test | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index 60fdae7..f68f13a 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause import random -from flaky import flaky from jinja2 import Environment from .sendmessage import * @@ -13,18 +12,17 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -@flaky(max_runs=3, min_passes=2) #<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") + "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") + st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -35,17 +33,16 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -@flaky(max_runs=3, min_passes=2) #<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") + "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") message = mt.render(mark="<13>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -56,17 +53,16 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -@flaky(max_runs=3, min_passes=2) #<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:21:40 logver=52 devname=US-Corp_Main1 devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user="" srcip=172.30.16.119 srcport=53235 srcintf="Internal" dstip=114.112.67.75 dstport=80 dstintf="External-SDC" proto=6 service=HTTP hostname="popo.wan.ijinshan.com" profile="scan" action=passthrough reqtype=direct url="/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl" sentbyte=525 rcvdbyte=325 direction=outgoing msg="URL belongs to an allowed category in policy" method=domain cat=52 catdesc="Information Technology" def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") + "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_juniper.py b/tests/test_juniper.py index c69f6c4..c4bdb81 100644 --- a/tests/test_juniper.py +++ b/tests/test_juniper.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause from jinja2 import Environment -from flaky import flaky from .sendmessage import * from .splunkutils import * @@ -13,138 +12,17 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -# <165>1 2007-02-15T09:17:15.719Z router1 mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username="user"] User 'user' exiting configuration mode -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_junos_structured(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username=\"user\"] User 'user' exiting configuration mode\n") - message = mt.render(mark="<165>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:structured\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <165>1 2007-02-15T09:17:15.719Z idp1 RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time="1507845354" message-type="SIG" source-address="183.78.180.27" source-port="45610" destination-address="118.127.xx.xx" destination-port="80" protocol-name="TCP" service-name="SERVICE_IDP" application-name="HTTP" rule-name="9" rulebase-name="IPS" policy-name="Recommended" export-id="15229" repeat-count="0" action="DROP" threat-severity="HIGH" attack-name="TROJAN:ZMEU-BOT-SCAN" nat-source-address="0.0.0.0" nat-source-port="0" nat-destination-address="172.xx.xx.xx" nat-destination-port="0" elapsed-time="0" inbound-bytes="0" outbound-bytes="0" inbound-packets="0" outbound-packets="0" source-zone-name="sec-zone-name-internet" source-interface-name="reth0.XXX" destination-zone-name="dst-sec-zone1-outside" destination-interface-name="reth1.xxx" packet-log-id="0" alert="no" username="N/A" roles="N/A" message="-"] -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_junos_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time=\"1507845354\" message-type=\"SIG\" source-address=\"183.78.180.27\" source-port=\"45610\" destination-address=\"118.127.xx.xx\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"9\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"15229\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"TROJAN:ZMEU-BOT-SCAN\" nat-source-address=\"0.0.0.0\" nat-source-port=\"0\" nat-destination-address=\"172.xx.xx.xx\" nat-destination-port=\"0\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"sec-zone-name-internet\" source-interface-name=\"reth0.XXX\" destination-zone-name=\"dst-sec-zone1-outside\" destination-interface-name=\"reth1.xxx\" packet-log-id=\"0\" alert=\"no\" username=\"N/A\" roles=\"N/A\" message=\"-\"]") - message = mt.render(mark="<165>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:idp:structured\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <165>1 2010-06-23T18:05:55 10.209.83.9 Jnpr Syslog 23414 1 [syslog@juniper.net dayId="20100623" recordId="0" timeRecv="2010/06/23 18:05:55" timeGen="2010/06/23 18:05:51" domain="" devDomVer2="0" device_ip="10.209.83.9" cat="Config" attack="" srcZn="NULL" srcIntf="" srcAddr="0.0.0.0" srcPort="0" natSrcAddr="NULL" natSrcPort="0" dstZn="NULL" dstIntf="NULL" dstAddr="0.0.0.0" dstPort="0" natDstAddr="NULL" natDstPort="0" protocol="IP" ruleDomain="" ruleVer="0" policy="" rulebase="NONE" ruleNo="0" action="NONE" severity="INFO" alert="no" elaspedTime="0" inbytes="0" outbytes="0" totBytes="0" inPak="0" outPak="0" totPak="0" repCount="0" packetData="no" varEnum="0" misc="Interaface eth2,eth3 is in Normal State" user="NULL" app="NULL" uri="NULL"] -# -# -# -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} Jnpr Syslog 23414 [syslog@juniper.net dayId=\"20100623\" recordId=\"0\" timeRecv=\"2010/06/23 18:05:55\" timeGen=\"2010/06/23 18:05:51\" domain=\"\" devDomVer2=\"0\" device_ip=\"10.209.83.9\" cat=\"Config\" attack=\"\" srcZn=\"NULL\" srcIntf=\"\" srcAddr=\"0.0.0.0\" srcPort=\"0\" natSrcAddr=\"NULL\" natSrcPort=\"0\" dstZn=\"NULL\" dstIntf=\"NULL\" dstAddr=\"0.0.0.0\" dstPort=\"0\" natDstAddr=\"NULL\" natDstPort=\"0\" protocol=\"IP\" ruleDomain=\"\" ruleVer=\"0\" policy=\"\" rulebase=\"NONE\" ruleNo=\"0\" action=\"NONE\" severity=\"INFO\" alert=\"no\" elaspedTime=\"0\" inbytes=\"0\" outbytes=\"0\" totBytes=\"0\" inPak=\"0\" outPak=\"0\" totPak=\"0\" repCount=\"0\" packetData=\"no\" varEnum=\"0\" misc=\"Interaface eth2,eth3 is in Normal State\" user=\"NULL\" app=\"NULL\" uri=\"NULL\"]") - message = mt.render(mark="<165>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:idp:structured\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - # <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_junos_fw_structured(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_FLOW - RT_FLOW_SESSION_CREATE_LS [junos@2636.1.1.1.2.26 logical-system-name=\"test-lsys\" source-address=\"10.10.10.100\" source-port=\"4206\" destination-address=\"10.20.20.15\" destination-port=\"445\" service-name=\"junos-smb\" nat-source-address=\"10.10.10.100\" nat-source-port=\"4206\" nat-destination-address=\"10.20.20.15\" nat-destination-port=\"445\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"123\" source-zone-name=\"TEST1\" destination-zone-name=\"TEST2\" session-id-32=\"14285714\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth1.100\"]") - message = mt.render(mark="<23>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall:structured\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - - -# <23> Mar 18 17:56:52 RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION="URL Permitted" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY="Enhanced_Information_Technology" REASON="BY_PRE_DEFINED" PROFILE="UTM-Wireless-Profile" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_utm_standard(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION=\"URL Permitted\" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY=\"Enhanced_Information_Technology\" REASON=\"BY_PRE_DEFINED\" PROFILE=\"UTM-Wireless-Profile\" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A") - message = mt.render(mark="<23>", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail def test_juniper_nsm_standard(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} jnpnsm-{{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") message = mt.render(mark="<134>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:nsm\" | head 2") + st = env.from_string("search index=netids host=\"jnpnsm-{{ host }}\" sourcetype=\"juniper:nsm\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -157,18 +35,16 @@ def test_juniper_nsm_standard(record_property, setup_wordlist, get_host_key, set # THE LOG SAMPLE BELOW IS IMPLIED FROM THE JUNIPER DOCS; need to obtain a real sample. # <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [IDP NAME], [IDP IP], predefined, rule, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [IDP NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail def test_juniper_nsm_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [IDP NAME], [IDP IP], predefined, rule, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [IDP NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} jnpnsmidp-{{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [IDP NAME], [IDP IP], predefined, rule, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [IDP NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") message = mt.render(mark="<134>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:nsm:idp\" | head 2") + st = env.from_string("search index=netids host=\"jnpnsmidp-{{ host }}\" sourcetype=\"juniper:nsm:idp\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -180,68 +56,17 @@ def test_juniper_nsm_idp_standard(record_property, setup_wordlist, get_host_key, assert resultCount == 1 # <23> Apr 24 12:30:05 cs-loki3 RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message - -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail - -def test_juniper_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message -") - message = mt.render(mark="<23>", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:idp\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <23> Nov 18 09:56:58 INTERNET-ROUTER RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192 -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail - -def test_juniper_firewall_standard(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192") - message = mt.render(mark="<23>", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[] - Session timed out for xxx@xxx.xxx.xxx/Users (session:00000000) due to inactivity (last access at 13:59:31 2013/02/27). Idle session identified during routine system scan. -# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied. -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail - -def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, setup_splunk): +# <23> Mar 18 17:56:52 [FW IP] [FW Model]: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time="2009-03-18 16:07:06" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1 +def test_juniper_netscreen(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} Juniper: {% now 'utc', '%Y-%m-%d %H:%M:%S' %} - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied.") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} jnpns-{{ host }} ns204: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time=\"2009-03-18 16:07:06\" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1\n") message = mt.render(mark="<23>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:sslvpn\" | head 2") + st = env.from_string("search index=netfw host=\"jnpns-{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -252,19 +77,18 @@ def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, assert resultCount == 1 +# <23> Apr 24 12:30:05 cs-loki3 RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message - # <23> Mar 18 17:56:52 [FW IP] [FW Model]: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time="2009-03-18 16:07:06" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1 -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_netscreen(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_netscreen_singleport(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} ns204: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time=\"2009-03-18 16:07:06\" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1\n") message = mt.render(mark="<23>", host=host) - sendsingle(message) + sendsingle(message, host="sc4s-juniper") - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_juniper_junos_rfc3164.py b/tests/test_juniper_junos_rfc3164.py new file mode 100644 index 0000000..3ec13ff --- /dev/null +++ b/tests/test_juniper_junos_rfc3164.py @@ -0,0 +1,97 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +# <23> Mar 18 17:56:52 RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION="URL Permitted" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY="Enhanced_Information_Technology" REASON="BY_PRE_DEFINED" PROFILE="UTM-Wireless-Profile" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A +def test_juniper_utm_standard(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION=\"URL Permitted\" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY=\"Enhanced_Information_Technology\" REASON=\"BY_PRE_DEFINED\" PROFILE=\"UTM-Wireless-Profile\" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A") + message = mt.render(mark="<23>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <23> Nov 18 09:56:58 INTERNET-ROUTER RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192 +def test_juniper_firewall_standard(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192") + message = mt.render(mark="<23>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[] - Session timed out for xxx@xxx.xxx.xxx/Users (session:00000000) due to inactivity (last access at 13:59:31 2013/02/27). Idle session identified during routine system scan. +# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied. +def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} Juniper: {% now 'utc', '%Y-%m-%d %H:%M:%S' %} - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied.") + message = mt.render(mark="<23>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:sslvpn\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + + +def test_juniper_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message -") + message = mt.render(mark="<23>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:idp\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_juniper_junos_rfc5124.py b/tests/test_juniper_junos_rfc5124.py new file mode 100644 index 0000000..b9ff921 --- /dev/null +++ b/tests/test_juniper_junos_rfc5124.py @@ -0,0 +1,105 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +# <165>1 2007-02-15T09:17:15.719Z router1 mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username="user"] User 'user' exiting configuration mode +# @pytest.mark.xfail +def test_juniper_junos_structured(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username=\"user\"] User 'user' exiting configuration mode\n") + message = mt.render(mark="<165>1", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:structured\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <165>1 2007-02-15T09:17:15.719Z idp1 RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time="1507845354" message-type="SIG" source-address="183.78.180.27" source-port="45610" destination-address="118.127.xx.xx" destination-port="80" protocol-name="TCP" service-name="SERVICE_IDP" application-name="HTTP" rule-name="9" rulebase-name="IPS" policy-name="Recommended" export-id="15229" repeat-count="0" action="DROP" threat-severity="HIGH" attack-name="TROJAN:ZMEU-BOT-SCAN" nat-source-address="0.0.0.0" nat-source-port="0" nat-destination-address="172.xx.xx.xx" nat-destination-port="0" elapsed-time="0" inbound-bytes="0" outbound-bytes="0" inbound-packets="0" outbound-packets="0" source-zone-name="sec-zone-name-internet" source-interface-name="reth0.XXX" destination-zone-name="dst-sec-zone1-outside" destination-interface-name="reth1.xxx" packet-log-id="0" alert="no" username="N/A" roles="N/A" message="-"] +# @pytest.mark.xfail +def test_juniper_junos_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time=\"1507845354\" message-type=\"SIG\" source-address=\"183.78.180.27\" source-port=\"45610\" destination-address=\"118.127.xx.xx\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"9\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"15229\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"TROJAN:ZMEU-BOT-SCAN\" nat-source-address=\"0.0.0.0\" nat-source-port=\"0\" nat-destination-address=\"172.xx.xx.xx\" nat-destination-port=\"0\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"sec-zone-name-internet\" source-interface-name=\"reth0.XXX\" destination-zone-name=\"dst-sec-zone1-outside\" destination-interface-name=\"reth1.xxx\" packet-log-id=\"0\" alert=\"no\" username=\"N/A\" roles=\"N/A\" message=\"-\"]") + message = mt.render(mark="<165>1", host=host) + + sendsingle(message) + + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:idp:structured\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <165>1 2010-06-23T18:05:55 10.209.83.9 Jnpr Syslog 23414 1 [syslog@juniper.net dayId="20100623" recordId="0" timeRecv="2010/06/23 18:05:55" timeGen="2010/06/23 18:05:51" domain="" devDomVer2="0" device_ip="10.209.83.9" cat="Config" attack="" srcZn="NULL" srcIntf="" srcAddr="0.0.0.0" srcPort="0" natSrcAddr="NULL" natSrcPort="0" dstZn="NULL" dstIntf="NULL" dstAddr="0.0.0.0" dstPort="0" natDstAddr="NULL" natDstPort="0" protocol="IP" ruleDomain="" ruleVer="0" policy="" rulebase="NONE" ruleNo="0" action="NONE" severity="INFO" alert="no" elaspedTime="0" inbytes="0" outbytes="0" totBytes="0" inPak="0" outPak="0" totPak="0" repCount="0" packetData="no" varEnum="0" misc="Interaface eth2,eth3 is in Normal State" user="NULL" app="NULL" uri="NULL"] +# +# +# +# @pytest.mark.xfail +def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} Jnpr Syslog 23414 [syslog@juniper.net dayId=\"20100623\" recordId=\"0\" timeRecv=\"2010/06/23 18:05:55\" timeGen=\"2010/06/23 18:05:51\" domain=\"\" devDomVer2=\"0\" device_ip=\"10.209.83.9\" cat=\"Config\" attack=\"\" srcZn=\"NULL\" srcIntf=\"\" srcAddr=\"0.0.0.0\" srcPort=\"0\" natSrcAddr=\"NULL\" natSrcPort=\"0\" dstZn=\"NULL\" dstIntf=\"NULL\" dstAddr=\"0.0.0.0\" dstPort=\"0\" natDstAddr=\"NULL\" natDstPort=\"0\" protocol=\"IP\" ruleDomain=\"\" ruleVer=\"0\" policy=\"\" rulebase=\"NONE\" ruleNo=\"0\" action=\"NONE\" severity=\"INFO\" alert=\"no\" elaspedTime=\"0\" inbytes=\"0\" outbytes=\"0\" totBytes=\"0\" inPak=\"0\" outPak=\"0\" totPak=\"0\" repCount=\"0\" packetData=\"no\" varEnum=\"0\" misc=\"Interaface eth2,eth3 is in Normal State\" user=\"NULL\" app=\"NULL\" uri=\"NULL\"]") + message = mt.render(mark="<165>1", host=host) + + sendsingle(message) + + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:idp:structured\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos +# @pytest.mark.xfail +def test_juniper_junos_fw_structured(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_FLOW - RT_FLOW_SESSION_CREATE_LS [junos@2636.1.1.1.2.26 logical-system-name=\"test-lsys\" source-address=\"10.10.10.100\" source-port=\"4206\" destination-address=\"10.20.20.15\" destination-port=\"445\" service-name=\"junos-smb\" nat-source-address=\"10.10.10.100\" nat-source-port=\"4206\" nat-destination-address=\"10.20.20.15\" nat-destination-port=\"445\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"123\" source-zone-name=\"TEST1\" destination-zone-name=\"TEST2\" session-id-32=\"14285714\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth1.100\"]") + message = mt.render(mark="<23>1", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall:structured\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index 5a601f0..2d40c8f 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause import random -from flaky import flaky from jinja2 import Environment from .sendmessage import * @@ -13,9 +12,7 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) - #<190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0 -@flaky(max_runs=3, min_passes=2) def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -25,7 +22,7 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -38,7 +35,6 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): # <190>Oct 30 09:46:17 1,2012/10/30 09:46:17,01606001116,THREAT,url,1,2012/04/10 04:39:55,192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,"litetopdetect.cn/index.php",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html -@flaky(max_runs=3, min_passes=2) def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -48,7 +44,7 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:threat\" | head 2") + st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"pan:threat\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_poc.py b/tests/test_poc.py deleted file mode 100644 index 37dcdef..0000000 --- a/tests/test_poc.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019 Splunk, Inc. -# -# Use of this source code is governed by a BSD-2-clause-style -# license that can be found in the LICENSE-BSD2 file or at -# https://opensource.org/licenses/BSD-2-Clause -import random - -from flaky import flaky -from jinja2 import Environment - -from .sendmessage import * -from .splunkutils import * - -env = Environment(extensions=['jinja2_time.TimeExtension']) - - -@flaky(max_runs=3, min_passes=2) -def test_defaultroute(record_property, setup_wordlist, setup_splunk): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - - mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %}.000z {{ host }} sc4s_default[0]: test\n") - message = mt.render(mark="<111>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main \"{{ host }}\" sourcetype=\"syslog:fallback\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py new file mode 100644 index 0000000..0dddfac --- /dev/null +++ b/tests/test_symantec_proxy.py @@ -0,0 +1,34 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +# <134>1 2019-08-21T17:42:08.000z "sample_logs bluecoat[0]:SPLV5.1 c-ip=192.0.0.6 cs-bytes=6269 cs-categories="unavailable" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent="ocspd/1.0.3" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name="Explicit HTTP" service.group="Standard" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name="10.0.0.6-sample_logs" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url="http://randomserver:8000/en-US/app/examples/" +def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{host}} bluecoat[0]: SPLV5.1 c-ip=192.0.0.6 cs-bytes=6269 cs-categories=\"unavailable\" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent=\"ocspd/1.0.3\" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name=\"Explicit HTTP\" service.group=\"Standard\" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name=\"10.0.0.6-sample_logs\" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url=\"http://randomserver:8000/en-US/app/examples/\"") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"bluecoat:proxysg:access:kv\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# diff --git a/tests/wait-for b/tests/wait-for new file mode 100755 index 0000000..bf6a071 --- /dev/null +++ b/tests/wait-for @@ -0,0 +1,79 @@ +#!/bin/sh + +TIMEOUT=15 +QUIET=0 + +echoerr() { + if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi +} + +usage() { + exitcode="$1" + cat << USAGE >&2 +Usage: + $cmdname host:port [-t timeout] [-- command args] + -q | --quiet Do not output any status messages + -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit "$exitcode" +} + +wait_for() { + for i in `seq $TIMEOUT` ; do + nc -z "$HOST" "$PORT" > /dev/null 2>&1 + + result=$? + if [ $result -eq 0 ] ; then + if [ $# -gt 0 ] ; then + exec "$@" + fi + exit 0 + fi + sleep 1 + done + echo "Operation timed out" >&2 + exit 1 +} + +while [ $# -gt 0 ] +do + case "$1" in + *:* ) + HOST=$(printf "%s\n" "$1"| cut -d : -f 1) + PORT=$(printf "%s\n" "$1"| cut -d : -f 2) + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -t) + TIMEOUT="$2" + if [ "$TIMEOUT" = "" ]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + break + ;; + --help) + usage 0 + ;; + *) + echoerr "Unknown argument: $1" + usage 1 + ;; + esac +done + +if [ "$HOST" = "" -o "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 +fi + +wait_for "$@" \ No newline at end of file