From cb8ebded433cff243aef4f1cde7bfb9aea9b7120 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Tue, 27 Aug 2019 15:09:11 -0400 Subject: [PATCH 1/7] Release/0.4.0 (#67) v0.4.0 - Add source support for Cisco NX - Add source support for Symantec Proxy SG and ASG (Formerly bluecoat) - Add support for mapping network source IP OR parsed host to specific vendor product where MSG parsing is impossible - Code cleanup and simplification - Begin using SEMVER for releases --- .circleci/config.yml | 49 ++++- .gitignore | 8 +- .idea/misc.xml | 10 - .idea/modules.xml | 8 - .idea/sc4s.iml | 13 -- .idea/splunk-connect-for-syslog.iml | 19 -- .idea/vcs.xml | 10 - CHANGELOG | 6 + docker-compose-ci.yml | 21 +- docker-compose-debug.yml | 56 +++++ docker-compose.yml | 31 ++- docs/gettingstarted.md | 58 ++++- docs/sources.md | 119 +++++++++-- package/Dockerfile | 2 +- package/etc/conf.d/blocks/b_parsers.conf | 69 ------ .../conf.d/conflib/_common/rfc_syslog.conf | 27 +++ .../_common/templates.conf} | 6 +- .../etc/conf.d/conflib/_common/utility.conf | 4 + .../vendor_product_by_source_context.conf | 9 + .../_splunk}/splunk_context.conf | 2 +- .../conf.d/conflib/_splunk/splunkfields.conf | 23 ++ .../etc/conf.d/conflib/vendor_cisco/asa.conf | 3 + .../etc/conf.d/conflib/vendor_cisco/ios.conf | 8 + .../conf.d/conflib/vendor_cisco/nx-os.conf | 3 + .../conflib/vendor_fortinet/fortios.conf | 3 + .../conf.d/conflib/vendor_juniper/junos.conf | 8 + .../conf.d/conflib/vendor_juniper/legacy.conf | 19 ++ .../etc/conf.d/conflib/vendor_pan/panos.conf | 3 + .../vendor_symantec/bluecoat_proxy.conf | 3 + .../{d_destinations.conf => splunk_hec.conf} | 8 +- .../filters/cisco_asa_tradditional.conf | 24 --- package/etc/conf.d/filters/juniper_junos.conf | 65 ------ .../log_paths/P_rfc3164-juniper_nsm_idp.conf | 15 ++ package/etc/conf.d/log_paths/internal.conf | 35 +++ .../log_paths/p_rfc3164-cisco-nx-os.conf | 14 ++ .../conf.d/log_paths/p_rfc3164-cisco_asa.conf | 21 ++ .../p_rfc3164-cisco_ios.conf} | 13 +- .../p_rfc3164-fortinet_fortios.conf} | 12 +- .../log_paths/p_rfc3164-juniper-idp.conf | 15 ++ .../log_paths/p_rfc3164-juniper-junos.conf | 24 +++ .../p_rfc3164-juniper_netscreen.conf | 14 ++ .../log_paths/p_rfc3164-juniper_nsm.conf | 19 ++ .../p_rfc3164-paloalto-panos.conf} | 36 ++-- .../p_rfc5424_noversion-cisco_asa.conf | 16 ++ .../p_rfc_5424_noversion-symantec-proxy.conf | 16 ++ .../p_rfc_5424_strict-juniper-junos.conf | 27 +++ package/etc/conf.d/log_paths/zfallback.conf | 11 + package/etc/conf.d/sources/network.conf | 104 +++++---- .../splunk_index.csv | 48 +++-- .../vendor_product_by_source.conf | 29 +++ .../vendor_product_by_source.csv | 6 + package/etc/syslog-ng.conf | 61 +----- .../apps}/SA-syslog-ng/default/indexes.conf | 21 ++ test-with-compose.sh | 5 +- tests/Dockerfile | 4 +- tests/entrypoint.sh | 23 ++ tests/requirements.txt | 1 - tests/sendmessage.py | 4 +- tests/splunkutils.py | 2 +- tests/test_cisco_asa.py | 11 +- tests/test_cisco_ios.py | 4 +- tests/test_cisco_nx_os.py | 55 +++++ tests/test_common.py | 51 +++++ tests/test_fortinet_ngfw.py | 16 +- tests/test_juniper.py | 200 ++---------------- tests/test_juniper_junos_rfc3164.py | 97 +++++++++ tests/test_juniper_junos_rfc5124.py | 105 +++++++++ tests/test_palo_alto.py | 8 +- tests/test_poc.py | 35 --- tests/test_symantec_proxy.py | 34 +++ tests/wait-for | 79 +++++++ 71 files changed, 1323 insertions(+), 665 deletions(-) delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/sc4s.iml delete mode 100644 .idea/splunk-connect-for-syslog.iml delete mode 100644 .idea/vcs.xml create mode 100644 CHANGELOG create mode 100644 docker-compose-debug.yml delete mode 100644 package/etc/conf.d/blocks/b_parsers.conf create mode 100644 package/etc/conf.d/conflib/_common/rfc_syslog.conf rename package/etc/conf.d/{templates/t_templates.conf => conflib/_common/templates.conf} (92%) create mode 100644 package/etc/conf.d/conflib/_common/utility.conf create mode 100644 package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf rename package/etc/conf.d/{blocks => conflib/_splunk}/splunk_context.conf (74%) create mode 100644 package/etc/conf.d/conflib/_splunk/splunkfields.conf create mode 100644 package/etc/conf.d/conflib/vendor_cisco/asa.conf create mode 100644 package/etc/conf.d/conflib/vendor_cisco/ios.conf create mode 100644 package/etc/conf.d/conflib/vendor_cisco/nx-os.conf create mode 100644 package/etc/conf.d/conflib/vendor_fortinet/fortios.conf create mode 100644 package/etc/conf.d/conflib/vendor_juniper/junos.conf create mode 100644 package/etc/conf.d/conflib/vendor_juniper/legacy.conf create mode 100644 package/etc/conf.d/conflib/vendor_pan/panos.conf create mode 100644 package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf rename package/etc/conf.d/destinations/{d_destinations.conf => splunk_hec.conf} (92%) delete mode 100644 package/etc/conf.d/filters/cisco_asa_tradditional.conf delete mode 100644 package/etc/conf.d/filters/juniper_junos.conf create mode 100644 package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf create mode 100644 package/etc/conf.d/log_paths/internal.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf rename package/etc/conf.d/{filters/cisco_ios.conf => log_paths/p_rfc3164-cisco_ios.conf} (60%) rename package/etc/conf.d/{filters/fortinet_fgt.conf => log_paths/p_rfc3164-fortinet_fortios.conf} (80%) create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf rename package/etc/conf.d/{filters/paloalto_networks.conf => log_paths/p_rfc3164-paloalto-panos.conf} (69%) create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf create mode 100644 package/etc/conf.d/log_paths/zfallback.conf rename package/etc/{context => context-local}/splunk_index.csv (64%) create mode 100644 package/etc/context-local/vendor_product_by_source.conf create mode 100644 package/etc/context-local/vendor_product_by_source.csv rename splunk/{ => etc/apps}/SA-syslog-ng/default/indexes.conf (53%) create mode 100755 tests/entrypoint.sh create mode 100644 tests/test_cisco_nx_os.py create mode 100644 tests/test_common.py create mode 100644 tests/test_juniper_junos_rfc3164.py create mode 100644 tests/test_juniper_junos_rfc5124.py delete mode 100644 tests/test_poc.py create mode 100644 tests/test_symantec_proxy.py create mode 100755 tests/wait-for diff --git a/.circleci/config.yml b/.circleci/config.yml index aa9293b..4a82547 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,16 +120,21 @@ jobs: docker container create --name dummy \ -v sc4s-tests:/work/tests \ -v sc4s-results:/work/test-results \ + -v splunk-etc:/work/splunk-etc \ registry.access.redhat.com/ubi7/ubi docker cp tests/ dummy:/work/tests/ + docker cp ./splunk/etc/* dummy:/work/splunk-etc/ docker rm dummy -# - run: -# name: Docker Compose build -# command: | -# docker-compose build + - run: + name: Docker Compose build + command: | + docker-compose build test - run: name: Docker Compose up - command: docker-compose -f ./docker-compose-ci.yml up --abort-on-container-exit + command: | + docker-compose -f ./docker-compose-ci.yml up -d splunk + sleep 70 + docker-compose -f ./docker-compose-ci.yml up --abort-on-container-exit - run: name: Collect test results from Docker Compose command: | @@ -204,6 +209,31 @@ jobs: name: Docker push tag command: docker push $IMAGE_NAME:edge + publish-version: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false + environment: + IMAGE_NAME: rfaircloth/scs + steps: + - checkout + + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker tag image + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker tag $IMAGE_NAME:$CIRCLE_SHA1 splunk/scs:$SEMVER + - run: + name: Docker push tag + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker push splunk/scs:$SEMVER publish-latest: environment: IMAGE_NAME: rfaircloth/scs @@ -256,6 +286,15 @@ workflows: branches: only: - develop + - publish-version: + requires: + - dgoss + - test-unit + filters: + branches: + only: + - master + - /^release\/.*/ - publish-latest: requires: - dgoss diff --git a/.gitignore b/.gitignore index 68dbac4..59343f4 100644 --- a/.gitignore +++ b/.gitignore @@ -378,11 +378,5 @@ fabric.properties !/package/scripts/splunkmetrics.sh !/package/scripts/ .ecs -======= /test-results/ -/.idea/workspace.xml -/.idea/tasks.xml -!/package/scripts/switch_transport.sh -!/package/scripts/splunkmetrics.sh -!/package/scripts/ ->>>>>>> master +/.idea/ \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 7e25d8b..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 4e16e75..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/sc4s.iml b/.idea/sc4s.iml deleted file mode 100644 index bda388e..0000000 --- a/.idea/sc4s.iml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/splunk-connect-for-syslog.iml b/.idea/splunk-connect-for-syslog.iml deleted file mode 100644 index 14d7789..0000000 --- a/.idea/splunk-connect-for-syslog.iml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 6310177..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..4b84880 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,6 @@ +0.4.0 + - Add source support for Cisco NX + - Add source support for Symantec Proxy SG and ASG (Formerly bluecoat) + - Add support for mapping network source IP OR parsed host to specific vendor product where MSG parsing is impossible + - Code cleanup and simplification + - Begin using SEMVER for releases \ No newline at end of file diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index b7dbd53..5201e5b 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -39,6 +39,23 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + sc4s-juniper: + image: rfaircloth/scs:${CIRCLE_SHA1} + hostname: sc4s-juniper + ports: + - "514" + stdin_open: true + tty: true + links: + - splunk + environment: + - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} + - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} + - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + - SYSLOG_PRESUME_FILTER=juniper_netscreen # logging: # driver: splunk # options: @@ -62,7 +79,7 @@ services: - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng + - splunk-etc:/opt/splunk/etc # logging: # driver: splunk # options: @@ -76,3 +93,5 @@ volumes: external: true sc4s-results: external: true + splunk-etc: + external: true diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml new file mode 100644 index 0000000..55cb296 --- /dev/null +++ b/docker-compose-debug.yml @@ -0,0 +1,56 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +version: "3.2" +services: + + sc4s: + image: splunk/scs:latest + build: + context: ./package + args: + RH_ORG: ${RH_ORG} + RH_ACTIVATION: ${RH_ACTIVATION} + entrypoint: + - "tail" + - "-f" + - "/dev/null" + hostname: sc4s + ports: + - "514" + - "601" + - "514/udp" + - "5514" + - "5514/udp" + stdin_open: true + tty: true + links: + - splunk + environment: + - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} + - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} + - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + splunk: + image: splunk/splunk:latest + hostname: splunk + ports: + - "8000:8000" + - "8088:8088" + - "8089:8089" + environment: + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} + - SPLUNK_START_ARGS=${SPLUNK_START_ARGS} + - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} + - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} + - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} + volumes: + - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng diff --git a/docker-compose.yml b/docker-compose.yml index fd562ee..b0135dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ services: links: - splunk - sc4s + - sc4s-juniper volumes: - sc4s-tests:/work/tests - sc4s-results:/work/test-results @@ -44,6 +45,32 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + sc4s-juniper: + image: splunk/scs:latest + build: + context: ./package +# cache_from: +# - registry.access.redhat.com/ubi7/ubi +# - registry.access.redhat.com/rhel7/rhel +# - rfaircloth/scs:edge + args: + RH_ORG: ${RH_ORG} + RH_ACTIVATION: ${RH_ACTIVATION} + hostname: sc4s-juniper + ports: + - "514" + stdin_open: true + tty: true + links: + - splunk + environment: + - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} + - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} + - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + - SYSLOG_PRESUME_FILTER=juniper_netscreen # logging: # driver: splunk # options: @@ -67,7 +94,7 @@ services: - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng + - splunk-etc:/opt/splunk/etc # logging: # driver: splunk # options: @@ -81,3 +108,5 @@ volumes: external: true sc4s-results: external: true + splunk-etc: + external: true \ No newline at end of file diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index d1e71cc..8abfe3b 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -22,11 +22,11 @@ services: image: splunk/scs:latest hostname: sc4s ports: - - "514" - - "601" - - "514/udp" - - "5514" - - "5514/udp" + - "514:514" + - "601:601" + - "514:514/udp" + - "5514:5514" + - "5514:5514/udp" stdin_open: true tty: true environment: @@ -36,16 +36,25 @@ services: - SPLUNK_DEFAULT_INDEX= - SPLUNK_METRICS_INDEX=em_metrics volumes: - - splunk_index.csv:/opt/syslog-ng/etc/context/splunk_index.csv + - ./sc4s/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv + - ./sc4s/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv ``` -* Download the latest context.csv file to the current directory +## Configure index destinations for Splunk +* Download the latest context.csv file to a subdirectory sc4s below the docker-compose.yml file created above ```bash -wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context/splunk_index.csv +wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` - * Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. +## Configure sources by source IP or host name +* This step is required even if not used +* Download the latest vendor_product_by_source.conf file to a subdirectory sc4s below the docker-compose.yml file created above +```bash +wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf +``` +* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. + * Start SC4S ```bash @@ -56,3 +65,34 @@ docker stack deploy --compose-file docker-compose.yml sc4s ## Scale out Additional hosts can be deployed for syslog collection from additional network zones and locations + + +## Single Source Technology instance - Alpha + +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. In the following configration example a dedicated port is opened (6514) for legacy juniper netscreen devices + +This approach is "alpha" and subject to change + +```yaml +version: "3" +services: + sc4s-juniper-netscreen: + image: splunk/scs:latest + hostname: sc4s-juniper-netscreen + ports: + - "6514:514" + - "6514:514/udp" + stdin_open: true + tty: true + environment: + - SPLUNK_HEC_URL=https://foo:8088/services/collector/event + - SPLUNK_HEC_TOKEN= + - SPLUNK_CONNECT_METHOD=hec + - SPLUNK_DEFAULT_INDEX= + - SPLUNK_METRICS_INDEX=em_metrics + - SYSLOG_PRESUME_FILTER=f_juniper_netscreen + volumes: + - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +``` \ No newline at end of file diff --git a/docs/sources.md b/docs/sources.md index 3fd78da..24e9d72 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -8,7 +8,7 @@ | Product Manual | https://www.cisco.com/c/en/us/td/docs/security/asa/asa82/configuration/guide/config/monitor_syslog.html | -## Sourcetypes +### Sourcetypes | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| @@ -16,11 +16,11 @@ | cisco:pix | Not supported | | cisco:fwsm | Not supported | -## Filter type +### Filter type MSG Parse: This filter parses message content -## Setup and Configuration +### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index as required. @@ -31,7 +31,7 @@ MSG Parse: This filter parses message content * device-id is hostname and included * timestamp is included -## Verification +### Verification Use the following search to validate events are present @@ -39,6 +39,63 @@ Use the following search to validate events are present index= sourcetype=cisco:asa ``` +Verify timestamp, and host values match as expected + +## Product - IOS and NX-OS based equipment + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/1467/ | +| IOS Manual | https://www.cisco.com/c/en/us/td/docs/switches/lan/catalyst2960/software/release/12-2_55_se/configuration/guide/scg_2960/swlog.html | +| NX-OS Manual | https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/6-x/system_management/configuration/guide/b_Cisco_Nexus_9000_Series_NX-OS_System_Management_Configuration_Guide/sm_5syslog.html| +| Cisco ACI | https://community.cisco.com/legacyfs/online/attachments/document/technote-aci-syslog_external-v1.pdf | +| Cisco WLC & AP | https://www.cisco.com/c/en/us/support/docs/wireless/4100-series-wireless-lan-controllers/107252-WLC-Syslog-Server.html#anc8 | +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cisco:ios | This source type is also used for NX-OS, ACI and WLC product lines | + +### Filter type + +* Cisco IOS products can be identified by message parsing alone +* Cisco NX OS, WLC, and ACI products must be identified by host or ip assignment update the filter `f_cisco_nx_os` as required + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index as required. +* IOS Follow vendor configuration steps per Product Manual above ensure: + * Ensure a reliable NTP server is set and synced + * Log Level is 6 "Informational" + * Protocol is TCP/IP + * permit-hostdown is on + * device-id is hostname and included + * timestamp is included +* NX-OS Follow vendor configuration steps per Product Manual above ensure: + * Ensure a reliable NTP server is set and synced + * Log Level is 6 "Informational" user may select alternate levels by module based on use cases + * Protocol is TCP/IP + * device-id is hostname and included + * timestamp is included and milisecond accuracy selected +* ACI Logging configuration of the ACI product often varies by use case. + * Ensure NTP sync is configured and active + * Ensure proper host names are configured +* WLC + * Ensure NTP sync is configured and active + * Ensure proper host names are configured + * For security use cases per AP logging is required + + +### Verification + +Use the following search to validate events are present, for NX-OS, WLC and ACI products ensure each host filter condition is verified + +``` +index= sourcetype=cisco:ios | stats count by host +``` + + Verify timestamp, and host values match as expected # Vendor - Fortinet @@ -51,7 +108,7 @@ Verify timestamp, and host values match as expected | Product Manual | https://docs.fortinet.com/product/fortigate/6.2 | -## Sourcetypes +### Sourcetypes | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| @@ -60,11 +117,11 @@ Verify timestamp, and host values match as expected | fgt_utm | None | | fgt_event | None -## Filter type +### Filter type MSG Parse: This filter parses message content -## Setup and Configuration +### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index as required. @@ -103,7 +160,7 @@ end ``` -## Verification +### Verification An active firewall will generate frequent events, in addition fortigate has the ability to test logging functionality using a build in command @@ -117,7 +174,7 @@ Verify timestamp, and host values match as expected index= (sourcetype=fgt_log OR sourcetype=fgt_traffic OR sourcetype=fgt_utm) ``` -###UTM Message type +### UTM Message type ![FortiGate UTM message](FortiGate_utm.png) @@ -140,7 +197,7 @@ Verify timestamp, and host values match as expected | Product Manual | https://docs.paloaltonetworks.com/pan-os/9-0/pan-os-admin/monitoring/use-syslog-for-monitoring/configure-syslog-monitoring.html | -## Sourcetypes +### Sourcetypes | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| @@ -148,11 +205,11 @@ Verify timestamp, and host values match as expected | pan:traffic | None | | pan:threat | None | -## Filter type +### Filter type MSG Parse: This filter parses message content -## Setup and Configuration +### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index as required. @@ -161,10 +218,46 @@ MSG Parse: This filter parses message content * Select IETF Format * Ensure the format of the event is not customized -## Verification +### Verification An active firewall will generate frequent events use the following search to validate events are present per source device ``` index= sourcetype=pan:*| stats count by host +``` + +# Vendor - Symantec + +## Product - ProxySG/ASG (Bluecoat) + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2758/ | +| Product Manual | https://support.symantec.com/us/en/article.tech242216.html | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| bluecoat:proxysg:access:kv | Requires version 3.6 | + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index as required. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Verification + +An active proxy will generate frequent events use the following search to validate events are present per source device + +``` +index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` \ No newline at end of file diff --git a/package/Dockerfile b/package/Dockerfile index 2f8698e..4e3b4c4 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -83,7 +83,7 @@ COPY goss.yaml /etc/goss.yaml COPY --from=0 /opt/syslog-ng /opt/syslog-ng COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d -COPY etc/context /opt/syslog-ng/etc/context +COPY etc/context-local /opt/syslog-ng/etc/context-local COPY sbin/entrypoint.sh /sbin/entrypoint.sh RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer diff --git a/package/etc/conf.d/blocks/b_parsers.conf b/package/etc/conf.d/blocks/b_parsers.conf deleted file mode 100644 index 8aae8e9..0000000 --- a/package/etc/conf.d/blocks/b_parsers.conf +++ /dev/null @@ -1,69 +0,0 @@ -#this parser can be used for "almost compliant" 5424 messages -block parser p_RFC5424-lite(prefix(".rfc5424-lite.")) { - channel { - rewrite { - # Remove Version Identifier - subst('(<\d+>)\d+' "$1" value("MSG")); - - }; - parser { - # by this time this message will parse with the BSD 3164 parser - syslog-parser(time-zone(`default-timezone`)); - # syslog-parser(); - - # extract name-value pairs. - # kv-parser(prefix("`prefix`")); - }; - }; -}; - -block parser p_test-parser(prefix(".testparse.")) { - channel { - rewrite { - # Put your rewrite rule here;use "MSG" as no-parse() puts entire payload there - # subst('(>.{12})\s' "${1}T" value("MSG")); - }; - parser { - # Don't initially parse so full output can be seen - # syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); - # syslog-parser(); - # syslog-parser(time-zone(`default-timezone`)); - # date-parser(format("%Y-%m-%d %H:%M:%S") template("$(substr ${MESSAGE} 0 19)") time-zone(`default-timezone`)); - }; - }; -}; - -#Used by fallback to log everything we can -block rewrite r_set_splunk(sourcetype(`splunk-sourcetype`) - index(`splunk-index`) - template(`splunk-default-template`) - vendor("unassigned") - category("") - filename("messages") - ) { - set("`index`", value(".splunk.index")); - set("`sourcetype`", value(".splunk.sourcetype")); - set("`template`", value(".splunk.template")); - set("`vendor`", value(".splunk.vendor")); - set("`category`", value(".splunk.category")); - set("`filename`", value(".splunk.filename")); - set($FACILITY, value("fields.syslog_facility")); - set($LEVEL, value("fields.syslog_severity")); - set($LOGHOST, value("fields.syslog_server")); - }; - -#Used when for normal filters index and sourcetype must be set in a prior re-write or context -block rewrite r_set_splunk_basic( - template(`splunk-default-template`) - vendor("unassigned") - category("") - filename("messages") - ) { - set("`template`", value(".splunk.template")); - set("`vendor`", value(".splunk.vendor")); - set("`category`", value(".splunk.category")); - set("`filename`", value(".splunk.filename")); - set($FACILITY, value("fields.syslog_facility")); - set($LEVEL, value("fields.syslog_severity")); - set($LOGHOST, value("fields.syslog_server")); - }; diff --git a/package/etc/conf.d/conflib/_common/rfc_syslog.conf b/package/etc/conf.d/conflib/_common/rfc_syslog.conf new file mode 100644 index 0000000..1809034 --- /dev/null +++ b/package/etc/conf.d/conflib/_common/rfc_syslog.conf @@ -0,0 +1,27 @@ +filter f_rfc5424_strict{ + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); +}; +filter f_rfc5424_noversion{ + message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); +}; +rewrite set_rfcnonconformant{ + set("rfc5424_nonconform" value("fields.sc4s_syslog_format")); +}; +rewrite set_rfc5424_strict{ + set("rfc5424_strict" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_strict{ + match("rfc5424_strict" value("fields.sc4s_syslog_format")) +}; +rewrite set_rfc5424_noversion{ + set("rfc5424_noversion" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_noversion{ + match("rfc5424_noversion" value("fields.sc4s_syslog_format")) +}; +rewrite set_rfc3164{ + set("rfc3164" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc3164{ + match("rfc3164" value("fields.sc4s_syslog_format")) +}; \ No newline at end of file diff --git a/package/etc/conf.d/templates/t_templates.conf b/package/etc/conf.d/conflib/_common/templates.conf similarity index 92% rename from package/etc/conf.d/templates/t_templates.conf rename to package/etc/conf.d/conflib/_common/templates.conf index 0190f1f..b07a8a5 100644 --- a/package/etc/conf.d/templates/t_templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -48,5 +48,9 @@ template t_hdr_sdata_msg { # =============================================================================================== template t_JSON { - template("$(format-json --scope all-nv-pairs)"); + template("$(format-json --scope all-nv-pairs + --exclude fields.* + --exclude .splunk.* + --exclude HOST --exclude HOSTFROMgit m + )"); }; diff --git a/package/etc/conf.d/conflib/_common/utility.conf b/package/etc/conf.d/conflib/_common/utility.conf new file mode 100644 index 0000000..8b986c3 --- /dev/null +++ b/package/etc/conf.d/conflib/_common/utility.conf @@ -0,0 +1,4 @@ + +rewrite set_metadata_presume { + set("$(env SYSLOG_PRESUME_FILTER)" value("fields.sc4s_presume")); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf new file mode 100644 index 0000000..cb238de --- /dev/null +++ b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf @@ -0,0 +1,9 @@ +block parser vendor_product_by_source() { + add-contextual-data( + selector(filters("/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf")), + database("context-local/vendor_product_by_source.csv") + ignore-case(yes) + prefix("fields.") + ); +}; + diff --git a/package/etc/conf.d/blocks/splunk_context.conf b/package/etc/conf.d/conflib/_splunk/splunk_context.conf similarity index 74% rename from package/etc/conf.d/blocks/splunk_context.conf rename to package/etc/conf.d/conflib/_splunk/splunk_context.conf index e2e0af9..e2d1fbb 100644 --- a/package/etc/conf.d/blocks/splunk_context.conf +++ b/package/etc/conf.d/conflib/_splunk/splunk_context.conf @@ -1,7 +1,7 @@ block parser p_add_context_splunk(key("syslogng-fallback")) { add-contextual-data( selector("`key`"), - database("context/splunk_index.csv"), + database("context-local/splunk_index.csv"), prefix(".splunk.") ); }; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf new file mode 100644 index 0000000..7d96599 --- /dev/null +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf @@ -0,0 +1,23 @@ +#Used to set index fields we will always use +rewrite r_set_splunk_basic { + set("`index`", value(".splunk.index")); + + set("`sourcetype`", value(".splunk.sourcetype")); + set("`source`", value(".splunk.source")); + set($FACILITY, value("fields.sc4s_syslog_facility")); + set($LEVEL, value("fields.sc4s_syslog_severity")); + set($LOGHOST, value("fields.sc4s_syslog_server")); + set($SOURCEIP, value("fields.sc4s_fromhostip")); +}; + +#rewrite + +#Used by fallback to log everything we can +block rewrite r_set_splunk( + template(`splunk-default-template`) + ) { + set("`template`", value("fields.sc4s_template")); + }; + + + diff --git a/package/etc/conf.d/conflib/vendor_cisco/asa.conf b/package/etc/conf.d/conflib/vendor_cisco/asa.conf new file mode 100644 index 0000000..bbd3157 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_cisco/asa.conf @@ -0,0 +1,3 @@ +filter f_cisco_asa { + message('^%ASA-\d+-\d{1,10}: '); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/vendor_cisco/ios.conf b/package/etc/conf.d/conflib/vendor_cisco/ios.conf new file mode 100644 index 0000000..09a37ed --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_cisco/ios.conf @@ -0,0 +1,8 @@ +rewrite set_metadata_vendor_product_cisco_ios{ + set("cisco_ios" value(".metadata.vendor_product")); +}; + +filter f_cisco_ios{ + match("cisco_ios", value(".metadata.vendor_product") type(glob)); +}; + diff --git a/package/etc/conf.d/conflib/vendor_cisco/nx-os.conf b/package/etc/conf.d/conflib/vendor_cisco/nx-os.conf new file mode 100644 index 0000000..2f9cacc --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_cisco/nx-os.conf @@ -0,0 +1,3 @@ +filter f_cisco_nx_os { + match("cisco_nx_os", value("fields.sc4s_vendor_product") type(glob) ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/vendor_fortinet/fortios.conf b/package/etc/conf.d/conflib/vendor_fortinet/fortios.conf new file mode 100644 index 0000000..6c08a87 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_fortinet/fortios.conf @@ -0,0 +1,3 @@ +filter f_fortinet_fortios { + message('devid=\"?F[G|W|6K].+type=\"?(traffic|utm|event)'); +}; diff --git a/package/etc/conf.d/conflib/vendor_juniper/junos.conf b/package/etc/conf.d/conflib/vendor_juniper/junos.conf new file mode 100644 index 0000000..49ae1de --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_juniper/junos.conf @@ -0,0 +1,8 @@ +filter f_juniper_junos { + match('^\[junos@2636' value("SDATA")) + or match('syslog@juniper.net' value("SDATA")) +}; + +filter f_juniper_junos_legacy { + program("RT_IDP|RT_FLOW|RT_IDS|RT_UTM|Juniper"); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/vendor_juniper/legacy.conf b/package/etc/conf.d/conflib/vendor_juniper/legacy.conf new file mode 100644 index 0000000..8ec73a3 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_juniper/legacy.conf @@ -0,0 +1,19 @@ + + +filter f_juniper_nsm { + match("^juniper_nsm$", value("fields.sc4s_vendor_product")); + +}; +filter f_juniper_nsm_idp { + match("juniper_nsm_idp", value("fields.sc4s_vendor_product") type(glob) ); + +}; + +filter f_juniper_netscreen { + match("juniper_netscreen", value("fields.sc4s_vendor_product") type(glob) ); + +}; + +filter f_juniper_idp { + match("juniper_idp", value("fields.sc4s_vendor_product") type(glob) ); +}; diff --git a/package/etc/conf.d/conflib/vendor_pan/panos.conf b/package/etc/conf.d/conflib/vendor_pan/panos.conf new file mode 100644 index 0000000..da0cb04 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_pan/panos.conf @@ -0,0 +1,3 @@ +filter f_paloalto_panos { + message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf b/package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf new file mode 100644 index 0000000..afaff11 --- /dev/null +++ b/package/etc/conf.d/conflib/vendor_symantec/bluecoat_proxy.conf @@ -0,0 +1,3 @@ +filter f_symantec_bluecoat_proxy { + program("bluecoat") +}; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/d_destinations.conf b/package/etc/conf.d/destinations/splunk_hec.conf similarity index 92% rename from package/etc/conf.d/destinations/d_destinations.conf rename to package/etc/conf.d/destinations/splunk_hec.conf index 5c59aee..81f3187 100644 --- a/package/etc/conf.d/destinations/d_destinations.conf +++ b/package/etc/conf.d/destinations/splunk_hec.conf @@ -27,7 +27,7 @@ destination d_hec { password("`SPLUNK_HEC_TOKEN`") persist-name("splunk") disk-buffer(mem-buf-length(15000) - disk-buf-size(20000) + disk-buf-size(200000) reliable(no) dir("/opt/syslog-ng/var/data/disk-buffer/")) tls(peer-verify(no) @@ -41,11 +41,11 @@ destination d_hec { ) body('$(format-json time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} + host=${HOST} + source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$(template ${.splunk.template} $(template `splunk-default-template`)) + event=$(template ${fields.sc4s_template} $(template `splunk-default-template`)) fields.*)') ); }; diff --git a/package/etc/conf.d/filters/cisco_asa_tradditional.conf b/package/etc/conf.d/filters/cisco_asa_tradditional.conf deleted file mode 100644 index 57d25ce..0000000 --- a/package/etc/conf.d/filters/cisco_asa_tradditional.conf +++ /dev/null @@ -1,24 +0,0 @@ -log { - source(s_default-ports); - - filter {tags("cisco:asa") - or message('^.{10,60} : %ASA-\d+-\d{1,10}: ') - ;}; - - #Cisco ASA can send 3264 or 5424 either way we are good with basic parsing - parser { - #basic parsing - syslog-parser(time-zone(`default-timezone`)); - }; - - #set the source type based on program field and lookup index from the splunk context csv - parser {p_add_context_splunk(key("cisco:asa")); }; - - #rewrite the final message for splunk json - #sourcetype and index are set in the filter defaults won't be used - rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- - - destination(d_hec); #--HEC-- - - flags(flow-control,final); -}; diff --git a/package/etc/conf.d/filters/juniper_junos.conf b/package/etc/conf.d/filters/juniper_junos.conf deleted file mode 100644 index 3a205e4..0000000 --- a/package/etc/conf.d/filters/juniper_junos.conf +++ /dev/null @@ -1,65 +0,0 @@ -# =============================================================================================== -# Juniper Standard and Structured logging -# =============================================================================================== - -log { - - source(s_default-ports); - -# For newer (structured) format; RFC 5424 compliant. The existing TA will not work for this format; these filters -# are included for future TA development. The exception is NSM IDP, for which we're awaiting a data -# sample to confirm that it follows the RFC 5424 format. -# If it is determined that NSM IDP ("syslog@juniper.net") messages are indeed 3164 format, remove the check for -# the "syslog@juniper.net" message below. It will then be caught by the standard section following. - -junction { - channel { - filter { tags("juniper_structured") or message('junos@2636') or message('syslog@juniper.net'); }; - parser { syslog-parser(flags(syslog-protocol)); }; - if (tags("juniper_JunOS_IDP") or program('RT_IDP')) { - parser {p_add_context_splunk(key("juniper:junos:idp:structured")); }; - } elif (tags("juniper_JunOS_Firewall") or program('RT_FLOW|RT_IDS|RT_UTM')) { - parser {p_add_context_splunk(key("juniper:junos:firewall:structured")); }; - } - elif (tags("juniper_idp") or program('Jnpr')) { - parser {p_add_context_splunk(key("juniper:idp:structured")); }; - } - else { - parser {p_add_context_splunk(key("juniper:structured")); }; - }; - rewrite { r_set_splunk_basic(template("t_hdr_sdata_msg"))}; - destination(d_hec); #--HEC-- - flags(final); - }; - -# For legacy "standard" format. The existing Juniper TA will work for these data sources. -# Remove the message filter below when tag support is implemented. - - channel { - filter { tags("juniper_standard") or message('RT_IDP:|RT_FLOW:|RT_IDS:|RT_UTM:|Juniper:|traffic\,\s+traffic\s+log|syslog@juniper.net|predefined\,\s+|NetScreen\s+device_id\=') }; -# filter { tags("juniper_standard") }; - parser { syslog-parser(time-zone(`default-timezone`)); }; - if (program('RT_IDP')) { - parser {p_add_context_splunk(key("juniper:junos:idp")); }; - } elif (program('RT_FLOW|RT_IDS|RT_UTM')) { - parser {p_add_context_splunk(key("juniper:junos:firewall")); }; - } elif (program('Juniper')) { - parser {p_add_context_splunk(key("juniper:sslvpn")); }; - } elif (message('traffic\,\s+traffic\s+log')) { - parser {p_add_context_splunk(key("juniper:nsm")); }; - } elif (message('syslog@juniper.net')) { - parser {p_add_context_splunk(key("juniper:idp")); }; - } elif (message('predefined\,\s+')) { - parser {p_add_context_splunk(key("juniper:nsm:idp")); }; - } elif (message('NetScreen\s+device_id\=')) { - parser {p_add_context_splunk(key("juniper:netscreen")); }; - } - else { - parser {p_add_context_splunk(key("juniper:legacy")); }; - }; - rewrite { r_set_splunk_basic(template("t_standard")) }; - destination(d_hec); #--HEC-- - flags(final); - }; - }; -}; diff --git a/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf b/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf new file mode 100644 index 0000000..377820c --- /dev/null +++ b/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf @@ -0,0 +1,15 @@ +log { + + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_nsm_idp); + + parser { + p_add_context_splunk(key("juniper:nsm:idp")); + }; + rewrite { + r_set_splunk(template("t_standard")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; diff --git a/package/etc/conf.d/log_paths/internal.conf b/package/etc/conf.d/log_paths/internal.conf new file mode 100644 index 0000000..879925f --- /dev/null +++ b/package/etc/conf.d/log_paths/internal.conf @@ -0,0 +1,35 @@ +log { + source(s_internal); + + if (match("^Log statistics; " value("MESSAGE"))) { + + parser {p_add_context_splunk(key("syslog-ng:metrics")); }; + rewrite { + subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', + ' + {"time": "$S_UNIXTIME.$S_MSEC", + "event": "metric", + "host": "$HOST", + "index": "${.splunk.index}", + "source": "internal", + "sourcetype": "${.splunk.sourcetype}", + "fields": { + "source_name": "${SourceName}", + "source_instance": "${SourceInstance}", + "state": "${State}", + "type": "${Type}", + "_value": ${Number}, + "metric_name": "syslogng.${SourceId}" + } + } + ', + value("MESSAGE") flags("utf8" "global") + ); + }; + destination(d_hecmetrics); #--HEC-- + } else { + parser {p_add_context_splunk(key("syslog-ng:events")); }; + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- + destination(d_hec); #--HEC-- + }; +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf new file mode 100644 index 0000000..e9f018c --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf @@ -0,0 +1,14 @@ +log { + + source(s_default-ports); + filter(f_cisco_nx_os); + + parser { + p_add_context_splunk(key("cisco:nx-os")); + }; + rewrite { + r_set_splunk(template("t_hdr_msg")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf new file mode 100644 index 0000000..428cb0f --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf @@ -0,0 +1,21 @@ + +# =============================================================================================== +# Cisco ASA (Firewall) +# =============================================================================================== + +log { + source(s_default-ports); + + filter(f_is_rfc3164); + filter(f_cisco_asa); + #set the source type based on program field and lookup index from the splunk context csv + + parser {p_add_context_splunk(key("cisco:asa")); }; + + #Using the 5424 parser the message content is all we need + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco_ios.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf similarity index 60% rename from package/etc/conf.d/filters/cisco_ios.conf rename to package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf index cac0bb7..780130e 100644 --- a/package/etc/conf.d/filters/cisco_ios.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf @@ -1,3 +1,4 @@ + # =============================================================================================== # Cisco IOS (Route/Switch) # =============================================================================================== @@ -5,12 +6,16 @@ log { source(s_default-ports); + filter(f_cisco_ios); - parser { cisco-parser(); }; - parser {p_add_context_splunk(key("cisco:ios")); }; - rewrite { r_set_splunk_basic(template("t_msg_only"))}; #--HEC-- + parser { + p_add_context_splunk(key("cisco:ios")); + }; + rewrite { + r_set_splunk(template("t_msg_only")) #--HEC-- + }; destination(d_hec); #--HEC-- - flags(flow-control,final); + flags(flow-control); }; diff --git a/package/etc/conf.d/filters/fortinet_fgt.conf b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf similarity index 80% rename from package/etc/conf.d/filters/fortinet_fgt.conf rename to package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf index d6205b5..d81c565 100644 --- a/package/etc/conf.d/filters/fortinet_fgt.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf @@ -1,12 +1,9 @@ log { source(s_default-ports); - - filter {tags("fgt_log") - or message('devid=\"?F[G|W|6K].+type=\"?(traffic|utm|event)'); - }; + filter(f_is_rfc3164); + filter(f_fortinet_fortios); parser { - syslog-parser(); kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone(`default-timezone`)); }; @@ -26,8 +23,9 @@ log { #rewrite the final message for splunk json #sourcetype and index are set in the filter defaults won't be used - rewrite {r_set_splunk_basic(template("t_standard")) }; #--HEC-- + rewrite {r_set_splunk(template("t_standard")) }; #--HEC-- destination(d_hec); #--HEC-- - flags(flow-control,final); + flags(flow-control); }; + diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf new file mode 100644 index 0000000..4b84403 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf @@ -0,0 +1,15 @@ +log { + + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_idp); + + parser { + p_add_context_splunk(key("juniper:idp")); + }; + rewrite { + r_set_splunk(template("t_standard")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf new file mode 100644 index 0000000..e689cf6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf @@ -0,0 +1,24 @@ +# =============================================================================================== +# Juniper Standard and Structured logging +# =============================================================================================== + +log { + + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_junos_legacy); + + if (program('RT_IDP')) { + parser {p_add_context_splunk(key("juniper:junos:idp")); }; + } elif (program('RT_FLOW|RT_IDS|RT_UTM')) { + parser {p_add_context_splunk(key("juniper:junos:firewall")); }; + } elif (program('Juniper')) { + parser {p_add_context_splunk(key("juniper:sslvpn")); }; + } else { + parser {p_add_context_splunk(key("juniper:legacy")); }; + }; + + rewrite { r_set_splunk(template("t_standard")) }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf new file mode 100644 index 0000000..efd046c --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf @@ -0,0 +1,14 @@ +log { + + source(s_default-ports); + filter(f_juniper_netscreen); + + parser { + p_add_context_splunk(key("juniper:netscreen")); + }; + rewrite { + r_set_splunk(template("t_standard")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf new file mode 100644 index 0000000..04810cc --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf @@ -0,0 +1,19 @@ +# =============================================================================================== +# Juniper NSM +# =============================================================================================== + +log { + + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_nsm); + + parser { + p_add_context_splunk(key("juniper:nsm")); + }; + rewrite { + r_set_splunk(template("t_standard")) + }; + destination(d_hec); #--HEC-- + flags(flow-control); +}; diff --git a/package/etc/conf.d/filters/paloalto_networks.conf b/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf similarity index 69% rename from package/etc/conf.d/filters/paloalto_networks.conf rename to package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf index a3e1016..956f747 100644 --- a/package/etc/conf.d/filters/paloalto_networks.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf @@ -1,22 +1,21 @@ - - log { source(s_default-ports); + filter(f_is_rfc3164); + filter(f_paloalto_panos); - filter {tags("pan:log") - or message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),') - ;}; - - # the palo message does not include a program value in the header unfortunatly + # the palo message does not include a program value in the header unfortunatly # the use of colon in the message tricks the RFC3164 parser we will re-write the message # so the parser will not incorrectly break it apart. # while we are at it we will save the mesage type into the program field so parser can find it rewrite { - subst('^([^ ]+ \w+ \d{2} \d{2}:\d{2}:\d{2} [^ ]+)( \d+,[^,]+,[^,]+,)([^,]+)(.*)', '$1 $3:$2$3$4', value("MESSAGE") ); + set("${LEGACY_MSGHDR}${MESSAGE}" value("MESSAGE")); + unset(value("LEGACY_MSGHDR")); + unset(value("PROGRAM")); + }; parser { #basic parsing - syslog-parser(time-zone(`default-timezone`)); + #we need to actual even time from the field GeneratedTime use csv parser to get it out csv-parser( columns('FUTURE_USE', 'ReceiveTime', 'SerialNumber', 'Type', 'Subtype', 'FUTURE_USE2', 'GeneratedTime') @@ -33,19 +32,19 @@ log { }; #set the source type based on program field and lookup index from the splunk context csv - if (program('THREAT')) { + if (message(',\d+,THREAT')) { parser {p_add_context_splunk(key("pan:threat")); }; - } elif (program('TRAFFIC')) { + } elif (message(',\d+,TRAFFIC')) { parser {p_add_context_splunk(key("pan:traffic")); }; - } elif (program('SYSTEM')) { + } elif (message(',\d+,SYSTEM')) { parser {p_add_context_splunk(key("pan:system")); }; - } elif (program('CONFIG')) { + } elif (message(',\d+,CONFIG')) { parser {p_add_context_splunk(key("pan:config")); }; - } elif (program('HIPWATCH')) { + } elif (message(',\d+,HIPWATCH')) { parser {p_add_context_splunk(key("pan:hipwatch")); }; - } elif (program('CORRELATION')) { + } elif (message(',\d+,CORRELATION')) { parser {p_add_context_splunk(key("pan:correlation")); }; - } elif (program('USERID')) { + } elif (message(',\d+,USERID')) { parser {p_add_context_splunk(key("pan:userid")); }; } else { parser {p_add_context_splunk(key("pan:log")); }; @@ -53,9 +52,10 @@ log { #rewrite the final message for splunk json #sourcetype and index are set in the filter defaults won't be used - rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- destination(d_hec); #--HEC-- - flags(flow-control,final); + flags(flow-control); }; + diff --git a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf new file mode 100644 index 0000000..295669a --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf @@ -0,0 +1,16 @@ +log { + source(s_default-ports); + filter(f_is_rfc5424_noversion); + filter(f_cisco_asa); + + #set the source type based on program field and lookup index from the splunk context csv + + parser {p_add_context_splunk(key("cisco:asa")); }; + + #Using the 5424 parser the message content is all we need + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf new file mode 100644 index 0000000..4d2b8e4 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf @@ -0,0 +1,16 @@ +log { + source(s_default-ports); + filter(f_is_rfc5424_noversion); + filter(f_symantec_bluecoat_proxy); + + #set the source type based on program field and lookup index from the splunk context csv + + parser {p_add_context_splunk(key("bluecoat:proxysg:access:kv")); }; + + #Using the 5424 parser the message content is all we need + rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf new file mode 100644 index 0000000..c49854b --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf @@ -0,0 +1,27 @@ +# =============================================================================================== +# Juniper Structured logging +# =============================================================================================== + +log { + + source(s_default-ports); + filter(f_is_rfc5424_strict); + filter(f_juniper_junos); + + if (program('RT_IDP')) { + parser {p_add_context_splunk(key("juniper:junos:idp:structured")); }; + } elif (program('RT_FLOW|RT_IDS|RT_UTM')) { + parser {p_add_context_splunk(key("juniper:junos:firewall:structured")); }; + } + elif (program('Jnpr')) { + parser {p_add_context_splunk(key("juniper:idp:structured")); }; + } + else { + parser {p_add_context_splunk(key("juniper:structured")); }; + }; + + rewrite { r_set_splunk(template("t_hdr_sdata_msg"))}; + destination(d_hec); #--HEC-- + + +}; diff --git a/package/etc/conf.d/log_paths/zfallback.conf b/package/etc/conf.d/log_paths/zfallback.conf new file mode 100644 index 0000000..4b1b5d5 --- /dev/null +++ b/package/etc/conf.d/log_paths/zfallback.conf @@ -0,0 +1,11 @@ +log { + source(s_default-ports); + + parser { + p_add_context_splunk(key("syslog-ng:fallback")); + }; + + rewrite { r_set_splunk(template("t_JSON"))}; #--HEC-- + destination(d_hec); #--HEC-- + flags(flow-control,fallback); +}; diff --git a/package/etc/conf.d/sources/network.conf b/package/etc/conf.d/sources/network.conf index e3ed6a9..70a5f7b 100644 --- a/package/etc/conf.d/sources/network.conf +++ b/package/etc/conf.d/sources/network.conf @@ -11,51 +11,65 @@ # source s_network-defaults { default-network-drivers() }; source s_default-ports { - `splunk-udp-driver` ( - transport("udp") - port(`udp-listening-port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=`udp-listening-port`/udp") -# teardown("firewall-cmd --permanent --remove-port=`udp-listening-port`/udp") -# ) - so-rcvbuf(`splunk-rcvbuf`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); + channel { + source { + `splunk-udp-driver` ( + transport("udp") + port(`udp-listening-port`) + ip-protocol(`ip-version`) + so-rcvbuf(`splunk-rcvbuf`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); - `splunk-tcp-driver` ( - transport("tcp") - port(`tcp-listening-port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=`tcp-listening-port`/tcp") -# teardown("firewall-cmd --permanent --remove-port=`tcp-listening-port`/tcp") -# ) - max-connections(`splunk-max-connections`) - log-iw-size(`splunk-window-size`) - log-fetch-limit(`splunk-fetch-limit`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - }; + `splunk-tcp-driver` ( + transport("tcp") + port(`tcp-listening-port`) + ip-protocol(`ip-version`) + max-connections(`splunk-max-connections`) + log-iw-size(`splunk-window-size`) + log-fetch-limit(`splunk-fetch-limit`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); + }; + #TODO: #60 Remove this function with enhancement + rewrite(set_metadata_presume); + rewrite(set_rfcnonconformant); + rewrite(r_set_splunk_basic); -source s_all-ports { - `splunk-udp-driver` ( transport("udp") port(514) flags(no-parse) persist-name("example-persist-name1")); - `splunk-udp-driver` ( transport("udp") port(919) flags(no-parse)); - `splunk-udp-driver` ( transport("udp") port(920) flags(no-parse)); - `splunk-udp-driver` ( transport("udp") port(921) flags(no-parse)); - - `splunk-tcp-driver` ( transport("tcp") port(919) flags(no-parse)); - `splunk-tcp-driver` ( transport("tcp") port(920) flags(no-parse)); - `splunk-tcp-driver` ( transport("tcp") port(921) flags(no-parse)); - }; + if { + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_strict); + } elif { + filter(f_rfc5424_noversion); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_noversion); + } elif { + parser {cisco-parser()}; + rewrite(set_metadata_vendor_product_cisco_ios); + } else { + parser { + syslog-parser(time-zone(`default-timezone`) flags(store-raw-message)); + }; + rewrite(set_rfc3164); + }; + parser { + vendor_product_by_source(); + }; + }; +}; diff --git a/package/etc/context/splunk_index.csv b/package/etc/context-local/splunk_index.csv similarity index 64% rename from package/etc/context/splunk_index.csv rename to package/etc/context-local/splunk_index.csv index e495491..e3d7f9d 100644 --- a/package/etc/context/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -1,46 +1,50 @@ -cisco:asa,index,main +bluecoat:proxysg:access:kv,index,netproxy +bluecoat:proxysg:access:kv,sourcetype,bluecoat:proxysg:access:kv +cisco:asa,index,netfw cisco:asa,sourcetype,cisco:asa -cisco:ios,index,main +cisco:ios,index,netops cisco:ios,sourcetype,cisco:ios +cisco:nx-os,index,netops +cisco:nx-os,sourcetype,cisco:ios fgt_event,sourcetype,fgt_event -fgt_event,index,main +fgt_event,index,netops fgt_log,sourcetype,fgt_log -fgt_log,index,main +fgt_log,index,netops fgt_traffic,sourcetype,fgt_traffic -fgt_traffic,index,main +fgt_traffic,index,netfw fgt_utm,sourcetype,fgt_utm -fgt_utm,index,main +fgt_utm,index,netids juniper:structured,index,main juniper:structured,sourcetype,juniper:structured -juniper:junos:firewall:structured,index,main +juniper:junos:firewall:structured,index,netfw juniper:junos:firewall:structured,sourcetype,juniper:junos:firewall:structured -juniper:junos:idp:structured,index,main +juniper:junos:idp:structured,index,netids juniper:junos:idp:structured,sourcetype,juniper:junos:idp:structured -juniper:junos:firewall,index,main +juniper:junos:firewall,index,netfw juniper:junos:firewall,sourcetype,juniper:junos:firewall -juniper:junos:idp,index,main +juniper:junos:idp,index,netids juniper:junos:idp,sourcetype,juniper:junos:idp -juniper:nsm:idp,index,main +juniper:nsm:idp,index,netids juniper:nsm:idp,sourcetype,juniper:nsm:idp -juniper:sslvpn,index,main +juniper:sslvpn,index,netfw juniper:sslvpn,sourcetype,juniper:sslvpn -juniper:netscreen,index,main +juniper:netscreen,index,netfw juniper:netscreen,sourcetype,netscreen:firewall -juniper:idp,index,main +juniper:idp,index,netids juniper:idp,sourcetype,juniper:idp -juniper:idp:structured,index,main +juniper:idp:structured,index,netids juniper:idp:structured,sourcetype,juniper:idp:structured -juniper:nsm,index,main +juniper:nsm,index,netids juniper:nsm,sourcetype,juniper:nsm -juniper:legacy,index,main +juniper:legacy,index,netfw juniper:legacy,sourcetype,juniper:legacy -pan:traffic,index,main -pan:threat,index,main -pan:system,index,main -pan:config,index,main +pan:traffic,index,netfw +pan:threat,index,netproxy +pan:system,index,netops +pan:config,index,netops pan:hipwatch,index,main pan:correlation,index,main -pan:userid,index,main +pan:userid,index,netops pan:unknown,index,main pan:traffic,sourcetype,pan:traffic pan:threat,sourcetype,pan:threat diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/context-local/vendor_product_by_source.conf new file mode 100644 index 0000000..4da1866 --- /dev/null +++ b/package/etc/context-local/vendor_product_by_source.conf @@ -0,0 +1,29 @@ +@version: 3.22 +#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a +#bug/limitation of selector files. The better syntax should be as follows +#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; + +filter f_test_test { + host("testvp-*" type(glob)) + or match("test_test" value("fields.sc4s_presume")) +}; +filter f_juniper_nsm { + host("jnpnsm-*" type(glob)) + or match("juniper_nsm" value("fields.sc4s_presume")) +}; +filter f_juniper_nsm_idp { + host("jnpnsmidp-*" type(glob)) + or match("juniper_nsm_idp" value("fields.sc4s_presume")) +}; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) + or match("juniper_idp" value("fields.sc4s_presume")) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob) ) + or match("juniper_netscreen" value("fields.sc4s_presume")) +}; +filter f_cisco_nx_os { + host("csconx-*" type(glob) ) + or match("cisco_nx_os" value("fields.sc4s_presume")) +}; \ No newline at end of file diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/context-local/vendor_product_by_source.csv new file mode 100644 index 0000000..5193616 --- /dev/null +++ b/package/etc/context-local/vendor_product_by_source.csv @@ -0,0 +1,6 @@ +f_test_test,sc4s_vendor_product,"test_test" +f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" +f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" +f_juniper_idp,sc4s_vendor_product,"juniper_idp" +f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" \ No newline at end of file diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 565b79e..27e1a6e 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -48,7 +48,7 @@ options { # =============================================================================================== # Splunk metadata (HEC/Kafka transport only) # =============================================================================================== -@define splunk-sourcetype "syslog:fallback" +@define splunk-sourcetype "syslog-ng:fallback" @define splunk-index "main" # =============================================================================================== @@ -74,61 +74,16 @@ options { # =============================================================================================== @include "scl.conf" @include "conf.d/templates/*.conf" +@include "conf.d/conflib/*.conf" +@include "conf.d/conflib/*/*.conf" +@include "conf.d/conflib/blocks/*.conf" +@include "conf.d/conflib/blocks/*/*.conf" + @include "conf.d/sources/*.conf" -@include "conf.d/blocks/*.conf" @include "conf.d/destinations/*.conf" -log { - source(s_internal); - - if (match("^Log statistics; " value("MESSAGE"))) { - - parser {p_add_context_splunk(key("syslog-ng:metrics")); }; - rewrite { - subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', - ' - {"time": "$S_UNIXTIME.$S_MSEC", - "event": "metric", - "host": "$HOST", - "index": "${.splunk.index}", - "source": "internal", - "sourcetype": "${.splunk.sourcetype}", - "fields": { - "source_name": "${SourceName}", - "source_instance": "${SourceInstance}", - "state": "${State}", - "type": "${Type}", - "_value": ${Number}, - "metric_name": "syslogng.${SourceId}" - } - } - ', - value("MESSAGE") flags("utf8" "global") - ); - }; - destination(d_hecmetrics); #--HEC-- - } else { - parser {p_add_context_splunk(key("syslog-ng:events")); }; - rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- - destination(d_hec); #--HEC-- - }; - flags(final); -}; - - -@include "conf.d/prefilters/*.conf" -@include "conf.d/filters/*.conf" +@include "conf.d/log_paths/*.conf" +@include "conf.d/plugin/*/log_paths/*.conf" -@include "conf.d/plugin/*/blocks/*.conf" -@include "conf.d/plugin/*/prefilters/*.conf" -@include "conf.d/plugin/*/filters/*.conf" - -log { - source(s_default-ports); - parser {p_add_context_splunk(key("syslog-ng:fallback")); }; - rewrite { r_set_splunk(template("t_JSON"))}; #--HEC-- - destination(d_hec); #--HEC-- - flags(flow-control,fallback); -}; diff --git a/splunk/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf similarity index 53% rename from splunk/SA-syslog-ng/default/indexes.conf rename to splunk/etc/apps/SA-syslog-ng/default/indexes.conf index e795078..f8dbfb2 100644 --- a/splunk/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -19,3 +19,24 @@ thawedPath = $SPLUNK_DB/test/thaweddb homePath = $SPLUNK_DB/test2/db coldPath = $SPLUNK_DB/test2/colddb thawedPath = $SPLUNK_DB/test2/thaweddb + + +[netfw] +homePath = $SPLUNK_DB/netfw/db +coldPath = $SPLUNK_DB/netfw/colddb +thawedPath = $SPLUNK_DB/netfw/thaweddb + +[netids] +homePath = $SPLUNK_DB/netids/db +coldPath = $SPLUNK_DB/netids/colddb +thawedPath = $SPLUNK_DB/netids/thaweddb + +[netops] +homePath = $SPLUNK_DB/netops/db +coldPath = $SPLUNK_DB/netops/colddb +thawedPath = $SPLUNK_DB/netops/thaweddb + +[netproxy] +homePath = $SPLUNK_DB/netproxy/db +coldPath = $SPLUNK_DB/netproxy/colddb +thawedPath = $SPLUNK_DB/netproxy/thaweddb \ No newline at end of file diff --git a/test-with-compose.sh b/test-with-compose.sh index 3f08cbf..b569b7f 100755 --- a/test-with-compose.sh +++ b/test-with-compose.sh @@ -8,16 +8,19 @@ mkdir test-results docker volume create sc4s-tests docker volume create sc4s-results +docker volume create splunk-etc docker container create --name dummy \ -v sc4s-tests:/work/tests \ -v sc4s-results:/work/test-results \ + -v splunk-etc:/work/splunk-etc \ registry.access.redhat.com/ubi7/ubi docker cp tests/ dummy:/work/tests/ +docker cp ./splunk/etc/* dummy:/work/splunk-etc/ docker rm dummy docker-compose build -docker-compose up --abort-on-container-exit --exit-code-from test +docker-compose up --abort-on-container-exit --exit-code-from test docker container create --name dummy \ -v sc4s-tests:/work/tests \ diff --git a/tests/Dockerfile b/tests/Dockerfile index cb630f3..ddb8c55 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -13,5 +13,7 @@ COPY requirements.txt / RUN pip3 install -r /requirements.txt RUN mkdir -p /work/tests RUN mkdir -p /work/test-results/functional +COPY entrypoint.sh / +COPY wait-for /bin/ #WORKDIR /work -CMD cd /work;python -m pytest --junitxml=/work/test-results/functional/functional.xml +CMD /entrypoint.sh \ No newline at end of file diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh new file mode 100755 index 0000000..646dd56 --- /dev/null +++ b/tests/entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/sh +echo Check for sc4s +wait-for sc4s:514 -t 0 -- echo scs is up +echo check for splunk web +wait-for splunk:8000 -t 0 -- echo splunkweb is up +echo check for splunk mgmt +wait-for splunk:8089 -t 0 -- echo splunkmgmt is up +echo check for splunk hec +wait-for splunk:8088 -t 0 -- echo splunkhec is up + +sleep 30 + +echo Check for sc4s +wait-for sc4s:514 -t 0 -- echo scs is up +echo check for splunk web +wait-for splunk:8000 -t 0 -- echo splunkweb is up +echo check for splunk mgmt +wait-for splunk:8089 -t 0 -- echo splunkmgmt is up +echo check for splunk hec +wait-for splunk:8088 -t 0 -- echo splunkhec is up + + +cd /work;python -m pytest --junitxml=/work/test-results/functional/functional.xml diff --git a/tests/requirements.txt b/tests/requirements.txt index 149b7a8..cc706aa 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,4 +11,3 @@ jinja2 jinja2-time http://dev.splunk.com/goto/sdk-python flake8 -flaky diff --git a/tests/sendmessage.py b/tests/sendmessage.py index 6388c78..6b591ad 100644 --- a/tests/sendmessage.py +++ b/tests/sendmessage.py @@ -8,9 +8,9 @@ from time import sleep -def sendsingle(message): +def sendsingle(message, host="sc4s", port=514): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_address = ('sc4s', 514) + server_address = (host, port) tried = 0 while True: diff --git a/tests/splunkutils.py b/tests/splunkutils.py index 7227ae9..bb77a58 100644 --- a/tests/splunkutils.py +++ b/tests/splunkutils.py @@ -33,5 +33,5 @@ def splunk_single(service, search): break else: tried += 1 - sleep(1) + sleep(5) return resultCount, eventCount diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index 308356a..7203f32 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause import random -from flaky import flaky from jinja2 import Environment from .sendmessage import * @@ -17,17 +16,16 @@ # Apr 15 2017 00:21:14 192.168.12.1 : %ASA-5-111010: User 'john', running 'CLI' from IP 0.0.0.0, executed 'dir disk0:/dap.xml' # Apr 15 2017 00:22:27 192.168.12.1 : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:81.24.28.226 dst inside:72.142.17.10 (type 3, code 0) on outside interface. Original IP payload: udp src 72.142.17.10/40998 dst 194.153.237.66/53. # Apr 15 2017 00:22:42 192.168.12.1 : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 -@flaky(max_runs=3, min_passes=2) def test_cisco_asa_tradditional(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") message = mt.render(mark="<111>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:asa\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -40,17 +38,16 @@ def test_cisco_asa_tradditional(record_property, setup_wordlist, setup_splunk): # <166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 -@flaky(max_runs=3, min_passes=2) def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/5424 to outside:72.142.18.38/23\n") + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} : %ASA-3-005424: TCP access denied by ACL from 179.236.133.160/5424 to outside:72.142.18.38/23\n") message = mt.render(mark="<166>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:asa\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-005424\"| head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index f729df9..13ec3f5 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause from jinja2 import Environment -from flaky import flaky from .sendmessage import * from .splunkutils import * @@ -14,7 +13,6 @@ # <190>30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated -@flaky(max_runs=3, min_passes=2) def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key @@ -24,7 +22,7 @@ def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") + st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_cisco_nx_os.py b/tests/test_cisco_nx_os.py new file mode 100644 index 0000000..27b1155 --- /dev/null +++ b/tests/test_cisco_nx_os.py @@ -0,0 +1,55 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +# Nov 1 14:07:58 excal-113 %MODULE-5-MOD_OK: Module 1 is online +def test_cisco_nx_os(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} csconx-{{ host }} %MODULE-5-MOD_OK: Module 1 is online") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netops host=\"csconx-{{ host }}\" sourcetype=\"cisco:ios\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Nov 1 14:07:58 excal-113 %MODULE-5-MOD_OK: Module 1 is online +# @pytest.mark.xfail +#def test_cisco_nx_os_singleport(record_property, setup_wordlist, get_host_key, setup_splunk): +# host = get_host_key +# +# mt = env.from_string( +# "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} %MODULE-5-MOD_OK: Module 1 is online") +# message = mt.render(mark="<23>", host=host) +# +# sendsingle(message, host="sc4s-nx-os") +# +# st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") +# search = st.render(host=host) +# +# resultCount, eventCount = splunk_single(setup_splunk, search) +# +# record_property("host", host) +# record_property("resultCount", resultCount) +# record_property("message", message) +# +# assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_common.py b/tests/test_common.py new file mode 100644 index 0000000..24cb44a --- /dev/null +++ b/tests/test_common.py @@ -0,0 +1,51 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +def test_defaultroute(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4sdefault[0]: test\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"syslog-ng:fallback\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_tag(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} sc4sdefault[0]: test\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"testvp-{{ host }}\" sourcetype=\"syslog-ng:fallback\" sc4s_vendor_product=test_test | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index 60fdae7..f68f13a 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause import random -from flaky import flaky from jinja2 import Environment from .sendmessage import * @@ -13,18 +12,17 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -@flaky(max_runs=3, min_passes=2) #<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") + "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") + st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -35,17 +33,16 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -@flaky(max_runs=3, min_passes=2) #<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") + "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") message = mt.render(mark="<13>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -56,17 +53,16 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -@flaky(max_runs=3, min_passes=2) #<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:21:40 logver=52 devname=US-Corp_Main1 devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user="" srcip=172.30.16.119 srcport=53235 srcintf="Internal" dstip=114.112.67.75 dstport=80 dstintf="External-SDC" proto=6 service=HTTP hostname="popo.wan.ijinshan.com" profile="scan" action=passthrough reqtype=direct url="/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl" sentbyte=525 rcvdbyte=325 direction=outgoing msg="URL belongs to an allowed category in policy" method=domain cat=52 catdesc="Information Technology" def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") + "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") message = mt.render(mark="<13>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_juniper.py b/tests/test_juniper.py index c69f6c4..c4bdb81 100644 --- a/tests/test_juniper.py +++ b/tests/test_juniper.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause from jinja2 import Environment -from flaky import flaky from .sendmessage import * from .splunkutils import * @@ -13,138 +12,17 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -# <165>1 2007-02-15T09:17:15.719Z router1 mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username="user"] User 'user' exiting configuration mode -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_junos_structured(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username=\"user\"] User 'user' exiting configuration mode\n") - message = mt.render(mark="<165>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:structured\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <165>1 2007-02-15T09:17:15.719Z idp1 RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time="1507845354" message-type="SIG" source-address="183.78.180.27" source-port="45610" destination-address="118.127.xx.xx" destination-port="80" protocol-name="TCP" service-name="SERVICE_IDP" application-name="HTTP" rule-name="9" rulebase-name="IPS" policy-name="Recommended" export-id="15229" repeat-count="0" action="DROP" threat-severity="HIGH" attack-name="TROJAN:ZMEU-BOT-SCAN" nat-source-address="0.0.0.0" nat-source-port="0" nat-destination-address="172.xx.xx.xx" nat-destination-port="0" elapsed-time="0" inbound-bytes="0" outbound-bytes="0" inbound-packets="0" outbound-packets="0" source-zone-name="sec-zone-name-internet" source-interface-name="reth0.XXX" destination-zone-name="dst-sec-zone1-outside" destination-interface-name="reth1.xxx" packet-log-id="0" alert="no" username="N/A" roles="N/A" message="-"] -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_junos_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time=\"1507845354\" message-type=\"SIG\" source-address=\"183.78.180.27\" source-port=\"45610\" destination-address=\"118.127.xx.xx\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"9\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"15229\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"TROJAN:ZMEU-BOT-SCAN\" nat-source-address=\"0.0.0.0\" nat-source-port=\"0\" nat-destination-address=\"172.xx.xx.xx\" nat-destination-port=\"0\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"sec-zone-name-internet\" source-interface-name=\"reth0.XXX\" destination-zone-name=\"dst-sec-zone1-outside\" destination-interface-name=\"reth1.xxx\" packet-log-id=\"0\" alert=\"no\" username=\"N/A\" roles=\"N/A\" message=\"-\"]") - message = mt.render(mark="<165>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:idp:structured\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <165>1 2010-06-23T18:05:55 10.209.83.9 Jnpr Syslog 23414 1 [syslog@juniper.net dayId="20100623" recordId="0" timeRecv="2010/06/23 18:05:55" timeGen="2010/06/23 18:05:51" domain="" devDomVer2="0" device_ip="10.209.83.9" cat="Config" attack="" srcZn="NULL" srcIntf="" srcAddr="0.0.0.0" srcPort="0" natSrcAddr="NULL" natSrcPort="0" dstZn="NULL" dstIntf="NULL" dstAddr="0.0.0.0" dstPort="0" natDstAddr="NULL" natDstPort="0" protocol="IP" ruleDomain="" ruleVer="0" policy="" rulebase="NONE" ruleNo="0" action="NONE" severity="INFO" alert="no" elaspedTime="0" inbytes="0" outbytes="0" totBytes="0" inPak="0" outPak="0" totPak="0" repCount="0" packetData="no" varEnum="0" misc="Interaface eth2,eth3 is in Normal State" user="NULL" app="NULL" uri="NULL"] -# -# -# -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} Jnpr Syslog 23414 [syslog@juniper.net dayId=\"20100623\" recordId=\"0\" timeRecv=\"2010/06/23 18:05:55\" timeGen=\"2010/06/23 18:05:51\" domain=\"\" devDomVer2=\"0\" device_ip=\"10.209.83.9\" cat=\"Config\" attack=\"\" srcZn=\"NULL\" srcIntf=\"\" srcAddr=\"0.0.0.0\" srcPort=\"0\" natSrcAddr=\"NULL\" natSrcPort=\"0\" dstZn=\"NULL\" dstIntf=\"NULL\" dstAddr=\"0.0.0.0\" dstPort=\"0\" natDstAddr=\"NULL\" natDstPort=\"0\" protocol=\"IP\" ruleDomain=\"\" ruleVer=\"0\" policy=\"\" rulebase=\"NONE\" ruleNo=\"0\" action=\"NONE\" severity=\"INFO\" alert=\"no\" elaspedTime=\"0\" inbytes=\"0\" outbytes=\"0\" totBytes=\"0\" inPak=\"0\" outPak=\"0\" totPak=\"0\" repCount=\"0\" packetData=\"no\" varEnum=\"0\" misc=\"Interaface eth2,eth3 is in Normal State\" user=\"NULL\" app=\"NULL\" uri=\"NULL\"]") - message = mt.render(mark="<165>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:idp:structured\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - # <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_junos_fw_structured(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_FLOW - RT_FLOW_SESSION_CREATE_LS [junos@2636.1.1.1.2.26 logical-system-name=\"test-lsys\" source-address=\"10.10.10.100\" source-port=\"4206\" destination-address=\"10.20.20.15\" destination-port=\"445\" service-name=\"junos-smb\" nat-source-address=\"10.10.10.100\" nat-source-port=\"4206\" nat-destination-address=\"10.20.20.15\" nat-destination-port=\"445\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"123\" source-zone-name=\"TEST1\" destination-zone-name=\"TEST2\" session-id-32=\"14285714\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth1.100\"]") - message = mt.render(mark="<23>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall:structured\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - - -# <23> Mar 18 17:56:52 RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION="URL Permitted" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY="Enhanced_Information_Technology" REASON="BY_PRE_DEFINED" PROFILE="UTM-Wireless-Profile" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_utm_standard(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION=\"URL Permitted\" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY=\"Enhanced_Information_Technology\" REASON=\"BY_PRE_DEFINED\" PROFILE=\"UTM-Wireless-Profile\" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A") - message = mt.render(mark="<23>", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail def test_juniper_nsm_standard(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} jnpnsm-{{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") message = mt.render(mark="<134>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:nsm\" | head 2") + st = env.from_string("search index=netids host=\"jnpnsm-{{ host }}\" sourcetype=\"juniper:nsm\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -157,18 +35,16 @@ def test_juniper_nsm_standard(record_property, setup_wordlist, get_host_key, set # THE LOG SAMPLE BELOW IS IMPLIED FROM THE JUNIPER DOCS; need to obtain a real sample. # <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [IDP NAME], [IDP IP], predefined, rule, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [IDP NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail def test_juniper_nsm_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [IDP NAME], [IDP IP], predefined, rule, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [IDP NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} jnpnsmidp-{{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [IDP NAME], [IDP IP], predefined, rule, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [IDP NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") message = mt.render(mark="<134>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:nsm:idp\" | head 2") + st = env.from_string("search index=netids host=\"jnpnsmidp-{{ host }}\" sourcetype=\"juniper:nsm:idp\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -180,68 +56,17 @@ def test_juniper_nsm_idp_standard(record_property, setup_wordlist, get_host_key, assert resultCount == 1 # <23> Apr 24 12:30:05 cs-loki3 RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message - -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail - -def test_juniper_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message -") - message = mt.render(mark="<23>", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:idp\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <23> Nov 18 09:56:58 INTERNET-ROUTER RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192 -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail - -def test_juniper_firewall_standard(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192") - message = mt.render(mark="<23>", host=host) - - sendsingle(message) - - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[] - Session timed out for xxx@xxx.xxx.xxx/Users (session:00000000) due to inactivity (last access at 13:59:31 2013/02/27). Idle session identified during routine system scan. -# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied. -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail - -def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, setup_splunk): +# <23> Mar 18 17:56:52 [FW IP] [FW Model]: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time="2009-03-18 16:07:06" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1 +def test_juniper_netscreen(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} Juniper: {% now 'utc', '%Y-%m-%d %H:%M:%S' %} - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied.") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} jnpns-{{ host }} ns204: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time=\"2009-03-18 16:07:06\" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1\n") message = mt.render(mark="<23>", host=host) sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:sslvpn\" | head 2") + st = env.from_string("search index=netfw host=\"jnpns-{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -252,19 +77,18 @@ def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, assert resultCount == 1 +# <23> Apr 24 12:30:05 cs-loki3 RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message - # <23> Mar 18 17:56:52 [FW IP] [FW Model]: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time="2009-03-18 16:07:06" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1 -@flaky(max_runs=3, min_passes=2) -# @pytest.mark.xfail -def test_juniper_netscreen(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_netscreen_singleport(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} ns204: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time=\"2009-03-18 16:07:06\" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1\n") message = mt.render(mark="<23>", host=host) - sendsingle(message) + sendsingle(message, host="sc4s-juniper") - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_juniper_junos_rfc3164.py b/tests/test_juniper_junos_rfc3164.py new file mode 100644 index 0000000..3ec13ff --- /dev/null +++ b/tests/test_juniper_junos_rfc3164.py @@ -0,0 +1,97 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +# <23> Mar 18 17:56:52 RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION="URL Permitted" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY="Enhanced_Information_Technology" REASON="BY_PRE_DEFINED" PROFILE="UTM-Wireless-Profile" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A +def test_juniper_utm_standard(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION=\"URL Permitted\" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY=\"Enhanced_Information_Technology\" REASON=\"BY_PRE_DEFINED\" PROFILE=\"UTM-Wireless-Profile\" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A") + message = mt.render(mark="<23>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <23> Nov 18 09:56:58 INTERNET-ROUTER RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192 +def test_juniper_firewall_standard(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192") + message = mt.render(mark="<23>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[] - Session timed out for xxx@xxx.xxx.xxx/Users (session:00000000) due to inactivity (last access at 13:59:31 2013/02/27). Idle session identified during routine system scan. +# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied. +def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} Juniper: {% now 'utc', '%Y-%m-%d %H:%M:%S' %} - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied.") + message = mt.render(mark="<23>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:sslvpn\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + + +def test_juniper_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message -") + message = mt.render(mark="<23>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:idp\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_juniper_junos_rfc5124.py b/tests/test_juniper_junos_rfc5124.py new file mode 100644 index 0000000..b9ff921 --- /dev/null +++ b/tests/test_juniper_junos_rfc5124.py @@ -0,0 +1,105 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +# <165>1 2007-02-15T09:17:15.719Z router1 mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username="user"] User 'user' exiting configuration mode +# @pytest.mark.xfail +def test_juniper_junos_structured(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username=\"user\"] User 'user' exiting configuration mode\n") + message = mt.render(mark="<165>1", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:structured\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <165>1 2007-02-15T09:17:15.719Z idp1 RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time="1507845354" message-type="SIG" source-address="183.78.180.27" source-port="45610" destination-address="118.127.xx.xx" destination-port="80" protocol-name="TCP" service-name="SERVICE_IDP" application-name="HTTP" rule-name="9" rulebase-name="IPS" policy-name="Recommended" export-id="15229" repeat-count="0" action="DROP" threat-severity="HIGH" attack-name="TROJAN:ZMEU-BOT-SCAN" nat-source-address="0.0.0.0" nat-source-port="0" nat-destination-address="172.xx.xx.xx" nat-destination-port="0" elapsed-time="0" inbound-bytes="0" outbound-bytes="0" inbound-packets="0" outbound-packets="0" source-zone-name="sec-zone-name-internet" source-interface-name="reth0.XXX" destination-zone-name="dst-sec-zone1-outside" destination-interface-name="reth1.xxx" packet-log-id="0" alert="no" username="N/A" roles="N/A" message="-"] +# @pytest.mark.xfail +def test_juniper_junos_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time=\"1507845354\" message-type=\"SIG\" source-address=\"183.78.180.27\" source-port=\"45610\" destination-address=\"118.127.xx.xx\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"9\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"15229\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"TROJAN:ZMEU-BOT-SCAN\" nat-source-address=\"0.0.0.0\" nat-source-port=\"0\" nat-destination-address=\"172.xx.xx.xx\" nat-destination-port=\"0\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"sec-zone-name-internet\" source-interface-name=\"reth0.XXX\" destination-zone-name=\"dst-sec-zone1-outside\" destination-interface-name=\"reth1.xxx\" packet-log-id=\"0\" alert=\"no\" username=\"N/A\" roles=\"N/A\" message=\"-\"]") + message = mt.render(mark="<165>1", host=host) + + sendsingle(message) + + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:idp:structured\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <165>1 2010-06-23T18:05:55 10.209.83.9 Jnpr Syslog 23414 1 [syslog@juniper.net dayId="20100623" recordId="0" timeRecv="2010/06/23 18:05:55" timeGen="2010/06/23 18:05:51" domain="" devDomVer2="0" device_ip="10.209.83.9" cat="Config" attack="" srcZn="NULL" srcIntf="" srcAddr="0.0.0.0" srcPort="0" natSrcAddr="NULL" natSrcPort="0" dstZn="NULL" dstIntf="NULL" dstAddr="0.0.0.0" dstPort="0" natDstAddr="NULL" natDstPort="0" protocol="IP" ruleDomain="" ruleVer="0" policy="" rulebase="NONE" ruleNo="0" action="NONE" severity="INFO" alert="no" elaspedTime="0" inbytes="0" outbytes="0" totBytes="0" inPak="0" outPak="0" totPak="0" repCount="0" packetData="no" varEnum="0" misc="Interaface eth2,eth3 is in Normal State" user="NULL" app="NULL" uri="NULL"] +# +# +# +# @pytest.mark.xfail +def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} Jnpr Syslog 23414 [syslog@juniper.net dayId=\"20100623\" recordId=\"0\" timeRecv=\"2010/06/23 18:05:55\" timeGen=\"2010/06/23 18:05:51\" domain=\"\" devDomVer2=\"0\" device_ip=\"10.209.83.9\" cat=\"Config\" attack=\"\" srcZn=\"NULL\" srcIntf=\"\" srcAddr=\"0.0.0.0\" srcPort=\"0\" natSrcAddr=\"NULL\" natSrcPort=\"0\" dstZn=\"NULL\" dstIntf=\"NULL\" dstAddr=\"0.0.0.0\" dstPort=\"0\" natDstAddr=\"NULL\" natDstPort=\"0\" protocol=\"IP\" ruleDomain=\"\" ruleVer=\"0\" policy=\"\" rulebase=\"NONE\" ruleNo=\"0\" action=\"NONE\" severity=\"INFO\" alert=\"no\" elaspedTime=\"0\" inbytes=\"0\" outbytes=\"0\" totBytes=\"0\" inPak=\"0\" outPak=\"0\" totPak=\"0\" repCount=\"0\" packetData=\"no\" varEnum=\"0\" misc=\"Interaface eth2,eth3 is in Normal State\" user=\"NULL\" app=\"NULL\" uri=\"NULL\"]") + message = mt.render(mark="<165>1", host=host) + + sendsingle(message) + + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:idp:structured\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos +# @pytest.mark.xfail +def test_juniper_junos_fw_structured(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_FLOW - RT_FLOW_SESSION_CREATE_LS [junos@2636.1.1.1.2.26 logical-system-name=\"test-lsys\" source-address=\"10.10.10.100\" source-port=\"4206\" destination-address=\"10.20.20.15\" destination-port=\"445\" service-name=\"junos-smb\" nat-source-address=\"10.10.10.100\" nat-source-port=\"4206\" nat-destination-address=\"10.20.20.15\" nat-destination-port=\"445\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"123\" source-zone-name=\"TEST1\" destination-zone-name=\"TEST2\" session-id-32=\"14285714\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth1.100\"]") + message = mt.render(mark="<23>1", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall:structured\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index 5a601f0..2d40c8f 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -5,7 +5,6 @@ # https://opensource.org/licenses/BSD-2-Clause import random -from flaky import flaky from jinja2 import Environment from .sendmessage import * @@ -13,9 +12,7 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) - #<190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0 -@flaky(max_runs=3, min_passes=2) def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -25,7 +22,7 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -38,7 +35,6 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): # <190>Oct 30 09:46:17 1,2012/10/30 09:46:17,01606001116,THREAT,url,1,2012/04/10 04:39:55,192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,"litetopdetect.cn/index.php",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html -@flaky(max_runs=3, min_passes=2) def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -48,7 +44,7 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:threat\" | head 2") + st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"pan:threat\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_poc.py b/tests/test_poc.py deleted file mode 100644 index 37dcdef..0000000 --- a/tests/test_poc.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019 Splunk, Inc. -# -# Use of this source code is governed by a BSD-2-clause-style -# license that can be found in the LICENSE-BSD2 file or at -# https://opensource.org/licenses/BSD-2-Clause -import random - -from flaky import flaky -from jinja2 import Environment - -from .sendmessage import * -from .splunkutils import * - -env = Environment(extensions=['jinja2_time.TimeExtension']) - - -@flaky(max_runs=3, min_passes=2) -def test_defaultroute(record_property, setup_wordlist, setup_splunk): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - - mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %}.000z {{ host }} sc4s_default[0]: test\n") - message = mt.render(mark="<111>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=main \"{{ host }}\" sourcetype=\"syslog:fallback\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py new file mode 100644 index 0000000..0dddfac --- /dev/null +++ b/tests/test_symantec_proxy.py @@ -0,0 +1,34 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +# <134>1 2019-08-21T17:42:08.000z "sample_logs bluecoat[0]:SPLV5.1 c-ip=192.0.0.6 cs-bytes=6269 cs-categories="unavailable" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent="ocspd/1.0.3" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name="Explicit HTTP" service.group="Standard" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name="10.0.0.6-sample_logs" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url="http://randomserver:8000/en-US/app/examples/" +def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{host}} bluecoat[0]: SPLV5.1 c-ip=192.0.0.6 cs-bytes=6269 cs-categories=\"unavailable\" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent=\"ocspd/1.0.3\" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name=\"Explicit HTTP\" service.group=\"Standard\" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name=\"10.0.0.6-sample_logs\" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url=\"http://randomserver:8000/en-US/app/examples/\"") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"bluecoat:proxysg:access:kv\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# diff --git a/tests/wait-for b/tests/wait-for new file mode 100755 index 0000000..bf6a071 --- /dev/null +++ b/tests/wait-for @@ -0,0 +1,79 @@ +#!/bin/sh + +TIMEOUT=15 +QUIET=0 + +echoerr() { + if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi +} + +usage() { + exitcode="$1" + cat << USAGE >&2 +Usage: + $cmdname host:port [-t timeout] [-- command args] + -q | --quiet Do not output any status messages + -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit "$exitcode" +} + +wait_for() { + for i in `seq $TIMEOUT` ; do + nc -z "$HOST" "$PORT" > /dev/null 2>&1 + + result=$? + if [ $result -eq 0 ] ; then + if [ $# -gt 0 ] ; then + exec "$@" + fi + exit 0 + fi + sleep 1 + done + echo "Operation timed out" >&2 + exit 1 +} + +while [ $# -gt 0 ] +do + case "$1" in + *:* ) + HOST=$(printf "%s\n" "$1"| cut -d : -f 1) + PORT=$(printf "%s\n" "$1"| cut -d : -f 2) + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -t) + TIMEOUT="$2" + if [ "$TIMEOUT" = "" ]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + break + ;; + --help) + usage 0 + ;; + *) + echoerr "Unknown argument: $1" + usage 1 + ;; + esac +done + +if [ "$HOST" = "" -o "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 +fi + +wait_for "$@" \ No newline at end of file From 3ea9127551ebbed5131576ca9af1a2091a093e1b Mon Sep 17 00:00:00 2001 From: mkarlstrand-splunk <49571555+mkarlstrand-splunk@users.noreply.github.com> Date: Sat, 31 Aug 2019 08:20:48 -0700 Subject: [PATCH 2/7] Update CONTRIBUTING.md (#74) --- CONTRIBUTING.md | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 85546be..21e497a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,21 +1,27 @@ # Contributing ### Past / Active(marked as *) Contributors -rfaircloth* -mbonsack* -rfipro* -Hurricane Labs* +Splunk Connect for Syslog is developed by Splunkers and the open-source community. -## Code of conduct +We want to give extra special thanks to Hurricane Labs our seed community contributor + +We thank all of our contributors! + +[https://github.com/splunk/splunk-connect-for-syslog/graphs/contributors] + +For the detailed history of contributions of a given file, try + +git blame file +to see line-by-line credits and -###Contributor License Agreement +git log --follow file +to see the change log even across renames and rewrites. + + +## Code of conduct -At the moment, we can only accept pull requests submitted from either: -* Splunk employees or -* Individuals that have signed our contribution agreement -If you wish to be a contributing member of our community, please see the agreement [for individuals](https://www.splunk.com/goto/individualcontributions) or [for organizations](https://www.splunk.com/goto/contributions). ### Our Pledge @@ -93,7 +99,11 @@ available at [http://contributor-covenant.org/version/1/4][version] ## Filing issues -Use project issue tracker +Please file issues in this project with clear description of the problem. + +## Project Discussion Forums + +splunk-usergroups #splunk_connect_for_syslog ## Contributing code or data From 0656fb427a95697c372cd7fc2279439409392a64 Mon Sep 17 00:00:00 2001 From: mkarlstrand-splunk <49571555+mkarlstrand-splunk@users.noreply.github.com> Date: Sat, 31 Aug 2019 08:37:27 -0700 Subject: [PATCH 3/7] Update gettingstarted.md (#73) Adding base system spec --- docs/gettingstarted.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 8abfe3b..a3c3c4f 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -1,7 +1,7 @@ # Pre-req -* Linux host with Docker 19.x or newer with Docker Swarm enabled +* Linux host with 4-6 cores & 2-4 GB memory (Docker container should be allocated 2 cores 2 GB of memory) and Docker 19.x or newer with Docker Swarm enabled * [Getting Started](https://docs.docker.com/get-started/) * A Splunk index for metrics typically "em_metrics" * One or more Splunk indexes for events collected by SC4S @@ -95,4 +95,4 @@ services: - SYSLOG_PRESUME_FILTER=f_juniper_netscreen volumes: - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv -``` \ No newline at end of file +``` From be12e528f2272c51d7e651e616067ee2853771fa Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Fri, 20 Sep 2019 14:32:45 -0400 Subject: [PATCH 4/7] Release/0.5.0 (#86) * New documentation for podman, docker and swarm! * Support for trusted certs for Splunk * Support for product specific TCP/UDP listeners --- .circleci/config.yml | 26 +- .env.template | 2 +- README.md | 2 + demo-with-compose.sh | 32 +- docker-compose-ci.yml | 42 +-- docker-compose-debug.yml | 1 + docker-compose-demo.yml | 68 ++++ docker-compose-perf.yml | 20 +- docker-compose.yml | 57 +--- docs/Configuration.md | 9 + docs/SC4S deployment.png | Bin 0 -> 41688 bytes docs/gettingstarted.md | 173 +++++----- docs/gettingstarted/docker-swarm-general.md | 160 +++++++++ docs/gettingstarted/docker-swarm-rhel7.md | 185 +++++++++++ docs/gettingstarted/docker-systemd-general.md | 164 +++++++++ docs/gettingstarted/podman-systemd-general.md | 164 +++++++++ docs/sources.md | 312 +++++++++++++++++- package/Dockerfile | 19 +- .../etc/conf.d/conflib/_common/network.conf | 10 + .../conf.d/conflib/_common/network.conf.tmpl | 93 ++++++ .../etc/conf.d/conflib/_common/templates.conf | 21 +- .../vendor_product_by_source_context.conf | 2 +- .../conf.d/conflib/_splunk/splunkfields.conf | 27 +- .../conf.d/conflib/vendor_juniper/junos.conf | 5 +- .../conf.d/conflib/vendor_juniper/legacy.conf | 5 +- .../etc/conf.d/destinations/splunk_hec.conf | 77 +---- .../conf.d/destinations/splunk_hec.conf.tmpl | 42 +++ .../destinations/splunk_hec_metrics.conf | 6 + .../destinations/splunk_hec_metrics.conf.tmpl | 20 ++ .../log_paths/P_rfc3164-juniper_nsm_idp.conf | 15 - package/etc/conf.d/log_paths/internal.conf | 10 +- .../log_paths/p_rfc3164-cisco-nx-os.conf | 14 - .../conf.d/log_paths/p_rfc3164-cisco_asa.conf | 20 +- .../log_paths/p_rfc3164-cisco_asa.conf.tmpl | 26 ++ .../conf.d/log_paths/p_rfc3164-cisco_ios.conf | 21 +- .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 25 ++ .../log_paths/p_rfc3164-cisco_nx-os.conf | 14 + .../log_paths/p_rfc3164-cisco_nx-os.conf.tmpl | 25 ++ .../log_paths/p_rfc3164-fortinet_fortios.conf | 37 +-- .../p_rfc3164-fortinet_fortios.conf.tmpl | 45 +++ .../log_paths/p_rfc3164-juniper-idp.conf | 15 - .../log_paths/p_rfc3164-juniper-junos.conf | 24 -- .../log_paths/p_rfc3164-juniper_idp.conf | 17 + .../log_paths/p_rfc3164-juniper_idp.conf.tmpl | 25 ++ .../log_paths/p_rfc3164-juniper_junos.conf | 17 + .../p_rfc3164-juniper_junos.conf.tmpl | 41 +++ .../p_rfc3164-juniper_netscreen.conf | 25 +- .../p_rfc3164-juniper_netscreen.conf.tmpl | 28 ++ .../log_paths/p_rfc3164-juniper_nsm.conf | 24 +- .../log_paths/p_rfc3164-juniper_nsm.conf.tmpl | 25 ++ .../log_paths/p_rfc3164-juniper_nsm_idp.conf | 21 ++ .../p_rfc3164-juniper_nsm_idp.conf.tmpl | 24 ++ .../log_paths/p_rfc3164-paloalto-panos.conf | 61 ---- .../log_paths/p_rfc3164-paloalto_panos.conf | 17 + .../p_rfc3164-paloalto_panos.conf.tmpl | 76 +++++ .../p_rfc5424_noversion-cisco_asa.conf | 20 +- .../p_rfc5424_noversion-cisco_asa.conf.tmpl | 27 ++ .../p_rfc_5424_noversion-symantec-proxy.conf | 16 - .../p_rfc_5424_noversion-symantec_proxy.conf | 10 + ...fc_5424_noversion-symantec_proxy.conf.tmpl | 28 ++ .../p_rfc_5424_strict-juniper-junos.conf | 27 -- .../p_rfc_5424_strict-juniper_junos.conf | 14 + .../p_rfc_5424_strict-juniper_junos.conf.tmpl | 43 +++ package/etc/conf.d/log_paths/zfallback.conf | 4 +- package/etc/conf.d/sources/network.conf | 75 +---- package/etc/conf.d/sources/network.conf.tmpl | 72 ++++ package/etc/context-local/splunk_index.csv | 95 ++---- package/etc/syslog-ng.conf | 33 +- package/sbin/entrypoint.sh | 10 - test-with-compose.sh | 20 +- tests/Dockerfile | 2 + tests/conftest.py | 6 +- tests/sendmessage.py | 6 +- tests/test_juniper_junos_rfc3164.py | 26 +- tests/test_juniper_junos_rfc5124.py | 27 +- ...test_juniper.py => test_juniper_legacy.py} | 33 +- tests/test_juniper_sslvpn.py | 34 ++ 77 files changed, 2223 insertions(+), 841 deletions(-) create mode 100644 docker-compose-demo.yml create mode 100644 docs/Configuration.md create mode 100644 docs/SC4S deployment.png create mode 100644 docs/gettingstarted/docker-swarm-general.md create mode 100644 docs/gettingstarted/docker-swarm-rhel7.md create mode 100644 docs/gettingstarted/docker-systemd-general.md create mode 100644 docs/gettingstarted/podman-systemd-general.md create mode 100644 package/etc/conf.d/conflib/_common/network.conf create mode 100644 package/etc/conf.d/conflib/_common/network.conf.tmpl create mode 100644 package/etc/conf.d/destinations/splunk_hec.conf.tmpl create mode 100644 package/etc/conf.d/destinations/splunk_hec_metrics.conf create mode 100644 package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf delete mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf delete mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl create mode 100644 package/etc/conf.d/sources/network.conf.tmpl delete mode 100755 package/sbin/entrypoint.sh rename tests/{test_juniper.py => test_juniper_legacy.py} (69%) create mode 100644 tests/test_juniper_sslvpn.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 5c02739..8d7ddba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -168,13 +168,13 @@ jobs: path: /clair-reports publish-common: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false environment: IMAGE_NAME: rfaircloth/scs - docker: - - image: circleci/buildpack-deps:stretch steps: - - setup_remote_docker: - docker_layer_caching: true + - checkout - run: name: Docker Login command: docker login -u $DOCKER_USER -p $DOCKER_PASS @@ -183,10 +183,14 @@ jobs: command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - run: name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$CIRCLE_BRANCH + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER - run: name: Docker push tag - command: docker push $IMAGE_NAME:$CIRCLE_BRANCH + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker push $IMAGE_NAME:$SEMVER publish-edge: environment: @@ -273,11 +277,11 @@ workflows: requires: - dgoss - test-unit - filters: - branches: - only: - - master - - develop +# filters: +# branches: +# only: +# - master +# - develop - publish-edge: requires: - dgoss diff --git a/.env.template b/.env.template index 58e599a..8d0cafa 100644 --- a/.env.template +++ b/.env.template @@ -15,7 +15,7 @@ SPLUNK_HEC_URL=https://splunk:8088/services/collector/event SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=metrics +SPLUNK_METRICS_INDEX=em_metrics SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download SPLUNKBASE_USERNAME=username SPLUNKBASE_PASSWORD=password diff --git a/README.md b/README.md index e0d3259..9e4aa53 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ cd splunk-connect-for-syslog cp .env.template .env ``` +- Update the splunkbase username and password in .env this allows the splunk container to install required add-ons for the demo + - Start the demo environment ```bash diff --git a/demo-with-compose.sh b/demo-with-compose.sh index 86c0fbf..3e0f693 100755 --- a/demo-with-compose.sh +++ b/demo-with-compose.sh @@ -4,19 +4,35 @@ #Use of this source code is governed by a BSD-2-clause-style #license that can be found in the LICENSE-BSD2 file or at #https://opensource.org/licenses/BSD-2-Clause - +WAITON=${1:-test} +compose=${2:-docker-compose-demo.yml} +echo $WAITON $compose mkdir test-results -docker volume create sc4s-tests +docker-compose down +docker volume rm sc4s-results +docker volume rm splunk-etc + +docker volume create sc4s-results +docker volume create splunk-etc docker container create --name dummy \ - -v sc4s-tests:/work/tests \ + -v sc4s-results:/work/test-results \ + -v splunk-etc:/work/splunk-etc \ registry.access.redhat.com/ubi7/ubi -docker cp tests/ dummy:/work/tests/ +docker cp ./splunk/etc/* dummy:/work/splunk-etc/ docker rm dummy -docker-compose pull -docker-compose up - -EXIT=$0 +docker-compose -f $compose pull +docker-compose -f $compose up -d splunk +docker-compose -f $compose up -d sc4s +sleep 60 +docker-compose -f $compose build +docker-compose -f $compose up +docker container create --name dummy \ + -v sc4s-results:/work/test-results \ + registry.access.redhat.com/ubi7/ubi +docker cp dummy:/work/test-results/functional test-results +docker rm dummy +EXIT=$0 diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 5201e5b..c33122c 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -14,7 +14,6 @@ services: - splunk - sc4s volumes: - - sc4s-tests:/work/tests - sc4s-results:/work/test-results environment: - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} @@ -26,8 +25,8 @@ services: - "514" - "601" - "514/udp" - - "5514" - - "5514/udp" + - "5000" + - "5000/udp" stdin_open: true tty: true links: @@ -39,31 +38,8 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - sc4s-juniper: - image: rfaircloth/scs:${CIRCLE_SHA1} - hostname: sc4s-juniper - ports: - - "514" - stdin_open: true - tty: true - links: - - splunk - environment: - - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - - SYSLOG_PRESUME_FILTER=juniper_netscreen -# logging: -# driver: splunk -# options: -# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 -# splunk-url: https://splunk:8088/services/collector/event -# splunk-index: main -# splunk-insecureskipverify: true -# splunk-verify-connection: false + - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 splunk: image: splunk/splunk:latest hostname: splunk @@ -80,17 +56,7 @@ services: - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - splunk-etc:/opt/splunk/etc -# logging: -# driver: splunk -# options: -# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 -# splunk-url: https://splunk:8088/services/collector/event -# splunk-index: main -# splunk-insecureskipverify: true -# splunk-verify-connection: false volumes: - sc4s-tests: - external: true sc4s-results: external: true splunk-etc: diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml index 55cb296..9b885f3 100644 --- a/docker-compose-debug.yml +++ b/docker-compose-debug.yml @@ -38,6 +38,7 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no splunk: image: splunk/splunk:latest hostname: splunk diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml new file mode 100644 index 0000000..5e4c200 --- /dev/null +++ b/docker-compose-demo.yml @@ -0,0 +1,68 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +version: "3.2" +services: + test: + build: ./tests + links: + - splunk + - sc4s + - sc4s-juniper + volumes: + - sc4s-tests:/work/tests + - sc4s-results:/work/test-results + environment: + - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} + + sc4s: + image: splunk/scs:latest + hostname: sc4s + ports: + - "514" + - "601" + - "514/udp" + - "5514" + - "5514/udp" + stdin_open: true + tty: true + links: + - splunk + environment: + - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} + - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} + - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + splunk: + image: splunk/splunk:latest + hostname: splunk + ports: + - "8000:8000" + - "8088:8088" + - "8089:8089" + environment: + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} + - SPLUNK_START_ARGS=${SPLUNK_START_ARGS} + - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} + - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} + - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} + volumes: + - splunk-etc:/opt/splunk/etc + +volumes: + sc4s-tests: + external: true + sc4s-results: + external: true + splunk-etc: + external: true diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml index 67f3844..9831278 100644 --- a/docker-compose-perf.yml +++ b/docker-compose-perf.yml @@ -31,14 +31,9 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} -# logging: -# driver: splunk -# options: -# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 -# splunk-url: https://splunk:8088/services/collector/event -# splunk-index: main -# splunk-insecureskipverify: true -# splunk-verify-connection: false + - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + splunk: image: splunk/splunk:latest hostname: splunk @@ -53,14 +48,6 @@ services: - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} -# logging: -# driver: splunk -# options: -# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 -# splunk-url: https://splunk:8088/services/collector/event -# splunk-index: main -# splunk-insecureskipverify: true -# splunk-verify-connection: false egbundles: image: rfaircloth/scs:egb-edge hostname: egbundles @@ -74,4 +61,3 @@ services: links: - egbundles - sc4s - diff --git a/docker-compose.yml b/docker-compose.yml index b0135dc..c637ff6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,12 +13,13 @@ services: links: - splunk - sc4s - - sc4s-juniper volumes: - - sc4s-tests:/work/tests - sc4s-results:/work/test-results environment: + - SPLUNK_USER=admin - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} + - SPLUNK_HOST=splunk + - SYSLOG_HOST=sc4s sc4s: image: splunk/scs:latest @@ -32,8 +33,8 @@ services: - "514" - "601" - "514/udp" - - "5514" - - "5514/udp" + - "5000" + - "5000/udp" stdin_open: true tty: true links: @@ -45,40 +46,8 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - sc4s-juniper: - image: splunk/scs:latest - build: - context: ./package -# cache_from: -# - registry.access.redhat.com/ubi7/ubi -# - registry.access.redhat.com/rhel7/rhel -# - rfaircloth/scs:edge - args: - RH_ORG: ${RH_ORG} - RH_ACTIVATION: ${RH_ACTIVATION} - hostname: sc4s-juniper - ports: - - "514" - stdin_open: true - tty: true - links: - - splunk - environment: - - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - - SYSLOG_PRESUME_FILTER=juniper_netscreen -# logging: -# driver: splunk -# options: -# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 -# splunk-url: https://splunk:8088/services/collector/event -# splunk-index: main -# splunk-insecureskipverify: true -# splunk-verify-connection: false + - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 splunk: image: splunk/splunk:latest hostname: splunk @@ -95,18 +64,8 @@ services: - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - splunk-etc:/opt/splunk/etc -# logging: -# driver: splunk -# options: -# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 -# splunk-url: https://splunk:8088/services/collector/event -# splunk-index: main -# splunk-insecureskipverify: true -# splunk-verify-connection: false volumes: - sc4s-tests: - external: true sc4s-results: external: true splunk-etc: - external: true \ No newline at end of file + external: true diff --git a/docs/Configuration.md b/docs/Configuration.md new file mode 100644 index 0000000..c8380d1 --- /dev/null +++ b/docs/Configuration.md @@ -0,0 +1,9 @@ + +| Variable | Values | Description | +|----------|---------------|-------------| +| SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list | +| SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | +| SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | +| SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE | comma separated list | Open SSL cipher suite list | +| SC4S_DEST_SPLUNK_HEC_SSL_VERSION | comma separated list | Open SSL version list | +| SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | diff --git a/docs/SC4S deployment.png b/docs/SC4S deployment.png new file mode 100644 index 0000000000000000000000000000000000000000..a9614bdc960e1c7f3bc03ba6144ae3e6db4c3231 GIT binary patch literal 41688 zcmZ_$2Rzm9|38jL;zVX5S;tB#dy{=Q$KGUT&to2YhpZf%BI{5@$ler%l3B7s_7-K& z-*p}~9=t=RZsYIYqHhx*~x1ux3+6nvyZ-bxe+Th21@Q;_rj90)+@HTiU z&dW!Y|ClF9hDcrHs%~Rbk_k1FxOz9j(DHxV5FD3-%E?J9if(ct=4P zbjrmqEX>2jD}Wudwz-YDyZ!$(gMHH)Wp4XtB61GCdIB~swhAx@S1o5>FLPxi7&G=e zu3$uUYyT`OxZdrL43J~n=Nu+moc=CmEa=kLUw-|aW)6z|Gg0!pl8nCee%l->$$5~x_Wpk zXz-Y;h+`)hu+?EcLwPv^2F< zl`Zw;Ve+cVBK97(nqaLJ6lEOEXxqxk$ZN@4foBaY9USCb`2@9G_-zzr{S?fRIy#OpU3V9_kiMp@ z3rt%@6>g{?s%D`is^#nErLF8`sikb;q~oQagK~Ga(e-i^Qgt!VwBd1+HIP-ow&>0e zhdYDcBD$6$UOxAoyxl>sa)!dznud;wKG-gO_?#4Fbu={uomH&86l5LkWdsz2MU)H- zVF;wMf<4Sb*-yz%-BLlpOI1(a(7@H+%iWS!UQW@{P?KK~=6qktT+!V}!O_mb70&D6 zrllt2VW{l|H|JI0@lmj_^7HnV_p#T60mGNI_T&>3bTrUag7JE}sJd&rDY)q>S$cUZ zTf^)vJw#pZ+v=z&8X}PT&U`wCNKZ$8VDSoGc8=Ow!Zvzpj^=7w3bLB6E`|shVOIku z9RYhKSJ0}bo+|js>*TDVr-lTlici)R=A_Oqq$LPPI`P}Xt#vgtoxPoG6)^vrTyyWFzf(9bmGOC(Bz;YG9`IJ=`b`$Ya@ltRS z)YrCk$1bvutf)3Z*+I`j&dttK&kpJ4$K&nj%5Nc~DXa$`aIh4SvG>ui^iqn4HR8q4uYT!thtygc~}T)I`AvVS&0bxxvL=g&E<7re0&yaI+iGB z9U)I^2Ob0d`$BH|4qD*rUPv!DD{F*}i;b$NEWfrkTn}j@g7nZ5bawR=@zn9v_jNYZ zFjV%|;?q@-_w^Fi)j+5#sG@|FypSk9FFRLd1Ptz~sqSE>tBq37k#)PTZ)@lcT$QYW zhoZ27fQq}bzLP4Ctg5w=sD_-Xg0-->sEVP!l7XGIo05YoOvw-Cs%8?^V0U<<#B*H+_$pzR6?rjA}vMjd^~vhm7QUJ zR`U1xd8~E#^nGK8Th2+xMaI@iMAuM2M?gzX)6?9? zz(CVM$=yZOSs7y}&P9;N@Vz(Seu$eBE!y z2mbyWW%%W-)%Nu-T!3Csgvsdmn69^6Zr0U3JU@@pF1kJoleW0U%yAhbqsUD1-B|OK zPa3NEtnj)DXK%DpcOI1B*(I4a^3;cf4!M}?%p(0)(9s#N&{@=Z`TkLz%Tl)griA~G zs{OH!!&f6~t=BiF93}c~OKV1KrH-qzYh+AJOtP%&?;2FTn+iHc9&dG!B6nAZH@lAg zU!Jc`wKQ>>G=4Fxb4D5=Yjne;luyl$_cr4Eb;3i)$Z%z)ArKrUjXRJiv(z6e#Fa2i zj_)Fk|77;sz$F%iAl5)*q<{y&a#vL*aM!;>HL zOYZznYYFan@5lBUtYPincjXqsjj?mC6DQ(iMI5+yU_nCUv}Cg%(kN%J-gp z;B6Se1aB9eC200>*W7hSlRD?DNv{cHU;2ny@Z=$G-W0NK(!xnsm*~Gy$>uWQ&gyT+ zN5K#Rnl@4VQV6ztm5Qvw+?OaeTIk4R_5++IYLTYKDU5&C6ih}shK#JO*x6Pm6*leO z!`VsdXMtdo^51N*^^{P{6|4;P3C}3nd2Qa|v^GZDxW73BC{9Ef;Dl zCX7~?;_=(DgJ;Bx%tF>i%FU)a%UyyQ0=8tDjyIc-VBSLJ?Rc~Tb_qZHJ@fwCRVdy) zun}j%R7v0UE0UIvf3!^}+77*yxn4IN4-Hw$VQoC!dW+U3WENBy@%8gih^ z_n*f!f0o1`cIM)xoCA;d1hwin8rRCUeT+J4FoCOs`H>QPqn4XH3fKNkQx2O0Wc*Q9RO+`m;oX^{60O zt+KIEtZ6aBv3SWot`2pw-PLr3;VQe@D>v5p*1u1M1~3ErJq;=iH@kFmu*HABcBUh8 zBAPw>exSNz^LxeQn`71XpVt%N3@`tmr-1RK8V5gG&~H`TSsgAZMNZTT1)l|^KT>6* z!K1o+|H0|;zHm~sOT$9ihwFKI|7UgxV6ZWGoiTSVca#St+KC+u$dT6g{#q1O3{|Rx zyfUim%W@U*-ZXX`F4nE=Eb{!n1)&6oBkw`%#*5sOkJQnd19@Q;+SOn&D`?7M!kAvW zFDRwcSig_C_SX5!@P8XF%>nvTk{1@>VTQDEC}Y}wW&Ie9B4=4I%10~Z6bd>I=A7-Y zW6yu@6W;vJ#DxEBr7|MqK-cfMhr7)o4d!7nRhSdxKy8B3^phxc>*rsi>X|cG&jkiroLSzrZ%-e!aoeqC4jPcypBmC=Lig-a+l_N9Y z`wFZ_>s9@=WhPCH7j<$NR_2dvT3?P=B-f5JWm#^ba<>75g6QEWvUnr`a%LFDqvdup>!wJepUe>;-y z1d-yS4*}H^vrpx8(b=LNFR)I$lQ)v%P&&kNJYMR|NBn0u%deS)=KnSoyV87L%_To^ z&g-LL=LcOaqAGj!|lA@zc|*^Jm@Vr@#WfB@Tn=RPD7fZV*ew z$?U=sLpbd-M?B%&wBhtwRX7gQcz;b~>&?Zu=0^l{uF|hIiroK=6dS5MmilT$4u=UC z1HRV}=@}tYiI!(VX_3{MHv_g0hVhecsuqJlwB$5Ypcs9f#IEgVxP1NOSAV%nn|7g^ zbN*ai>n>;0`PngY>-$@*#dkVS1N(P6-l+4=+&()RX%n&|4LIA1lo|yw{>NF)I(6IG zZaH$q#JjYutl`*FxD|GhJqwkz(Xf>51=5L|gU#cVe-D;M732-g?05YwOb=JnC+JE_ z;Qr)D&|X!}7&)A7@b1^S&bTUIm~XdBcov>{^tloI9Q5*txWyT{NAJI0(VFQCvXWA8 z67}@;myWYpxp%;XY_olBOuEqY8=v_5Ro*d#*yNsW#|KwS1|Er22CO_!ALp$5W>MaH zR4m?_k-`jIDSA?J{Q>msc=}ujvRymjyfz*%p9o){m_DnO0LxlC9dta#-+#}t?chE4 zH^JcZOO2M)*dBFf z2mJi7`9SbtK-*!j%eU#wSf!3q2%cJ&n4nJZn|Mfo+7q6IQYdek-$)fm=<|-MymY2tO-WDbK+xBKG@HBvA zSx7$vdIKNV8nqFLMpuLyil;t7nv& zHfl4)Vdcu{TvI^AlV4oKrBBIjbC3e5;za(+w9SIQ4ZOlG()%Iasr# zZJ45ij#dgy{PARu_kT=ZpkW@|x+Ov#3NK@MvcKKMNIU4{LlyD;HXX8qSQxeOi}|}e zX2yg}*NK+TDvHS1{q$Y*Rb=O58a2T;Eue4DGm08z&g~SBt-X#moOzJ_dcrc* zaGbPua9Cl-Pb|?D);(zrj_|k}3n7dAIP`?Pkx*)Ce1@<%bZ*uqBC2|bz@=laF>C7m zIbuRJyt`lIg}sAo+;+!1oNsP}DE2JnvV;CVx2DQ`!8R(xj9eg`tZjeFpR1nWFucf* z+%AgWepiIu0wjxE7IDru+Se29S~EcYJ-%H~9=PFH>$mIReQxB^vVhooOTg$feLS^q zTtLhB>6BDrM?)g|#U?g2+2-m-x@{u_yrdW3&g{3Hw-DW%a_t~S@8pY(x^>ZS?0i-i z#!dKg>HHyV+M{2B>-BNd^l2YR(^pRldMhlu#2#KbKUz@_UOnq$3LPa<>|`&K5qzV3 zDS&L@!oh=!NFTh`A)J-%65_q-;PXc8j;4O4UJZ)kpg;$o%rS13V(6k5IiG4*u z{sKzRbO8bnZKim@IzTWzW*$lZnEM>&#q=QvAyBE8f{ScE|P=`m*lotvrolF%l_2vvpBKwTIAYE0RsZ%@pk+*%fmBO0SaQ8OY2RWjU|cD z?0!hI!IsS|^6vame!S#(X!oKVK5-?n*d=K2K?kGIn}_Fc>HeG2gf8p^^e7FM8L>)I zN&79YaFQD<1jvdKD(1c=Jg>(=E6QfY>|a1Glt}yP%3^8VRO8rjhnL3leO^1C4{y$1 zMeW|%HQ4``6CiHj2puQRl2Fhuqz85L4%$JuBA~2R$x&NoXFsXsNP0%R)oZtagAi%J zE4W6Y69a1lK>>a2lepY&MG`N0R5Vc!i41|s+1!3k!Q=CUI(bIGJEc`hicy`nAJn?RlOV{LY+;YpXRU}y}u-S2S>lP+kYRxsH-f4iVm0+ zV`(~Gi*SFrpC28|TNdZ(=gT@Z%H17%0W^X=@RTS7|iarHFa2Oc-sn9m z7UQKvyLq{(eCEn+tx6>(kM$nu_YE4b!M@SVy5({?=hqY*t9M>MO1ckqEd z2PNw+=?E2CN`BAazdkc%VbhcO?S~TQkiw6??4CX7&C1 zjj7oLbO3O(o5=NtL|m>{Bcgmf_Vc^cFg{K2!{pEp->)UAVc5UC_>&sX)8hzzCof%C z`vte*t=O8QnhOKTo&C7SuD z0acQnWbB2;{TDI^+5OkAWXO?9d?9xT>I(U;8eK;*rFO5E%q7yQSej+-8n?-nu+X9! z{bdAZ4EA9~LkvGKPT$8-&MZy>vXt4xXdXoIzQ;0(AsVMEJ_vi0x|@{3#u8? zOhtwIL}pb$Z7TCarR9W?o+$c?(RG39-h%G-%Q92ZV%=uY!{#wfSV0H+!X*3O(&BqC z1nLh>#7P|9T^kV&!FT^SvMX|M?b+q0c%gFLuk@jV1<7H?ewCkJsD+%3w4E(Dvv6eT zyj9rCWRkmmH^E+D;enD3vD7zWHr#-6Yz^p9eRP#b|J8lX!m7#S81`uWGcvBj4ozv}*wob%{D+df zG1m@p<_t}Qn|%;yUn0557twz+yP)HjK1?iLf;;CAGyuJfmk`umSk}18!7_Kr{qj7) z@VvfVHq!utUT0P&H^EGlA}*KKgvfVK=V$1J6$=D?2R<&LJ6cvTG72SmWMKNIJdyqh zfzn4vS8bEf;M#AD*29x7ZkNm-vnvr19;UJkMm4xJuDm$XVbK(N(U>F?lJycYJ(g;S z=DEY1rNa19_m4~ICcrhN%t%hTy~VN9Oy;;<60lRTB+u05&YEHN&h4Ic>w_M&@H6~( zeR`5>lxbQ?Xm>nuanl~He-3FFp36gMQtY{1vbwKm%t`gD#E+u)awuy>>t^HHtb@_6 z-sMosc!{sAAmTRtS|n1|P9h;+;qZK%LH;}K4*kZYK^MX0rE$U`Wnzl0u>`o17Q<1T zbm^s_XzbREri!ihdafk8qR=pHd-a+D=U;1p`r&<)&ugK2UykW&9T!adU2n3b8_x>d zqtn5FM-g0oRtqT@t>>bOk<3G5>mL6e{Q%YABHS=w3rbZvhmASh4jKl5BA4j_WAKSv z%0_V+2lE^9+?}~cw9fY{huwb<*DEc9;?HnDF@F~(j0>C9f4Z^g*P*;!g3p?9$9z2M zd+cuYn77n9Lv9mu4T(#0@3poh;AKDHBj((aWe}OX?fcqF?>a zP7aCDNuSErYRnHoWU;x3f918lSzyhpjv!Y07({x6CiUO_j$Vxae2W(67?aNRT>HvJ7)MuVynSbdERid+Am}f~P!lCRaH~2J`<#OE@8>$- zgQU5?tbUfT31o6rOKx!n0Ru7wqBMP>1Psd)zjC$Waz~DZVATJT>TWVz^ZbGASZsB; z?tW6NND&u{t%qtHN5-G0n`Y3*JAN9f>X)3To$_;}758$2_J<+FC=*HWO1Y^1tQ;K+ znGhI#^nvaXK@?-q@ZIQ6d9_yBdp@s;=|p%=8AhH&cu)Bj0!XC;Kvd{A*#xPmC8-z(S2&|Eo96vf2+gBah1OVepC zTWG&8w^+T$I1He*P(^{-X8``#8-z2C0gUEHhkyyiCERCFMUv$^<;{FCtE+28Fzkd1 zO!7Yd^+(Sk@2-I}sDDy?bTOAA=voYHZkAa1+XJi_N^HNoIt<2QxDF7UJ#nJHJ%%yE?zKzG#w9c0)gv?+Wa7(|1Jttk0%KzZ{;yCG={DFwGXn>(kU@=t1 z>ZRGkr5qQ4V^#y;b^>5|v)j%14q5knE8`d>#uegjSU)g~qev}~gEii0c`o#4U2+W} ztmF|~d#WcK8B=Hg9xGtnwmV3rP=m{$b{Y-8e03eL8u6XKZWJw)HEp`jH@Hl^SS^;E?CIg__SMqkiT0Gj9 zT=M2JtnN0KUnG7#Zk=vIu`Qy6bzjk_yc^g4oFfQ1LXNU=lz#ipIlVD)&^4myJ!94! zUOa56i-y*^Oi#7#i?;oY6P|J%y=&I`&AbX=25hTtU$6!UYtoMUlVnR@f)v~)H2Fvj z@O1IIj1vw$ij4-5z#|m=Isplt9h||pfIeXB?X@c#PSawxeRP?Ej%nfomwu|?iEy%U zT~;vOeQ`;6jOmYWJCudSX)bF)V$DuTQ&|smbrE32Q*LXo16;1AFx&4VJ}JX`kb}&s z4A;g>?ZO*pKLGqDy;YnQb$7!jjC+ouqYkZ>?i5HC=ad7 zOC9x;bvw+PM;!;R{QsS|5NWFX0oC1S2f8PgVr~>B%d?;+T7~RlD*~Z!zY8jX<;?XmVo&$VxfOA%68VxE;CIUbWA_TS&!sCt1ip(-qpBVR%7*1Y-~1|LDJLzq6P7XDXV1FS(z8PU$N;=rS-zc)Ho+5BE_%D z->^ITDxPfYx4-XZ$Mubo6&{hB)gO$T{*tt;#nrmiP85e}1MqrXzvS_!zpPv8@$D1; z?Y6^7fW=R~I^MicNxFSt6AN&s#pcN%; znm-VfOV#!>Ol!`OPadP(Z~zMQi3yjsjm+!bKc*(rxJ*`KUFF^k+Iz2k<)mw*X(_pp zL?T~M!Y`6`*I;%ra{u9SivZP`!GX6+kPkN|A~@|x%Dv4a{&$@W?SXtzq7*|gMw_L= z@BCtIY>sVYzJ)uSgbfHAPZ!IB7oXbR2z<%JEpFz}%+0sZU9tcC1>M*lLe$IwR2GxV z%BrO;J_-}b#i|fUr~`{ekLI$2Wme%H4D;(tCi#&T@}+#uL@Ze`tl01DaEW$hrwI^& zh(gj2aZO}zZ-0Bxa~1)>-Is}+zG>3$<0@9Eab=;Bc$5)n(uwAgyA)ab3?wUNf^5L= z9Iq8bGWc~{mopK7v~6}>aCp>etzWAOFhT{II?bToR=}sN9N7V`-9@)i4ZDJ5cW?p7 z-HHXm6$A|BA$T&G4#J!!$|+;wgUJL< z-gcyk-EJe&{(`VD9_O~xJvX8BmyOfDOZ1V%1Q$ZE3}M3U zW9wI-2wYK+)?7yItLweG8r`P^YDn9pBy_Vv!Ps4h4D81hnOB!SD2&eZnDaiUS$UBu z_Nw|a@7<(EbzmW~UD(O8G2GPae0)fFX*CuqE5kkG>?Sel<(5m{D)-Z$-%bs+_V3G zjA4;Iwt%?2Ho@SS5QBnn0#Q}U=>H8X`7xkpJF z2mIVuc_nHt)Pp`w(X)d&6a4t-tL%&4v>QN?Cw(6*0ZHt$O-DkUM7ua* zGf08zP@*35WRtfG7E{jhnT>9B9hM(t;;|oM#Rh)~z@iN57H5DB=)Qy-BfEs5)m{om zpjI1MTkxOj$|mAB>6M!z0nb=-_-m<2z=`7n^!032Fianl&58ecr8>EDDcb=cC0q79{lj7f%7USqVbt_+uJ= zhkjq*yCA;FNo(2>hl`MtMWe4y(tXXcQH0m93_W|YzfzbpE>CTe(vjU4sSND3EExp| z3(-SY1h#Y}gW$;LU3~jI_98Qsf}ZTPVcWfpB(wwh%2<^x?HJUiH$!(#j(PWt^g7QQ zkQprq(P8aELmYC8&_p~iE2WTmP?0HT#;b($DTGMZ*sIL2<|9}an;)&m&i6l~%u>lm z{nko1QuR$MgplJp4c3C#&D8>rXHovF`CNuTZR~hjW!uOMk=w%lvXB|X5uk(#A%1jv zv|F{%lWyDbzyNF2^4Lzv0~*KJfm4tS9kM=`gWgF(i$v`J1qjll$;*?VxX6kqbgjH? zDvi^~2Gk@yX)~325UcArT53t0wiC#@8f5L*3x{mbkYJHxBU9pX#^%{pUl&^{#6|O$ zw}SZ(EKAvqd!JILGWyNUOeYJ+k+K?)K8a+KbUQFQGz5?zMhy`dz)OMDIjc<6W}W6-ji^TM>Ld2l;u!wzH`vBO02d6p6Ssnk#? zMd=I~Vdz~&F-#01IbsfwRCPq$%{-JgjkkUAuw*CvLiczcGnlzO9(2&M3te+nO(c&z z+D0AX5tZ|l-*y~0wW@Kblm7tKD7*iJA<2|zHcv4vnIVoXxhasJAlcUs8)M?)uUpgH)k2bGC?4e1aif^ zOhV!vy!xb=4Eump58YD52#?$4ORx zsW^3d_>{-|p~UIV=Uto1>);@F?g5R4z*8<51eAElP@1P5}&W8~g&M=ugh0)T86pE$V{9Wd~UIF_2b z7RX*29DUp*0ClzjcD#j!3+O;^?bCbbdyJsAYfB5B`|BOlF6MrT1LzUF^_S6Mc{5fboHd<2_941@PR7 z^59cPN08T`Ix`w{{N{s}Xv&!Jilc%16ddfcHv;VsaNY|aqm&EMCeb!0_uU=VM{a&= z5DGf+$>g_HTb`1m?k?;ptsUn;(JM}2;pO`D=)H{zVc$2hNqZ5<<2M&c1%4|0+Le(9 zzTJoH`{90k@JUDDnB!_lnv#(oWuq3JXYhSsxgE@f$E)Psl?4~$7@E45dNY?WEK~O3 zSY9)44U3J$eR}~&X~J)F*d)XRP-W;SZ3OjhRnF-rw8(7K8&7Nu+B^LbELEAJ3G&~T z8J5;SKz6wAC6qp7={?^q52g{VcsmQk2n{A~@2~CdC0$LMtBRGH!S}j#N%$FRNO3b4 zqNb}2>Huga)<^aR^~4S{5Bjcrz&phbjIaA*B9c_9;kc>uu}$Svfv zG{as*!A2ewcFnk_vC;RWbJtSc5MO`Yf&2CmWw;y! zs|N$pr<|trUZ-UoQKfyLeX5ir5q%f*GdZVznw&WJdnkc1heXsKaF@%>Z;fN;LlD() z%;0HaWa`BS35Hh~?u~K*Q6AhO`fk2>_z5Y{)X|gu0*v_AfpaPOy4g0nS*R4W5WHAayNv#~gXmONO+)i2U%|J~JvZb>uJ zlQe@Wo3UYiGbC_Ab<;+uERg)%mIcWVrKdt%1~Y|8K0Ra2gh3jL(xSp?jE|*KjY~U9 zAoGf-)$=VKE~!z#dfIN*8wt%w;|)%30`c1}xgBDD`fN`aj=p_D636u}gRV39u*dA1 zO=ltt{6;026W~X!f`J|gjQ~|uZ{jkLT{RW);U((?e37Ai)_f0kv6Dt?OvAN30%29iR-I?7_#QA36hK29b3vZJhonVYy+t zof^Uu2&&c7Da;yhzyJz*7Srt!Wmg?ThQOxOtQ!61nP4FVhUsB~iEh$f9h8`HyRzs6 zKi|S=X*r>nw3W*5t|uv3JvpBX=Qc$^V-n=FHjuy%a~ZBDF1rNG(E{%E0-W!*T}om$ zncz*ale6O=;y?R8m?!ivB6a}ny@B~M?-60=?@qG*A}Ant>xX&Aqv4JVnvd|PshIZ? z_Kchx-qX`GgWPY4rR#!A-K1Ba8lk2a^9NW4)1dGqD%tiCy5#t!brOw)ta;>V8@yPxw~KK8;@)7LXCL<_B5LEs_(+I)n-VFY;K~% z&A7SoeE&^ah6IiRrc&v+E zM@2mKCFU=bC)W8a^G}!gev+Tey~or;--p;m4#)T>+Nod7>9~D6JzUxuTUgYYWZv$A zVfcqlR)mB~l1LDg8FmP+cO)x#Qq7dFMYqu6n~+!03XUoTUt{O}BnX(~V`RcJmw7Wx z5d!2!^Q2Bwmf9>*2o`8wy0Lq7EQP-nd7L0#dN=nazRSG(RutWIQ3JV-i{HcCS#OvK zR+3>V5rEGjvw*B_)J<=qf9w*{(u$*&zgIq{gky37_YCw2-ckDx*vKV_&ZZkrOtI^) z&ppRoQRMvq$CrIa`#8$sDii#w>U@?zbeeM2x`UlYzCCGKRu#oydg1daD#240Dh9H4 zG>P)C4h={xQ)ddBh{(Dz1%RvA<3SAsP~830H`8$G1B()^fH(mwDFv|3I1jp6Tp4*< z6{~A;0)D34y3eTN7H&CIn&s-A{Y-Cj0#eqtQ(z>hB>ech?!8}LYER-zLO3<;FdYSb zEEedmPZ8UqOV8#YUH~a5N9Z*Nd@zj>j}l|eNrynVkC5&*mS(p_2{e%>OpR*uAcwB2 z#mb4v%}uou?4=(L`c_>|vh(8ws6YF@+d1R;0lctBA^TxLL@m=I?8El8p+05x`xGY6 zvgBZf@%&5)#9)Sf3R1%lQ<(>A26}XNcpYJQl}cZ(TddJnYeA^z)4PibmxyuMcB!0E zVOvR?B<^e13P=*HcX}w9N0x)5@#I99o(G?+x1-%Q#cn z#T@5^i~1W)Ay)LDa;cgMJFGNg6*D`1S}$ynirRXhpr>gZ^@Q2e=VZG}&b-G)lzEa& zd7j|?bKCxGwaS_2jdT03cMXxoLr%GObO+R`n9#)G2pBZDn-yA2W*J1vGz}CN#vdh3 z@!P3Z1A&&zBqeh%<4!_AWlZkA0^!`XE&1YyE_PWJ!girLbNuY=_`Ovo#icLQ?gD~i zoF5EKhZGE~B(lbkBOW9%zK>Dz7w{@?t-O(lx{600$n*YKJoDf{{N*a_=K0tAh zcicjEShFxuP*i%miZL7T_~1a9%Z9Gwm`ZlrwNpQwWo9lVXTcka1SR6iTC!msg0=B0 zT>8qHZCuC$?#z({>4a++iFuD6U>eD^g;vNlxu_$+P9iI!od4uC2#P{~V&V9hQMYT7^r;W968l=2pW2I*^3Pu($}Ty<7UQf*z)V zX`;{rtcwYHJS^)9QnE117vcfX1h3h_lt8O0=|UBusP=Xnk)MPF5~%kCI*kP%AP^hadF{6MhE-64 z9MojELMLpH`OOSBh_!XJR-Ip=*0$UKES=4=>Tkhg>xDp_v{1`TF@c5kjMKEIpOW3*ZY!odOyVK?L^I zI+4oB1s$Z@*k?U2;@!aA1DYS(W?P4mfT7dD_yW#X1qU?hsb9+d0pri3O=*}9-2hcU zVM`>RlbInaeGnvmI#MuEMTmwfo9D?4SMlx7gEbxw%x09BTS1+=lp94POd2KF+PXv{ zhrUf7Q}tQ@X|;eoW~T1cT?;L{<^9bKo+%X^^ue;3e@s#E!OmZ1(lv4bW^jTozcry_ zBAZJ^wnaNybm{KAr5~8In#=TjdnGFyy$nQ6^fuhw_Vw*jAa2_Ov3dy;@9;Jn=ntE2 z3AjHiQciHA@|-AEhs<9C3;t`o>hP!X?*lF7l9YtPV-hlFX27a!u?h3$vU4hm#TBrV zG78doA}=r;{;%c+0(IkfEO!1f1#}Sy@wCEy zSynzkWoc7kS+`10xX!)#))$*umxf`gJ+JK0mH7TRK6 z-gw%D#EYh zUVVSD3DoX;fE}735noRrY_<*utb||pNTaG_FJ)QvTkZ?aZoIZY4 zpGl(!0we!YWPLL4zl$|J$wk32i)Tq_C5SSc7P=iBUJppxzERhDfX{rjVk~@XuSLq> z`9*UNa5H06Mox)Eim8u#V8r@!vOv%RkSvIgAngrchXVsEolwX-ww0prqIcxG%byG5 z=|NfU1xix*tm~HuZFbMcyDU(O ztc9tbCmWhqW0clnWj|^b69HwD!0~c^{QA*adH;P<>k(tM1Xh9_fEYa$W9OhReCdI} zzZ`lUTXo=T=vuD>)9xavxUoeRr8Pu;i8opaG!p3olK!Tn)shWx@nGOW6iq0Dgx@HQ z?KAU^NYeO*=Uc;YUek`cGVuuMM(hqxp0ccED1b%I&v$^p*jAz*Cg{xNwfz53l*Kc~ zcZfsp;Q4(>V*3i@xL=y+-`A_QB=c!N!li+SBgwj}he%ZO)MXf1ROsN03yW?{5aLCta3d1doiSaGozP6qSg6GfW&S5&I* zW9YgJcFcr`{=(MXe86oRY_?n(XV4oWXWK?_{M-hdc%$9U!%IX??ctZb4rU^4UlSzr z9{#^9B_E9(vxDHrZMsCoVg-l}eP}`UdtGlG;zcU3``^5fe)L32(1)gGeZL$OMKdbr zfZh%j?g|hsG}#`wsPOEA;PEj4B1B(xH{<9vtyd~LzmD=VUc$dFs>P4>E&L`~MsRtI z$3)GF=iR6R77~mm@-|#sLPpXOVzJbhMK9wt-5Tg8UivYuWxJD>BohFC;!b;@ismrV z$XcAeK0RDvTnY*O2n?Z-?*!#7Mh3hLT@3@ z0rOw~qEjxf%46|U1*p#&*EAj1qvVIf%v zV*I*X@SLu9Ov3sjWz+6gqYa=+t^{|nnsN*i{1TM?SHHY`bnd@VlTJ)4^a-fIC~IT+ zea(#LB$6L=FMnf6=zO5CDU%*n0qlaF=v8qw9d56p*eolheWJTSWIxcS)4iN0m=v(` z>rP;*okNSKg$eSvnbkTONVwS~=Pu0wzA%}F5hEnf=f+r53f!X3LlE^-a|``Hhl4nj z;g|o&$zExQoISDIVkb;B$~2`GAfIR_29Hm?UqSPq;?e{izm9^tVT;Ut&9{8siK=H_ zy0eiY(iQx?b|r|3>>BQkB~7N6eo4Uo#a0-KX7*wk3*4mcf4;l0#eW50ZJO{RGye_2 zx*6vmE&FBXGJ6$cHkpSnUvP|IFW1ofnjU<9sm(qCEcp{{bbP#2mQHc6_0f>~a+Sj1 zGDw^5qgH2t_r^IHG`L%{7>~1^0Afl*G5mr35Sx>Z|XDCvFE)%u(&_nl@9Qi@@W$= zQmhDEC?>{xV_dN4u(1__lSoV)-Hb*3r~Guztg>O;e=aI+99^0=+95i>dje3mN#x4V z^Icdw3*`wQD&vh4n$rET3N1qz)0!6g^|BOz@|=SX*gB_9w&dy?@Uy&t?72Oo@c1KC zqx4X-gg$RKzfq!m+t{S}4pQ}3{=J7!nFsst{Zh$C(E55tSNj&!QId1r{74q#ZZ#3x zNN%4g;qR4?$c&OZe5<=j$WVm1+nnP*qW|;wo}wtnTbX?uQXKMlyCan~xPZu`QhCkg z*WJ2HlD+Y+B86!%;DCO`q0&qmQT7E@;q#p)lHd)6+GYR@$2kqMJDp3+kY8*gC_gA~ z*iNm1E)?N0PG$z-!3x556Tikk@X?)fh?{Z9T`wo@B@GriY~avkJ<5w@a33S+X6j*I zo?X8R{b58nyFb6z0$?PBcBMY0+SHWc8>nGhBuO{H zRgyH4i`oz5Z!-^kdwWOo$=%y<%3H-~0VtGt@-=mJP=UElTAj>X?L7rIpe9qIgT0nM(ID-AL)~1@Fss%0QSe%jrTR9ssvYiEJP+x+Qiy<%C-F~&p z@;A>$;M{gKYkD!`oRjm+U0vd~7si=vfC!RZI?JtKte zgP?))4EO%=>$DLEJ@jjzl~xe&MgkP8_6?<8+@~6eH>4|3{KV25Z?y>5MF@1Apn=FF zaghZr(tfnNmj3R#vV7+JmllA3|IaN+4UYAg&k!lnL7-tsa(_2gRV=QEck-!(btASI zS;}}W$LF{fi5bxBefVN+@C4x%^iB!$qwPEho~=kgV2+g(&oOG!*w0{9xTCJp_@i{L zZ*XWT?mc-LM12I>44NQaOf93rU6F?N&l#=3KU?0tvj)%ZvK>;t&>N5X6n`HfVBrN} z_Opdpa3X}-)o%TFo$}-7o9}W-2XaT!JB+o^?vRyzUuo#7Z?eEmL#dN}>5LIl`MlG8 z%Y-|EQYZPkafjppxCV~I>C+S5S`c925P~b$zfVuxX!{A+3pcAq{c=;1Ccq6@-ALpi z@(02cMNvM9aI1g|cRP`|9>xFHpO{x z%#hUsE|>19Brr-WY)zuC*zO$>=%c{70&#@5Z@u(o!V)g=-ldFrjm?9q_6Aa*?0+HE zVkjVmdCQMJ)Z~peZ3}Ag1fm56G@U)TDx)WmT};&kDAqWVC69Lsw8#`17>7ysotEi~ zN?PboS7>N%FyU(U=0H>(YB@?7x1m`(M~O9V*VVaoOiLEw3)j<%nLZwBl1B%DdbAWs zf)ohrOuQ!r8C)Fe^{Uk+{B~Y|&YqF8lA8+;^CZ%3Oz(dA z*LWXIHHTA>wqyhipG4tzy>N(j2nxSBIAZEw8k@ppT+d^wuKSy-`NzMe8wy|y>+_RP zFl%)OoOE;9TpGL~$+Mau9hU<=|5L>AXbHp58Ka>SNW95__?;}qJj9M7dzKZDt$Ote zj^Nekk3_=0AI!yj)7-nM_UwA)qXXyIqdWuqKwyj`BgLZP5FN4j*`GRVaN}+v6PKE= z@dEOBBc(>Rcl67uLG1Zj3u4A<7(=CeZ+yZVT~{NvUo1S);KpkmF{9)+wR_v|2vxq8 znv*>duulTSc`pOF$IVmM<*rUgIBcu^{Lp+StLE-DoQ7*9_r%}v=#L*1mUayk5sgjpFCaBEc*lG(lAA)60Nf+b1KPfO z?6)-(scfxU?SHu+MbM0#E6}eVE1An`Ec<=@^KVHZZYVO6A~k?HCsCW~B$$ zan;rlu-)7pN4tSN)`hLuS7jyEcn|wZUf;Jrg~S@~ZpK=iMsW~s$YW0JFqugnDmjvq zLEz5N1h~cc06AX$lHhl(Ht9Js24oZy-_J`TQFCbmQc19Bvx%TZcrvRWNS8aE--Cqo z&{0*N?a>ACcZnx|-aM9E2Y3eSlxPNHw0Ee>#gD!5Nvs+DHzylCds51}g_%WL zTmgXSMH}>@XCWT!Sc0A|9ZXQ$u_nl6X=7_~k0JU9DekcL%shxyTd``_^28V(0$H+J8v%Y92#`(TjD? zUlB3PXKQNYh~j00d)|1NIAc%gi-Y1;E|jij9&vnazbWx7`h4s-V&)_{ofO&)8o1E zq07>nLCd)m=PIbMRG;Ll^Am_nZkQ&vF75;q$Rjg0v3JbGFIKAOF7FTbc$sEtJSnyW z(bhInt_wVSkwT^-^tVq8Qe>`z@JjZcclDOE+Wb$O*6ZSa_g{%-NTj=XeT_E1tPLxM z;WSw?+Y4>_kEoxOEVrfH)>>#he_Sw~*wl9=pb>bKmPJSIQ^4UbL5?q(Mrd2ea5?x> zCFp#WUT?(?dwA;GM$X$U$bBgz581d_ zz!$a~d7a44*m~{229qf0w9Xxsu6Koj$VOSz!YfQ{y{(+|(Zs8vm@H}au=2425`PIc zf`eN$llR2MD7t9)(ZvwfzbGrqjmJ`Ezn^^0xro#jL-vj)X&P+y-MFi#DdKWs&99lf zf}JEaR?S)r`|=-a(;&d*;Al~aHg>hdQrxnR8lS`Zs@tC#k`K)hm1qWeb(=9 zpRN#XfDHF>vZ*ZgLPjNogL`nw0+`DZ>`zXh6cxGA$;238Ye2==4j#n_<{d!J#D-)r zmrZZpG+|E>b#}UO)AD}l14&Cv4U9z3_n#GP|1}cxj22cIH>9?jLwwR4M0DL5slRiw z%hegnzahaxu_ZfWRfKUQX!#?3MZJX}xeVcnbB%jrD_(=(r~gG%%sR)2u7tqgqKHL1 zA_+oV*K})FW9*UQKljoc7&T2j_cm<@?Jl&|xo*(e(gQMu9wGzqdnWFMZbcYZ7GiNs zy~#u!X0Et>#4;^^bvjN2kr~TA%YH4-hdF%ey`R(+>9-h0bHao9CVNsG&W+MG5K9^Q z>6W}0VVDgEL$9r$fh{O?ix{wf7JmPJ0o7S=g2_aNqpM$H)tK)Hi3{8d_! zN>+KVK4ZAslX{arGW+&(n#0)@luH}Gf9*RynT+5-%C7eiijQ68YKySRnEWb7dv!W? zM^e%YKeA!5ykoQjuVo?RojjivRSAQx9&c2t8XOa{JPWqA>vzjO!k_%T)7kP&=87Q^ zoy$+(WnbmZyi$`bqq|(c+VUB8X&q|*HehvimP zbaC6Q<@!Gca(BUW$+{PnVhlA)TMBXvx0eFnJ*&1Aqt_VV0lflqa4I6gU8~%epjo88f6%9 zc6aJ}o*aW2R8jWALr1p8_FezaQg(nu_%@j6IQeUshh5J9q>R=XUlx?a$*xZ~F(j}J z7m9biG-JU~L|q)*Q&Y~k9@*tUqi5sGToPd}K)%iuf787xWHbz8yF!9HvE&l%R_ zpLP461m91j9!F?zcC;~r07cMDd^)Dz&6P>Q{_L(Vo%FXzeX3~lr^XYAyO)(0j|699 zWMn0|Et!KUu|ZKU5GP$MOE4F_bhga5D-h*sa-#tP-zCIVI`}GurGtnE5Ah-)HnRyI z%PZYr%-f6Ha%3(((=9ZSUHE!ENc@25<*qFlx@&nTD^-n7av4Xh_3IfO)PXR5SxGLw z@$O7^R8P7hI+uWtf(s&QE-7K1oD>kxcrs zWp1+F^GjTd<=dAdb_ic_(VY;R|1w%u{fzK@z`n~n(-dq)ZSvTUakC~dF`}Cage0sX zbZPgsKp%l4#KSS!ouU>Hm~=fw(T-O5DpvzR|^^OLtn<3G5uNP0%pjI1Y zX^|3^GfJI@VnJ;jNiiqa_=x(BH;P+TN~$uUit!NJ%(RQ$tT&^jpdPtmPiV3WiJtO) zRq^^%zOw3QfsoJcn#>>Xi{sjqNc3s7N{tMzRXwk$uxZsbOF6%Luqs3*7N}BDH-Fvf zV(USDbsnFt=}^x#$KPM?G!ZZaQ^)PCtKVh9&?@^SYen)?EOmvSgEB~|2oY;Ee@M%v zJ7UT%mois>m4Rbz8SnX1^C6eJH+&)lZ8$QW`zXmo`aZ0En6_t1W8aX<0K~&1#D+8m zFphHqgpbP7ZYIHKdYjIK@^`p&I5Cxa?Apm;dzl&_o5Ib8XFCsCND_5Sz1J$8#JGP

G?q z*#*OCh59xfnjI3uheX`#8L|=)XL;&8+%SoYJfleQa~LE!bA8h2{j8FN6^}P^vdzhI zxnv3}a6sMwhjv|a?`59;I{BCA-e-hlPDysvVKjF!O4thFQvXI09*sWIPE-TU~p|M?g86HkDbS#Xk30j{2^`oyxlGcvzx*^elU~{LS@H;lSR7J zA#xeJg_6a$k3zqN_PEs9mMtWNVeBO5))W@O@s~Gvrk{NALk{JT9{xlpNid6FGiDpD zpu6}(U0|aWU9OqD%*}K3nT#nyn?PQ$m83AvI6);4t6fp!m34lOZ>XgP&la5-e8Y}! zjM?M*&zZnR8rpE>3MMKvEq{YwvlPy?8lBBNO_}jqCY`A}Ye2eNk<=djZ9a#o^K!7P zDd=BzUwGi+%8U-!eHFWO~Uc%fAbuh&f z0X!vnah~Zqwspr-zs@^K`ZM*aWEa!Hp`5vGF<|F!5X1(dyBhi@<=}@km)8%%Pp%g= zQpH~OEpZ5Q1-K2Imm9(yM0KxT4(6^*6J))W`DJ4K=W68_?C9{XPP7DQtwbzDQuRl% zI_r#cAH-QspJZ1J@RIuW@q0OzSeLwHbJ>I%rPaoHc=~ng>1h>$Gm(rEQHK%c@1sip_Q@N(Fu_%Ma@k!}}e zJVPo+MgQ*4-X|BLIP+*uP5qz3Ay<9RH{9>^@Qe^gDba+!|u^4U+*e$ zxES-LEk(LSb99Mi;+A~zAa$=H27`SNDU0&WaT)$Jl&4f$7J+r2{w7-23JeA8-wcIF zobS1!2=g`vTlO%ldwHZ3MQEMB({6O5X)BX-pb*8F-fum6W=vSN1q`l9 zP0Z-DstVjReoBX_F`!I|TciuPL{d*Wb2Wg-!FG?SjrZvbuCwqk`#YPHGg{puAfj+0 zUGX{pwDsn(Hlhu^On}QOJ3-OK(b@fRBRXIc(!9Z9`x4oytJU#6pA04{cxZ2z^SoNJ z%r6a$_>~!-toc|~d*#nJveiwZhuWUO9o7Ox6c=R$_2g6Qw{Ku*RIwhyuP*l;My@@hITn(4 z@N^2Rn-1od$+&op?&Lzs`t&(VI}G|=5T=4DFf;)5q-2$S3fAQw4~$5UYeoo}LGS|9 ztpX&jGD}Q9Z;5Rg7BxR5wiDACNKgIy-8wd?g}yI{I!cCWvga0!fPCydX5#NIQ0+9< zwQSmp-1Kax)PGNi^Px7&VXNaT7VGW&+<4Bej_t-LrgclM&Gl^Qyj}Wf%Eo)~tRo%O zVfC$a_AffosvVK$_(^r?%pV#1PX~DXMm>fD*K5vTZDuVd z2}J{EbZ?uc3-v2*iJ;~49;d=5sbXi`2Y#I2a?OAm=Y7KA%_yh+T78{Icz7vy;{yz$ zLZuiFiAqb!Tefg~>ZG?$+0RRld?SlDsvi9@5YWoDe>`(eSw1V9G zPepYK4ESO^Z2=v*#sXIM4*WGr>%6&Ehd$WMB7a6{J{&fBKY{DCrT=t44=Wye&BrF< zc1X{AY+)Vzs|wNF#>Zj!>7tv z`tjJCD67!5)A$Bg@SeT7!g&sXant{l`{Inf*dI7qa+%rw z>rXBj-d8Xr{wyK>+|c9JtcF03k^q-0hW2Gkq{>F$J*qHVD8xR5Y^eNAAWQi zIH~tXdkbOLqIZKE`dNC-7bR4?&0OULy`SDL(O)80hkQPth#n6lGSgiHC72!W1QYr; z?oP<_`B^($ogUWsOo*Qq!sK{1jX8-^m#I&Md=?JY;c8cS9-s4_+zqg%kdlASQIchK z$oA4+BI~m#iR}!pporFOU1^^7PxiNhX`;<Z3&Q!<=kS^F1o!Pan9!gZc$owIP)_(=7 z+8)a#b|EC2E>$(kA!b4U(16GvQh4@ERts|1gFm@vQK*Wo`A3_j&#^r2taw&8nCzyl zobfro6izfWZ)BggznM7?C}?4d%EkV^!eAxr&;C;`{gr| z?uk={lHQ8z6A<@vPZxbo&X&v&hPB7ry_I<@Rs9R=vlD4(g@NX#;iqf05B7?9elH}J zt44+8MQVFW{X7;&O*AMy;}Pns^peO3CeD;$Ay(nOFK{L*D*T{Ao^PlvXf%{<4Lo>b z7S_pbv!8|UVj}%fT7d+auVx)&`vREe*km&5WAx1ngWW2EBlo^TpvKp;G1pVV+706W z#}Rff3Qt}*#(`Jq%O(ND#pbeCfyFzSWkg^#b08M2D>&6LnR2GH`!;nSFbzAu5%Ojq* zUTn1Vb-8_fkU%RDB_?M}W!Sh(1y*PYdxb2kinP!pDzw?R;nLh?f*_AdjX;f$)? z_;g7|p-H+tIi_ht10*kS|;ln7_;vp+j)cUBYx4w z^JV$7EN>;K-5SSk#?BxG3)&`yyQ&m>N<8&^7UEyMD2Z$+Xo60^pcfTzdXaxvdUnL% zOPk}O&uc(xNc~kR;WqzwsRUW%ye+)^n~D;hcN37p%Zr(5QHlXFI-%0Us)DNwL=kP? zaC&h9;O5Rdkwm5ZHz$IwG!K;X(DG@8pz^r!2T~dCiRXWBkh~XScbK>Rj(E~6v19-W zIV(FU437X1Q~q)Fzn~w<@hxRlMB@ee`D-7}a?Gq=i^$3x$&_4rbPD7o-_`HJ#P~rJ z?1rqr9t=;s{9Cz;MW1z&8DbKR?~wfq{UJJ`a7FjOmIR{1m$eq87!3#- zyK~tJ`}W9BJ?|YLAKZ(az7Rs1@_R6ASIx6C>B;r^gH@Rfi-_mQ@*zM62DkUW)@uR_ znYMna$R|8WRfE02r^*=}iiwIZwMn5yNv?x&D$whTfk;npf4j!htIC$rBuM zVL#8&i+qLYNLez-uelG%4b%P{6?*~^rR=^_b$$Cc3p1XZo1JkV<{-#O6d#aMUyu`% zXRY@#v7-CbPZB$rzpI9Tym5A3n#vhBPeR(+12Hoq*i_jU7s$is`0p$r?r(%2`1mW^ z$$qYvrDR|7f}h?MEyq<`Qse7hUb!cpaj^jrCWL}80eD6&*TZ(nF9v@aKv1?anKVE~ zreskr1@eg4Jpwzu?;s|aX|sn+Exi-?S{~)ppQ}ihg9@8ZbQW{JwEQPR3C2zjk~^MS z-0<+1XZea5U??Wsq2$Uh-e^oacYynUVvpEO*?tK!|FY15ciz#YR?mH}9_&26d2=ch zx2URKZ#8+7Wx2v9HR3IPx>|jsN&MbGJVRN8d5_HcMQN0%^YS9mLqNJ8x~FY$B8b?a zuJxG?dI6Bv8*<;=uKmy|<8oM)rnEYvxlG=+vGZ-OcXPF?eJ^ruEn%}sW`mi2gZY0( zk|=~XG>@e5nuUQvwu3$q-&NKw;?LxqB<-7+rY22r{D&8hZ1wciwVxEZCw?>-qI(}1 zS?|-2y)dHU`BE7-X>yel9wP9?n_!3$a9MruM}tJ2%+5j;s?V$5CH33oP}>>?Ho;_? zKKtWWqke|Ve~k2lIUDJ}T?QIih6I#ODFUo)2VQM4CoRZb4I6kjD`CF{zt1b4ub5Gf zSNeE>LylutQt({DWdMFI0nSL_Mg83oz5mkuQ7XCX=2gB9vVar=*&5di(&2d;-+K%_ zps5LqanQ>(cW~U7+{mnPVU+?L2o|#9vGAP+!W-M2on-9tm-u<_%4#q ziCIw>Qst-8Qs!(9=uG0??s;Hh_C-5`WsmTGwbjIpLO&{!wqJyQA$?8q1@V= z&&=QbM)HpqAnZ;9vZXbR!IrOo{A|$}xUC@!_fW0PA zhrjsSh2qogM&SpuL>Cy3%qy z>kNf6C&H#xjh#R0$JWp?;3fn`p&64T}s3nPQPY$kryTfvJfJfdX}9J;X@wgKht%6?{)5NI>_u3h-4yZ9DqT zRlhq$IA`6U0*{YTSCmE%Rt-?~ai1E+NsgDh$W~-3S9{ct<(8e7)!;1fxGUVHP}RE# z+56;Zdj?u+`@Zpsw=>7PQs|G%=tpW3h&}8`1HDdJQoCi;HGI4|sk>{_z3);)$21VZPe2vgeJ}j;xE;ujMZK|zZHlidvq_!*3gUEG_q*~>e=mbld+*SV32?m>d)r%X9X?iX@sOda6 zt_xygXIl|^OT|n&L0??$QC_bX6s;FF@}wmIdV(}VsChlKz5o>;9(I+v2cJe^G z2C@*7#0rPe|8ACWSBKZ0Vc1SgN=4IcsMJWnVg5_!5JSK#+t5F3mKyX@jHCp7vr`gU zj@p;t*B&z$Z+G%UhAuE&(y2G9eQ7)i)y^gP4M0m9fd-c3zSist^Zx-yxv`=lOxP!I zxnsBhY%**p2o8v;wB-8W6|#<<<*H6Y5AfWb4?s}BJ z3_P_~qb9r)59~kS5t_7nHGKMX{rzE85d_SAMCJ$Z6OS9zVZbJ87jAk%EqDK%@y$YRiG6y& zS`%(bE+FgoT7bk|{7d{3xSCty-3TUFz_jzieW1-U`vcf+)jp~CdIb=^TO)sg7E5>$ z81>N>dQlg=6HY6Cnxn?+3IN@?iVzEhKIK^QWme8eb?k0wQFGWe^Hg{k0oQdCL>K>E zD3^lN7rfA)BY%M&Y#8Ba*~DTzbYiB2jC8)%q4F-ko8%08@LY^;dfh0-v{L{SxnYqK zUxg{%yz-N*Hn=ppANM@^vdngcN(~jow#2?1_HVLE{KwSxm!F|hAC8c!oj{S1{cV-) z)04+3(MD2?_yl}%=o4yMnuyFgU)rGdoa|vlVdeWg^i<1Dw>||{UFf=^Z{NlL=KHYL z_QAj9e0OJFb_nYH61w`suNZ|7X02Yttaa55mPacjNUVNC8p9~6``jU8M{E>`|ga)&|W4k;{T3cd8jqchb-LeegB7Vz^zBNt9X9MYob@{n$ZXU0RqAq+E9a32&95Rd?pZys`8&UC!i~osb zX%Ba;$$3t6@E=^3{BMur*L3)&zDA(hi0~8TF8*RmumF}3K^gO%CBlbawi{*Jg|Dwb zne9yocMY4<*#QY5g8$VOqEQR=vp9W`Cm217y)rexe3I1U&YcC{k9A zR7PtRvnXO|8K#-&n!7<@s0&i{MF40NeZ`I$ULioZjqXrtCT2 zWniWxJ6U#*QUrJF2IX|6WSt58vHc9&iC{o#GINIS6hd0+YGPT&#=IuiF?Mch9( zxt_suGm?pfk*o}o9YX*~vjmGu6&Ol%rhU@ZG}TVl=mK@C0Q5ANy%3)FC|W7Tf4*cR z%@LfeuzR~q_vgknAOwB~Q$c(scCdque zpm5l8!YoY}Jk zeeipFPEX2jKMHkuz|-o2xGWHA#l^IJI8QtQAX9`VXd}L5$dK&AZE@ZXd1+aaw zJ&VZU{ocKCND|9rZuEw`u1jG=RO=J)4a-6N>K|fqgxTWZMtWXYkJSuAFP_V^V*tqe zAXD*nclz2L1>F1A3Xo9$DoHObc4qOfsRLKBu>yOUMyda1_`Q2qizC zYlWT;Pe8b%(qhg=tAE`;pHT`{8fF3wt%OIRKIy)oX-=|&HUOuMxNn8p0N)Eai(HhG zCA|6%fJ6Y)wayD5ZF(XO61znla~RRR*ncP_KgOX>{^8Hd8Y4Joy+x!Rptu*_?~8js z&CPWGlw9KRg5mKE^y-5(K>v&a0QUJF2gF(oK`jUmMt)JgJSzIO$mi`*dsR`8UoCV& zfTcnq{wBqcpiRmF{!gTR(LxF|kYFtWclB}s9U(RUmJ~ASM##K(q?uVo?L)lXAARt_ zbD9bjyjSbAB=rnVGUoFR#%Ga@;>6=3?F1QiLX%aHu-dkE6dN|YR~xhy9|yC7%PW$ZeQ+2qrT5yIKvYo@PJE8nF>b(wbOKSJXoj-u zL;#H?P#p?bd#5$L0B1HjTMN?jT5}4Lg`0myr`p7L=ykd4=m5EQ$q7&uqJr4;iqWL< zn~ML1S(>f>SP_a~mTG8v8_WL|bHAr}ze=8m$O^tck+OEW?KxAeuAs(OL$T@bGzR(p zf=xpQW3|{@;drZG>HmT*LRx^*%|Fq6iQah>bQ?~0)5kr5U{eIe_YR#hkq!huyZ>3$Ytl=O?*QuYHkU3jjbNWJPW}T!(e?M*DloiA_<`}w1pwyiyZy=z z`%0n9Qrjkev_^(Bl@Mi|ecNzY^y>I37gTs^$DnsYU=deh0A>#gg=LU27`W;Le#YX4 z^A^NeNtNEHPySe;o2X`Ns8^F%wc? z)V|xHiQI*!2o9p=5lPFZ!Pk>Fpy$(f@8iR}q%( z!xxb$TN${}A&o)shqBm15Cq$&F@u1S_pey8S(5lqnak=gN;GcS3FM^(;S$p5b*CQA7#pgJ7F*Th2F%LuM)DKCAopAM##(}pMaf`as!7J0fj%K)G+aCso-n@!Tm;q&h2XqC_PH1v zk{Oj2+jzfszAn7ca)*^7CDco?J{FvDF(+g5SZVhV5 zxa%Zjkw!=1s;Ssbrsok&K%pGBYGe8bvJzN@ic*lw?0hKuHGX~-a?&d|o};e#c?ig5 zz@G4wCVQ6gBCd*c?t}I3Kh%;(p#r0vs7LZPM#7-mrn6|6d3k>n|8K-f2uGUBO?~0N z04v#y{|m5^bxCaVi?lwyINPv-==4>5ZI}am<_(~+#_~PlC292}=3Yd?nlkz< zZwtxA$qQa0`9Cl$)@xa3>4d&`ZDum?qkFOWjGYl?)H((iZr|L}q=_UPYy_gj{{Uez z_Bch^8HSFI;ZgejHFFZ@tjy;P%75iy&>x@=& zH>O^!Wu$niE7@c>Ss#)7Ji<@UXuhnAH9q&Tg`3!f;^FdV4DJFsxzu1M_YN@O<4&0{ zVIR2~0WlrNl!u6x@1L!2L=Qo~!?;ITLKH}gs2Y*GLS=+j0t5-CqE%!O(rLlzeV5k~ zZtBYGk-n(4)65wL*tT9QrR&_~Ho*Wl@Py9u#Th9$yke>Olv^@NgMSW{7lYl<3gwaS z{ITM>;=bj$kjo|!^+y67VWl>-*g8MnpUPAPE9e-=S^(B*1qDq{< z_Id*spXtb=?$IqgC*-FehV9q+Pi#wQEXrW>`&4VV^LA{MJ!;?+L&gHOSolUt%O#G^?RKFsr7GlOS3C8o4gI zBDFhlY|Zj4mf>uf?9s=Sd78F@I8Olm@(o2;;YxId(&eXznR_dxQ*|=h4UDE3)X!mP z0=%dwYXft8c08+NHDXLnM->WdT^4m7lB|gqPw0>@s>XQjTLX7J9{1Z!>LQ z=9b*g)l8jFcbs=)ijX5nWbU0=$MQPwDt@YXajv$3W-@uCQ zzIkluZia8ArFFb^m6%0C`3eaU(Zy!QgtN~Ts4jCz$xvRW=~ggTe*L+DFVBa7Lh96w zi`3HO)PA71@uNVI@kz!g;8R}3JygdX?=2o5*C9n6=T_dajj<>7S9ZF`n#@*ecaneX zrqmBBCDP1OoVUw$WeURyv!grlEGTds4?9wEAB$5L~n!+!RHj0AxqtArbk;B=N- zwd3ULjNW^`UgQzjeVJRTBj@Zn-vSBz+%?Q!6vR}JAL`-Fc-Qi5WMa7*kl@8WB^wZl1?i&Y++ zTwGj+ZH$<8h!F@pmQRym}DddM#9nf<9-M_^x47a(iUHglL=V z#W2r+{+kY09%$q&Uoh5i9J=aHud0caAXo>+PVX*noYBQ#j0hL+`43(>oEefArC1JA zfq6bQ7t>#**{I^vWQh3MEahTs!sY8}@W^e5*l3I1{JesvP#IUbkylijV`qZ?(p_1W zwgFtWbFJ0|{?R9bDpvV?t=|6iXTp&`BbY0BIT9n%RzdmmeZcit=1ab&97Ee-o?*ha zN>`Ezgwg|&n8L2=kmHfJslvF;&DEW@)r^{F>{cY&iz$k5yVIZ4~W!a+L1@D;%?qdTdd$Y{8_BT>gb**0|S~h(>?2@?-;%pcDAMb1TDQ{^KvZDuMwayFP1=2wXY&&X$tWhDhUx?jml$k?dyPWBK4u3Z1yr$3Zy z3wx@u55Ng!3Z59#waFIlH|dYz+w;2(6Dqyu&bxiqVA4Mat|R;DxpppBzmOgP2qHf8 zl3*e@y4^(>9v3>`@Bdj5gb(~$-JLu%ya+3J5pwQFV|4YH5tSNOZRZFL;R6cuoE@kK z&O)2o8&AF!UXSI_YOl~G`(YCw?&pVJ;k0zGOKvAe^(M7%G9?`H=j62}jwvIsPuR1R zKd7o*Er^z!r6ckHZqWx%u^1r{~xaNTwT`0Y~_D4tVrtp~!Qtds-%o6>6Tm&^m(`5~Ja5IDTb# zxteF#Xj3@Y_2WqK6dXh<(H>SP4Pif)FWu`G0LBcxAzvf_t-xRB#arn)0)=g#ROpBL z_FMr!T>1scPw;7+PYhGckml7oa^f>@q^&$^?t#ODeJ5CDyCC&`kn)bGavZP?yx@)H zh3q-e3vFq5S#2>e855%V=M9nDrjw*GC!B{9whCkbRE&&XSpY6oy}O3- zCcceKy7S_Y5zMdH@gT=HFjX*cK-gO5(MhVkmYAN=yMQYdSBmpzQZp`KA67A96fW*dQ^xP zx+CxcdYTtL3FZ7RW-qQo3hqXxZ0HSWQ^FxBd*>m|mH_#YB*=pA2J>lJ@j$1Bc6&hXU#$hAM_~T(Z^)}k zAZrYLay5~=DyG*cO)_qu!Cy1X12sQnF;Za`e!dajqb?KT1L&x@F#a1ddXII>Hcl5N z3RuyOf6SYMN#F}PXNLzl=&$sV*dVvP(UrO%_zTYa))c1V#sp>1s#hDe@U3XU)c#fe z{^~^2%P!xOgj`8*F7yA_EtDdoxwv~qX$5W}ft_LRmotalEfR-C2c%I2u`#R&Dl6ml zDsT?;h+lfZR}Jm{Djv2FyPFfQcSfUNnSiAdK-az2cS5yOqUO!JobP3B@y0T9;M0pT zfyhb_&c*ytF6KkBnBr>kTMu}apuNV0Pd{>0F7ZOIhyDidU$ybP)m6h-`M z9C!`bZzWF8%BjNWMNPA#7}j$lf59Up2A;cgIy*%7kMZ%*q0D5bl3(`)=nfp~IF>K* zUoRSsA7x$?;!kj(ID@N^XjMJU!z7|MBI5Y_?l2wLuwJWYE6!^#!<*^?YnEkm!lhGk z$y?;nJoIZPpE^g&$Hwwa-YJ0iydJ}EfF&z#&U|?#ePyqa8-whB|6cwZ`xz7* zTxQ!VQr%MEp1QlfE(1j#yZJUf{X??2_qb+(c9&*}!Prsrk?_&V!7>~)1D_e~1RvUG zG{kZ0bVIqf7^?}-JX?J+<}os}@yK8@F5yj!S)w;7>Wy1T?LpI!;k}yTy=i=WX@3rY z=n0LDjp#@lf4}2TN5hdM+siY&Y8(bRcJk?C`nr$`D*q9#*)p{2m4zPFlTLMl1!s)j z|1s>PD7-kBX8IJR2OPT@B(Yee27Lu>wOF8()fG98Mbq)L6Up2dcFM5EjzH>O7v1-= zXYyz4klVoR^%CD=m}2k`PQ;f@E^|gFq3Gy?;^w8Nb!zCIa`0SVS?7w>d)onxuwk&e z1b=Y##(B8-FKGd`cw|YdU=LP#`1JdjP_gEf&>&e$ZTpRH)I8GLclp+b%jIHzVLv=S zgv4d3%j|>d=q2v?V1Z6g=}Ej=GKb$^*$49R9aCst(^1Szs8ox<1`ID@^_XCtpZ02l zomucccQ4Ylq4(B1qPz;SNWKSD_)uv>4+LTn!($d`TvKp`^DgM4S z_cHJ?<=+oP;p_2SwxeX!u=+jH-F*%#pnfG;ng%yLt0b9`B7FmwE^1r|)$0dA^G@(I zBBdH7VZlB`I3J>ENxpktGfOwNH8av&XD>F9?KVS?cZ44CmU;UXztc%uq{n*%lXU0L zkIy{k6^2!65@4E;juM*+c%L|N>9+!uPaF!Qk*8tAKSShy9Udp~>A@4*>qD$IM)yilVrZ8H#7`g>s`V`T`(%qbd@&Y^Hv56WhatHTm~5X=nKs^@d!HP+JsoxJ8(bhU5V<@Bt27E zppo7F_be8}{4_(}d^dKkucSO`9`e@l0K%ZHvOzL(V;8ObpIrI#8uC%I2tgoMutv-7 z%KPg4)*{<{gTDufTt3c+q1Sv@tEM7me5-oB;Kh|aaN1Qy_JIwHzdm*0cjXy9pghk) zg@g|>uJ=dMRJMKfvf(3-j{LUr_iZ%Fe^`Kzsl8S!s>tvWrWZjZ8|}xjrn)rNM#|P@ zLxp;5`R8?Ty(UMs$skiVS0kex83YLeeAy2L$hG^yJ)cVYUd}`5E*{*@iC~HD*oTW- zAh3&vk2w*tJHlp|^7B6}#nUQ$^U(jgf16+*8o2*hrRX@RHwk-M?G&Dq&1W=fovbR1 znoifAOL{I&fUswXiHXl91?v>u<1e^vl7y_0#0|H&{`!7VXnboKSV=zKB}mxOQl#i2 z0E}*zDb(o&vL#v!@sUNGm97}L@0zdGuS|8DL?f49sJ+GZ_vKRf)>!27dqltX;Bx&E zgPToB88GVSzL_(RCP7b_;S@E}%VP1@1C(Wi`gznJ9L zj+S~b;vqMO{mgW4^A}|ujDQ@{T(c9%C7r)u~qK29PP)%FPH!1+;` z$b7?C_FiZR0nNY?praG!9doEl4?^Tl>RSUwdX@}28s?9~J)C11vTEQYE>pD0AlDuQ zL_Rl+<>-aN247v#JJ`E^hVxC{gu-L)MPa)v&xvaTQN^fSchCI8FOGjN5@-|){;(`O z_C&&xiC65MjBwbI?^Rh^K#*C#`U0*lf~edTpcKlXwER&~@`A{RP4$}lU#mCDS2?qH zytEO9{&U@hf3AxuKgjpT z!e9Ny_jG*pa$W`nH$D)Z1(2{D^mA+#Q?H-f<_`vhM7eSP^`02tv2M3 znbm2*>_KoESjxXY{n|fT0yf!B2$hw0*Jmh683&M&5}GrtN8%7X^3^~I5oDSP+E1Et z1|Q_dV=G8nVOn;Edu=ic?!T(h$E<#lCQDF^<+wP`p`!VIE%a%8iN_3QU`$xwbApkU zGwK6>51gws;{dYr9B8Drm4N5CRh~iE4pX49+8EA0$}Hq{@U9x#z;!~VKivPM0!!Ox zF9vIo-}QgJFj(osB=V8@vK+s!rGakXh?+;3!O%0R35YwCUR$b5v$zj{=P|tn7=-WA zb`O=xkp~H5kr+VUxDOFFP@)|9^WHC|>I}~-IDnP53qnv!Dvc|n1ml2@Zc>m3JrTP= zM7g>!=F3E?{g%OBbM*rQ{^yL|bx!ZumQd}f!*<0@KwNf0$Z)omE`OR3FU$fm^WTx+ zFyypmBAZhBhH*DP@!wymH&L@(=Gtor{Sv}G7meXZa+=?08zVo0@F$Rk7>*>D25Gfg z#wmts919Yl-Ae*ArLcNeiL7{UH!{Bcy_Fw@8kT!TXP5s+nb)H$ zBEwbYyZi^{FIc=pQ?SSmZC-dP6!b2z%zW^~zs=_cV888|qvPGQ`}OK1 z)hwt5oj5u%m(_<5j-O*K&zq+49g2>e4A8|-F|;r0{d?G4RDUAX6Q~u1Rf9p@YAPTg zAO{X#+Q}=mE4QR;3Sk9vwXKzDE#cM$>1qpv+6e;%I<;e_BU@<@fL%8BY&owJ50U0j zKtj(9#NM;eMBSt}?z&eYV$1G?uf~j4z|tH#-IZstS=X(}HoXrVCFe1y8-kKpLs?p+ z1glIg$={=ZWn{;zI`w`w@ku9iap?k7%iYC2$!Dh%kWSady8}OtkDM1|;P0!B^jy>P z*~p$0kh~Bwr4t8;uBl1HcFhaaO7;CW?tiRn{7@klHxUsm-Om0$m==3_GV42!9gF-TwVKr8$9z4cM3SI$Bg41R>k5<+@6Qlj=|Qoi9| zFNyIn!H%;#%*Vib{}d2ky&X!|(e;WwlM5}cdXMX!C-d982xucNO ztZB~XwKB2rI9;3E_ebY5D(Oc4fTLb?#?-+3(6PV#k&-O#m z-Pd#R&OOQHTN?>I@gNQD_?oZPfg+c&LFm`t<*0!&Aqa2r2CHgn=jV8zaLx4_dbzh# z3X{V@=qm;e@mb)MSM`o46HDzrV>tN0m|hlO+$}t_f)q6@e!Q>W1sGB%!c5*sS@CG) z7LqgNwcs5#dx!UHw7LRf<042qgt!80qZ4GlmdIh|Xfs!Q_^ms+H}xQo*BXgEh%;!d z-QELg@w@P=p6pn5jhwr9a}8X|XB^6hzx-?C#WM2ZBScmn@=2K(MD#K)$I(&JX9aX{ z>^78w%TyC1H`5Lm&9-|UodR8vhxJFlQ@;eBl*oNOV|P1MO!zhS7zpNEw%Rhf48%!T zQ7^~O0LHQ-*PRYyPrx!erR_AcEDqnc(-^A$P-EE$KA zGt0&!SQ8@lXgJ<4JnXWwf?jo9C0<*c8%0k&3+A1<=zi0QION4DPDj?FF>BCiwRGt# z1l0zrR7v{m@s@isHN=#AQw+yL1{)wGNPzR?2^SkXQZm(@;gL0GLdNtpZ zh998ZT@pwco(yXiwL4`K$=Tc7+6ML9lg6$@3}q=$R#=aT^us2#Z=0I4vAGA)QKfGK><1$serCvo zSW@E!%_Au>x3d(7Uo}*Rp%33;g@D#Je3);y*phvzM$(QC<^<#gp?G?!vU%Yh6E1ch zLi^!uA~$L(DymDTBh8baKv`WtsLc`S^RD9h@w-r|zhB%>S)@~)6$7h-*r^6d&0pUE zhH7OKxnNt5sO5doX;%nS)+Td<52BTju-}`!!{9ba>62z-V?Q%M0nf z?ey{bWoju{;fv_(&yjGc?T=1G61GUqcImBWLtDrP%)cQ3uOtbPQOj05-{I3DTYn%# z64rm3{<^8!YPLz4#=Gkht10TBH@akWTk}C%RX`=HZ+FX}tmEH{Gfu(|pJ00DZIVMf z1QV_*EhQiDP!GjkoS<6|P7@+GZ%(iZQNW{zQo2e#<5SuoK>n~Ey3pKTW7^xHOl#G# zNkw47xnu7rx*O*Vq1uiHAG*518q>7Edm#!H@d($0Ss-%1{U~YBh}}#1Q~MqwvP4K~ zjntrw^k$Gl!~0MD&Q-)F$(6yP6h1AH@Bh@V42wt4DtaqzDEZZu9nBJX)QnD*w3e)n zD-|~!?S&}Y>9#~_vyfCvx0iY|-U2p?DO5!jt|X3LSnht;!6Zj?2SCnM7MHM%Zb{lCLh+O@TEcPa6?0WnIFs`zJh^rScJ3~o*R0=u z+?dcGF^m6EiS8}?MTa#D{zMT8p}Fa6imowy*Lnmhnu6Q$yE_hBlg#m@U=#e8Jcb5n zg;Y6`w(HIs5S70PCuHjm@of|PFcj(6>}nDtB_hK?(Z!5seU^<&B%>}Y@6)VDg^S{o zG*J$-sGxI~v@3~{ZJ8!F(^@{KTolCM_;P90P`@F-ZuTPI$%yc4jP?Wj?C=j>LN4ql z=C9Mf7qZ})VaZHFM zB*qov)})Q1TH~l;%FViBqM}i9M!CyPC0DM9hHBUvp%ppSXhOmiYjb9wueI8J{a(L$ z&7a@T_jta~_xU{U_j6&M<8ysbJey2UCs9?{zMo|>)wxXx!M=y;xw_iB*?7V09SN}$ z_?%#cY4;uF!Ly|F(W?VccDy&T8`=|N6&-S1)Odq@2?D}QtXM4f5dpuSOx&(CyEN1c z`!UW3($!Ko6RcA*pIgjY>S)|#PioED?ucfnW7Dn*m|*hlDDK+apW1K*p)j|of@WT- z@`2B)R=UPY?ne#n;GR=#7DoE^fQHtVXqm=KE5hj0auLg-Jj{@pJo{5tGqA6p^@}Ll zS!Q5moa5q`<02B^UeB|-yIkph*z~ffTA%~HRIM#Ema5uhlo&nSa{cu-{zpvDcKMhD z)4c;byssGRp^pT#4#_ikqtBREB~==oTqCm9h=(C`^R>iAzZ6d|?=4wT&)HE?wSp?F zt@_1i{B3zgQi%9f1E+gaa zw9b@vaY-I6aVZ9$RQhB=mcj3#bLjal9|D1Qd0RW~t^MUAR7_9IiwFAT321$3P=k917e z$U2=Q5~IjO>)e;i+^lbi-@1217PZgw$(Z?O=F0xl89R7}8fnHqCAhf;@>lRtIg*^w zO~&2|g4aLBW32wjD4amX$U0E*+ijn(if*omsqt|@4YiH(1??`bXBocpz?XuB>mnf{ z9Mr1l!=~-hr1zf@u#)Wh0-Iv*EBz0=tb3B-93xW%Q&;zgdC zo#7ygV*7^ri0O2#Ks&11kziqsmD^=w7)_yp&;yY;Lr?7t-Q9IZA}TB6)Lo*s)g9Zr z-_?_ke93Bd#>w4Wt}`#K%B_<|;wgh>0q$Eai#3fH zn+_$vi`I)&Ke=xk_1HUt4ikIZQYKAZr~=)~caekCKzz1zQm$cpSbFz%OAUQy<@jp^EuKSf9PSNrAg z6D8It`ZaxFqdwvF-ca8EA)${sbn&*>zK=U((h581`AK3Y-jX{`3t4+(ns~xknJQh@ z&b$mGF-1L7&tL^cQY?{#8_L0wXXwP4K}jyuokup~szMyVI`!{O99Ma52*NYWdc-g&6M3ZHTg zs!J3>Au-Os{|#74XhK}quk>s&IoT~I&)s~?HZ7umv(3u0$2ZJIa9;T(>mEcnD(J8Q z<-M+7&B++#tm@;>3Whzm;peb&&83nJeG`+TQiOHpLz45WKs<4wWe~x^C1yEV-^U-i zha|&kzO~Ji5FDn?F*0%e(lpzR`&gES`-lrGJsJDuF8CHw)mqFOVvPOAw_-O{e{xyU zsqQ|y#Ld&d>Pdq?ilsTDVYnW4fE4NV7z1MnjPn-YF*K+{TYrL{yerRDZq@6Z;*xldQo{mMc(c}3R>P8tbW>M>x zi5oxYQ$mAG#LKe)PxZXMXWtDo7v-F#-*~UDlg!{^9eccFL@xH%FPA~GsFH9ge<@obx>O7~QPc}xfI zQB-i~+v}#88c>89v@D-L_$hH5ra7b`wJQjEwJEWY50Z|xi~Cpjw{_H^MQF`@(|5z` zaeR5Eo|2r1SHmlm9xaRwW0nEneIoYD<&c)1OTeJ20^CuBg1OjNbPbz--m5;c@C8|p zl3anu8BId+>Id-B>mZHqk}AwAcZRwf5@c2)lqOp2VHBiktnIf8h|2#i?mBUtY^iP} z{w>a1eF1Lr*9^%viB}MFyxvRKIJJH$>UWzKlE2JmTt|3bJ|Bqil15*oBChKY2vMop zbMP%P+l+6Qw)(#%e*z>w^fIuTl3b2gYn|$^Nr&ieI*n%74QbtMlU)fZ!|hybV6m#8 zmJ*ZOqw>!ydkYJv85c^I6%;}GFp9YBq9&AZB8*X>>8?9(0iFFjqKkZSlE}rTUj!^{eeZ{cs{XCp_3sX0v&uK%fX>9fLLMy6ra`E8Qr<( zGB@!X30gk?`DzDi0aC+5_B_8sxc0ZAhoj~#4JdQOuax;@tl$)%&)4Mz65wqTw6!4t z68#g)Q!Fu=-{|;+vt_=&itGaYU7Z)du-R#7Hze2&5|qzvpEvwnR=au(COKCd zHhMvIC=I~%)K!YvQW5mMe75=J95+Z3rmGI~X6&mNVrj%-cHW>*7-u13H7LhLH83Fk9%SywbF zyH_8(c)rf}{R`kyEbvc}S=5j0tq*NT2s)g>DM`advYpXUwddX_BXQXqPiw(;7L=8# t$TCfu+qylGkKKPx{@atFq+r#I`4Z8%#l@L9|Cr4IKXYROzSPk1;$OsP7x(}G literal 0 HcmV?d00001 diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index a3c3c4f..a5ea9ab 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -1,98 +1,83 @@ +# Getting Started -# Pre-req - -* Linux host with 4-6 cores & 2-4 GB memory (Docker container should be allocated 2 cores 2 GB of memory) and Docker 19.x or newer with Docker Swarm enabled - * [Getting Started](https://docs.docker.com/get-started/) -* A Splunk index for metrics typically "em_metrics" -* One or more Splunk indexes for events collected by SC4S -* Splunk HTTP event collector enabled with a token dedicated for SC4S - * [Splunk Enterprise](http://dev.splunk.com/view/event-collector/SP-CAAAE6Q) - * [Splunk Enterprise Cloud](http://docs.splunk.com/Documentation/Splunk/7.3.1/Data/UsetheHTTPEventCollector#Configure_HTTP_Event_Collector_on_managed_Splunk_Cloud) -* A network load balancer (NLB) configured for round robin. Note: Special consideration may be required when more advanced products are used. The optimal configuration of the load balancer will round robin each http POST request (not each connection) - -# Setup - -* Create a directory on the server for configuration -* Create a docker-compose.yml file based on the following template - -```yaml -version: "3" -services: - sc4s: - image: splunk/scs:latest - hostname: sc4s - ports: - - "514:514" - - "601:601" - - "514:514/udp" - - "5514:5514" - - "5514:5514/udp" - stdin_open: true - tty: true - environment: - - SPLUNK_HEC_URL=https://foo:8088/services/collector/event - - SPLUNK_HEC_TOKEN= - - SPLUNK_CONNECT_METHOD=hec - - SPLUNK_DEFAULT_INDEX= - - SPLUNK_METRICS_INDEX=em_metrics - volumes: - - ./sc4s/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv - - ./sc4s/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv -``` - -## Configure index destinations for Splunk -* Download the latest context.csv file to a subdirectory sc4s below the docker-compose.yml file created above -```bash -wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv -``` -* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. - -## Configure sources by source IP or host name -* This step is required even if not used -* Download the latest vendor_product_by_source.conf file to a subdirectory sc4s below the docker-compose.yml file created above -```bash -wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf -``` -* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. - -* Start SC4S - -```bash -docker stack deploy --compose-file docker-compose.yml sc4s -``` - - -## Scale out +Splunk Connect for Syslog is a containerized distribution of syslog-ng with a configuration framework +designed to simplify getting syslog data into Splunk Enterprise and Splunk Cloud. Our approach is +to provide a runtime-agnostic solution allowing customers to deploy using the container runtime +environment of choice. -Additional hosts can be deployed for syslog collection from additional network zones and locations +# Planning Deployment + +Syslog is an overloaded term that refers to multiple message formats AND optionally a wire protocol for +transmission of events between computer systems over UDP, TCP, or TLS. The protocol is designed to minimize +overhead on the sender favoring performance over reliability. This fundamental choice means any instability +or resource constraint will cause data to be lost in transmission. + +* When practical and cost effective considering the importance of completeness as a requirement, place the sc4s +instance in the same VLAN as the source device. + +* Avoid crossing a Wireless network, WAN, Firewall, Load Balancer, or inline IDS. +* When High Availability of a single instance of SC4S is required, implement multi node clustering of the container +environment. +* Avoid TCP except where the source is unable to contain the event to a single UDP packet. +* Avoid TLS except where the event may cross a untrusted network. + + +# Implementation + +## Splunk Setup + +### Create Indexes + +SC4S is pre-configured to map each sourcetype to a typical index, for new installations best practice is to create the following +indexes in Splunk. The indexes can be customized easily if desired. If using defaults create the following indexes on Splunk: + +* netauth +* netfw +* netids +* netops +* netproxy +* netipam +* em_metrics (ensure this is created as a metrics index) + +### Install Related Splunk Apps + +Install the following: + +* [Splunk App for Infrastructure](https://splunkbase.splunk.com/app/3975/) +* [Splunk Add-on for Infrastructure](https://splunkbase.splunk.com/app/4217/) +* [Splunk Metrics Workspace](https://splunkbase.splunk.com/app/4192/) *NOTE Included in Splunk 7.3.0 and above* + +### Configure the Splunk HTTP Event Collector + +- Set up the Splunk HTTP Event Collector with the HEC endpoints behind a load balancer (VIP) configured for https round robin *WITHOUT* sticky +session. Alternatively, a list of HEC endpoint URLs can be configured in SC4S if no load balancer is in place. In either case, it is +recommended that SC4S traffic be sent to HEC endpoints configured directly on the indexers rather than an intermediate tier of HWFs. +- Create a HEC token that will be used by SC4S and ensure the token has access to place events in main, em_metrics, and all indexes used as +event destinations +- Refer to [Splunk Cloud](http://docs.splunk.com/Documentation/Splunk/7.3.1/Data/UsetheHTTPEventCollector#Configure_HTTP_Event_Collector_on_managed_Splunk_Cloud) +or [Splunk Enterprise](http://dev.splunk.com/view/event-collector/SP-CAAAE6Q) for specific HEC configuration instructions based on your +Splunk type. + +## Implement a Container Runtime and SC4S + +### Prerequisites + +* Linux host with Docker (CE 19.x or greater with Docker Swarm) or Podman enabled, depending on runtime choice (below). +* A network load balancer (NLB) configured for round robin. Note: Special consideration may be required when more advanced products are used. The optimal configuration of the load balancer will round robin each http POST request (not each connection). + +### Select a Container Runtime and SC4S Configuration + +| Container and Orchestration | Notes | +|-----------------------------|-------| +| [Podman + systemd single node](gettingstarted/podman-systemd-general.md) | First choice for RedHat 7.x and 8.x, second choice for Debian and Ubuntu (packages provided via PPA) | +| [Docker CE + systemd single node](gettingstarted/docker-systemd-general.md) | First choice for Debian, Ubuntu, and CentOS distributions with limited existing docker experience | +| [Docker CE + Swarm single node](gettingstarted/docker-swarm-general.md) | Option for Debian, Ubuntu, and CentOS desiring swarm orchestration | +| [Docker CE + Swarm single node RHEL 7.7](gettingstarted/docker-swarm-rhel7.md) | Option for RedHat 7.7 desiring swarm orchestration | + + +# Scale out + +Additional hosts can be deployed for syslog collection from additional network zones and locations -## Single Source Technology instance - Alpha - -For certain source technologies message categorization by content is impossible to support collection -of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using -an alternate port. In the following configration example a dedicated port is opened (6514) for legacy juniper netscreen devices - -This approach is "alpha" and subject to change - -```yaml -version: "3" -services: - sc4s-juniper-netscreen: - image: splunk/scs:latest - hostname: sc4s-juniper-netscreen - ports: - - "6514:514" - - "6514:514/udp" - stdin_open: true - tty: true - environment: - - SPLUNK_HEC_URL=https://foo:8088/services/collector/event - - SPLUNK_HEC_TOKEN= - - SPLUNK_CONNECT_METHOD=hec - - SPLUNK_DEFAULT_INDEX= - - SPLUNK_METRICS_INDEX=em_metrics - - SYSLOG_PRESUME_FILTER=f_juniper_netscreen - volumes: - - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv -``` +![SC4S deployment diagram](SC4S%20deployment.png) \ No newline at end of file diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md new file mode 100644 index 0000000..d7beaa7 --- /dev/null +++ b/docs/gettingstarted/docker-swarm-general.md @@ -0,0 +1,160 @@ + +# Install Docker CE and Swarm + +Refer to [Getting Started](https://docs.docker.com/get-started/) + +# SC4S Configuration + +* Create a directory on the server for configuration. This should be available to all administrators, for example: +``/opt/sc4s/`` +* Create a docker-compose.yml file in the directory created above, based on the following template: + +```yaml +version: "3.7" +services: + sc4s: + image: splunk/scs:latest + ports: + - target: 514 + published: 514 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host + - target: 514 + published: 514 + protocol: udp +#Comment the following line out if using docker-compose + mode: host + env_file: + - /opt/sc4s/env_file + volumes: +#Uncomment the following line if overriding index destinations +# - ./context-local/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +#Uncomment the following lines if using a host or network based filter and log_path +# - ./context-local/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv +# - ./context-local/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf + +``` + +## Configure the SC4S environment + +Create the following file ``/opt/sc4s/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +``` + +## Configure index destinations for Splunk + +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow +customization of index destinations. + +* Create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above. From this directory, +execute the command below to download the index context file: + +```bash +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv +``` +* Edit splunk_index.csv review the index configuration and revise as required for sourcetypes utilized in your environment. + +## Configure sources by source IP or host name + +Legacy sources and non-standard-compliant source require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. + +* If not already done in the step immediately above, create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``) +created in the first step above. From this directory, execute the commands below to download the vendor context files: + +```bash +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv +``` +* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. + +# Start SC4S + +```bash +docker stack deploy --compose-file docker-compose.yml sc4s +``` + + +## Scale out + +Additional hosts can be deployed for syslog collection from additional network zones and locations. + + +# Single Source Technology instance + +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. +Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. + +Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. + +In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify the individual ports or a +range as appropriate for your network. + +* Modify the unit file ``/opt/sc4s/docker-compose.yml`` +```yaml +version: "3.7" +services: + sc4s: + image: splunk/scs:latest + ports: + - target: 514 + published: 514 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host + - target: 514 + published: 514 + protocol: udp +#Comment the following line out if using docker-compose + mode: host + - target: 5000-5021 + published: 5000-5021 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host + - target: 5000-5021 + published: 5000-5021 + protocol: udp +#Comment the following line out if using docker-compose + mode: host + env_file: + - /opt/sc4s/env_file + volumes: +#Uncomment the following line if overriding index destinations +# - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +#Uncomment the following lines if using a host or network based filter and log_path +# - ./sc4s-juniper/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv +# - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf +``` + +Modify the following file ``/opt/sc4s/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 +#Uncomment the following line if using untrusted SSL certificates +#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` + +* Start SC4S. + +```bash +docker stack deploy --compose-file docker-compose.yml sc4s +``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md new file mode 100644 index 0000000..e57bdb5 --- /dev/null +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -0,0 +1,185 @@ + +# Install Docker CE and Swarm + +*Warning* this method of installing docker on RHEL does not appear to be supported: + +## Enable required repositories +```bash +subscription-manager repos --enable=rhel-7-server-rpms +subscription-manager repos --enable=rhel-7-server-extras-rpms +subscription-manager repos --enable=rhel-7-server-optional-rpms +yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo +``` + +## Enable EPEL +```bash +yum install wget -y +cd /tmp +wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +rpm -Uvh epel-release-latest-7.noarch.rpm +``` + +## Install required packages and enable docker +```bash +yum install docker-ce -y +systemctl enable docker.service +systemctl start docker.service +``` + +## Initialize Docker Swarm +```bash +sudo docker swarm init +``` + +# SC4S Configuration + +* Create a directory on the server for configuration. This should be available to all administrators, for example: +``/opt/sc4s/`` +* Create a docker-compose.yml file in the directory created above, based on the following template: + +```yaml +version: "3.7" +services: + sc4s: + image: splunk/scs:latest + ports: + - target: 514 + published: 514 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host + - target: 514 + published: 514 + protocol: udp +#Comment the following line out if using docker-compose + mode: host + env_file: + - /opt/sc4s/env_file + volumes: +#Uncomment the following line if overriding index destinations +# - ./context-local/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +#Uncomment the following lines if using a host or network based filter and log_path +# - ./context-local/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv +# - ./context-local/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf + +``` + +## Configure the SC4S environment + +Create the following file ``/opt/sc4s/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +``` + + +## Configure index destinations for Splunk + +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow +customization of index destinations. + +* Create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above. From this directory, +execute the command below to download the index context file: + +```bash +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv +``` +* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. + +## Configure sources by source IP or host name + +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. + +* If not already done in the step immediately above, create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``) +created in the first step above. From this directory, execute the commands below to download the vendor context files: + +```bash +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv +``` +* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. + +# Start SC4S + +```bash +sudo docker stack deploy --compose-file docker-compose.yml sc4s +``` + +## Scale out + +Additional hosts can be deployed for syslog collection from additional network zones and locations. + + +# Single Source Technology instance + +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. +Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. + +In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify the individual ports or a +range as appropriate for your network. + +* Modify the unit file ``/opt/sc4s/docker-compose.yml`` +```yaml +version: "3.7" +services: + sc4s: + image: splunk/scs:latest + ports: + - target: 514 + published: 514 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host + - target: 514 + published: 514 + protocol: udp +#Comment the following line out if using docker-compose + mode: host + - target: 5000-5021 + published: 5000-5021 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host + - target: 5000-5021 + published: 5000-5021 + protocol: udp +#Comment the following line out if using docker-compose + mode: host + env_file: + - /opt/sc4s/env_file + volumes: +#Uncomment the following line if overriding index destinations +# - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +#Uncomment the following lines if using a host or network based filter and log_path +# - ./sc4s-juniper/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv +# - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf +``` + +Modify the following file ``/opt/sc4s/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 +#Uncomment the following line if using untrusted SSL certificates +#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` +* Start SC4S. + +```bash +docker stack deploy --compose-file docker-compose.yml sc4s +``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md new file mode 100644 index 0000000..1ab3909 --- /dev/null +++ b/docs/gettingstarted/docker-systemd-general.md @@ -0,0 +1,164 @@ + +# Install Docker CE + +Refer to [Getting Started](https://docs.docker.com/get-started/) + +# Setup + +* Create a systemd unit file use to start the container with the host os. ``/lib/systemd/system/sc4s.service`` + +*NOTE*: In a future release, the mechanism to support mounted volumes (the three -v arguments in the unit file below) will change. For now, do not change these arguments, and ensure that the three files they reference are downloaded to the proper directory per the configuration instructions below. + +*NOTE-2*: Replace the URL and HEC tokens with the appropriate values for our environment + +```ini +[Unit] +Description=SC4S Container +After=network.service +Requires=network.service + +[Service] +Environment="SC4S_IMAGE=splunk/sc4s:latest" + +#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" +#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started +#Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +#Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + +TimeoutStartSec=0 +Restart=always +ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE +ExecStartPre=/usr/bin/docker run \ + --env-file=/opt/sc4s/default/env_file \ + "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \ + --name SC4S_preflight --rm \ + $SC4S_IMAGE -s +ExecStart=/usr/bin/docker run -p 514:514 \ + --env-file=/opt/sc4s/default/env_file \ + "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \ + --name SC4S \ + --rm \ +$SC4S_IMAGE +``` + +## Configure the SC4S environment + +Create the following file ``/opt/sc4s/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +#Uncomment the following line if using untrusted SSL certificates +#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` + +*NOTE*: While optional, until the final mechanism for mounted volumes in containers is released, do _not_ skip the download (wget) steps in +the configuration steps below. Ensure these three files are in place in the directory specified, even if there is no intent to modify them. + +## Configure index destinations for Splunk + +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow +customization of index destinations. + +* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` arguments) +match the directory you created above. From this directory, execute the following to download the latest index context file: + +```bash +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv +``` +* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. + +## Configure sources by source IP or host name + +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. + +* If not already done in the step immeidately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory +references in the unit file above (the ``-v`` arguments) match the directory you created above. From this directory, execute the following to +download the latest vendor context files: + +```bash +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv +``` +* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. + +# Start SC4S + +```bash +sudo systemctl daemon-reload +sudo systemctl enable sc4s +sudo systemctl start sc4s +``` + + +# Single Source Technology instance + +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. + +Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. + +In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify the individual ports or a +range as appropriate for your network. + +* Modify the unit file ``/lib/systemd/system/sc4s.service`` +```ini +[Unit] +Description=SC4S Container +After=network.service +Requires=network.service + +[Service] +Environment="SC4S_IMAGE=splunk/scs:latest" + +#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" +#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started +#Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +#Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + +TimeoutStartSec=0 +Restart=always +ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE +ExecStartPre=/usr/bin/docker run \ + --env-file=/opt/sc4s/default/env_file \ + "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \ + --name SC4S_preflight --rm \ + $SC4S_IMAGE -s +ExecStart=/usr/bin/docker run -p 514:514 -p 5000-5020:5000-5020 \ + --env-file=/opt/sc4s/default/env_file \ + "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \ + --name SC4S \ + --rm \ +$SC4S_IMAGE + +``` + +Modify the following file ``/opt/sc4s/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 +#Uncomment the following line if using untrusted SSL certificates +#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` + +* Restart SC4S. + +```bash +sudo systemctl restart sc4s +``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md new file mode 100644 index 0000000..4693995 --- /dev/null +++ b/docs/gettingstarted/podman-systemd-general.md @@ -0,0 +1,164 @@ + +# Install podman + +Refer to [Installation](https://podman.io/getting-started/installation) + +# Setup + +* Create a systemd unit file use to start the container with the host os. ``/lib/systemd/system/sc4s.service`` + +*NOTE*: In a future release, the mechanism to support mounted volumes (the three -v arguments in the unit file below) will change. For now, do not change these arguments, and ensure that the three files they reference are downloaded to the proper directory per the configuration instructions below. + +*NOTE-2*: Replace the URL and HEC tokens with the appropriate values for our environment + +```ini +[Unit] +Description=SC4S Container +After=network.service +Requires=network.service + +[Service] +Environment="SC4S_IMAGE=splunk/sc4s:latest" + +#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" +#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started +#Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +#Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + +TimeoutStartSec=0 +Restart=always +ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE +ExecStartPre=/usr/bin/podman run \ + --env-file=/opt/sc4s/default/env_file \ + "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \ + --name SC4S_preflight --rm \ + $SC4S_IMAGE -s +ExecStart=/usr/bin/podman run -p 514:514 \ + --env-file=/opt/sc4s/default/env_file \ + "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \ + --name SC4S \ + --rm \ +$SC4S_IMAGE + +``` + +*NOTE*: While optional, until the final mechanism for mounted volumes in containers is released, do _not_ skip the download (wget) steps in +the configuration steps below. Ensure these three files are in place in the directory specified, even if there is no intent to modify them. + +## Configure the SC4S environment + +Create the following file ``/opt/sc4s/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +#Uncomment the following line if using untrusted SSL certificates +#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` + +## Configure index destinations for Splunk + +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow +customization of index destinations. + +* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` arguments) +match the directory you created above. From this directory, execute the following to download the latest index context file: + +```bash +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv +``` +* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. + +## Configure sources by source IP or host name + +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. + +* If not already done in the step immeidately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory +references in the unit file above (the ``-v`` arguments) match the directory you created above. From this directory, execute the following to +download the latest vendor context files: + +```bash +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv +``` +* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. + +# Start SC4S + +```bash +sudo systemctl daemon-reload +sudo systemctl enable sc4s +sudo systemctl start sc4s +``` + +# Single Source Technology instance + +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. + +Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. + +In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify the individual ports or a +range as appropriate for your network. + +* Modify the unit file ``/lib/systemd/system/sc4s.service`` +```ini +[Unit] +Description=SC4S Container +After=network.service +Requires=network.service + +[Service] +Environment="SC4S_IMAGE=splunk/scs:latest" + +#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" +#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started +#Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +#Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + +TimeoutStartSec=0 +Restart=always +ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE +ExecStartPre=/usr/bin/podman run \ + --env-file=/opt/sc4s/default/env_file \ + "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \ + --name SC4S_preflight --rm \ + $SC4S_IMAGE -s +ExecStart=/usr/bin/podman run -p 514:514 -p 5000-5020:5000-5020 \ + --env-file=/opt/sc4s/default/env_file \ + "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \ + --name SC4S \ + --rm \ +$SC4S_IMAGE + +``` + +Modify the following file ``/opt/sc4s/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 +#Uncomment the following line if using untrusted SSL certificates +#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` + +* Restart SC4S. + +```bash +sudo systemctl restart sc4s +``` diff --git a/docs/sources.md b/docs/sources.md index 24e9d72..363bbe5 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -16,6 +16,13 @@ | cisco:pix | Not supported | | cisco:fwsm | Not supported | +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_asa | cisco:asa | netfw | none | + + ### Filter type MSG Parse: This filter parses message content @@ -23,7 +30,7 @@ MSG Parse: This filter parses message content ### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. -* Review and update the splunk_index.csv file and set the index as required. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. * Follow vendor configuration steps per Product Manual above ensure: * Log Level is 6 "Informational" * Protocol is TCP/IP @@ -31,6 +38,13 @@ MSG Parse: This filter parses message content * device-id is hostname and included * timestamp is included +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_JUNIPER_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format | + ### Verification Use the following search to validate events are present @@ -50,21 +64,30 @@ Verify timestamp, and host values match as expected | NX-OS Manual | https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/6-x/system_management/configuration/guide/b_Cisco_Nexus_9000_Series_NX-OS_System_Management_Configuration_Guide/sm_5syslog.html| | Cisco ACI | https://community.cisco.com/legacyfs/online/attachments/document/technote-aci-syslog_external-v1.pdf | | Cisco WLC & AP | https://www.cisco.com/c/en/us/support/docs/wireless/4100-series-wireless-lan-controllers/107252-WLC-Syslog-Server.html#anc8 | + ### Sourcetypes | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| | cisco:ios | This source type is also used for NX-OS, ACI and WLC product lines | +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_ios | cisco:ios | netops | none | +| cisco_nx_os | cisco:ios | netops | none | + ### Filter type * Cisco IOS products can be identified by message parsing alone * Cisco NX OS, WLC, and ACI products must be identified by host or ip assignment update the filter `f_cisco_nx_os` as required + ### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. -* Review and update the splunk_index.csv file and set the index as required. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. * IOS Follow vendor configuration steps per Product Manual above ensure: * Ensure a reliable NTP server is set and synced * Log Level is 6 "Informational" @@ -78,14 +101,20 @@ Verify timestamp, and host values match as expected * Protocol is TCP/IP * device-id is hostname and included * timestamp is included and milisecond accuracy selected -* ACI Logging configuration of the ACI product often varies by use case. +* ACI Logging configuration of the ACI product often varies by use case. * Ensure NTP sync is configured and active * Ensure proper host names are configured -* WLC +* WLC * Ensure NTP sync is configured and active * Ensure proper host names are configured * For security use cases per AP logging is required +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_IOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CISCO_NX_OS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification @@ -112,10 +141,21 @@ Verify timestamp, and host values match as expected | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| fgt_log | The catch all sourcetype is not used | +| fgt_log | The catch all sourcetype is not used | | fgt_traffic | None | | fgt_utm | None | -| fgt_event | None +| fgt_event | None + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| fortinet_fortios_traffic | fgt_traffic | netops | none | +| fortinet_fortios_utm | fgt_utm | netids | none | +| fortinet_fortios_event | fgt_event | netops | none | +| fortinet_fortios_log | fgt_log | netops | none | + ### Filter type @@ -124,7 +164,7 @@ MSG Parse: This filter parses message content ### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. -* Review and update the splunk_index.csv file and set the index as required. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. * Refer to the admin manual for specific details of configuration to send Reliable syslog using RFC 3195 format, a typical logging configuration will include the following features. ``` @@ -160,9 +200,15 @@ end ``` +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + ### Verification -An active firewall will generate frequent events, in addition fortigate has the ability to test logging functionality using a build in command +An active firewall will generate frequent events, in addition fortigate has the ability to test logging functionality using a built in command ``` diag log test @@ -171,7 +217,7 @@ diag log test Verify timestamp, and host values match as expected ``` -index= (sourcetype=fgt_log OR sourcetype=fgt_traffic OR sourcetype=fgt_utm) +index= (sourcetype=fgt_log OR sourcetype=fgt_traffic OR sourcetype=fgt_utm) ``` ### UTM Message type @@ -185,7 +231,206 @@ index= (sourcetype=fgt_log OR sourcetype=fgt_traffic OR sourcetype ###Event Message Type ![FortiGate Event message](FortiGate_event.png) -Verify timestamp, and host values match as expected +Verify timestamp, and host values match as expected + +# Vendor - Juniper + +## Product - Juniper JunOS + +| Ref | Link | +|-------------------|-------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2847/ | +| JunOS TechLibrary | https://www.juniper.net/documentation/en_US/junos/topics/example/syslog-messages-configuring-qfx-series.html | + +### Sourcetypes + +| sourcetype | notes | +|--------------------------|------------------------------------------------------------------| +| juniper:junos:firewall | None | +| juniper:junos:idp | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------------------|------------------------|----------------|---------------| +| juniper_junos_flow | juniper:junos:firewall | netfw | none | +| juniper_junos_idp | juniper:junos:idp | netids | none | +| juniper_junos_utm | juniper:junos:firewall | netfw | none | + +### Filter type + +* MSG Parse: This filter parses message content + + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index as required. +* Follow vendor configuration steps per referenced Product Manual + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_JUNIPER_JUNOS_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format| +| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined using 5424 format | + +### Verification + +Use the following search to validate events are present; for Juniper JunOS ensure each host filter condition is verified + +``` +index= sourcetype=juniper:junos:firewall | stats count by host +index= sourcetype=juniper:junos:idp | stats count by host +``` + +Verify timestamp, and host values match as expected + +## Product - Juniper NSM + +| Ref | Link | +|----------------|-------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2847/ | +| NSM syslog KB | http://kb.juniper.net/InfoCenter/index?page=content&id=KB11810 | + +### Sourcetypes + +| sourcetype | notes | +|------------------|-----------------------------------------------------------------------| +| juniper:nsm | None | +| juniper:nsm:idp | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|------------------------|---------------------|----------------|---------------| +| juniper_nsm | juniper:nsm | netfw | none | +| juniper_nsm_idp | juniper:nsm:idp | netids | none | + +### Filter type + +* Juniper NSM products must be identified by host or ip assignment. Update the filter `f_juniper_nsm` or `f_juniper_nsm_idp` as required + + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index as required. +* Follow vendor configuration steps per Product Manual + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_JUNIPER_NSM_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_JUNIPER_NSM_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +Use the following search to validate events are present; for Juniper NSM ensure each host filter condition is verified + +``` +index= sourcetype=juniper:nsm | stats count by host +index= sourcetype=juniper:nsm:idp | stats count by host +``` + +Verify timestamp, and host values match as expected + +## Product - Juniper Netscreen + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2847/ | +| Netscreen Manual | http://kb.juniper.net/InfoCenter/index?page=content&id=KB4759 | + +### Sourcetypes + +| sourcetype | notes | +|-------------------------|------------------------------------------------------------------------------------------------| +| netscreen:firewall | None | +| juniper:idp | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|------------------------|---------------------|----------------|---------------| +| juniper_netscreen | netscreen:firewall | netfw | none | +| juniper_idp | juniper:idp | netfw | none | + +### Filter type + +* Juniper Netscreen products must be identified by host or ip assignment. Update the filter `f_juniper_netscreen` or `f_juniper_idp` as required + + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index as required. +* Follow vendor configuration steps per Product Manual + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +Use the following search to validate events are present; for Juniper Netscreen products ensure each host filter condition is verified + +``` +index= sourcetype=netscreen:firewall | stats count by host +index= sourcetype=juniper:idp | stats count by host +``` + +Verify timestamp, and host values match as expected + +## Product - Juniper SSLVPN + +| Ref | Link | +|------------------|-------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2847/ | +| Pulse Secure KB | https://kb.pulsesecure.net/articles/Pulse_Secure_Article/KB22227 | + +### Sourcetypes + +| sourcetype | notes | +|------------------|-----------------------------------------------------------------------| +| juniper:sslvpn | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|------------------------|---------------------|----------------|---------------| +| juniper_sslvpn | juniper:sslvpn | netfw | none | + +### Filter type + +* MSG Parse: This filter parses message content + + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index as required. +* Follow vendor configuration steps per referenced Product Manual + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +Use the following search to validate events are present; for Juniper SSL VPN ensure each host filter condition is verified + +``` +index= sourcetype=juniper:sslvpn | stats count by host +``` + +Verify timestamp, and host values match as expected # Vendor - PaloAlto @@ -204,6 +449,22 @@ Verify timestamp, and host values match as expected | pan:log | None | | pan:traffic | None | | pan:threat | None | +| pan:system | None | +| pan:config | None | +| pan:hipwatch | None | +| pan:correlation | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| pan_log | pan:log | netops | none | +| pan_traffic | pan:traffic | netfw | none | +| pan_threat | pan:threat | netproxy | none | +| pan_system | pan:system | netops | none | +| pan_config | pan:config | netops | none | +| pan_hipwatch | pan:hipwatch | netops | none | +| pan_correlation | pan:correlation | netops | none | ### Filter type @@ -212,15 +473,21 @@ MSG Parse: This filter parses message content ### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. -* Review and update the splunk_index.csv file and set the index as required. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. * Refer to the admin manual for specific details of configuration * Select TCP or SSL transport option * Select IETF Format * Ensure the format of the event is not customized +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + ### Verification -An active firewall will generate frequent events use the following search to validate events are present per source device +An active firewall will generate frequent events. Use the following search to validate events are present per source device ``` index= sourcetype=pan:*| stats count by host @@ -240,7 +507,14 @@ index= sourcetype=pan:*| stats count by host | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| bluecoat:proxysg:access:kv | Requires version 3.6 | +| bluecoat:proxysg:access:kv | Requires version TA 3.6 | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| bluecoat_proxy | bluecoat:proxysg:access:kv | netops | none | + ### Filter type @@ -249,15 +523,21 @@ MSG Parse: This filter parses message content ### Setup and Configuration * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. -* Review and update the splunk_index.csv file and set the index as required. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. * Refer to the Splunk TA documentation for the specific customer format required for proxy configuration * Select TCP or SSL transport option * Ensure the format of the event is customized per Splunk documentation +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + ### Verification -An active proxy will generate frequent events use the following search to validate events are present per source device +An active proxy will generate frequent events. Use the following search to validate events are present per source device ``` index= sourcetype=bluecoat:proxysg:access:kv | stats count by host -``` \ No newline at end of file +``` diff --git a/package/Dockerfile b/package/Dockerfile index 4e3b4c4..091e2cb 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -67,11 +67,9 @@ RUN cd /tmp ;\ rm epel-release-latest-7.noarch.rpm ;\ rpm --import https://packages.confluent.io/rpm/5.2/archive.key ;\ yum install gcc tzdata libdbi libsecret libxml2 sqlite tcp_wrappers librdkafka \ - rh-python36 rh-python36-python-tools libcurl ivykis scl-utils tcp_wrappers-libs curl -y;\ + rh-python36 rh-python36-python-tools libcurl ivykis scl-utils tcp_wrappers-libs curl wget -y;\ echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh ;\ - source scl_source enable rh-python36 ;\ - pip install dumb-init - + source scl_source enable rh-python36 ENV DEBCONF_NONINTERACTIVE_SEEN=true ENV SPLUNK_CONNECT_METHOD=hec @@ -81,15 +79,18 @@ RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install COPY goss.yaml /etc/goss.yaml COPY --from=0 /opt/syslog-ng /opt/syslog-ng + +RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.5.0/gomplate_linux-amd64-slim && \ + chmod 755 /usr/local/bin/gomplate + + COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d COPY etc/context-local /opt/syslog-ng/etc/context-local -COPY sbin/entrypoint.sh /sbin/entrypoint.sh RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer -RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V +RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t -RUN mkdir -p /var/log/syslog-ng/data/disk-buffer EXPOSE 514 EXPOSE 601/tcp @@ -97,6 +98,6 @@ EXPOSE 6514/tcp ENV SPLUNK_CONNECT_METHOD=UF -ENTRYPOINT ["/sbin/entrypoint.sh", "--"] +ENTRYPOINT ["/opt/syslog-ng/sbin/syslog-ng", "-F"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file +HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate diff --git a/package/etc/conf.d/conflib/_common/network.conf b/package/etc/conf.d/conflib/_common/network.conf new file mode 100644 index 0000000..ae13635 --- /dev/null +++ b/package/etc/conf.d/conflib/_common/network.conf @@ -0,0 +1,10 @@ +@module confgen context(source) name(gen_source_dedicated_port) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/conflib/_common/network.conf.tmpl") + +block source dedicated_port(udp_port(no) tcp_port(no) tls_port(no) flags("no-parse")) { + gen_source_dedicated_port( + udp_port(`udp_port`) + tcp_port(`tcp_port`) + tls_port(`tls_port`) + flags(`flags`) + ) +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/network.conf.tmpl b/package/etc/conf.d/conflib/_common/network.conf.tmpl new file mode 100644 index 0000000..1592248 --- /dev/null +++ b/package/etc/conf.d/conflib/_common/network.conf.tmpl @@ -0,0 +1,93 @@ +# =============================================================================================== +# source definition for remote devices +# =============================================================================================== + +# =============================================================================================== +# Defaults for the default-network-drivers() source: +# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic +# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic +# 6514 TCP, for TLS-encrypted traffic +# =============================================================================================== + + channel { + source { +{{ if ne (getenv "confgen_udp_port") "no" }} + syslog ( + transport("udp") + port({{getenv "confgen_udp_port"}}) + ip-protocol(4) + so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "425984"}}) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); +{{ end }} +{{ if ne (getenv "confgen_tcp_port") "no" }} + network ( + transport("tcp") + port({{getenv "confgen_tcp_port"}}) + ip-protocol(4) + max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) + log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) + log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); +{{ end }} + }; + #TODO: #60 Remove this function with enhancement + rewrite(set_metadata_presume); + rewrite(set_rfcnonconformant); + rewrite(r_set_splunk_default); + +{{ if eq (getenv "confgen_parser") "rfc5424_strict" }} + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_strict); +{{ else if eq (getenv "confgen_parser") "rfc5424_noversion" }} + filter(f_rfc5424_noversion); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_noversion); +{{ else if eq (getenv "confgen_parser") "cisco_parser" }} + parser {cisco-parser()}; + rewrite(set_metadata_vendor_product_cisco_ios); +{{ else if eq (getenv "confgen_parser") "rfc3164" }} + parser { + syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); + }; + rewrite(set_rfc3164); +{{ else }} + if { + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_strict); + } elif { + filter(f_rfc5424_noversion); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_noversion); + } elif { + parser {cisco-parser()}; + rewrite(set_metadata_vendor_product_cisco_ios); + } else { + parser { + syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); + }; + rewrite(set_rfc3164); + }; +{{ end }} + }; diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index b07a8a5..5efd586 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -44,13 +44,28 @@ template t_hdr_sdata_msg { }; # =============================================================================================== -# JSON; for JSON pretty-printing +# JSON; for JSON pretty-printing (for debugging) # =============================================================================================== - + template t_JSON { template("$(format-json --scope all-nv-pairs --exclude fields.* --exclude .splunk.* - --exclude HOST --exclude HOSTFROMgit m + --exclude HOST + --exclude HOST_FROM + )"); + }; + +# =============================================================================================== +# JSON; for JSON pretty-printing (for RFC5424) +# =============================================================================================== + +template t_JSON_5424 { + template("$(format-json --scope all-nv-pairs + --exclude fields.* + --exclude .splunk.* + --exclude HOST + --exclude HOST_FROM + --exclude RAWMSG )"); }; diff --git a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf index cb238de..f3789a7 100644 --- a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf +++ b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf @@ -1,6 +1,6 @@ block parser vendor_product_by_source() { add-contextual-data( - selector(filters("/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf")), + selector(filters("`syslog-ng-sysconfdir`/context-local/vendor_product_by_source.conf")), database("context-local/vendor_product_by_source.csv") ignore-case(yes) prefix("fields.") diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf index 7d96599..68e9a01 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf @@ -1,23 +1,26 @@ -#Used to set index fields we will always use -rewrite r_set_splunk_basic { +#Used to set indexed fields we will always use to global defaults +rewrite r_set_splunk_default { set("`index`", value(".splunk.index")); - set("`sourcetype`", value(".splunk.sourcetype")); - set("`source`", value(".splunk.source")); + set("`splunk-sourcetype`", value(".splunk.sourcetype")); + set("`splunk-source`:$LOGHOST", value(".splunk.source")); + set("`splunk-template`", value(".splunk.template")); set($FACILITY, value("fields.sc4s_syslog_facility")); set($LEVEL, value("fields.sc4s_syslog_severity")); set($LOGHOST, value("fields.sc4s_syslog_server")); set($SOURCEIP, value("fields.sc4s_fromhostip")); }; - -#rewrite - -#Used by fallback to log everything we can -block rewrite r_set_splunk( - template(`splunk-default-template`) +#used by each log-path to set index and sourcetype which may be +#overridden by user defined values +block rewrite r_set_splunk_dest_default( + index() + sourcetype() + template(`splunk-template`) ) { - set("`template`", value("fields.sc4s_template")); - }; + set("`index`", value(".splunk.index")); + set("`sourcetype`", value(".splunk.sourcetype")); + set("`template`", value(".splunk.template")); +}; diff --git a/package/etc/conf.d/conflib/vendor_juniper/junos.conf b/package/etc/conf.d/conflib/vendor_juniper/junos.conf index 49ae1de..4c259e3 100644 --- a/package/etc/conf.d/conflib/vendor_juniper/junos.conf +++ b/package/etc/conf.d/conflib/vendor_juniper/junos.conf @@ -1,8 +1,7 @@ -filter f_juniper_junos { +filter f_juniper_junos_structured { match('^\[junos@2636' value("SDATA")) - or match('syslog@juniper.net' value("SDATA")) }; -filter f_juniper_junos_legacy { +filter f_juniper_junos_standard { program("RT_IDP|RT_FLOW|RT_IDS|RT_UTM|Juniper"); }; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/vendor_juniper/legacy.conf b/package/etc/conf.d/conflib/vendor_juniper/legacy.conf index 8ec73a3..27b3f52 100644 --- a/package/etc/conf.d/conflib/vendor_juniper/legacy.conf +++ b/package/etc/conf.d/conflib/vendor_juniper/legacy.conf @@ -14,6 +14,7 @@ filter f_juniper_netscreen { }; -filter f_juniper_idp { - match("juniper_idp", value("fields.sc4s_vendor_product") type(glob) ); + filter f_juniper_idp { + match("juniper_idp", value("fields.sc4s_vendor_product") type(glob)) + or match('^\[syslog@juniper' value("SDATA")) }; diff --git a/package/etc/conf.d/destinations/splunk_hec.conf b/package/etc/conf.d/destinations/splunk_hec.conf index 81f3187..81db7ee 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf +++ b/package/etc/conf.d/destinations/splunk_hec.conf @@ -1,76 +1,5 @@ - - -# =============================================================================================== -# Direct connection to Splunk via HEC -# =============================================================================================== - -# =============================================================================================== -# Be sure to adjust batch paramaters below to suit scale/environment -# Set workers to the number of indexers or HWF HEC endpoints -# If validated certs are used, uncomment relevant lines in the tls() block below -# and change peer-verify() to "yes" -# =============================================================================================== +@module confgen context(destination) name(gen_splunk_hec) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/destinations/splunk_hec.conf.tmpl") destination d_hec { - http( - url("`SPLUNK_HEC_URL`") - method("POST") - log-fifo-size(`splunk-log-fifo-size`) - workers(`SYSLOGNG_HEC_WORKERS`) - batch-lines(1000) - batch-bytes(2048Kb) - batch-timeout(3) - timeout(15) - user_agent("syslog-ng User Agent") - user("syslog-ng") - headers("Connection: close") - password("`SPLUNK_HEC_TOKEN`") - persist-name("splunk") - disk-buffer(mem-buf-length(15000) - disk-buf-size(200000) - reliable(no) - dir("/opt/syslog-ng/var/data/disk-buffer/")) - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=${HOST} - source=${.splunk.source} - sourcetype=${.splunk.sourcetype} - index=${.splunk.index} - event=$(template ${fields.sc4s_template} $(template `splunk-default-template`)) - fields.*)') - ); - }; - -destination d_hecmetrics { - http( - url("`SPLUNK_HEC_URL`") - method("POST") - batch-lines(5) - batch-bytes(512Kb) - batch-timeout(1) - timeout(15) - user_agent("syslog-ng User Agent") - user("syslog-ng") - password("`SPLUNK_HEC_TOKEN`") - persist-name("splunk_metrics") - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$MESSAGE') - ); - }; + gen_splunk_hec() +}; diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl new file mode 100644 index 0000000..441c5a9 --- /dev/null +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -0,0 +1,42 @@ + http( + url("{{getenv "SPLUNK_HEC_URL"}}") + method("POST") + log-fifo-size({{getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) + workers({{getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}}) + batch-lines({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_LINES" "1000"}}) + batch-bytes({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_BYTES" "4096kb"}}) + batch-timeout({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_TIMEOUT" "1"}}) + timeout({{getenv "SC4S_DEST_SPLUNK_HEC_TIMEOUT" "30"}}) + user_agent("sc4s/1.0 (events)") + user("sc4s") + headers("{{getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}") + password("{{getenv "SPLUNK_HEC_TOKEN"}}") + persist-name("splunk_hec") + +{{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE" "yes") "yes" }} + disk-buffer( +{{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes" }} + mem-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}}) + reliable(yes) +{{else}} + mem-buf-length({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}}) + reliable(no) +{{ end }} +{{ end }} + disk-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}}) + dir("/opt/syslog-ng/var/data/disk-buffer/") + ) + tls(peer-verify({{getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) +{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}} cipher-suite("{{getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}} +{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}} ssl-version("{{getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}} + ca-file("{{getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}") + ) + body('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=${HOST} + source=${.splunk.source} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template ${fields.sc4s_template} $(template "t_standard")) + fields.*)') + ); diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf b/package/etc/conf.d/destinations/splunk_hec_metrics.conf new file mode 100644 index 0000000..1183f94 --- /dev/null +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf @@ -0,0 +1,6 @@ +@module confgen context(destination) name(gen_splunk_hec_metrics) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/destinations/splunk_hec_metrics.conf.tmpl") + + +destination d_hecmetrics { + gen_splunk_hec_metrics() +}; diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl new file mode 100644 index 0000000..7b8d9a1 --- /dev/null +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl @@ -0,0 +1,20 @@ + http( + url("{{getenv "SPLUNK_HEC_URL"}}") + method("POST") + batch-lines(50) + batch-bytes(1024Kb) + batch-timeout(1) + timeout(15) + user_agent("sc4s/1.0 (internal metrics)") + user("sc4s") + headers("{{getenv "SC4S_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}") + password("{{getenv "SPLUNK_HEC_TOKEN"}}") + persist-name("splunk_metrics") + + tls(peer-verify({{getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) +{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}} cipher-suite("{{getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}} +{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}} ssl-version("{{getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}} + ca-file("{{getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}") + ) + body('$MESSAGE') + ); diff --git a/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf b/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf deleted file mode 100644 index 377820c..0000000 --- a/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf +++ /dev/null @@ -1,15 +0,0 @@ -log { - - source(s_default-ports); - filter(f_is_rfc3164); - filter(f_juniper_nsm_idp); - - parser { - p_add_context_splunk(key("juniper:nsm:idp")); - }; - rewrite { - r_set_splunk(template("t_standard")) - }; - destination(d_hec); #--HEC-- - flags(flow-control); -}; diff --git a/package/etc/conf.d/log_paths/internal.conf b/package/etc/conf.d/log_paths/internal.conf index 879925f..743c15c 100644 --- a/package/etc/conf.d/log_paths/internal.conf +++ b/package/etc/conf.d/log_paths/internal.conf @@ -3,7 +3,9 @@ log { if (match("^Log statistics; " value("MESSAGE"))) { - parser {p_add_context_splunk(key("syslog-ng:metrics")); }; + rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:metrics"), index("em_metrics")) }; + + parser {p_add_context_splunk(key("syslog-ng_metrics")); }; rewrite { subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', ' @@ -28,8 +30,10 @@ log { }; destination(d_hecmetrics); #--HEC-- } else { - parser {p_add_context_splunk(key("syslog-ng:events")); }; - rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- + rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:events"), index("_internal"), template("t_msg_only"))}; + + parser {p_add_context_splunk(key("syslog-ng_events")); }; + destination(d_hec); #--HEC-- }; }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf deleted file mode 100644 index e9f018c..0000000 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf +++ /dev/null @@ -1,14 +0,0 @@ -log { - - source(s_default-ports); - filter(f_cisco_nx_os); - - parser { - p_add_context_splunk(key("cisco:nx-os")); - }; - rewrite { - r_set_splunk(template("t_hdr_msg")) - }; - destination(d_hec); #--HEC-- - flags(flow-control); -}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf index 428cb0f..f97b906 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf @@ -1,21 +1,13 @@ - # =============================================================================================== # Cisco ASA (Firewall) # =============================================================================================== -log { - source(s_default-ports); - - filter(f_is_rfc3164); - filter(f_cisco_asa); - #set the source type based on program field and lookup index from the splunk context csv +@module confgen context(log) name(gen_log_rfc_3164-cisco-asa) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl") - parser {p_add_context_splunk(key("cisco:asa")); }; - - #Using the 5424 parser the message content is all we need - rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- - - destination(d_hec); #--HEC-- +log { + #Default source + gen_log_rfc_3164-cisco-asa( + sourcedefault(yes) + ); - flags(flow-control); }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl new file mode 100644 index 0000000..a8e9b29 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -0,0 +1,26 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_cisco_asa); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT" }}) + ) + }; +{{ end }} + + #set the source type based on program field and lookup index from the splunk context csv + #Using the 5424 parser the message content is all we need + + rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("cisco_asa")); }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf index 780130e..2f7a3bb 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf @@ -3,19 +3,12 @@ # Cisco IOS (Route/Switch) # =============================================================================================== -log { - - source(s_default-ports); - filter(f_cisco_ios); +@module confgen context(log) name(gen_log_rfc_3164-cisco-ios) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl") +log { + #Default source + gen_log_rfc_3164-cisco-ios( + sourcedefault(yes) + ); - parser { - p_add_context_splunk(key("cisco:ios")); - }; - rewrite { - r_set_splunk(template("t_msg_only")) #--HEC-- - }; - destination(d_hec); #--HEC-- - - flags(flow-control); -}; +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl new file mode 100644 index 0000000..67fc196 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -0,0 +1,25 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_cisco_ios); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_CISCO_IOS_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_CISCO_IOS_UDP_PORT" }}) + + ) + }; +{{ end }} + + rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("cisco_ios")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf new file mode 100644 index 0000000..04297c6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf @@ -0,0 +1,14 @@ + +# =============================================================================================== +# Cisco IOS (Route/Switch) +# =============================================================================================== + +@module confgen context(log) name(gen_log_rfc_3164-cisco-nx-os) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-cisco-nx-os( + sourcedefault(yes) + ); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl new file mode 100644 index 0000000..a6c1e20 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl @@ -0,0 +1,25 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_cisco_nx_os); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT" }}) + + ) + }; +{{ end }} + + rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))}; + parser { + p_add_context_splunk(key("cisco_nx_os")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf index d81c565..c22dceb 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf @@ -1,31 +1,14 @@ -log { - source(s_default-ports); - filter(f_is_rfc3164); - filter(f_fortinet_fortios); - - parser { - kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); - date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone(`default-timezone`)); - }; - rewrite { set("${.kv.devname}", value("HOST")); }; +# =============================================================================================== +# Fortigate FortiOS +# =============================================================================================== - #set the source type based on program field and lookup index from the splunk context csv - if (match("traffic" value(".kv.type"))) { - parser {p_add_context_splunk(key("fgt_traffic")); }; - } elif (match("utm" value(".kv.type"))) { - parser {p_add_context_splunk(key("fgt_utm")); }; - } elif (match("event" value(".kv.type"))) { - parser {p_add_context_splunk(key("fgt_event")); }; - } else { - parser {p_add_context_splunk(key("fgt_log")); }; - }; +@module confgen context(log) name(gen_log_rfc_3164-fortigate-fortios) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl") - #rewrite the final message for splunk json - #sourcetype and index are set in the filter defaults won't be used - rewrite {r_set_splunk(template("t_standard")) }; #--HEC-- - destination(d_hec); #--HEC-- - - flags(flow-control); -}; +log { + #Default source + gen_log_rfc_3164-fortigate-fortios( + sourcedefault(yes) + ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl new file mode 100644 index 0000000..6684b30 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -0,0 +1,45 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_fortinet_fortios); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT" }}) + + ) + }; +{{ end }} + + parser { + kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); + date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); + }; + + rewrite { set("${.kv.devname}", value("HOST")); }; + + #set the source type based on program field and lookup index from the splunk context csv + + + if (match("traffic" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"), template("t_standard"))}; + parser {p_add_context_splunk(key("fortinet_fortios_traffic")); }; + } elif (match("utm" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"), template("t_standard"))}; + parser {p_add_context_splunk(key("fortinet_fortios_utm")); }; + } elif (match("event" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"), template("t_standard"))}; + parser {p_add_context_splunk(key("fortinet_fortios_event")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"), template("t_standard"))}; + parser {p_add_context_splunk(key("fortinet_fortios_log")); }; + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf deleted file mode 100644 index 4b84403..0000000 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf +++ /dev/null @@ -1,15 +0,0 @@ -log { - - source(s_default-ports); - filter(f_is_rfc3164); - filter(f_juniper_idp); - - parser { - p_add_context_splunk(key("juniper:idp")); - }; - rewrite { - r_set_splunk(template("t_standard")) - }; - destination(d_hec); #--HEC-- - flags(flow-control); -}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf deleted file mode 100644 index e689cf6..0000000 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf +++ /dev/null @@ -1,24 +0,0 @@ -# =============================================================================================== -# Juniper Standard and Structured logging -# =============================================================================================== - -log { - - source(s_default-ports); - filter(f_is_rfc3164); - filter(f_juniper_junos_legacy); - - if (program('RT_IDP')) { - parser {p_add_context_splunk(key("juniper:junos:idp")); }; - } elif (program('RT_FLOW|RT_IDS|RT_UTM')) { - parser {p_add_context_splunk(key("juniper:junos:firewall")); }; - } elif (program('Juniper')) { - parser {p_add_context_splunk(key("juniper:sslvpn")); }; - } else { - parser {p_add_context_splunk(key("juniper:legacy")); }; - }; - - rewrite { r_set_splunk(template("t_standard")) }; - destination(d_hec); #--HEC-- - flags(flow-control); -}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf new file mode 100644 index 0000000..0509425 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf @@ -0,0 +1,17 @@ +@module confgen context(log) name(gen_log_rfc_3164-juniper-idp) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-juniper-idp( + sourcedefault(yes) + ); + +}; + +log { + #Specific source + gen_log_rfc_3164-juniper-idp( + sourcedefault(no) + ); + +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl new file mode 100644 index 0000000..9589805 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -0,0 +1,25 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc5424_strict); + filter(f_juniper_idp); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT" }}) + ); + }; +{{ end }} + + rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"), template("t_hdr_sdata_msg"))}; + parser { + p_add_context_splunk(key("juniper_idp")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf new file mode 100644 index 0000000..9c9a9b6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf @@ -0,0 +1,17 @@ +@module confgen context(log) name(gen_log_rfc_3164-juniper-junos) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-juniper-junos( + sourcedefault(yes) + ); + +}; + +log { + #Specific source + gen_log_rfc_3164-juniper-junos( + sourcedefault(no) + ); + +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl new file mode 100644 index 0000000..3850ad4 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -0,0 +1,41 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_junos_standard); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT" }}) + + ) + }; +{{ end }} + + if (program('RT_IDP')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; + parser {p_add_context_splunk(key("juniper_idp")); }; + } elif (program('RT_FLOW')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"), template("t_standard"))}; + parser {p_add_context_splunk(key("juniper_junos_flow")); }; + } elif (program('RT_IDS')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; + parser {p_add_context_splunk(key("juniper_junos_ids")); }; + } elif (program('RT_UTM')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"), template("t_standard"))}; + parser {p_add_context_splunk(key("juniper_junos_utm")); }; + } elif (program('Juniper')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"), template("t_standard"))}; + parser {p_add_context_splunk(key("juniper_sslvpn")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"), template("t_standard"))}; + parser {p_add_context_splunk(key("juniper_legacy")); }; + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf index efd046c..83de2bd 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf @@ -1,14 +1,17 @@ +@module confgen context(log) name(gen_log_rfc_3164-juniper-netscreen) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-juniper-netscreen( + sourcedefault(yes) + ); + +}; + log { + #Specific source + gen_log_rfc_3164-juniper-netscreen( + sourcedefault(no) + ); - source(s_default-ports); - filter(f_juniper_netscreen); - - parser { - p_add_context_splunk(key("juniper:netscreen")); - }; - rewrite { - r_set_splunk(template("t_standard")) - }; - destination(d_hec); #--HEC-- - flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl new file mode 100644 index 0000000..a800328 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -0,0 +1,28 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_juniper_netscreen); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT" }}) + ) + }; +{{ end }} + + rewrite { + r_set_splunk_dest_default(sourcetype("netscreen:firewall"), + index("netfw"), + template("t_standard")) + }; + + parser { + p_add_context_splunk(key("juniper_netscreen")); + }; + + destination(d_hec); #--HEC-- + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf index 04810cc..b552088 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf @@ -2,18 +2,20 @@ # Juniper NSM # =============================================================================================== +@module confgen context(log) name(gen_log_rfc_3164-juniper-nsm) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl") + log { + #Default source + gen_log_rfc_3164-juniper-nsm( + sourcedefault(yes) + ); + +}; - source(s_default-ports); - filter(f_is_rfc3164); - filter(f_juniper_nsm); +log { + #Specific source + gen_log_rfc_3164-juniper-nsm( + sourcedefault(no) + ); - parser { - p_add_context_splunk(key("juniper:nsm")); - }; - rewrite { - r_set_splunk(template("t_standard")) - }; - destination(d_hec); #--HEC-- - flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl new file mode 100644 index 0000000..e637dc4 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -0,0 +1,25 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_nsm); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT" }}) + ) + }; +{{ end }} + + rewrite { r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"), template("t_standard"))}; + + parser { + p_add_context_splunk(key("juniper_nsm")); + }; + + destination(d_hec); #--HEC-- + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf new file mode 100644 index 0000000..ecfa966 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf @@ -0,0 +1,21 @@ +# =============================================================================================== +# Juniper NSM IDP +# =============================================================================================== + +@module confgen context(log) name(gen_log_rfc_3164-juniper-nsm-idp) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-juniper-nsm-idp( + sourcedefault(yes) + ); + +}; + +log { + #Specific source + gen_log_rfc_3164-juniper-nsm-idp( + sourcedefault(no) + ); + +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl new file mode 100644 index 0000000..2974ff2 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -0,0 +1,24 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_nsm_idp); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT" }}) + ) + }; +{{ end }} + + rewrite {r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"), template("t_standard"))}; + parser { + p_add_context_splunk(key("juniper_idp")); + }; + + destination(d_hec); #--HEC-- + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf b/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf deleted file mode 100644 index 956f747..0000000 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf +++ /dev/null @@ -1,61 +0,0 @@ -log { - source(s_default-ports); - filter(f_is_rfc3164); - filter(f_paloalto_panos); - - # the palo message does not include a program value in the header unfortunatly - # the use of colon in the message tricks the RFC3164 parser we will re-write the message - # so the parser will not incorrectly break it apart. - # while we are at it we will save the mesage type into the program field so parser can find it - rewrite { - set("${LEGACY_MSGHDR}${MESSAGE}" value("MESSAGE")); - unset(value("LEGACY_MSGHDR")); - unset(value("PROGRAM")); - - }; - parser { - #basic parsing - - #we need to actual even time from the field GeneratedTime use csv parser to get it out - csv-parser( - columns('FUTURE_USE', 'ReceiveTime', 'SerialNumber', 'Type', 'Subtype', 'FUTURE_USE2', 'GeneratedTime') - prefix(".pan.") - delimiters(',') - ); - #2012/04/10 04:39:55 - #parse the date - date-parser( - format("%Y/%m/%d %H:%M:%S") - template("${.pan.GeneratedTime}") - time-zone("Universal") - ); - }; - - #set the source type based on program field and lookup index from the splunk context csv - if (message(',\d+,THREAT')) { - parser {p_add_context_splunk(key("pan:threat")); }; - } elif (message(',\d+,TRAFFIC')) { - parser {p_add_context_splunk(key("pan:traffic")); }; - } elif (message(',\d+,SYSTEM')) { - parser {p_add_context_splunk(key("pan:system")); }; - } elif (message(',\d+,CONFIG')) { - parser {p_add_context_splunk(key("pan:config")); }; - } elif (message(',\d+,HIPWATCH')) { - parser {p_add_context_splunk(key("pan:hipwatch")); }; - } elif (message(',\d+,CORRELATION')) { - parser {p_add_context_splunk(key("pan:correlation")); }; - } elif (message(',\d+,USERID')) { - parser {p_add_context_splunk(key("pan:userid")); }; - } else { - parser {p_add_context_splunk(key("pan:log")); }; - }; - - #rewrite the final message for splunk json - #sourcetype and index are set in the filter defaults won't be used - rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- - - destination(d_hec); #--HEC-- - - flags(flow-control); -}; - diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf new file mode 100644 index 0000000..7eccabe --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf @@ -0,0 +1,17 @@ +@module confgen context(log) name(gen_log_rfc_3164-palalto-panos) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-palalto-panos( + sourcedefault(yes) + ); + +}; + +log { + #Specific source + gen_log_rfc_3164-palalto-panos( + sourcedefault(no) + ); + +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl new file mode 100644 index 0000000..58e98e9 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -0,0 +1,76 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_paloalto_panos); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT" }}) + + ) + }; +{{ end }} + + # the palo message does not include a program value in the header unfortunatly + # the use of colon in the message tricks the RFC3164 parser we will re-write the message + # so the parser will not incorrectly break it apart. + # while we are at it we will save the mesage type into the program field so parser can find it + rewrite { + set("${LEGACY_MSGHDR}${MESSAGE}" value("MESSAGE")); + unset(value("LEGACY_MSGHDR")); + unset(value("PROGRAM")); + + }; + parser { + #basic parsing + + #we need to actual even time from the field GeneratedTime use csv parser to get it out + csv-parser( + columns('FUTURE_USE', 'ReceiveTime', 'SerialNumber', 'Type', 'Subtype', 'FUTURE_USE2', 'GeneratedTime') + prefix(".pan.") + delimiters(',') + ); + #2012/04/10 04:39:55 + #parse the date + date-parser( + format("%Y/%m/%d %H:%M:%S") + template("${.pan.GeneratedTime}") + time-zone("Universal") + ); + }; + + #set the source type based on program field and lookup index from the splunk context csv + if (message(',\d+,THREAT')) { + rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("pan_threat")); }; + } elif (message(',\d+,TRAFFIC')) { + rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("pan_traffic")); }; + } elif (message(',\d+,SYSTEM')) { + rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("pan_system")); }; + } elif (message(',\d+,CONFIG')) { + rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("pan_config")); }; + } elif (message(',\d+,HIPWATCH')) { + rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("pan_hipwatch")); }; + } elif (message(',\d+,CORRELATION')) { + rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("pan_correlation")); }; + } elif (message(',\d+,USERID')) { + rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("pan_userid")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("pan_log")); }; + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf index 295669a..55d4463 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf +++ b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf @@ -1,16 +1,14 @@ -log { - source(s_default-ports); - filter(f_is_rfc5424_noversion); - filter(f_cisco_asa); - - #set the source type based on program field and lookup index from the splunk context csv - parser {p_add_context_splunk(key("cisco:asa")); }; +# =============================================================================================== +# Cisco IOS ASA +# =============================================================================================== - #Using the 5424 parser the message content is all we need - rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- +@module confgen context(log) name(gen_log_rfc_5424_noversion-cisco-asa) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl") - destination(d_hec); #--HEC-- +log { + #Default source + gen_log_rfc_5424_noversion-cisco-asa( + sourcedefault(yes) + ); - flags(flow-control); }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl new file mode 100644 index 0000000..4aeacfe --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl @@ -0,0 +1,27 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc5424_noversion); + filter(f_cisco_asa); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_JUNIPER_CISCO_ASA_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_JUNIPER_CISCO_ASA_UDP_PORT" }}) + + ) + }; +{{ end }} + + #set the source type based on program field and lookup index from the splunk context csv + #Using the 5424 parser the message content is all we need + + rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("cisco_asa")); }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf deleted file mode 100644 index 4d2b8e4..0000000 --- a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf +++ /dev/null @@ -1,16 +0,0 @@ -log { - source(s_default-ports); - filter(f_is_rfc5424_noversion); - filter(f_symantec_bluecoat_proxy); - - #set the source type based on program field and lookup index from the splunk context csv - - parser {p_add_context_splunk(key("bluecoat:proxysg:access:kv")); }; - - #Using the 5424 parser the message content is all we need - rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC-- - - destination(d_hec); #--HEC-- - - flags(flow-control); -}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf new file mode 100644 index 0000000..d701ba5 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf @@ -0,0 +1,10 @@ + +@module confgen context(log) name(gen_log_rfc_5424_noversion-symantec_proxy) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl") + +log { + #Default source + gen_log_rfc_5424_noversion-symantec_proxy( + sourcedefault(yes) + ); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl new file mode 100644 index 0000000..b651236 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl @@ -0,0 +1,28 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc5424_noversion); + filter(f_symantec_bluecoat_proxy); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT" }}) + + ) + }; +{{ end }} + + #set the source type based on program field and lookup index from the splunk context csv. + #With the 5424 parser the message is the only portion needed in the template. + + rewrite { r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy"), template("t_msg_only")) }; + + parser {p_add_context_splunk(key("bluecoat_proxy")); }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf deleted file mode 100644 index c49854b..0000000 --- a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf +++ /dev/null @@ -1,27 +0,0 @@ -# =============================================================================================== -# Juniper Structured logging -# =============================================================================================== - -log { - - source(s_default-ports); - filter(f_is_rfc5424_strict); - filter(f_juniper_junos); - - if (program('RT_IDP')) { - parser {p_add_context_splunk(key("juniper:junos:idp:structured")); }; - } elif (program('RT_FLOW|RT_IDS|RT_UTM')) { - parser {p_add_context_splunk(key("juniper:junos:firewall:structured")); }; - } - elif (program('Jnpr')) { - parser {p_add_context_splunk(key("juniper:idp:structured")); }; - } - else { - parser {p_add_context_splunk(key("juniper:structured")); }; - }; - - rewrite { r_set_splunk(template("t_hdr_sdata_msg"))}; - destination(d_hec); #--HEC-- - - -}; diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf new file mode 100644 index 0000000..48365ea --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf @@ -0,0 +1,14 @@ +# =============================================================================================== +# Juniper Structured logging +# =============================================================================================== + + +@module confgen context(log) name(gen_log_rfc_5424_strict-juniper_junos) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl") + +log { + #Default source + gen_log_rfc_5424_strict-juniper_junos( + sourcedefault(yes) + ); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl new file mode 100644 index 0000000..bfa3fad --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl @@ -0,0 +1,43 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc5424_strict); + filter(f_juniper_junos_structured); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT" }}) + udp_port({{ getenv "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT" }}) + ) + }; +{{ end }} + + + if (program('RT_IDP')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; + parser {p_add_context_splunk(key("juniper_idp_structured")); }; + } elif (program('RT_FLOW')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) }; + parser {p_add_context_splunk(key("juniper_junos_flow_structured")); }; + } elif (program('RT_IDS')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; + parser {p_add_context_splunk(key("juniper_junos_ids_structured")); }; + } elif (program('RT_UTM')) { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) }; + parser {p_add_context_splunk(key("juniper_junos_utm_structured")); }; + } +# Legacy Netscreen IDP is handled in the "p_rfc3164-juniper-idp.conf" log path +# +# } elif (program('Jnpr')) { +# rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp:structured"), index("netids")) }; +# parser {p_add_context_splunk(key("juniper_junos_idp")); }; +# } + else { + rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops"), template("t_JSON_5424")) }; + parser {p_add_context_splunk(key("juniper_structured")); }; + }; + + destination(d_hec); #--HEC-- diff --git a/package/etc/conf.d/log_paths/zfallback.conf b/package/etc/conf.d/log_paths/zfallback.conf index 4b1b5d5..3b5b7fc 100644 --- a/package/etc/conf.d/log_paths/zfallback.conf +++ b/package/etc/conf.d/log_paths/zfallback.conf @@ -1,11 +1,11 @@ log { source(s_default-ports); + rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:fallback"), index("main"), template("t_JSON")) }; parser { - p_add_context_splunk(key("syslog-ng:fallback")); + p_add_context_splunk(key("syslog-ng_fallback")); }; - rewrite { r_set_splunk(template("t_JSON"))}; #--HEC-- destination(d_hec); #--HEC-- flags(flow-control,fallback); }; diff --git a/package/etc/conf.d/sources/network.conf b/package/etc/conf.d/sources/network.conf index 70a5f7b..599061d 100644 --- a/package/etc/conf.d/sources/network.conf +++ b/package/etc/conf.d/sources/network.conf @@ -1,75 +1,6 @@ -# =============================================================================================== -# source definition for remote devices -# =============================================================================================== +@module confgen context(source) name(gen_s_default_ports) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/sources/network.conf.tmpl") -# =============================================================================================== -# Defaults for the default-network-drivers() source: -# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic -# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic -# 6514 TCP, for TLS-encrypted traffic -# =============================================================================================== - -# source s_network-defaults { default-network-drivers() }; source s_default-ports { - channel { - source { - `splunk-udp-driver` ( - transport("udp") - port(`udp-listening-port`) - ip-protocol(`ip-version`) - so-rcvbuf(`splunk-rcvbuf`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - - `splunk-tcp-driver` ( - transport("tcp") - port(`tcp-listening-port`) - ip-protocol(`ip-version`) - max-connections(`splunk-max-connections`) - log-iw-size(`splunk-window-size`) - log-fetch-limit(`splunk-fetch-limit`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - }; - #TODO: #60 Remove this function with enhancement - rewrite(set_metadata_presume); - rewrite(set_rfcnonconformant); - rewrite(r_set_splunk_basic); - - if { - filter(f_rfc5424_strict); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_strict); - } elif { - filter(f_rfc5424_noversion); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_noversion); - } elif { - parser {cisco-parser()}; - rewrite(set_metadata_vendor_product_cisco_ios); - } else { - parser { - syslog-parser(time-zone(`default-timezone`) flags(store-raw-message)); - }; - rewrite(set_rfc3164); - }; - - parser { - vendor_product_by_source(); - }; - }; + gen_s_default_ports() }; + diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl new file mode 100644 index 0000000..0cb943d --- /dev/null +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -0,0 +1,72 @@ +# =============================================================================================== +# source definition for remote devices +# =============================================================================================== + +# =============================================================================================== +# Defaults for the default-network-drivers() source: +# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic +# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic +# 6514 TCP, for TLS-encrypted traffic +# =============================================================================================== + + channel { + source { + syslog ( + transport("udp") + port(514) + ip-protocol(4) + so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "425984"}}) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); + + network ( + transport("tcp") + port(514) + ip-protocol(4) + max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) + log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) + log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); + }; + #TODO: #60 Remove this function with enhancement + rewrite(set_metadata_presume); + rewrite(set_rfcnonconformant); + rewrite(r_set_splunk_default); + + if { + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_strict); + } elif { + filter(f_rfc5424_noversion); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_noversion); + } elif { + parser {cisco-parser()}; + rewrite(set_metadata_vendor_product_cisco_ios); + } else { + parser { + syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); + }; + rewrite(set_rfc3164); + }; + + parser { + vendor_product_by_source(); + }; + }; diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv index e3d7f9d..71bcd53 100644 --- a/package/etc/context-local/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -1,62 +1,33 @@ -bluecoat:proxysg:access:kv,index,netproxy -bluecoat:proxysg:access:kv,sourcetype,bluecoat:proxysg:access:kv -cisco:asa,index,netfw -cisco:asa,sourcetype,cisco:asa -cisco:ios,index,netops -cisco:ios,sourcetype,cisco:ios -cisco:nx-os,index,netops -cisco:nx-os,sourcetype,cisco:ios -fgt_event,sourcetype,fgt_event -fgt_event,index,netops -fgt_log,sourcetype,fgt_log -fgt_log,index,netops -fgt_traffic,sourcetype,fgt_traffic -fgt_traffic,index,netfw -fgt_utm,sourcetype,fgt_utm -fgt_utm,index,netids -juniper:structured,index,main -juniper:structured,sourcetype,juniper:structured -juniper:junos:firewall:structured,index,netfw -juniper:junos:firewall:structured,sourcetype,juniper:junos:firewall:structured -juniper:junos:idp:structured,index,netids -juniper:junos:idp:structured,sourcetype,juniper:junos:idp:structured -juniper:junos:firewall,index,netfw -juniper:junos:firewall,sourcetype,juniper:junos:firewall -juniper:junos:idp,index,netids -juniper:junos:idp,sourcetype,juniper:junos:idp -juniper:nsm:idp,index,netids -juniper:nsm:idp,sourcetype,juniper:nsm:idp -juniper:sslvpn,index,netfw -juniper:sslvpn,sourcetype,juniper:sslvpn -juniper:netscreen,index,netfw -juniper:netscreen,sourcetype,netscreen:firewall -juniper:idp,index,netids -juniper:idp,sourcetype,juniper:idp -juniper:idp:structured,index,netids -juniper:idp:structured,sourcetype,juniper:idp:structured -juniper:nsm,index,netids -juniper:nsm,sourcetype,juniper:nsm -juniper:legacy,index,netfw -juniper:legacy,sourcetype,juniper:legacy -pan:traffic,index,netfw -pan:threat,index,netproxy -pan:system,index,netops -pan:config,index,netops -pan:hipwatch,index,main -pan:correlation,index,main -pan:userid,index,netops -pan:unknown,index,main -pan:traffic,sourcetype,pan:traffic -pan:threat,sourcetype,pan:threat -pan:system,sourcetype,pan:system -pan:config,sourcetype,pan:config -pan:hipwatch,sourcetype,pan:hipwatch -pan:correlation,sourcetype,pan:correlation -pan:userid,sourcetype,pan:userid -pan:unknown,sourcetype,pan:unknown -syslog-ng:events,index,main -syslog-ng:events,sourcetype,syslog-ng:events -syslog-ng:fallback,index,main -syslog-ng:fallback,sourcetype,syslog-ng:fallback -syslog-ng:metrics,index,em_metrics -syslog-ng:metrics,sourcetype,httpevent \ No newline at end of file +#bluecoat_proxy,index,netproxy +#cisco_asa,index,netfw +#cisco_ios,index,netops +#cisco_nx_os,index,netops +#fortinet_fortios_event,index,netops +#fortinet_fortios_log,index,netops +#fortinet_fortios_traffic,index,netfw +#fortinet_fortios_utm,index,netids +#juniper_idp,index,netids +#juniper_structured,index,netops +#juniper_idp_structured,index,netids +#juniper_junos_flow_structured,index,netfw +#juniper_junos_ids_structured,index,netids +#juniper_junos_utm_structured,index,netfw +#juniper_junos_flow,index,netfw +#juniper_junos_ids,index,netids +#juniper_junos_utm,index,netfw +#juniper_sslvpn,index,netfw +#juniper_netscreen,index,netfw +#juniper_nsm,index,netfw +#juniper_nsm_idp,index,netids +#juniper_legacy,index,netops +#pan_traffic,index,netfw +#pan_threat,index,netproxy +#pan_system,index,netops +#pan_config,index,netops +#pan_hipwatch,index,main +#pan_correlation,index,main +#pan_userid,index,netauth +#pan_unknown,index,netops +#syslog-ng_events,index,_internal +#syslog-ng_fallback,index,main +#syslog-ng_metrics,index,em_metrics diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 27e1a6e..edee26f 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -33,40 +33,18 @@ options { on-error(fallback-to-string); }; -# =============================================================================================== -# Source driver defaults (which govern the parser used) -# =============================================================================================== -@define splunk-udp-driver "syslog" -@define splunk-tcp-driver "network" - -# =============================================================================================== -# Listening ports -# =============================================================================================== -@define udp-listening-port "514" -@define tcp-listening-port "514" - -# =============================================================================================== -# Splunk metadata (HEC/Kafka transport only) -# =============================================================================================== -@define splunk-sourcetype "syslog-ng:fallback" -@define splunk-index "main" # =============================================================================================== # Default message template # =============================================================================================== -@define splunk-default-template "t_standard" +@define splunk-template "t_standard" # =============================================================================================== # Data collection parameters, buffers, and Timezone # =============================================================================================== -@define ip-version 4 -@define splunk-max-connections 1000 -@define splunk-log-fifo-size 180000000 -@define splunk-fetch-limit 20000 -# make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit -@define splunk-window-size 20000000 -@define splunk-rcvbuf 425984 -@define default-timezone "GMT" +#TODO: Remove once release with this PR is produced by upstream +#https://github.com/balabit/syslog-ng/pull/2932 +@define syslog-ng-sysconfdir "/opt/syslog-ng/etc" # =============================================================================================== # Global modules and includes. All device-specific filters and destinations exist in conf.d @@ -84,6 +62,3 @@ options { @include "conf.d/log_paths/*.conf" @include "conf.d/plugin/*/log_paths/*.conf" - - - diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh deleted file mode 100755 index 5d37dd1..0000000 --- a/package/sbin/entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/opt/rh/rh-python36/root/usr/bin/dumb-init /bin/bash -#Copyright 2019 Splunk, Inc. -# -#Use of this source code is governed by a BSD-2-clause-style -#license that can be found in the LICENSE-BSD2 file or at -#https://opensource.org/licenses/BSD-2-Clause -#Run syslog -mkdir /opt/syslog-ng/var -rm /opt/syslog-ng/var/syslog-ng.ctl || true -/opt/syslog-ng/sbin/syslog-ng -F diff --git a/test-with-compose.sh b/test-with-compose.sh index b569b7f..25eeb4f 100755 --- a/test-with-compose.sh +++ b/test-with-compose.sh @@ -4,26 +4,32 @@ #Use of this source code is governed by a BSD-2-clause-style #license that can be found in the LICENSE-BSD2 file or at #https://opensource.org/licenses/BSD-2-Clause - +WAITON=${1:-test} +compose=${2:-docker-compose.yml} +echo $WAITON $compose mkdir test-results -docker volume create sc4s-tests +docker-compose down +docker volume rm sc4s-results +docker volume rm splunk-etc + docker volume create sc4s-results docker volume create splunk-etc docker container create --name dummy \ - -v sc4s-tests:/work/tests \ -v sc4s-results:/work/test-results \ -v splunk-etc:/work/splunk-etc \ registry.access.redhat.com/ubi7/ubi -docker cp tests/ dummy:/work/tests/ docker cp ./splunk/etc/* dummy:/work/splunk-etc/ docker rm dummy -docker-compose build -docker-compose up --abort-on-container-exit --exit-code-from test +docker-compose -f $compose pull +docker-compose -f $compose up -d splunk +docker-compose -f $compose up -d sc4s +sleep 60 +docker-compose -f $compose build +docker-compose -f $compose up --abort-on-container-exit --exit-code-from $WAITON docker container create --name dummy \ - -v sc4s-tests:/work/tests \ -v sc4s-results:/work/test-results \ registry.access.redhat.com/ubi7/ubi diff --git a/tests/Dockerfile b/tests/Dockerfile index ddb8c55..0bf3d12 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -15,5 +15,7 @@ RUN mkdir -p /work/tests RUN mkdir -p /work/test-results/functional COPY entrypoint.sh / COPY wait-for /bin/ +COPY ./* /work/tests/ +COPY ./data /work/tests/data #WORKDIR /work CMD /entrypoint.sh \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index ab4fd96..7777889 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,9 +33,13 @@ def get_host_key(setup_wordlist): @pytest.fixture def setup_splunk(): tried = 0 + username = os.getenv('SPLUNK_USER', "admin") + password = os.getenv('SPLUNK_PASSWORD', "Changed@11") + host = os.getenv('SPLUNK_HOST', "splunk") + port = os.getenv('SPLUNK_PORT', "8089") while True: try: - c = client.connect(username="admin", password="Changed@11", host="splunk", port="8089") + c = client.connect(username=username, password=password, host=host, port=port) break except ConnectionRefusedError: tried += 1 diff --git a/tests/sendmessage.py b/tests/sendmessage.py index 6b591ad..122381f 100644 --- a/tests/sendmessage.py +++ b/tests/sendmessage.py @@ -6,9 +6,11 @@ import socket from time import sleep +import os - -def sendsingle(message, host="sc4s", port=514): +def sendsingle(message, + host=os.getenv('SYSLOG_HOST', "sc4s"), + port=514): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_address = (host, port) diff --git a/tests/test_juniper_junos_rfc3164.py b/tests/test_juniper_junos_rfc3164.py index 3ec13ff..b127814 100644 --- a/tests/test_juniper_junos_rfc3164.py +++ b/tests/test_juniper_junos_rfc3164.py @@ -20,7 +20,7 @@ def test_juniper_utm_standard(record_property, setup_wordlist, get_host_key, set sendsingle(message) - st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -52,30 +52,6 @@ def test_juniper_firewall_standard(record_property, setup_wordlist, get_host_key assert resultCount == 1 -# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[] - Session timed out for xxx@xxx.xxx.xxx/Users (session:00000000) due to inactivity (last access at 13:59:31 2013/02/27). Idle session identified during routine system scan. -# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied. -def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} Juniper: {% now 'utc', '%Y-%m-%d %H:%M:%S' %} - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied.") - message = mt.render(mark="<23>", host=host) - - sendsingle(message) - - st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:sslvpn\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - - - def test_juniper_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key diff --git a/tests/test_juniper_junos_rfc5124.py b/tests/test_juniper_junos_rfc5124.py index b9ff921..2e0b4e1 100644 --- a/tests/test_juniper_junos_rfc5124.py +++ b/tests/test_juniper_junos_rfc5124.py @@ -22,7 +22,7 @@ def test_juniper_junos_structured(record_property, setup_wordlist, get_host_key, sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"juniper:structured\" | head 2") + st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"juniper:structured\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -55,31 +55,6 @@ def test_juniper_junos_idp_structured(record_property, setup_wordlist, get_host_ assert resultCount == 1 -# <165>1 2010-06-23T18:05:55 10.209.83.9 Jnpr Syslog 23414 1 [syslog@juniper.net dayId="20100623" recordId="0" timeRecv="2010/06/23 18:05:55" timeGen="2010/06/23 18:05:51" domain="" devDomVer2="0" device_ip="10.209.83.9" cat="Config" attack="" srcZn="NULL" srcIntf="" srcAddr="0.0.0.0" srcPort="0" natSrcAddr="NULL" natSrcPort="0" dstZn="NULL" dstIntf="NULL" dstAddr="0.0.0.0" dstPort="0" natDstAddr="NULL" natDstPort="0" protocol="IP" ruleDomain="" ruleVer="0" policy="" rulebase="NONE" ruleNo="0" action="NONE" severity="INFO" alert="no" elaspedTime="0" inbytes="0" outbytes="0" totBytes="0" inPak="0" outPak="0" totPak="0" repCount="0" packetData="no" varEnum="0" misc="Interaface eth2,eth3 is in Normal State" user="NULL" app="NULL" uri="NULL"] -# -# -# -# @pytest.mark.xfail -def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): - host = get_host_key - - mt = env.from_string( - "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} Jnpr Syslog 23414 [syslog@juniper.net dayId=\"20100623\" recordId=\"0\" timeRecv=\"2010/06/23 18:05:55\" timeGen=\"2010/06/23 18:05:51\" domain=\"\" devDomVer2=\"0\" device_ip=\"10.209.83.9\" cat=\"Config\" attack=\"\" srcZn=\"NULL\" srcIntf=\"\" srcAddr=\"0.0.0.0\" srcPort=\"0\" natSrcAddr=\"NULL\" natSrcPort=\"0\" dstZn=\"NULL\" dstIntf=\"NULL\" dstAddr=\"0.0.0.0\" dstPort=\"0\" natDstAddr=\"NULL\" natDstPort=\"0\" protocol=\"IP\" ruleDomain=\"\" ruleVer=\"0\" policy=\"\" rulebase=\"NONE\" ruleNo=\"0\" action=\"NONE\" severity=\"INFO\" alert=\"no\" elaspedTime=\"0\" inbytes=\"0\" outbytes=\"0\" totBytes=\"0\" inPak=\"0\" outPak=\"0\" totPak=\"0\" repCount=\"0\" packetData=\"no\" varEnum=\"0\" misc=\"Interaface eth2,eth3 is in Normal State\" user=\"NULL\" app=\"NULL\" uri=\"NULL\"]") - message = mt.render(mark="<165>1", host=host) - - sendsingle(message) - - st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:idp:structured\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - # <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos # @pytest.mark.xfail def test_juniper_junos_fw_structured(record_property, setup_wordlist, get_host_key, setup_splunk): diff --git a/tests/test_juniper.py b/tests/test_juniper_legacy.py similarity index 69% rename from tests/test_juniper.py rename to tests/test_juniper_legacy.py index c4bdb81..34de672 100644 --- a/tests/test_juniper.py +++ b/tests/test_juniper_legacy.py @@ -22,7 +22,7 @@ def test_juniper_nsm_standard(record_property, setup_wordlist, get_host_key, set sendsingle(message) - st = env.from_string("search index=netids host=\"jnpnsm-{{ host }}\" sourcetype=\"juniper:nsm\" | head 2") + st = env.from_string("search index=netfw host=\"jnpnsm-{{ host }}\" sourcetype=\"juniper:nsm\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -57,7 +57,7 @@ def test_juniper_nsm_idp_standard(record_property, setup_wordlist, get_host_key, # <23> Apr 24 12:30:05 cs-loki3 RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message - # <23> Mar 18 17:56:52 [FW IP] [FW Model]: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time="2009-03-18 16:07:06" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1 -def test_juniper_netscreen(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_netscreen_fw(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( @@ -77,16 +77,41 @@ def test_juniper_netscreen(record_property, setup_wordlist, get_host_key, setup_ assert resultCount == 1 +# <165>1 2010-06-23T18:05:55 10.209.83.9 Jnpr Syslog 23414 1 [syslog@juniper.net dayId="20100623" recordId="0" timeRecv="2010/06/23 18:05:55" timeGen="2010/06/23 18:05:51" domain="" devDomVer2="0" device_ip="10.209.83.9" cat="Config" attack="" srcZn="NULL" srcIntf="" srcAddr="0.0.0.0" srcPort="0" natSrcAddr="NULL" natSrcPort="0" dstZn="NULL" dstIntf="NULL" dstAddr="0.0.0.0" dstPort="0" natDstAddr="NULL" natDstPort="0" protocol="IP" ruleDomain="" ruleVer="0" policy="" rulebase="NONE" ruleNo="0" action="NONE" severity="INFO" alert="no" elaspedTime="0" inbytes="0" outbytes="0" totBytes="0" inPak="0" outPak="0" totPak="0" repCount="0" packetData="no" varEnum="0" misc="Interaface eth2,eth3 is in Normal State" user="NULL" app="NULL" uri="NULL"] +# +# +# +# @pytest.mark.xfail +def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} Jnpr Syslog 23414 [syslog@juniper.net dayId=\"20100623\" recordId=\"0\" timeRecv=\"2010/06/23 18:05:55\" timeGen=\"2010/06/23 18:05:51\" domain=\"\" devDomVer2=\"0\" device_ip=\"10.209.83.9\" cat=\"Config\" attack=\"\" srcZn=\"NULL\" srcIntf=\"\" srcAddr=\"0.0.0.0\" srcPort=\"0\" natSrcAddr=\"NULL\" natSrcPort=\"0\" dstZn=\"NULL\" dstIntf=\"NULL\" dstAddr=\"0.0.0.0\" dstPort=\"0\" natDstAddr=\"NULL\" natDstPort=\"0\" protocol=\"IP\" ruleDomain=\"\" ruleVer=\"0\" policy=\"\" rulebase=\"NONE\" ruleNo=\"0\" action=\"NONE\" severity=\"INFO\" alert=\"no\" elaspedTime=\"0\" inbytes=\"0\" outbytes=\"0\" totBytes=\"0\" inPak=\"0\" outPak=\"0\" totPak=\"0\" repCount=\"0\" packetData=\"no\" varEnum=\"0\" misc=\"Interaface eth2,eth3 is in Normal State\" user=\"NULL\" app=\"NULL\" uri=\"NULL\"]") + message = mt.render(mark="<165>1", host=host) + + sendsingle(message) + + st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:idp\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + # <23> Apr 24 12:30:05 cs-loki3 RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message - # <23> Mar 18 17:56:52 [FW IP] [FW Model]: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time="2009-03-18 16:07:06" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1 -def test_juniper_netscreen_singleport(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_netscreen_fw_singleport(record_property, setup_wordlist, get_host_key, setup_splunk): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} ns204: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time=\"2009-03-18 16:07:06\" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1\n") message = mt.render(mark="<23>", host=host) - sendsingle(message, host="sc4s-juniper") + sendsingle(message, port=5000) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host) diff --git a/tests/test_juniper_sslvpn.py b/tests/test_juniper_sslvpn.py new file mode 100644 index 0000000..e301be1 --- /dev/null +++ b/tests/test_juniper_sslvpn.py @@ -0,0 +1,34 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[] - Session timed out for xxx@xxx.xxx.xxx/Users (session:00000000) due to inactivity (last access at 13:59:31 2013/02/27). Idle session identified during routine system scan. +# <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied. +def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} Juniper: {% now 'utc', '%Y-%m-%d %H:%M:%S' %} - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied.") + message = mt.render(mark="<23>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:sslvpn\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From e6e5e34c1ead33c4e428f297855d89f5b24a6605 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 25 Sep 2019 17:39:53 -0400 Subject: [PATCH 5/7] Fix/rtd submodules (#94) (#95) Exclude submodules --- .readthedocs.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..2880d22 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,12 @@ +version: 2 + +# Build documentation with MkDocs +mkdocs: + configuration: mkdocs.yml + +# Optionally build your docs in additional formats such as PDF and ePub + +formats: all + +submodules: + exclude: all \ No newline at end of file From dc6f95ad338fbcd9b6cf8799b7b6429c335db3eb Mon Sep 17 00:00:00 2001 From: mbonsack Date: Sat, 28 Sep 2019 12:49:20 -0700 Subject: [PATCH 6/7] Fix/parser metadata (#100) * Parser/metdata update * Feature/proofpoint (#97) * Add Proofpoint PPS filter support * Fix parser/metdata/IOS filter --- docker-compose.yml | 2 +- docs/sources.md | 52 +++++++++++++++++ .../conf.d/conflib/_common/network.conf.tmpl | 0 .../{rfc_syslog.conf => syslog_format.conf} | 12 ++++ .../etc/conf.d/conflib/_common/utility.conf | 4 -- package/etc/conf.d/filters/cisco/ios.conf | 6 +- .../etc/conf.d/filters/proofpoint/pps.conf | 18 ++++++ .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_nx-os.conf.tmpl | 10 ++-- .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 32 +++++++++++ ..._rfc3164-proofpoint_pps_sendmail.conf.tmpl | 32 +++++++++++ package/etc/conf.d/sources/network.conf.tmpl | 4 +- package/etc/context-local/splunk_index.csv | 4 +- .../vendor_product_by_source.conf | 32 +++++++---- .../vendor_product_by_source.csv | 4 +- package/etc/templates/source_network.t | 6 +- .../apps/SA-syslog-ng/default/indexes.conf | 7 ++- tests/test_proofpoint.py | 56 +++++++++++++++++++ 18 files changed, 249 insertions(+), 34 deletions(-) delete mode 100644 package/etc/conf.d/conflib/_common/network.conf.tmpl rename package/etc/conf.d/conflib/_common/{rfc_syslog.conf => syslog_format.conf} (80%) delete mode 100644 package/etc/conf.d/conflib/_common/utility.conf create mode 100644 package/etc/conf.d/filters/proofpoint/pps.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl create mode 100644 tests/test_proofpoint.py diff --git a/docker-compose.yml b/docker-compose.yml index a1fc681..1765168 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - - SC4S_SOURCE_TLS_ENABLE=yes + - SC4S_SOURCE_TLS_ENABLE=no - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 volumes: diff --git a/docs/sources.md b/docs/sources.md index 363bbe5..19f26d1 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -493,6 +493,58 @@ An active firewall will generate frequent events. Use the following search to va index= sourcetype=pan:*| stats count by host ``` +# Vendor - Proofpoint + +## Product - Proofpoint Protection Server + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3080/ | +| Product Manual | https://proofpointcommunities.force.com/community/s/article/Remote-Syslog-Forwarding | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| pps_filter_log | | +| pps_mail_log | This sourcetype will conflict with sendmail itself, so will require that the PPS send syslog on a dedicated port or be uniquely identifiable with a hostname glob or CIDR block if this sourcetype is desired for PPS. | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| proofpoint_pps_filter | pps_filter_log | email | none | +| proofpoint_pps_sendmail | pps_mail_log | email | none | + + +### Filter type + +MSG Parse: This filter parses message content +* NOTE: This filter will simply parse the syslog message itself, and will _not_ perform the (required) re-assembly of related +messages to create meaningful final output. This will require follow-on processing in Splunk. + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Follow vendor configuration steps per referenced Product Manual + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT variable immediately below.| +| SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT variable immediately above. | + +### Verification + +One or two sourcetypes are included in Proofpoint PPS logs. The search below will surface both of them: + +``` +index= sourcetype=pps_*_log | stats count by host +``` + # Vendor - Symantec ## Product - ProxySG/ASG (Bluecoat) diff --git a/package/etc/conf.d/conflib/_common/network.conf.tmpl b/package/etc/conf.d/conflib/_common/network.conf.tmpl deleted file mode 100644 index e69de29..0000000 diff --git a/package/etc/conf.d/conflib/_common/rfc_syslog.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf similarity index 80% rename from package/etc/conf.d/conflib/_common/rfc_syslog.conf rename to package/etc/conf.d/conflib/_common/syslog_format.conf index 1809034..8c5794c 100644 --- a/package/etc/conf.d/conflib/_common/rfc_syslog.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -24,4 +24,16 @@ rewrite set_rfc3164{ }; filter f_is_rfc3164{ match("rfc3164" value("fields.sc4s_syslog_format")) +}; +rewrite set_cisco_ios{ + set("cisco_ios" value("fields.sc4s_syslog_format")); +}; +filter f_is_cisco_ios{ + match("cisco_ios" value("fields.sc4s_syslog_format")) +}; +rewrite set_no_parse{ + set("no_parse" value("fields.sc4s_syslog_format")); +}; +filter f_is_no_parse{ + match("no_parse" value("fields.sc4s_syslog_format")) }; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/utility.conf b/package/etc/conf.d/conflib/_common/utility.conf deleted file mode 100644 index 8b986c3..0000000 --- a/package/etc/conf.d/conflib/_common/utility.conf +++ /dev/null @@ -1,4 +0,0 @@ - -rewrite set_metadata_presume { - set("$(env SYSLOG_PRESUME_FILTER)" value("fields.sc4s_presume")); -}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco/ios.conf b/package/etc/conf.d/filters/cisco/ios.conf index 09a37ed..0e4c478 100644 --- a/package/etc/conf.d/filters/cisco/ios.conf +++ b/package/etc/conf.d/filters/cisco/ios.conf @@ -1,8 +1,6 @@ -rewrite set_metadata_vendor_product_cisco_ios{ - set("cisco_ios" value(".metadata.vendor_product")); -}; +# In general this will not be used; parser setting will override the need for this filter f_cisco_ios{ - match("cisco_ios", value(".metadata.vendor_product") type(glob)); + match("cisco_ios", value("fields.sc4s_vendor_product") type(glob)); }; diff --git a/package/etc/conf.d/filters/proofpoint/pps.conf b/package/etc/conf.d/filters/proofpoint/pps.conf new file mode 100644 index 0000000..6fd213c --- /dev/null +++ b/package/etc/conf.d/filters/proofpoint/pps.conf @@ -0,0 +1,18 @@ +# Proofpoint + +filter f_proofpoint_pps_filter { + match("proofpoint_pps_filter", value("fields.sc4s_vendor_product") type(glob)) or + ( + ( + match('^(background|cvt|filter|pps)_instance\d+$' value("PROGRAM") type("pcre")) or + match('^\/opt\/proofpoint\/pps-\d\.\d\.\d\.\d+\/\S' value("PROGRAM") type("pcre")) or + match('^queued-(alert|default|reinject|released)$' value("PROGRAM") type("pcre")) + ) and + match('^rprt\s' value(MESSAGE) type("pcre")) + ); +}; + +filter f_proofpoint_pps_sendmail { + match('sendmail' value("PROGRAM") type("pcre")) and + match("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product") type(glob)); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index 6518a0d..05b9e29 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -7,7 +7,7 @@ log { {{- if eq (.) "yes" }} source(s_default-ports); - filter(f_cisco_ios); + filter(f_is_cisco_ios); {{- end }} {{- if eq (.) "no" }} source (s_dedicated_port_CISCO_IOS); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl index 2caff20..03700cd 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl @@ -5,17 +5,17 @@ {{- end -}} {{ define "log_path" }} log { -{{- if eq (.) "yes"}} +{{- if eq (.) "yes" }} source(s_default-ports); filter(f_cisco_nx_os); -{{- end}} -{{- if eq (.) "no"}} +{{- end }} +{{- if eq (.) "no" }} source (s_dedicated_port_CISCO_NX_OS); -{{- end}} +{{- end }} rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))}; parser { - p_add_context_splunk(key("cisco_nx_os")); + p_add_context_splunk(key("cisco_nx_os")); }; destination(d_hec); #--HEC-- diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl new file mode 100644 index 0000000..fa89c6c --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl @@ -0,0 +1,32 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_proofpoint_pps_filter); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_PROOFPOINT_PPS_FILTER); +{{- end }} + + rewrite { r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("proofpoint_pps_filter")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for PROOFPOINT_PPS_FILTER traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl new file mode 100644 index 0000000..db0baab --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl @@ -0,0 +1,32 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "PROOFPOINT_PPS_SENDMAIL" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_proofpoint_pps_sendmail); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_PROOFPOINT_PPS_SENDMAIL); +{{- end }} + + rewrite { r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("proofpoint_pps_sendmail")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_SENDMAIL traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for PROOFPOINT_PPS_SENDMAIL traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index 42b4344..a9e8b78 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -81,7 +81,7 @@ source s_default-ports { rewrite(set_rfc5424_noversion); } elif { parser {cisco-parser()}; - rewrite(set_metadata_vendor_product_cisco_ios); + rewrite(set_cisco_ios); } else { parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); @@ -94,5 +94,5 @@ source s_default-ports { parser { vendor_product_by_source(); }; - }; + }; }; \ No newline at end of file diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv index 1df4cc1..566c436 100644 --- a/package/etc/context-local/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -28,6 +28,8 @@ #pan_correlation,index,main #pan_userid,index,netauth #pan_unknown,index,netops -#sc4s_events,index,_internal +#proofpoint_pps_filter,index,email +#proofpoint_pps_sendmail,index,email +#sc4s_events,index,main #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/context-local/vendor_product_by_source.conf index 4da1866..37e3412 100644 --- a/package/etc/context-local/vendor_product_by_source.conf +++ b/package/etc/context-local/vendor_product_by_source.conf @@ -4,26 +4,34 @@ #filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; filter f_test_test { - host("testvp-*" type(glob)) - or match("test_test" value("fields.sc4s_presume")) + host("testvp-*" type(glob)) or + netmask(192.168.100.1/24) }; filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) - or match("juniper_nsm" value("fields.sc4s_presume")) + host("jnpnsm-*" type(glob)) or + netmask(192.168.1.0/24) }; filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) - or match("juniper_nsm_idp" value("fields.sc4s_presume")) + host("jnpnsmidp-*" type(glob)) or + netmask(192.168.2.0/24) }; filter f_juniper_idp { - host("jnpidp-*" type(glob)) - or match("juniper_idp" value("fields.sc4s_presume")) + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) }; filter f_juniper_netscreen { - host("jnpns-*" type(glob) ) - or match("juniper_netscreen" value("fields.sc4s_presume")) + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) }; filter f_cisco_nx_os { - host("csconx-*" type(glob) ) - or match("cisco_nx_os" value("fields.sc4s_presume")) + host("csconx-*" type(glob)) or + netmask(192.168.5.0/24) +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; +filter f_proofpoint_pps_filter { + host("pps-*" type(glob)) or + netmask(192.168.7.0/24) }; \ No newline at end of file diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/context-local/vendor_product_by_source.csv index 5193616..3f90603 100644 --- a/package/etc/context-local/vendor_product_by_source.csv +++ b/package/etc/context-local/vendor_product_by_source.csv @@ -3,4 +3,6 @@ f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" -f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" \ No newline at end of file +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/templates/source_network.t b/package/etc/templates/source_network.t index 8651474..f804816 100644 --- a/package/etc/templates/source_network.t +++ b/package/etc/templates/source_network.t @@ -73,12 +73,14 @@ source s_dedicated_port_{{ .port_id}} { rewrite(set_rfc5424_noversion); {{- else if eq .parser "cisco_parser" }} parser {cisco-parser()}; - rewrite(set_metadata_vendor_product_cisco_ios); + rewrite(set_cisco_ios); {{- else if eq .parser "rfc3164" }} parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); }; rewrite(set_rfc3164); +{{- else if eq .parser "no_parse" }} + rewrite(set_no_parse); {{- else }} if {filter(f_rfc5424_strict); parser { @@ -93,7 +95,7 @@ source s_dedicated_port_{{ .port_id}} { rewrite(set_rfc5424_noversion); } elif { parser {cisco-parser()}; - rewrite(set_metadata_vendor_product_cisco_ios); + rewrite(set_cisco_ios); } else { parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index f8dbfb2..60d7cae 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -39,4 +39,9 @@ thawedPath = $SPLUNK_DB/netops/thaweddb [netproxy] homePath = $SPLUNK_DB/netproxy/db coldPath = $SPLUNK_DB/netproxy/colddb -thawedPath = $SPLUNK_DB/netproxy/thaweddb \ No newline at end of file +thawedPath = $SPLUNK_DB/netproxy/thaweddb + +[email] +homePath = $SPLUNK_DB/email/db +coldPath = $SPLUNK_DB/email/colddb +thawedPath = $SPLUNK_DB/email/thaweddb \ No newline at end of file diff --git a/tests/test_proofpoint.py b/tests/test_proofpoint.py new file mode 100644 index 0000000..4b04935 --- /dev/null +++ b/tests/test_proofpoint.py @@ -0,0 +1,56 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +# Apr 17 18:33:26 aplegw01 filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com +def test_proofpoint_pps_filter(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com\n") + message = mt.render(mark="<166>", host=host) + + sendsingle(message) + + st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"pps_filter_log\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Apr 17 18:35:26 aplegw02 sendmail[56106]: w3HMZPVT056101: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok: Message 22675962 accepted) +def test_proofpoint_pps_mail(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} pps-{{ host }} sendmail[195529]: w3HMZPVT056101: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok: Message 22675962 accepted)\n") + message = mt.render(mark="<166>", host=host) + + sendsingle(message) + + st = env.from_string("search index=email host=\"pps-{{ host }}\" sourcetype=\"pps_mail_log\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + From 18df1b633db182a0a17bf4f55858a950fb11802f Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Mon, 7 Oct 2019 22:04:23 -0400 Subject: [PATCH 7/7] Feature/don't upload p rs to releases (#126) * update ci --- .circleci/config.yml | 400 +++++++++++++----- .env.template | 6 +- .gitignore | 3 +- clair-scan.sh | 68 +++ docker-compose-ci.yml | 2 +- ...SC4S deployment.png => SC4Sdeployment.png} | Bin docs/gettingstarted.md | 3 +- docs/gettingstarted/docker-swarm-general.md | 40 +- docs/gettingstarted/docker-swarm-rhel7.md | 39 +- docs/gettingstarted/docker-systemd-general.md | 38 +- docs/gettingstarted/podman-systemd-general.md | 45 +- docs/performance.md | 20 +- package/Dockerfile | 9 +- .../conflib/_common/compliance_meta.conf | 4 +- .../vendor_product_by_source_context.conf | 4 +- .../conflib/_splunk/splunk_context.conf | 4 +- .../context}/microfocus_arcsight_source.csv | 0 .../local/{ => config}/destinations/README.md | 0 .../local/{ => config}/filters/README.md | 0 .../local/{ => config}/filters/example.conf | 0 .../local/{ => config}/log_paths/README.md | 0 .../{ => config}/log_paths/example.conf.tmpl | 0 .../local/{ => config}/sources/README.md | 0 .../context}/compliance_meta_by_source.conf | 0 .../context}/compliance_meta_by_source.csv | 0 .../local/context}/splunk_index.csv | 0 .../context}/vendor_product_by_source.conf | 0 .../context}/vendor_product_by_source.csv | 0 .../p_rfc3164_microfocus_arcsight.conf.tmpl | 2 +- .../compliance_meta_by_source.conf | 5 + .../compliance_meta_by_source.csv | 2 + .../etc/context_templates/splunk_index.csv | 40 ++ .../vendor_product_by_source.conf | 34 ++ .../vendor_product_by_source.csv | 8 + .../source_network.t | 0 .../etc/local_config/destinations/README.md | 1 + package/etc/local_config/filters/README.md | 1 + package/etc/local_config/filters/example.conf | 4 + package/etc/local_config/log_paths/README.md | 1 + .../local_config/log_paths/example.conf.tmpl | 76 ++++ package/etc/local_config/sources/README.md | 1 + package/etc/syslog-ng.conf | 9 +- package/sbin/entrypoint.sh | 9 +- 43 files changed, 662 insertions(+), 216 deletions(-) create mode 100755 clair-scan.sh rename docs/{SC4S deployment.png => SC4Sdeployment.png} (100%) rename package/etc/{context-local => conf.d/context}/microfocus_arcsight_source.csv (100%) rename package/etc/conf.d/local/{ => config}/destinations/README.md (100%) rename package/etc/conf.d/local/{ => config}/filters/README.md (100%) rename package/etc/conf.d/local/{ => config}/filters/example.conf (100%) rename package/etc/conf.d/local/{ => config}/log_paths/README.md (100%) rename package/etc/conf.d/local/{ => config}/log_paths/example.conf.tmpl (100%) rename package/etc/conf.d/local/{ => config}/sources/README.md (100%) rename package/etc/{context-local => conf.d/local/context}/compliance_meta_by_source.conf (100%) rename package/etc/{context-local => conf.d/local/context}/compliance_meta_by_source.csv (100%) rename package/etc/{context-local => conf.d/local/context}/splunk_index.csv (100%) rename package/etc/{context-local => conf.d/local/context}/vendor_product_by_source.conf (100%) rename package/etc/{context-local => conf.d/local/context}/vendor_product_by_source.csv (100%) create mode 100644 package/etc/context_templates/compliance_meta_by_source.conf create mode 100644 package/etc/context_templates/compliance_meta_by_source.csv create mode 100644 package/etc/context_templates/splunk_index.csv create mode 100644 package/etc/context_templates/vendor_product_by_source.conf create mode 100644 package/etc/context_templates/vendor_product_by_source.csv rename package/etc/{templates => go_templates}/source_network.t (100%) create mode 100644 package/etc/local_config/destinations/README.md create mode 100644 package/etc/local_config/filters/README.md create mode 100644 package/etc/local_config/filters/example.conf create mode 100644 package/etc/local_config/log_paths/README.md create mode 100644 package/etc/local_config/log_paths/example.conf.tmpl create mode 100644 package/etc/local_config/sources/README.md diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d7ddba..65ac936 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,58 +9,60 @@ version: 2.1 orbs: - clair_scanner: ovotech/clair-scanner@1 + snyk: snyk/snyk@0.0.8 + clair-scanner: ovotech/clair-scanner@1.5.0 jobs: build: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false environment: - IMAGE_NAME: rfaircloth/scs - docker: - - image: circleci/buildpack-deps:stretch + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci +# docker: +# - image: circleci/buildpack-deps:stretch steps: +# - setup_remote_docker: +# docker_layer_caching: true - checkout - run: git submodule sync - run: git submodule update --init --recursive - - setup_remote_docker: - docker_layer_caching: true + - run: mkdir /tmp/artifacts + - run: docker pull gittools/gitversion:latest-linux-netcoreapp2.1 - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Build Docker image command: docker build -f package/Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:$CIRCLE_SHA1 package - run: - name: Push Docker image + name: Push Docker image by hash command: docker push $IMAGE_NAME:$CIRCLE_SHA1 - build-egb: - environment: - IMAGE_NAME: rfaircloth/scs - docker: - - image: circleci/buildpack-deps:stretch - steps: - - checkout - - run: git submodule sync - - run: git submodule update --init --recursive - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Build Docker image - command: docker build -f perftests/bundlesrv/Dockerfile -t $IMAGE_NAME:egb-$CIRCLE_SHA1 perftests/bundlesrv - run: - name: Tag Docker image - command: docker tag $IMAGE_NAME:egb-$CIRCLE_SHA1 $IMAGE_NAME:egb-edge + name: Docker tag and push with version label + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + echo image $IMAGE_NAME:$SEMVER + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER + docker push $IMAGE_NAME:$SEMVER + - run: - name: Push Docker image - command: docker push $IMAGE_NAME:egb-$CIRCLE_SHA1 + name: Docker Save + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker save $IMAGE_NAME:$SEMVER | gzip -c > /tmp/artifacts/oci_container.tar.gz - run: - name: Push Docker image - command: docker push $IMAGE_NAME:egb-edge + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + + - store_artifacts: + path: /tmp/artifacts + dgoss: environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci GOSS_VER: v0.3.7 GOSS_OPTS: "--max-concurrent=1 --format junit" GOSS_FILES_STRATEGY: cp @@ -73,7 +75,7 @@ jobs: docker_layer_caching: true - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 @@ -96,6 +98,8 @@ jobs: - store_test_results: path: test-results test-unit: + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci docker: - image: circleci/buildpack-deps:stretch steps: @@ -106,7 +110,7 @@ jobs: - run: git submodule update --init --recursive - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker-compose pull @@ -148,140 +152,275 @@ jobs: when: always - store_test_results: path: test-results - test-scan_images: + test-scan-synk: + docker: + - image: 'circleci/buildpack-deps:stable' environment: - IMAGE_NAME: rfaircloth/scs - executor: clair_scanner/default + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci steps: - - clair_scanner/scan: - image: $IMAGE_NAME:$CIRCLE_SHA1 - whitelist: clair-whitelist.yml + - checkout + - setup_remote_docker: + docker_layer_caching: true - run: - command: | - mkdir -p /root/project/test-results - pip install -r requirements.txt - python clair_to_junit_parser.py "/clair-reports/$IMAGE_NAME:$CIRCLE_SHA1.json" --output test-results/results.xml - when: on_fail - - store_test_results: - path: test-results/results.xml - - store_artifacts: - path: /clair-reports - - publish-common: - machine: - image: ubuntu-1604:201903-01 - docker_layer_caching: true # default - false + name: Docker Login + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + - run: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - snyk/scan: + docker-image-name: $IMAGE_NAME:$CIRCLE_SHA1 + test-scan-clair: + docker: + - image: 'docker:stable' environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci steps: - checkout + - setup_remote_docker: + docker_layer_caching: true - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker tag image - command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER - - run: - name: Docker push tag + name: "Vulnerability scan" command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker push $IMAGE_NAME:$SEMVER + + - store_artifacts: + path: clair-reports + publish-edge: environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases docker: - - image: circleci/buildpack-deps:stretch + #- image: circleci/buildpack-deps:stretch + - image: circleci/golang:latest + steps: - setup_remote_docker: docker_layer_caching: true + - checkout + - run: + name: Create Directory + command: mkdir /tmp/artifacts - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - run: name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:edge + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:edge - run: name: Docker push tag - command: docker push $IMAGE_NAME:edge + command: docker push $PUBLIC_IMAGE_NAME:edge + + - run: + name: Docker Save + command: | + docker save $IMAGE_NAME:$CIRCLE_SHA1 | gzip -c > /tmp/artifacts/oci_container.tar.gz + - run: + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + + - run: + name: "Publish Release on GitHub" + command: | + go get -u github.com/tcnksm/ghr + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete edge /tmp/artifacts/ publish-version: machine: image: ubuntu-1604:201903-01 docker_layer_caching: true # default - false environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases steps: + - checkout - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + name: Create Directory + command: mkdir /tmp/artifacts - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + name: Docker Login + command: | + docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: name: Docker tag image command: | SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker tag $IMAGE_NAME:$CIRCLE_SHA1 splunk/scs:$SEMVER + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:$SEMVER + docker push $PUBLIC_IMAGE_NAME:$SEMVER + - run: - name: Docker push tag + name: Docker Save + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker save $PUBLIC_IMAGE_NAME:$SEMVER | gzip -c > /tmp/artifacts/oci_container.tar.gz + - run: + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + - run: + name: "Publish Release on GitHub" command: | + go get -u github.com/tcnksm/ghr SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker push splunk/scs:$SEMVER + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${SEMVER} /tmp/artifacts/ + + publish-latest: environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases docker: - - image: circleci/buildpack-deps:stretch + #- image: circleci/buildpack-deps:stretch + - image: circleci/golang:latest steps: - setup_remote_docker: docker_layer_caching: true + - checkout + - run: + name: Create Directory + command: mkdir /tmp/artifacts - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - run: name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 splunk/scs:latest + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:latest - run: name: Docker push tag - command: docker push splunk/scs:latest + command: docker push $PUBLIC_IMAGE_NAME:latest + + - run: + name: Docker Save + command: | + docker save $IMAGE_NAME:$CIRCLE_SHA1 | gzip -c > /tmp/artifacts/oci_container.tar.gz + - run: + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + + - run: + name: "Publish Release on GitHub" + command: | + go get -u github.com/tcnksm/ghr + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete latest /tmp/artifacts/ + + + dockerhub-version: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: splunk/scs + steps: + + - checkout + + - run: + name: Create Directory + command: mkdir artifacts + - run: + name: Docker Login + command: | + docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker tag image + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:$SEMVER + docker push $PUBLIC_IMAGE_NAME:$SEMVER + + dockerhub-edge: + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: splunk/scs + docker: + - image: circleci/buildpack-deps:stretch + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Create Directory + command: mkdir /tmp/artifacts + - run: + name: Docker Login + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker tag image + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:edge + - run: + name: Docker push tag + command: docker push $PUBLIC_IMAGE_NAME:edge + + dockerhub-latest: + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: splunk/scs + docker: + - image: circleci/buildpack-deps:stretch + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Create Directory + command: mkdir /tmp/artifacts + - run: + name: Docker Login + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker tag image + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:latest + - run: + name: Docker push tag + command: docker push $PUBLIC_IMAGE_NAME:latest + workflows: - build-publish: + version: 2 + build-branches: jobs: - build - - build-egb - dgoss: requires: - build - test-unit: requires: - build - - test-scan_images: + - test-scan-synk: requires: - build - - publish-common: - requires: - - dgoss - - test-unit -# filters: -# branches: -# only: -# - master -# - develop +#Clair scanner image is broken using synk for now +# - test-scan-clair: +# requires: +# - build - publish-edge: requires: - dgoss @@ -290,7 +429,7 @@ workflows: branches: only: - develop - - publish-version: + - publish-latest: requires: - dgoss - test-unit @@ -298,8 +437,15 @@ workflows: branches: only: - master - - /^release\/.*/ - - publish-latest: + - dockerhub-edge: + requires: + - dgoss + - test-unit + filters: + branches: + only: + - develop + - dockerhub-latest: requires: - dgoss - test-unit @@ -307,3 +453,53 @@ workflows: branches: only: - master + build-tags: + jobs: + - build: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + - dgoss: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - build + - test-unit: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - build + - test-scan-synk: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - build + - publish-version: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - dgoss + - test-unit + - dockerhub-version: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - dgoss + - test-unit \ No newline at end of file diff --git a/.env.template b/.env.template index 8d0cafa..c1f7af2 100644 --- a/.env.template +++ b/.env.template @@ -16,6 +16,6 @@ SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics -SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download -SPLUNKBASE_USERNAME=username -SPLUNKBASE_PASSWORD=password +#SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download +#SPLUNKBASE_USERNAME=username +#SPLUNKBASE_PASSWORD=password diff --git a/.gitignore b/.gitignore index 3ac4e78..4d40b52 100644 --- a/.gitignore +++ b/.gitignore @@ -382,4 +382,5 @@ fabric.properties /.idea/ tests/test_plugin_*.py -package/etc/conf.d/local/ \ No newline at end of file +# package/etc/conf.d/local/ +!package/etc/conf.d/local diff --git a/clair-scan.sh b/clair-scan.sh new file mode 100755 index 0000000..e890ce3 --- /dev/null +++ b/clair-scan.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + + set -e + + REPORT_DIR=clair-reports + mkdir $REPORT_DIR || true + + #DB=$(docker run -p 5432:5432 -d arminc/clair-db:latest) + docker run -p 5432:5432 -d --rm --name db arminc/clair-db:latest + #CLAIR=$(docker run -p 6060:6060 --link "$DB":postgres -d arminc/clair-local-scan:latest)' + sleep 30 + docker run -p 6060:6060 --link db:postgres -d --rm --name clair arminc/clair-local-scan:latest + #CLAIR_SCANNER=$(docker run -v /var/run/docker.sock:/var/run/docker.sock --link clair:clair --name clairscanner --rm -d ovotech/clair-scanner@sha256:53fe8e8ac63af330d2dfc63498d23d8825d07f916f7d230271176de06d12acd6 tail -f /dev/null) + + CLAIR_SCANNER=$(docker run --link clair:clair --name clairscanner --rm -d ovotech/clair-scanner@sha256:53fe8e8ac63af330d2dfc63498d23d8825d07f916f7d230271176de06d12acd6 tail -f /dev/null) + + #clair_ip=$(docker exec -it "$CLAIR" hostname -i | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+') + #scanner_ip=$(docker exec -it "$CLAIR_SCANNER" hostname -i | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+') + + docker cp "clair-whitelist.yml" "$CLAIR_SCANNER:/whitelist.yml" + WHITELIST="-w /whitelist.yml" + + function scan() { + echo Scanning $1 + local image=$1 + # replace forward-slashes and colons with underscores + munged_image=$(echo "$image" | sed 's/\//_/g' | sed 's/:/_/g') + sanitised_image_filename="${munged_image}.json" + local ret=0 + #--ip "$scanner_ip" \ + # + local docker_cmd=(docker exec -it "$CLAIR_SCANNER" clair-scanner \ + --clair=http://clair:6060 \ + -t "high" \ + --report "$REPORT_DIR/$sanitised_image_filename" \ + --log "$REPORT_DIR/log.json" --whitelist=${WHITELIST:+"-x"} + --reportAll=true \ + --exit-when-no-features=false \ + "$image") + + docker pull "$image" + + "${docker_cmd[@]}" 2>&1 || ret=$? + if [ $ret -eq 0 ]; then + echo "No unapproved vulnerabilities" + elif [ $ret -eq 1 ]; then + echo "Unapproved vulnerabilities found" + EXIT_STATUS=1 + elif [ $ret -eq 5 ]; then + echo "Image was not scanned, not supported." + EXIT_STATUS=1 + else + echo "Unknown clair-scanner return code $ret." + EXIT_STATUS=1 + fi + + docker cp "$CLAIR_SCANNER:/$sanitised_image_filename" "$REPORT_DIR/$sanitised_image_filename" || true + } + + EXIT_STATUS=0 + + scan "$IMAGE_NAME:$CIRCLE_SHA1" + + docker kill clairscanner + docker kill clair + docker kill db + + exit $EXIT_STATUS \ No newline at end of file diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index c33122c..b4bb66d 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -19,7 +19,7 @@ services: - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} sc4s: - image: rfaircloth/scs:${CIRCLE_SHA1} + image: ${IMAGE_NAME}:${CIRCLE_SHA1} hostname: sc4s ports: - "514" diff --git a/docs/SC4S deployment.png b/docs/SC4Sdeployment.png similarity index 100% rename from docs/SC4S deployment.png rename to docs/SC4Sdeployment.png diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 7a29b37..a43e906 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -21,6 +21,7 @@ instance in the same VLAN as the source device. environment. * Avoid TCP except where the source is unable to contain the event to a single UDP packet. * Avoid TLS except where the event may cross a untrusted network. +* Plan for appropriately sized hardware (see)[performance.md] # Implementation @@ -81,4 +82,4 @@ Splunk type. Additional hosts can be deployed for syslog collection from additional network zones and locations: -![SC4S deployment diagram](SC4S%20deployment.png) \ No newline at end of file +![SC4S deployment diagram](SC4Sdeployment.png) \ No newline at end of file diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 9fef61c..f9a6e12 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -35,21 +35,11 @@ services: - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the yml template -above (`volumes` declarations) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. -Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. +* NOTE: If you use the default `volumes` declarations as-is from the `docker-compose.yml` file template example, you must create and/or download all files and directories referenced in the file according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s Environment" section. Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. ## Configure the SC4S environment -Create the following file ``/opt/sc4s/env_file`` and add the environment variables below: - -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment - -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. - -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -62,32 +52,38 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment. + +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example above. + ## Configure index destinations for Splunk -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow -customization of index destinations. +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. -* Create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above. From this directory, +* Create a subdirectory called ``default`` in the directory that you created in the previous step (e.g. ``/opt/sc4s/``). Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the command below to download the index context file: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` -* Edit splunk_index.csv review the index configuration and revise as required for sourcetypes utilized in your environment. +* Edit splunk_index.csv to review the index configuration and revise as required for the sourcetypes utilized in your environment. ## Configure sources by source IP or host name -Legacy sources and non-standard-compliant source require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* If not already done in the step immediately above, create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) -created in the first step above. From this directory, execute the commands below to download the vendor context files: +* If not already done, create a subdirectory called ``default`` in the ``/opt/sc4s/`` directory. Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the following commands to download the vendor context files: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv ``` -* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. +* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. ## Start/Restart SC4S @@ -224,4 +220,4 @@ Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', cl ``` If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more -information. \ No newline at end of file +information. diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index eba329c..008dd65 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -63,21 +63,12 @@ services: - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the yml template -above (`volumes` declarations) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. -Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. +* NOTE: If you use the default `volumes` declarations as-is from the `docker-compose.yml` file template example, do create and/or download all files and directories referenced in the file according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s environment" section. Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. -## Configure the SC4S environment - -Create the following file ``/opt/sc4s/env_file`` and add the environment variables below: -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment - -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +## Configure the SC4S environment -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -90,32 +81,40 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment. + +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + + + ## Configure index destinations for Splunk -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow -customization of index destinations. +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. -* Create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above. From this directory, +* Create a subdirectory called ``default`` in the directory that you created in the previous step (e.g. ``/opt/sc4s/``). Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the command below to download the index context file: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` -* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. +* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment. ## Configure sources by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* If not already done in the step immediately above, create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) -created in the first step above. From this directory, execute the commands below to download the vendor context files: +* If not already done, create a subdirectory called ``default`` in the ``/opt/sc4s/`` directory. Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the following commands to download the vendor context files: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv ``` -* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. +* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. ## Start/Restart SC4S diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index d4cd5c4..0e1fc58 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -42,20 +42,11 @@ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp \ $SC4S_IMAGE ``` -* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the unit -file above (the `Environment` assignments) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. +* NOTE: If you use the default `Environment` assignments as-is from the `sc4s.service` unit file template example, do create and/or download all files and directories referenced in the file's Service stanza according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s environment" section. ## Configure the SC4S environment -Create the following file ``/opt/sc4s/default/env_file`` and add the environment variables below: - -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment - -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. - -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -68,33 +59,38 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example. + + ## Configure index destinations for Splunk -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow -customization of index destinations. +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. -* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` variables) -match the directory you created above. From this directory, execute the following to download the latest index context file: +* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest index context file: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` -* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. +* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment. ## Configure sources by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +apply to support such sources. To identify sources that require this step refer to the "sources" section of this documentation. -* If not already done in the step immediately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory -references in the unit file above (the ``-v`` variables) match the directory you created above. From this directory, execute the following to -download the latest vendor context files: +* If not already done, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest vendor context files: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv ``` -* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. +* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. ## Configure SC4S for systemd and start SC4S diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 74b6855..361ac12 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -20,8 +20,8 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" #Note Uncomment the following two lines for host and ip based source type mapping AND download the two file templates per getting started -#Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" -#Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" +Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" #Uncomment the following line if custom TLS certs are provided #Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -42,20 +42,11 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp \ $SC4S_IMAGE ``` -* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the unit -file above (the `Environment` assignments) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. +* NOTE: If you use the default `Environment` assignments as-is from the `sc4s.service` unit file template example, do create and/or download all files and directories referenced in the file's Service stanza according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s environment" section. ## Configure the sc4s environment -Create the following file ``/opt/sc4s/default/env_file`` and add the environment variables below: - -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment - -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. - -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -68,33 +59,37 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example. + + ## Configure index destinations for Splunk -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow -customization of index destinations. +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` variables) -match the directory you created above. From this directory, execute the following to download the latest index context file: +* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest index context file: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` -* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. +* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment. ## Configure sources by source IP or host name -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* If not already done in the step immediately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory -references in the unit file above (the ``-v`` variables) match the directory you created above. From this directory, execute the following to -download the latest vendor context files: +* If not already done, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest vendor context files: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv ``` -* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. +* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. ## Configure SC4S for systemd and start SC4S @@ -235,4 +230,4 @@ Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', cl ``` If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more -information. \ No newline at end of file +information. diff --git a/docs/performance.md b/docs/performance.md index 966a318..ea02c72 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -22,5 +22,21 @@ average rate = 9717.58 msg/sec, count=1749420, time=180.026, (average) msg size= ## Limitations -Splunk Enterprise's implementation of the http event collection server responds to the client with a status code 200 and fails to commit the events to disk during a rolling restart. In our testing, 20-30 events per indexer are lost. - +In our tests, if Splunk Enterprise’s implementation of the http event collection server responded to the client with a status code 200 and failed to commit the events to disk during a rolling restart, then 20-30 events per indexer were lost. + +## Guidance on sizing hardware + +The following reference deployment hardware specifications are based on Splunk performance testing results in Amazon Web Services. +The overall load on your deployment hardware will vary based on the percentage of events not handled by a filter or use of +exceptionally complex regex in filters. While we consider the following conservative, actual hardware performance will vary +due to network interface card, driver, kernel version, exact CPU, type of memory and configuration. SYSLOG is a fire +and forget protocol making it sensitive to performance. Given this it is highly recommended that you validate +performance with your hardware and production data samples. The syslog-ng loggen tool available in the SC4S container +and the commands above can be utilized in this effort. + +Deployment Size | Hardware Spec | Average EPS with average msg size 800 k +-- | -- | -- +Small | 2 X 3.1 ghz cores1 GB of memory | 2K msg/sec +Medium | 4 X 3.1 ghz cores2 GB of memory | 4.5K msg/sec +Large | 8 X 3.1 ghz cores4 GB of memory | 9K msg/sec +XL | 16 X 3.1 ghz cores8 GB of memory | 18K msg/sec diff --git a/package/Dockerfile b/package/Dockerfile index a5f67cc..7008a31 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -86,12 +86,13 @@ RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gompl COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d -COPY etc/templates /opt/syslog-ng/etc/templates -COPY etc/context-local /opt/syslog-ng/etc/context-local +COPY etc/go_templates /opt/syslog-ng/etc/go_templates +COPY etc/context_templates /opt/syslog-ng/etc/context_templates +COPY etc/local_config /opt/syslog-ng/etc/local_config + COPY sbin/entrypoint.sh / RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V -#RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t EXPOSE 514 EXPOSE 601/tcp @@ -101,4 +102,4 @@ ENV SPLUNK_CONNECT_METHOD=UF ENTRYPOINT ["/entrypoint.sh", "-F"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate +HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/compliance_meta.conf b/package/etc/conf.d/conflib/_common/compliance_meta.conf index 3529ae5..75479c6 100644 --- a/package/etc/conf.d/conflib/_common/compliance_meta.conf +++ b/package/etc/conf.d/conflib/_common/compliance_meta.conf @@ -1,7 +1,7 @@ parser compliance_meta_by_source { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/compliance_meta_by_source.conf")), - database("context-local/compliance_meta_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/compliance_meta_by_source.conf")), + database("conf.d/local/context/compliance_meta_by_source.csv") ignore-case(yes) ); }; diff --git a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf index f3789a7..44d8ff1 100644 --- a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf +++ b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf @@ -1,7 +1,7 @@ block parser vendor_product_by_source() { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/vendor_product_by_source.conf")), - database("context-local/vendor_product_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.conf")), + database("conf.d/local/context/vendor_product_by_source.csv") ignore-case(yes) prefix("fields.") ); diff --git a/package/etc/conf.d/conflib/_splunk/splunk_context.conf b/package/etc/conf.d/conflib/_splunk/splunk_context.conf index e2d1fbb..6fb181d 100644 --- a/package/etc/conf.d/conflib/_splunk/splunk_context.conf +++ b/package/etc/conf.d/conflib/_splunk/splunk_context.conf @@ -1,7 +1,7 @@ block parser p_add_context_splunk(key("syslogng-fallback")) { add-contextual-data( selector("`key`"), - database("context-local/splunk_index.csv"), + database("conf.d/local/context/splunk_index.csv"), prefix(".splunk.") ); -}; \ No newline at end of file +}; diff --git a/package/etc/context-local/microfocus_arcsight_source.csv b/package/etc/conf.d/context/microfocus_arcsight_source.csv similarity index 100% rename from package/etc/context-local/microfocus_arcsight_source.csv rename to package/etc/conf.d/context/microfocus_arcsight_source.csv diff --git a/package/etc/conf.d/local/destinations/README.md b/package/etc/conf.d/local/config/destinations/README.md similarity index 100% rename from package/etc/conf.d/local/destinations/README.md rename to package/etc/conf.d/local/config/destinations/README.md diff --git a/package/etc/conf.d/local/filters/README.md b/package/etc/conf.d/local/config/filters/README.md similarity index 100% rename from package/etc/conf.d/local/filters/README.md rename to package/etc/conf.d/local/config/filters/README.md diff --git a/package/etc/conf.d/local/filters/example.conf b/package/etc/conf.d/local/config/filters/example.conf similarity index 100% rename from package/etc/conf.d/local/filters/example.conf rename to package/etc/conf.d/local/config/filters/example.conf diff --git a/package/etc/conf.d/local/log_paths/README.md b/package/etc/conf.d/local/config/log_paths/README.md similarity index 100% rename from package/etc/conf.d/local/log_paths/README.md rename to package/etc/conf.d/local/config/log_paths/README.md diff --git a/package/etc/conf.d/local/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl similarity index 100% rename from package/etc/conf.d/local/log_paths/example.conf.tmpl rename to package/etc/conf.d/local/config/log_paths/example.conf.tmpl diff --git a/package/etc/conf.d/local/sources/README.md b/package/etc/conf.d/local/config/sources/README.md similarity index 100% rename from package/etc/conf.d/local/sources/README.md rename to package/etc/conf.d/local/config/sources/README.md diff --git a/package/etc/context-local/compliance_meta_by_source.conf b/package/etc/conf.d/local/context/compliance_meta_by_source.conf similarity index 100% rename from package/etc/context-local/compliance_meta_by_source.conf rename to package/etc/conf.d/local/context/compliance_meta_by_source.conf diff --git a/package/etc/context-local/compliance_meta_by_source.csv b/package/etc/conf.d/local/context/compliance_meta_by_source.csv similarity index 100% rename from package/etc/context-local/compliance_meta_by_source.csv rename to package/etc/conf.d/local/context/compliance_meta_by_source.csv diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/conf.d/local/context/splunk_index.csv similarity index 100% rename from package/etc/context-local/splunk_index.csv rename to package/etc/conf.d/local/context/splunk_index.csv diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/conf.d/local/context/vendor_product_by_source.conf similarity index 100% rename from package/etc/context-local/vendor_product_by_source.conf rename to package/etc/conf.d/local/context/vendor_product_by_source.conf diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/conf.d/local/context/vendor_product_by_source.csv similarity index 100% rename from package/etc/context-local/vendor_product_by_source.csv rename to package/etc/conf.d/local/context/vendor_product_by_source.csv diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl index 6170b56..4c89731 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl @@ -22,7 +22,7 @@ parser p_microfocus_arcsight_ts_end { parser p_microfocus_arcsight_source { add-contextual-data( selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), - database("context-local/microfocus_arcsight_source.csv") + database("conf.d/context/microfocus_arcsight_source.csv") ignore-case(yes) prefix(".splunk.") default-selector("unknown") diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf new file mode 100644 index 0000000..1d5acae --- /dev/null +++ b/package/etc/context_templates/compliance_meta_by_source.conf @@ -0,0 +1,5 @@ +@version: 3.23 +filter f_test_test { + host("something-*" type(glob)) or + netmask(192.168.100.1/24) +}; diff --git a/package/etc/context_templates/compliance_meta_by_source.csv b/package/etc/context_templates/compliance_meta_by_source.csv new file mode 100644 index 0000000..6608db0 --- /dev/null +++ b/package/etc/context_templates/compliance_meta_by_source.csv @@ -0,0 +1,2 @@ +#f_test_test,.splunk.index,"badindex" +#f_test_test,fields.compliance,"pci" diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv new file mode 100644 index 0000000..e93911a --- /dev/null +++ b/package/etc/context_templates/splunk_index.csv @@ -0,0 +1,40 @@ +#bluecoat_proxy,index,netproxy +#cef_ArcSight_ArcSight,index,netwaf +#cef_Incapsula_SIEMintegration,index,netwaf +#cef_Microsoft_Microsoft Windows,index,oswinsec +#cef_Microsoft_System or Application Event,index,oswin +#cisco_asa,index,netfw +#cisco_ios,index,netops +#cisco_nx_os,index,netops +#local_example,index,main +#fortinet_fortios_event,index,netops +#fortinet_fortios_log,index,netops +#fortinet_fortios_traffic,index,netfw +#fortinet_fortios_utm,index,netids +#juniper_idp,index,netids +#juniper_structured,index,netops +#juniper_idp_structured,index,netids +#juniper_junos_fw_structured,index,netfw +#juniper_junos_ids_structured,index,netids +#juniper_junos_utm_structured,index,netfw +#juniper_junos_fw,index,netfw +#juniper_junos_ids,index,netids +#juniper_junos_utm,index,netfw +#juniper_sslvpn,index,netfw +#juniper_netscreen,index,netfw +#juniper_nsm,index,netfw +#juniper_nsm_idp,index,netids +#juniper_legacy,index,netops +#pan_traffic,index,netfw +#pan_threat,index,netproxy +#pan_system,index,netops +#pan_config,index,netops +#pan_hipwatch,index,main +#pan_correlation,index,main +#pan_userid,index,netauth +#pan_unknown,index,netops +#proofpoint_pps_filter,index,email +#proofpoint_pps_sendmail,index,email +#sc4s_events,index,main +#sc4s_fallback,index,main +#sc4s_metrics,index,em_metrics diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf new file mode 100644 index 0000000..342c9f4 --- /dev/null +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -0,0 +1,34 @@ +@version: 3.23 + +filter f_test_test { + host("testvp-*" type(glob)) or + netmask(192.168.100.1/24) +}; +filter f_juniper_nsm { + host("jnpnsm-*" type(glob)) or + netmask(192.168.1.0/24) +}; +filter f_juniper_nsm_idp { + host("jnpnsmidp-*" type(glob)) or + netmask(192.168.2.0/24) +}; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) +}; +filter f_cisco_nx_os { + host("csconx-*" type(glob)) or + netmask(192.168.5.0/24) +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; +filter f_proofpoint_pps_filter { + host("pps-*" type(glob)) or + netmask(192.168.7.0/24) +}; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv new file mode 100644 index 0000000..3f90603 --- /dev/null +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -0,0 +1,8 @@ +f_test_test,sc4s_vendor_product,"test_test" +f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" +f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" +f_juniper_idp,sc4s_vendor_product,"juniper_idp" +f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/templates/source_network.t b/package/etc/go_templates/source_network.t similarity index 100% rename from package/etc/templates/source_network.t rename to package/etc/go_templates/source_network.t diff --git a/package/etc/local_config/destinations/README.md b/package/etc/local_config/destinations/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/destinations/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/filters/README.md b/package/etc/local_config/filters/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/filters/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/filters/example.conf b/package/etc/local_config/filters/example.conf new file mode 100644 index 0000000..047fdc8 --- /dev/null +++ b/package/etc/local_config/filters/example.conf @@ -0,0 +1,4 @@ + +filter f_local_example { + program(sc4splugin); +}; \ No newline at end of file diff --git a/package/etc/local_config/log_paths/README.md b/package/etc/local_config/log_paths/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/log_paths/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl new file mode 100644 index 0000000..a8ac264 --- /dev/null +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -0,0 +1,76 @@ +# LOCAL_EXAMPLE + +# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique +# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the +# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ + +# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), +# the template generator will build a custom source based on the value of one or more of the set variables. + +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} + +# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name +# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". +# "parser" can be customized on dedicated ports only +# "common" uses the same parser sequence as the default ports and is the most commonly used + +{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} + +# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path +# The ports used are based on the values of one or more of the environment variables set above. + +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { + +# The first time this template is used the log_path will be linked to the default port + +{{- if eq (.) "yes"}} + source(s_default-ports); + +# Filters should be updated to use the simplest and most effecient logic possible to discard +# the message from this path + + filter(f_is_rfc3164); + filter(f_local_example); +{{- end}} +{{- if eq (.) "no"}} + +# In the second pass through the template a link to the dedicated port is used. This +# normally does not require additional filters + +source (s_dedicated_port_LOCAL_EXAMPLE); +{{- end}} + +#Set a default sourcetype and index + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + +#using the key "local_example" find any cutomized index,source or sourcetype meta values + + parser {p_add_context_splunk(key("local_example")); }; + +# Any additional logic needed to process the event before sending to Splunk goes here + +# Send it to Splunk + + destination(d_hec); #--HEC-- + +# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will +# forward events to another system + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic + + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic + +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/local_config/sources/README.md b/package/etc/local_config/sources/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/sources/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index e4a0a78..d0422fb 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -62,8 +62,7 @@ options { @include "conf.d/destinations/*.conf" @include "conf.d/log_paths/*.conf" -@include "conf.d/local/filters/*.conf" -@include "conf.d/local/filters/*/*.conf" -@include "conf.d/local/sources/*.conf" -@include "conf.d/local/destinations/*.conf" -@include "conf.d/local/log_paths/*.conf" +@include "conf.d/local/config/filters/*.conf" +@include "conf.d/local/config/sources/*.conf" +@include "conf.d/local/config/destinations/*.conf" +@include "conf.d/local/config/log_paths/*.conf" diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index d100fc0..30537c7 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -7,10 +7,15 @@ do echo Templating conf for $d gomplate \ --input-dir=$d \ - --template t=etc/templates/ \ + --template t=etc/go_templates/ \ --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" done -echo syslog-ng started +mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ +mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ +cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ +cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ + +echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file