From b148a3fd43a0bff5576aec722f66726b92172c9c Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 16 Oct 2019 18:45:17 -0400 Subject: [PATCH] Release/1.0.0 (#164) * Support Cisco Meraki (#150) * Feature/improve startup time (#151) * Bump package/syslog-ng from `26c0fe2` to `f219fbb` (#155) * Fixes #156 (#157) Support forcepoint webprotect aka websense * Fixes #144 Add ZScaler support (#159) * Add persist path to docs (#162) * Update meraki.conf resolve error on match syntax --- README.md | 2 +- docker-compose.yml | 1 + docs/gettingstarted/byoe-rhel7.md | 20 +- docs/gettingstarted/docker-swarm-general.md | 27 ++- docs/gettingstarted/docker-swarm-rhel7.md | 28 ++- docs/gettingstarted/docker-systemd-general.md | 25 ++- docs/gettingstarted/podman-systemd-general.md | 27 ++- docs/sources.md | 174 +++++++++++++++++- .../conf.d/conflib/_common/syslog_format.conf | 9 + package/etc/conf.d/filters/cisco/meraki.conf | 22 +++ .../conf.d/filters/fortinet/webprotect.conf | 3 + package/etc/conf.d/filters/zscaler/nss.conf | 3 + package/etc/conf.d/local/context/README.md | 1 + .../context/compliance_meta_by_source.conf | 5 - .../context/compliance_meta_by_source.csv | 2 - .../etc/conf.d/local/context/splunk_index.csv | 40 ---- .../context/vendor_product_by_source.conf | 37 ---- .../context/vendor_product_by_source.csv | 8 - ...nf.tmpl => p_rfc3164-cisco_nxos.conf.tmpl} | 0 .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 36 ++++ ...> p_rfc3164-microfocus_arcsight.conf.tmpl} | 0 .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 75 ++++++++ ...> p_rfc5424-noversion_cisco_asa.conf.tmpl} | 0 ...fc5424-noversion_symantec_proxy.conf.tmpl} | 0 ... p_rfc5424-strict_juniper_junos.conf.tmpl} | 0 .../p_rfc5424_epoch-cisco_merkai.conf.tmpl | 42 +++++ package/etc/conf.d/sources/network.conf.tmpl | 3 + .../compliance_meta_by_source.conf | 8 +- .../etc/context_templates/splunk_index.csv | 1 + .../vendor_product_by_source.conf | 26 +-- .../vendor_product_by_source.csv | 1 + package/etc/go_templates/source_network.t | 6 + package/sbin/entrypoint.sh | 21 ++- tests/test_cisco_meraki.py | 35 ++++ tests/test_forcepoint_web.py | 35 ++++ tests/test_zscaler_proxy.py | 55 ++++++ 36 files changed, 623 insertions(+), 155 deletions(-) create mode 100644 package/etc/conf.d/filters/cisco/meraki.conf create mode 100644 package/etc/conf.d/filters/fortinet/webprotect.conf create mode 100644 package/etc/conf.d/filters/zscaler/nss.conf create mode 100644 package/etc/conf.d/local/context/README.md delete mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.conf delete mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.csv delete mode 100644 package/etc/conf.d/local/context/splunk_index.csv delete mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.conf delete mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.csv rename package/etc/conf.d/log_paths/{p_rfc3164-cisco_nx-os.conf.tmpl => p_rfc3164-cisco_nxos.conf.tmpl} (100%) create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl rename package/etc/conf.d/log_paths/{p_rfc3164_microfocus_arcsight.conf.tmpl => p_rfc3164-microfocus_arcsight.conf.tmpl} (100%) create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl rename package/etc/conf.d/log_paths/{p_rfc5424_noversion-cisco_asa.conf.tmpl => p_rfc5424-noversion_cisco_asa.conf.tmpl} (100%) rename package/etc/conf.d/log_paths/{p_rfc_5424_noversion-symantec_proxy.conf.tmpl => p_rfc5424-noversion_symantec_proxy.conf.tmpl} (100%) rename package/etc/conf.d/log_paths/{p_rfc_5424_strict-juniper_junos.conf.tmpl => p_rfc5424-strict_juniper_junos.conf.tmpl} (100%) create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl create mode 100644 tests/test_cisco_meraki.py create mode 100644 tests/test_forcepoint_web.py create mode 100644 tests/test_zscaler_proxy.py diff --git a/README.md b/README.md index c7c8559..64043d5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Splunk Connect for Syslog (SC4S) is a community project focused on reducing the ## Usage -For full usage instructions, please visit the Splunk Connect for Syslog documentation page. +For full usage instructions, please visit the Splunk Connect for Syslog [documentation pages over at readthedocs](https://splunk-connect-for-syslog.readthedocs.io/en/master/). ## Support diff --git a/docker-compose.yml b/docker-compose.yml index 1765168..a9e5e74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,7 @@ services: RH_ORG: ${RH_ORG} RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s + command: -det ports: - "514:514" - "601:601" diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 2567d89..47d2f7c 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -100,15 +100,17 @@ WantedBy=multi-user.target source scl_source enable rh-python36 cd /opt/syslog-ng -for d in $(find /opt/syslog-ng/etc -type d) -do - echo Templating conf for $d - gomplate \ - --input-dir=$d \ - --template t=etc/go_templates/ \ - --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ - --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" -done +#The following is no longer needed but retained as a comment just in case we run into command line length issues +#for d in $(find /opt/syslog-ng/etc -type d) +#do +# echo Templating conf for $d +# gomplate \ +# --input-dir=$d \ +# --template t=etc/go_templates/ \ +# --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ +# --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" +#done +gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index a1c0f72..420065f 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -5,8 +5,10 @@ Refer to [Getting Started](https://docs.docker.com/get-started/) # SC4S Configuration -* Create a directory on the server for local configurations. This should be available to all administrators, for example: +* Create a directory on the server for local configurations and disk buffering. This should be available to all +administrators, for example: ``/opt/sc4s/`` + * Create a docker-compose.yml file in the directory created above, based on the following template: ```yaml @@ -29,23 +31,35 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer # Uncomment the following line if custom TLS certs are provided # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new + set of files will be created in addition to the original ones. _The original ones will not be removed_. + If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created + again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +`docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. + ## Configure the SC4S environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -148,8 +162,9 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer #Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` * Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" @@ -222,7 +237,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index c06849b..e8ee2ac 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -33,8 +33,9 @@ sudo docker swarm init # SC4S Configuration -* Create a directory on the server for local configurations. This should be available to all administrators, for example: +* Create a directory on the server for local configurations and disk buffering. This should be available to all administrators, for example: ``/opt/sc4s/`` + * Create a docker-compose.yml file in the directory created above, based on the following template: ```yaml @@ -57,23 +58,35 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer # Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new +set of files will be created in addition to the original ones. _The original ones will not be removed_. +If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created +again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +`docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. + ## Configure the SC4S environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -178,8 +191,9 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer #Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` * Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" @@ -252,7 +266,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 32eb897..20b5a3c 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -20,6 +20,8 @@ Environment="SC4S_IMAGE=splunk/sc4s:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -35,23 +37,35 @@ ExecStartPre=/usr/bin/docker run \ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S --rm \ $SC4S_IMAGE ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new +set of files will be created in addition to the original ones. _The original ones will not be removed_. +If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created +again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +unit file above. Failure to do this will cause SC4S to abort at startup. + ## Configure the SC4S environment Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: @@ -134,6 +148,8 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -148,6 +164,7 @@ ExecStartPre=/usr/bin/docker run \ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S \ --rm \ $SC4S_IMAGE @@ -233,7 +250,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 394762d..a70cd2d 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -14,12 +14,14 @@ After=network.service Requires=network.service [Service] -Environment="SC4S_IMAGE=splunk/scs:latest" +Environment="SC4S_IMAGE=splunk/sc4s:latest" # Optional mount point for local overrides and configurations; see notes in docs Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -35,23 +37,35 @@ ExecStartPre=/usr/bin/podman run \ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S --rm \ $SC4S_IMAGE ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new +set of files will be created in addition to the original ones. _The original ones will not be removed_. +If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created +again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +unit file above. Failure to do this will cause SC4S to abort at startup. + ## Configure the sc4s environment Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: @@ -134,6 +148,8 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -148,6 +164,7 @@ ExecStartPre=/usr/bin/podman run \ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S \ --rm \ $SC4S_IMAGE @@ -233,7 +250,7 @@ podman logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/sources.md b/docs/sources.md index 12f4f3c..f7e85cb 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -41,12 +41,7 @@ MSG Parse: This filter parses message content * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. -* Follow vendor configuration steps per Product Manual above ensure: - * Log Level is 6 "Informational" - * Protocol is TCP/IP - * permit-hostdown is on - * device-id is hostname and included - * timestamp is included +* Follow vendor configuration steps per Product Manual above ### Options @@ -107,7 +102,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_JUNIPER_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | | SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format | ### Verification @@ -189,9 +184,106 @@ Use the following search to validate events are present, for NX-OS, WLC and ACI index= sourcetype=cisco:ios | stats count by host ``` +## Product - Meraki Product Line MR, MS, MX, MV + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3018/ | +| Product Manual | https://documentation.meraki.com/zGeneral_Administration/Monitoring_and_Reporting/Syslog_Server_Overview_and_Configuration | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| merkai | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_meraki | meraki | netfw | The current TA does not sub sourcetype or utilize source preventing segmenation into more appropriate indexes | + + +### Filter type + +IP, Netmask, Host or Port + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Follow vendor configuration steps per Product Manual above + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_MERAKI_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_MERAKI_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=merkai +``` Verify timestamp, and host values match as expected + +Verify timestamp, and host values match as expected + +# Vendor - Forcepoint + +## Product - Webprotect (Websense) + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2966/ | +| Product Manual | http://www.websense.com/content/support/library/web/v85/siem/siem.pdf | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| websense:cg:kv | None | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| forcepoint_webprotect | websense:cg:kv | netproxy | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the admin manual for specific details of configuration to send Reliable syslog using RFC 3195 format, a typical logging configuration will include the following features. + + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events, in addition WebProtect has the ability to test logging functionality using a built in command + + +``` +index= sourcetype=websense:cg:kv +``` + # Vendor - Fortinet ## Product - Fortigate @@ -804,3 +896,71 @@ An active proxy will generate frequent events. Use the following search to valid ``` index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` + + +# Vendor - Zscaler + +## Product - All Products + +The ZScaler product manual includes and extensive section of configuration for multiple Splunk TCP input ports around page +26. When using SC4S these ports are not required and should not be used. Simply configure all outputs from the NSS to utilize +the IP or host name of the SC4S instance and port 514 + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3865/ | +| Product Manual | https://community.zscaler.com/t/zscaler-splunk-app-design-and-installation-documentation/4728 | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| zscalernss-alerts | Requires format customization add ``\tvendor=Zscaler\tproduct=alerts`` immediately prior to the ``\n`` in the NSS Alert Web format. See Zscaler manual for more info. | +| zscalernss-dns | Requires format customization add ``\tvendor=Zscaler\tproduct=dns`` immediately prior to the ``\n`` in the NSS DNS format. See Zscaler manual for more info. | +| zscalernss-web | None | +| zscalernss-zpa-app | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | +| zscalernss-zpa-auth | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | +| zscalernss-zpa-connector | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth_connector`` immediately prior to the ``\n`` in the LSS Connector format. See Zscaler manual for more info. | +| zscalernss-fw | Requires format customization add ``\tvendor=Zscaler\tproduct=fw`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| zscalernss_alerts | zscalernss-alerts | main | none | +| zscalernss_dns | zscalernss-dns | netdns | none | +| zscalernss_fw | zscalernss-fw | netfw | none | +| zscalernss_web | zscalernss-web | netproxy | none | +| zscalernss-zpa-app | zscalernss_zpa-app | netids | none | +| zscalernss-zpa-auth | zscalernss_zpa_auth | netauth | none | +| zscalernss-zpa-connector | zscalernss_zpa_connector | netops | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_ZSCALER_NSS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=zscalernss-* | stats count by host +``` diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 8c5794c..5b69e71 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -4,6 +4,9 @@ filter f_rfc5424_strict{ filter f_rfc5424_noversion{ message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); }; +filter f_rfc5424_epochtime{ + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) ))'); +}; rewrite set_rfcnonconformant{ set("rfc5424_nonconform" value("fields.sc4s_syslog_format")); }; @@ -19,6 +22,12 @@ rewrite set_rfc5424_noversion{ filter f_is_rfc5424_noversion{ match("rfc5424_noversion" value("fields.sc4s_syslog_format")) }; +rewrite set_rfc5424_epochtime{ + set("rfc5424_epochtime" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_epochtime{ + match("rfc5424_epochtime" value("fields.sc4s_syslog_format")) +}; rewrite set_rfc3164{ set("rfc3164" value("fields.sc4s_syslog_format")); }; diff --git a/package/etc/conf.d/filters/cisco/meraki.conf b/package/etc/conf.d/filters/cisco/meraki.conf new file mode 100644 index 0000000..c0573b5 --- /dev/null +++ b/package/etc/conf.d/filters/cisco/meraki.conf @@ -0,0 +1,22 @@ +# Meraki + +filter f_cisco_meraki { + match("cisco_meraki", value("fields.sc4s_vendor_product") type(glob)) +}; + +parser p_cisco_meraki { + channel { + filter { + message( + #'(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+ )(?.*))' + '(?:(?:<(?\d{1,3})>(?[1-9][0-9]?) (?:(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) )(?.*))' + flags(store-matches) + ); + }; + parser { + date-parser(format('%s') + template("${EPOCH}")); + }; + }; + +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/fortinet/webprotect.conf b/package/etc/conf.d/filters/fortinet/webprotect.conf new file mode 100644 index 0000000..2d669e3 --- /dev/null +++ b/package/etc/conf.d/filters/fortinet/webprotect.conf @@ -0,0 +1,3 @@ +filter f_forcepoint_webprotect_kv { + program('vendor=[Ww]ebsense'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/zscaler/nss.conf b/package/etc/conf.d/filters/zscaler/nss.conf new file mode 100644 index 0000000..9ee4e1a --- /dev/null +++ b/package/etc/conf.d/filters/zscaler/nss.conf @@ -0,0 +1,3 @@ +filter f_zscaler_nss { + message('\tvendor=Zscaler\t'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/README.md b/package/etc/conf.d/local/context/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/context/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.conf b/package/etc/conf.d/local/context/compliance_meta_by_source.conf deleted file mode 100644 index f325b4f..0000000 --- a/package/etc/conf.d/local/context/compliance_meta_by_source.conf +++ /dev/null @@ -1,5 +0,0 @@ -@version: 3.24 -filter f_test_test { - host("something-*" type(glob)) or - netmask(192.168.100.1/24) -}; diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.csv b/package/etc/conf.d/local/context/compliance_meta_by_source.csv deleted file mode 100644 index 6608db0..0000000 --- a/package/etc/conf.d/local/context/compliance_meta_by_source.csv +++ /dev/null @@ -1,2 +0,0 @@ -#f_test_test,.splunk.index,"badindex" -#f_test_test,fields.compliance,"pci" diff --git a/package/etc/conf.d/local/context/splunk_index.csv b/package/etc/conf.d/local/context/splunk_index.csv deleted file mode 100644 index e93911a..0000000 --- a/package/etc/conf.d/local/context/splunk_index.csv +++ /dev/null @@ -1,40 +0,0 @@ -#bluecoat_proxy,index,netproxy -#cef_ArcSight_ArcSight,index,netwaf -#cef_Incapsula_SIEMintegration,index,netwaf -#cef_Microsoft_Microsoft Windows,index,oswinsec -#cef_Microsoft_System or Application Event,index,oswin -#cisco_asa,index,netfw -#cisco_ios,index,netops -#cisco_nx_os,index,netops -#local_example,index,main -#fortinet_fortios_event,index,netops -#fortinet_fortios_log,index,netops -#fortinet_fortios_traffic,index,netfw -#fortinet_fortios_utm,index,netids -#juniper_idp,index,netids -#juniper_structured,index,netops -#juniper_idp_structured,index,netids -#juniper_junos_fw_structured,index,netfw -#juniper_junos_ids_structured,index,netids -#juniper_junos_utm_structured,index,netfw -#juniper_junos_fw,index,netfw -#juniper_junos_ids,index,netids -#juniper_junos_utm,index,netfw -#juniper_sslvpn,index,netfw -#juniper_netscreen,index,netfw -#juniper_nsm,index,netfw -#juniper_nsm_idp,index,netids -#juniper_legacy,index,netops -#pan_traffic,index,netfw -#pan_threat,index,netproxy -#pan_system,index,netops -#pan_config,index,netops -#pan_hipwatch,index,main -#pan_correlation,index,main -#pan_userid,index,netauth -#pan_unknown,index,netops -#proofpoint_pps_filter,index,email -#proofpoint_pps_sendmail,index,email -#sc4s_events,index,main -#sc4s_fallback,index,main -#sc4s_metrics,index,em_metrics diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.conf b/package/etc/conf.d/local/context/vendor_product_by_source.conf deleted file mode 100644 index 37e3412..0000000 --- a/package/etc/conf.d/local/context/vendor_product_by_source.conf +++ /dev/null @@ -1,37 +0,0 @@ -@version: 3.22 -#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a -#bug/limitation of selector files. The better syntax should be as follows -#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; - -filter f_test_test { - host("testvp-*" type(glob)) or - netmask(192.168.100.1/24) -}; -filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) or - netmask(192.168.1.0/24) -}; -filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) or - netmask(192.168.2.0/24) -}; -filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) -}; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or - netmask(192.168.4.0/24) -}; -filter f_cisco_nx_os { - host("csconx-*" type(glob)) or - netmask(192.168.5.0/24) -}; -filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) -}; -filter f_proofpoint_pps_filter { - host("pps-*" type(glob)) or - netmask(192.168.7.0/24) -}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.csv b/package/etc/conf.d/local/context/vendor_product_by_source.csv deleted file mode 100644 index 3f90603..0000000 --- a/package/etc/conf.d/local/context/vendor_product_by_source.csv +++ /dev/null @@ -1,8 +0,0 @@ -f_test_test,sc4s_vendor_product,"test_test" -f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" -f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" -f_juniper_idp,sc4s_vendor_product,"juniper_idp" -f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" -f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" -f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" -f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl new file mode 100644 index 0000000..ac0cb48 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -0,0 +1,36 @@ +# Forcepoint Webprotect +{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_forcepoint_webprotect_kv); +{{- end}} +{{- if eq (.) "no"}} + source (s_dedicated_port_FORCEPOINT_WEBPROTECT); +{{- end}} + + rewrite { + subst(" [^ =]+\=\-", "", value("MESSAGE"), flags("global")); + }; + rewrite { r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg"))}; + parser {p_add_context_splunk(key("forcepoint_webprotect")); }; + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for FORCEPOINT_WEBPROTECT traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for FORCEPOINT_WEBPROTECT traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl new file mode 100644 index 0000000..7bd86fc --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -0,0 +1,75 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_zscaler_nss); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_ZSCALER_NSS); +{{- end }} + + rewrite { + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + }; + parser { + #basic parsing + kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}")); + }; + + if (match("alerts" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_alerts")); }; + } elif (match("dns" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_dns")); }; + } elif (match("fw" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_fw")); }; + } elif (match("NSS" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_web")); }; + } elif (match("audit" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zia_audit")); }; + } elif (match("sandbox" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zia_sandbox")); }; + } elif (match("zpa" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa")); }; + } elif (match("zpa_auth" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_auth")); }; + } elif (match("zpa_auth_connector" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_connector")); }; + } elif (match("zpa_bba" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_bba")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("zscaler_nss")); + }; + }; + + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for ZSCALER_NSS traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for ZSCALER_NSS traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl new file mode 100644 index 0000000..dbbb1f6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl @@ -0,0 +1,42 @@ +# Checkpoint Splunk format +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_default-ports); + filter(f_cisco_meraki); +{{- end}} +{{- if eq (.) "no"}} + source (s_dedicated_port_CISCO_MERAKI); +{{- end}} + + #parser { + # kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); + # date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); + # + # }; + + #rewrite { set("${.kv.hostname}", value("HOST")); }; + + rewrite { r_set_splunk_dest_default(sourcetype("meraki"), index("netfw"), template("t_hdr_msg"))}; + parser {p_add_context_splunk(key("cisco_meraki")); }; + + + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_MERAKI traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index a9e8b78..234e87c 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -82,6 +82,9 @@ source s_default-ports { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf index f325b4f..322b938 100644 --- a/package/etc/context_templates/compliance_meta_by_source.conf +++ b/package/etc/context_templates/compliance_meta_by_source.conf @@ -1,5 +1,5 @@ @version: 3.24 -filter f_test_test { - host("something-*" type(glob)) or - netmask(192.168.100.1/24) -}; +#filter f_test_test { +# host("something-*" type(glob)) or +# netmask(192.168.100.1/24) +#}; diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index ec4f02c..7a418e3 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -15,6 +15,7 @@ #cisco_ios,index,netops #cisco_nx_os,index,netops #local_example,index,main +#forcepoint_webprotect,index,netproxy #fortinet_fortios_event,index,netops #fortinet_fortios_log,index,netops #fortinet_fortios_traffic,index,netfw diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 0903ca3..57c73ac 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -4,6 +4,14 @@ filter f_test_test { host("testvp-*" type(glob)) or netmask(192.168.100.1/24) }; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) +}; filter f_juniper_nsm { host("jnpnsm-*" type(glob)) or netmask(192.168.1.0/24) @@ -12,23 +20,19 @@ filter f_juniper_nsm_idp { host("jnpnsmidp-*" type(glob)) or netmask(192.168.2.0/24) }; -filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) -}; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or +filter f_cisco_meraki { + host("testcm-*" type(glob)) or netmask(192.168.4.0/24) }; filter f_cisco_nx_os { host("csconx-*" type(glob)) or netmask(192.168.5.0/24) -}; -filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) }; filter f_proofpoint_pps_filter { host("pps-*" type(glob)) or netmask(192.168.7.0/24) -}; \ No newline at end of file +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 3f90603..2f85bc4 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -1,4 +1,5 @@ f_test_test,sc4s_vendor_product,"test_test" +f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index f804816..09ea120 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -74,6 +74,9 @@ source s_dedicated_port_{{ .port_id}} { {{- else if eq .parser "cisco_parser" }} parser {cisco-parser()}; rewrite(set_cisco_ios); +{{- else if eq .parser "cisco_meraki_parser" }} + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); {{- else if eq .parser "rfc3164" }} parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); @@ -96,6 +99,9 @@ source s_dedicated_port_{{ .port_id}} { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 30537c7..ff9a112 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -2,15 +2,18 @@ source scl_source enable rh-python36 cd /opt/syslog-ng -for d in $(find /opt/syslog-ng/etc -type d) -do - echo Templating conf for $d - gomplate \ - --input-dir=$d \ - --template t=etc/go_templates/ \ - --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ - --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" -done +#The following is no longer needed but retained as a comment just in case we run into command line length issues +#for d in $(find /opt/syslog-ng/etc -type d) +#do +# echo Templating conf for $d +# gomplate \ +# --input-dir=$d \ +# --template t=etc/go_templates/ \ +# --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ +# --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" +#done +gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ + mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ diff --git a/tests/test_cisco_meraki.py b/tests/test_cisco_meraki.py new file mode 100644 index 0000000..920714a --- /dev/null +++ b/tests/test_cisco_meraki.py @@ -0,0 +1,35 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<134>1 1563249630.774247467 devicename security_event ids_alerted signature=1:28423:1 priority=1 timestamp=1468531589.810079 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection +def test_cisco_meraki_security_event(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}1 {% now 'utc', '%s' %}.123456789 testcm-{{ host }} security_event ids_alerted signature=1:28423:1 priority=1 timestamp={% now 'utc', '%s' %}.123456 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection\n") + message = mt.render(mark="<134>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"testcm-{{ host }}\" sourcetype=\"meraki\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +#<134>1 Dec 6 08:41:44 192.168.1.1 1 1386337316.207232138 MX84 events Cellular connection up diff --git a/tests/test_forcepoint_web.py b/tests/test_forcepoint_web.py new file mode 100644 index 0000000..7ecf4cb --- /dev/null +++ b/tests/test_forcepoint_web.py @@ -0,0 +1,35 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<134>Oct 16 12:13:06 sourcehost2 vendor=Websense 9f product=Security product_version=7.7.0 action=permitted severity=7 category=755 user=LDAP://user7 OU=Users,OU=Beijing,DC=com/TEST\, TEST_NAME src_host=10.0.0.4 src_port=61435 dst_host=HOST-013 dst_ip=10.0.0.19 dst_port=25404 bytes_out=4074 bytes_in=12328 http_response=200 http_method=POST http_content_type=image/gif;charset=UTF-8 http_user_agent=Mozilla/3.0 (Windows; U; Windows NT 6.1; es-def; rv:1.7.0.11) Gecko/2009060215 Firefox/8.0.11 (.NET CLR 8.5.30729) http_proxy_status_code=200 reason=- disposition=2573 policy=role-8**Default role=4 duration=63 url=http://test_web.com/contents/content1.jpg +def test_forcepoint_webprotect_kv(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} vendor=Websense 9f product=Security product_version=7.7.0 action=permitted severity=7 category=755 user=LDAP://user7 OU=Users,OU=Beijing,DC=com/TEST\, TEST_NAME src_host=10.0.0.4 src_port=61435 dst_host=HOST-013 dst_ip=10.0.0.19 dst_port=25404 bytes_out=4074 bytes_in=12328 http_response=200 http_method=POST http_content_type=image/gif;charset=UTF-8 http_user_agent=Mozilla/3.0 (Windows; U; Windows NT 6.1; es-def; rv:1.7.0.11) Gecko/2009060215 Firefox/8.0.11 (.NET CLR 8.5.30729) http_proxy_status_code=200 reason=- disposition=2573 policy=role-8**Default role=4 duration=63 url=http://test_web.com/contents/content1.jpg unknownfield=-\n") + message = mt.render(mark="<134>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"websense:cg:kv\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +#<134>1 Dec 6 08:41:44 192.168.1.1 1 1386337316.207232138 MX84 events Cellular connection up diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py new file mode 100644 index 0000000..e0f7fb1 --- /dev/null +++ b/tests/test_zscaler_proxy.py @@ -0,0 +1,55 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +#Note the long white space is a \t +#2019-10-16 15:44:36 reason=Allowed event_id=6748427317914894361 protocol=HTTPS action=Allowed transactionsize=663 responsesize=65 requestsize=598 urlcategory=UK_ALLOW_Pharmacies serverip=216.58.204.70 clienttranstime=0 requestmethod=CONNECT refererURL=None useragent=Windows Windows 10 Enterprise ZTunnel/1.0 product=NSS location=UK_Wynyard_VPN->other ClientIP=192.168.0.38 status=200 user=first.last@example.com url=4171764.fls.doubleclick.net:443 vendor=Zscaler hostname=4171764.fls.doubleclick.net clientpublicIP=213.86.221.94 threatcategory=None threatname=None filetype=None appname=DoubleClick pagerisk=0 department=Procurement, Generics urlsupercategory=User-defined appclass=Sales and Marketing dlpengine=None urlclass=Bandwidth Loss threatclass=None dlpdictionaries=None fileclass=None bwthrottle=NO servertranstime=0 md5=None +def test_zscaler_proxy(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894361\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# +def test_zscaler_proxy_pri(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894362\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#