From 39faf36b0c2bed04a4d2cd22e452ffcf6298361f Mon Sep 17 00:00:00 2001 From: mkarlstrand-splunk <49571555+mkarlstrand-splunk@users.noreply.github.com> Date: Mon, 14 Oct 2019 14:50:06 -0700 Subject: [PATCH 01/10] Update gettingstarted.md --- docs/gettingstarted.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index a7fff9d..887e886 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -53,8 +53,8 @@ Install the following: ### Configure the Splunk HTTP Event Collector - Set up the Splunk HTTP Event Collector with the HEC endpoints behind a load balancer (VIP) configured for https round robin *WITHOUT* sticky -session. Alternatively, a list of HEC endpoint URLs can be configured in SC4S if no load balancer is in place. In either case, it is -recommended that SC4S traffic be sent to HEC endpoints configured directly on the indexers rather than an intermediate tier of HWFs. +session. Alternatively, a list of HEC endpoint URLs can be configured in SC4S (native Syslog-ng load balancing) if no load balancer is in place. In either case, it is +recommended that SC4S traffic be sent to HEC endpoints configured directly on the indexers rather than an intermediate tier of HWFs. Deployments with 10 or fewer Indexers and where HEC is used exclusively for syslog, the recommendation is to use the native load balancing. In all other scenarios the recommendation is to use an external load balacer. If utilizing the native load balancing, be sure to update the configuration when the number and/or names of the indexers change. - Create a HEC token that will be used by SC4S and ensure the token has access to place events in main, em_metrics, and all indexes used as event destinations. - Refer to [Splunk Cloud](http://docs.splunk.com/Documentation/Splunk/7.3.1/Data/UsetheHTTPEventCollector#Configure_HTTP_Event_Collector_on_managed_Splunk_Cloud) @@ -82,4 +82,4 @@ Splunk type. Additional hosts can be deployed for syslog collection from additional network zones and locations: -![SC4S deployment diagram](SC4Sdeployment.png) \ No newline at end of file +![SC4S deployment diagram](SC4Sdeployment.png) From 4fea07000524410f078341b3e0e1dd4ae5693b7d Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Tue, 15 Oct 2019 07:27:59 -0400 Subject: [PATCH 02/10] Misc Doc Updates From b148a3fd43a0bff5576aec722f66726b92172c9c Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 16 Oct 2019 18:45:17 -0400 Subject: [PATCH 03/10] Release/1.0.0 (#164) * Support Cisco Meraki (#150) * Feature/improve startup time (#151) * Bump package/syslog-ng from `26c0fe2` to `f219fbb` (#155) * Fixes #156 (#157) Support forcepoint webprotect aka websense * Fixes #144 Add ZScaler support (#159) * Add persist path to docs (#162) * Update meraki.conf resolve error on match syntax --- README.md | 2 +- docker-compose.yml | 1 + docs/gettingstarted/byoe-rhel7.md | 20 +- docs/gettingstarted/docker-swarm-general.md | 27 ++- docs/gettingstarted/docker-swarm-rhel7.md | 28 ++- docs/gettingstarted/docker-systemd-general.md | 25 ++- docs/gettingstarted/podman-systemd-general.md | 27 ++- docs/sources.md | 174 +++++++++++++++++- .../conf.d/conflib/_common/syslog_format.conf | 9 + package/etc/conf.d/filters/cisco/meraki.conf | 22 +++ .../conf.d/filters/fortinet/webprotect.conf | 3 + package/etc/conf.d/filters/zscaler/nss.conf | 3 + package/etc/conf.d/local/context/README.md | 1 + .../context/compliance_meta_by_source.conf | 5 - .../context/compliance_meta_by_source.csv | 2 - .../etc/conf.d/local/context/splunk_index.csv | 40 ---- .../context/vendor_product_by_source.conf | 37 ---- .../context/vendor_product_by_source.csv | 8 - ...nf.tmpl => p_rfc3164-cisco_nxos.conf.tmpl} | 0 .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 36 ++++ ...> p_rfc3164-microfocus_arcsight.conf.tmpl} | 0 .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 75 ++++++++ ...> p_rfc5424-noversion_cisco_asa.conf.tmpl} | 0 ...fc5424-noversion_symantec_proxy.conf.tmpl} | 0 ... p_rfc5424-strict_juniper_junos.conf.tmpl} | 0 .../p_rfc5424_epoch-cisco_merkai.conf.tmpl | 42 +++++ package/etc/conf.d/sources/network.conf.tmpl | 3 + .../compliance_meta_by_source.conf | 8 +- .../etc/context_templates/splunk_index.csv | 1 + .../vendor_product_by_source.conf | 26 +-- .../vendor_product_by_source.csv | 1 + package/etc/go_templates/source_network.t | 6 + package/sbin/entrypoint.sh | 21 ++- tests/test_cisco_meraki.py | 35 ++++ tests/test_forcepoint_web.py | 35 ++++ tests/test_zscaler_proxy.py | 55 ++++++ 36 files changed, 623 insertions(+), 155 deletions(-) create mode 100644 package/etc/conf.d/filters/cisco/meraki.conf create mode 100644 package/etc/conf.d/filters/fortinet/webprotect.conf create mode 100644 package/etc/conf.d/filters/zscaler/nss.conf create mode 100644 package/etc/conf.d/local/context/README.md delete mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.conf delete mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.csv delete mode 100644 package/etc/conf.d/local/context/splunk_index.csv delete mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.conf delete mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.csv rename package/etc/conf.d/log_paths/{p_rfc3164-cisco_nx-os.conf.tmpl => p_rfc3164-cisco_nxos.conf.tmpl} (100%) create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl rename package/etc/conf.d/log_paths/{p_rfc3164_microfocus_arcsight.conf.tmpl => p_rfc3164-microfocus_arcsight.conf.tmpl} (100%) create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl rename package/etc/conf.d/log_paths/{p_rfc5424_noversion-cisco_asa.conf.tmpl => p_rfc5424-noversion_cisco_asa.conf.tmpl} (100%) rename package/etc/conf.d/log_paths/{p_rfc_5424_noversion-symantec_proxy.conf.tmpl => p_rfc5424-noversion_symantec_proxy.conf.tmpl} (100%) rename package/etc/conf.d/log_paths/{p_rfc_5424_strict-juniper_junos.conf.tmpl => p_rfc5424-strict_juniper_junos.conf.tmpl} (100%) create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl create mode 100644 tests/test_cisco_meraki.py create mode 100644 tests/test_forcepoint_web.py create mode 100644 tests/test_zscaler_proxy.py diff --git a/README.md b/README.md index c7c8559..64043d5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Splunk Connect for Syslog (SC4S) is a community project focused on reducing the ## Usage -For full usage instructions, please visit the Splunk Connect for Syslog documentation page. +For full usage instructions, please visit the Splunk Connect for Syslog [documentation pages over at readthedocs](https://splunk-connect-for-syslog.readthedocs.io/en/master/). ## Support diff --git a/docker-compose.yml b/docker-compose.yml index 1765168..a9e5e74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,7 @@ services: RH_ORG: ${RH_ORG} RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s + command: -det ports: - "514:514" - "601:601" diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 2567d89..47d2f7c 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -100,15 +100,17 @@ WantedBy=multi-user.target source scl_source enable rh-python36 cd /opt/syslog-ng -for d in $(find /opt/syslog-ng/etc -type d) -do - echo Templating conf for $d - gomplate \ - --input-dir=$d \ - --template t=etc/go_templates/ \ - --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ - --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" -done +#The following is no longer needed but retained as a comment just in case we run into command line length issues +#for d in $(find /opt/syslog-ng/etc -type d) +#do +# echo Templating conf for $d +# gomplate \ +# --input-dir=$d \ +# --template t=etc/go_templates/ \ +# --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ +# --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" +#done +gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index a1c0f72..420065f 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -5,8 +5,10 @@ Refer to [Getting Started](https://docs.docker.com/get-started/) # SC4S Configuration -* Create a directory on the server for local configurations. This should be available to all administrators, for example: +* Create a directory on the server for local configurations and disk buffering. This should be available to all +administrators, for example: ``/opt/sc4s/`` + * Create a docker-compose.yml file in the directory created above, based on the following template: ```yaml @@ -29,23 +31,35 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer # Uncomment the following line if custom TLS certs are provided # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new + set of files will be created in addition to the original ones. _The original ones will not be removed_. + If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created + again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +`docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. + ## Configure the SC4S environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -148,8 +162,9 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer #Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` * Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" @@ -222,7 +237,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index c06849b..e8ee2ac 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -33,8 +33,9 @@ sudo docker swarm init # SC4S Configuration -* Create a directory on the server for local configurations. This should be available to all administrators, for example: +* Create a directory on the server for local configurations and disk buffering. This should be available to all administrators, for example: ``/opt/sc4s/`` + * Create a docker-compose.yml file in the directory created above, based on the following template: ```yaml @@ -57,23 +58,35 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer # Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new +set of files will be created in addition to the original ones. _The original ones will not be removed_. +If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created +again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +`docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. + ## Configure the SC4S environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -178,8 +191,9 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer #Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` * Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" @@ -252,7 +266,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 32eb897..20b5a3c 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -20,6 +20,8 @@ Environment="SC4S_IMAGE=splunk/sc4s:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -35,23 +37,35 @@ ExecStartPre=/usr/bin/docker run \ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S --rm \ $SC4S_IMAGE ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new +set of files will be created in addition to the original ones. _The original ones will not be removed_. +If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created +again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +unit file above. Failure to do this will cause SC4S to abort at startup. + ## Configure the SC4S environment Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: @@ -134,6 +148,8 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -148,6 +164,7 @@ ExecStartPre=/usr/bin/docker run \ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S \ --rm \ $SC4S_IMAGE @@ -233,7 +250,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 394762d..a70cd2d 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -14,12 +14,14 @@ After=network.service Requires=network.service [Service] -Environment="SC4S_IMAGE=splunk/scs:latest" +Environment="SC4S_IMAGE=splunk/sc4s:latest" # Optional mount point for local overrides and configurations; see notes in docs Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -35,23 +37,35 @@ ExecStartPre=/usr/bin/podman run \ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S --rm \ $SC4S_IMAGE ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new +set of files will be created in addition to the original ones. _The original ones will not be removed_. +If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created +again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +unit file above. Failure to do this will cause SC4S to abort at startup. + ## Configure the sc4s environment Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: @@ -134,6 +148,8 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -148,6 +164,7 @@ ExecStartPre=/usr/bin/podman run \ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S \ --rm \ $SC4S_IMAGE @@ -233,7 +250,7 @@ podman logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/sources.md b/docs/sources.md index 12f4f3c..f7e85cb 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -41,12 +41,7 @@ MSG Parse: This filter parses message content * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. -* Follow vendor configuration steps per Product Manual above ensure: - * Log Level is 6 "Informational" - * Protocol is TCP/IP - * permit-hostdown is on - * device-id is hostname and included - * timestamp is included +* Follow vendor configuration steps per Product Manual above ### Options @@ -107,7 +102,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_JUNIPER_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | | SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format | ### Verification @@ -189,9 +184,106 @@ Use the following search to validate events are present, for NX-OS, WLC and ACI index= sourcetype=cisco:ios | stats count by host ``` +## Product - Meraki Product Line MR, MS, MX, MV + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3018/ | +| Product Manual | https://documentation.meraki.com/zGeneral_Administration/Monitoring_and_Reporting/Syslog_Server_Overview_and_Configuration | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| merkai | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_meraki | meraki | netfw | The current TA does not sub sourcetype or utilize source preventing segmenation into more appropriate indexes | + + +### Filter type + +IP, Netmask, Host or Port + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Follow vendor configuration steps per Product Manual above + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_MERAKI_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_MERAKI_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=merkai +``` Verify timestamp, and host values match as expected + +Verify timestamp, and host values match as expected + +# Vendor - Forcepoint + +## Product - Webprotect (Websense) + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2966/ | +| Product Manual | http://www.websense.com/content/support/library/web/v85/siem/siem.pdf | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| websense:cg:kv | None | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| forcepoint_webprotect | websense:cg:kv | netproxy | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the admin manual for specific details of configuration to send Reliable syslog using RFC 3195 format, a typical logging configuration will include the following features. + + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events, in addition WebProtect has the ability to test logging functionality using a built in command + + +``` +index= sourcetype=websense:cg:kv +``` + # Vendor - Fortinet ## Product - Fortigate @@ -804,3 +896,71 @@ An active proxy will generate frequent events. Use the following search to valid ``` index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` + + +# Vendor - Zscaler + +## Product - All Products + +The ZScaler product manual includes and extensive section of configuration for multiple Splunk TCP input ports around page +26. When using SC4S these ports are not required and should not be used. Simply configure all outputs from the NSS to utilize +the IP or host name of the SC4S instance and port 514 + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3865/ | +| Product Manual | https://community.zscaler.com/t/zscaler-splunk-app-design-and-installation-documentation/4728 | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| zscalernss-alerts | Requires format customization add ``\tvendor=Zscaler\tproduct=alerts`` immediately prior to the ``\n`` in the NSS Alert Web format. See Zscaler manual for more info. | +| zscalernss-dns | Requires format customization add ``\tvendor=Zscaler\tproduct=dns`` immediately prior to the ``\n`` in the NSS DNS format. See Zscaler manual for more info. | +| zscalernss-web | None | +| zscalernss-zpa-app | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | +| zscalernss-zpa-auth | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | +| zscalernss-zpa-connector | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth_connector`` immediately prior to the ``\n`` in the LSS Connector format. See Zscaler manual for more info. | +| zscalernss-fw | Requires format customization add ``\tvendor=Zscaler\tproduct=fw`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| zscalernss_alerts | zscalernss-alerts | main | none | +| zscalernss_dns | zscalernss-dns | netdns | none | +| zscalernss_fw | zscalernss-fw | netfw | none | +| zscalernss_web | zscalernss-web | netproxy | none | +| zscalernss-zpa-app | zscalernss_zpa-app | netids | none | +| zscalernss-zpa-auth | zscalernss_zpa_auth | netauth | none | +| zscalernss-zpa-connector | zscalernss_zpa_connector | netops | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_ZSCALER_NSS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=zscalernss-* | stats count by host +``` diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 8c5794c..5b69e71 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -4,6 +4,9 @@ filter f_rfc5424_strict{ filter f_rfc5424_noversion{ message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); }; +filter f_rfc5424_epochtime{ + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) ))'); +}; rewrite set_rfcnonconformant{ set("rfc5424_nonconform" value("fields.sc4s_syslog_format")); }; @@ -19,6 +22,12 @@ rewrite set_rfc5424_noversion{ filter f_is_rfc5424_noversion{ match("rfc5424_noversion" value("fields.sc4s_syslog_format")) }; +rewrite set_rfc5424_epochtime{ + set("rfc5424_epochtime" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_epochtime{ + match("rfc5424_epochtime" value("fields.sc4s_syslog_format")) +}; rewrite set_rfc3164{ set("rfc3164" value("fields.sc4s_syslog_format")); }; diff --git a/package/etc/conf.d/filters/cisco/meraki.conf b/package/etc/conf.d/filters/cisco/meraki.conf new file mode 100644 index 0000000..c0573b5 --- /dev/null +++ b/package/etc/conf.d/filters/cisco/meraki.conf @@ -0,0 +1,22 @@ +# Meraki + +filter f_cisco_meraki { + match("cisco_meraki", value("fields.sc4s_vendor_product") type(glob)) +}; + +parser p_cisco_meraki { + channel { + filter { + message( + #'(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+ )(?.*))' + '(?:(?:<(?\d{1,3})>(?[1-9][0-9]?) (?:(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) )(?.*))' + flags(store-matches) + ); + }; + parser { + date-parser(format('%s') + template("${EPOCH}")); + }; + }; + +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/fortinet/webprotect.conf b/package/etc/conf.d/filters/fortinet/webprotect.conf new file mode 100644 index 0000000..2d669e3 --- /dev/null +++ b/package/etc/conf.d/filters/fortinet/webprotect.conf @@ -0,0 +1,3 @@ +filter f_forcepoint_webprotect_kv { + program('vendor=[Ww]ebsense'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/zscaler/nss.conf b/package/etc/conf.d/filters/zscaler/nss.conf new file mode 100644 index 0000000..9ee4e1a --- /dev/null +++ b/package/etc/conf.d/filters/zscaler/nss.conf @@ -0,0 +1,3 @@ +filter f_zscaler_nss { + message('\tvendor=Zscaler\t'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/README.md b/package/etc/conf.d/local/context/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/context/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.conf b/package/etc/conf.d/local/context/compliance_meta_by_source.conf deleted file mode 100644 index f325b4f..0000000 --- a/package/etc/conf.d/local/context/compliance_meta_by_source.conf +++ /dev/null @@ -1,5 +0,0 @@ -@version: 3.24 -filter f_test_test { - host("something-*" type(glob)) or - netmask(192.168.100.1/24) -}; diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.csv b/package/etc/conf.d/local/context/compliance_meta_by_source.csv deleted file mode 100644 index 6608db0..0000000 --- a/package/etc/conf.d/local/context/compliance_meta_by_source.csv +++ /dev/null @@ -1,2 +0,0 @@ -#f_test_test,.splunk.index,"badindex" -#f_test_test,fields.compliance,"pci" diff --git a/package/etc/conf.d/local/context/splunk_index.csv b/package/etc/conf.d/local/context/splunk_index.csv deleted file mode 100644 index e93911a..0000000 --- a/package/etc/conf.d/local/context/splunk_index.csv +++ /dev/null @@ -1,40 +0,0 @@ -#bluecoat_proxy,index,netproxy -#cef_ArcSight_ArcSight,index,netwaf -#cef_Incapsula_SIEMintegration,index,netwaf -#cef_Microsoft_Microsoft Windows,index,oswinsec -#cef_Microsoft_System or Application Event,index,oswin -#cisco_asa,index,netfw -#cisco_ios,index,netops -#cisco_nx_os,index,netops -#local_example,index,main -#fortinet_fortios_event,index,netops -#fortinet_fortios_log,index,netops -#fortinet_fortios_traffic,index,netfw -#fortinet_fortios_utm,index,netids -#juniper_idp,index,netids -#juniper_structured,index,netops -#juniper_idp_structured,index,netids -#juniper_junos_fw_structured,index,netfw -#juniper_junos_ids_structured,index,netids -#juniper_junos_utm_structured,index,netfw -#juniper_junos_fw,index,netfw -#juniper_junos_ids,index,netids -#juniper_junos_utm,index,netfw -#juniper_sslvpn,index,netfw -#juniper_netscreen,index,netfw -#juniper_nsm,index,netfw -#juniper_nsm_idp,index,netids -#juniper_legacy,index,netops -#pan_traffic,index,netfw -#pan_threat,index,netproxy -#pan_system,index,netops -#pan_config,index,netops -#pan_hipwatch,index,main -#pan_correlation,index,main -#pan_userid,index,netauth -#pan_unknown,index,netops -#proofpoint_pps_filter,index,email -#proofpoint_pps_sendmail,index,email -#sc4s_events,index,main -#sc4s_fallback,index,main -#sc4s_metrics,index,em_metrics diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.conf b/package/etc/conf.d/local/context/vendor_product_by_source.conf deleted file mode 100644 index 37e3412..0000000 --- a/package/etc/conf.d/local/context/vendor_product_by_source.conf +++ /dev/null @@ -1,37 +0,0 @@ -@version: 3.22 -#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a -#bug/limitation of selector files. The better syntax should be as follows -#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; - -filter f_test_test { - host("testvp-*" type(glob)) or - netmask(192.168.100.1/24) -}; -filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) or - netmask(192.168.1.0/24) -}; -filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) or - netmask(192.168.2.0/24) -}; -filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) -}; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or - netmask(192.168.4.0/24) -}; -filter f_cisco_nx_os { - host("csconx-*" type(glob)) or - netmask(192.168.5.0/24) -}; -filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) -}; -filter f_proofpoint_pps_filter { - host("pps-*" type(glob)) or - netmask(192.168.7.0/24) -}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.csv b/package/etc/conf.d/local/context/vendor_product_by_source.csv deleted file mode 100644 index 3f90603..0000000 --- a/package/etc/conf.d/local/context/vendor_product_by_source.csv +++ /dev/null @@ -1,8 +0,0 @@ -f_test_test,sc4s_vendor_product,"test_test" -f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" -f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" -f_juniper_idp,sc4s_vendor_product,"juniper_idp" -f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" -f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" -f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" -f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl new file mode 100644 index 0000000..ac0cb48 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -0,0 +1,36 @@ +# Forcepoint Webprotect +{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_forcepoint_webprotect_kv); +{{- end}} +{{- if eq (.) "no"}} + source (s_dedicated_port_FORCEPOINT_WEBPROTECT); +{{- end}} + + rewrite { + subst(" [^ =]+\=\-", "", value("MESSAGE"), flags("global")); + }; + rewrite { r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg"))}; + parser {p_add_context_splunk(key("forcepoint_webprotect")); }; + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for FORCEPOINT_WEBPROTECT traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for FORCEPOINT_WEBPROTECT traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl new file mode 100644 index 0000000..7bd86fc --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -0,0 +1,75 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_zscaler_nss); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_ZSCALER_NSS); +{{- end }} + + rewrite { + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + }; + parser { + #basic parsing + kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}")); + }; + + if (match("alerts" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_alerts")); }; + } elif (match("dns" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_dns")); }; + } elif (match("fw" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_fw")); }; + } elif (match("NSS" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_web")); }; + } elif (match("audit" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zia_audit")); }; + } elif (match("sandbox" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zia_sandbox")); }; + } elif (match("zpa" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa")); }; + } elif (match("zpa_auth" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_auth")); }; + } elif (match("zpa_auth_connector" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_connector")); }; + } elif (match("zpa_bba" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_bba")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("zscaler_nss")); + }; + }; + + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for ZSCALER_NSS traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for ZSCALER_NSS traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl new file mode 100644 index 0000000..dbbb1f6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl @@ -0,0 +1,42 @@ +# Checkpoint Splunk format +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_default-ports); + filter(f_cisco_meraki); +{{- end}} +{{- if eq (.) "no"}} + source (s_dedicated_port_CISCO_MERAKI); +{{- end}} + + #parser { + # kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); + # date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); + # + # }; + + #rewrite { set("${.kv.hostname}", value("HOST")); }; + + rewrite { r_set_splunk_dest_default(sourcetype("meraki"), index("netfw"), template("t_hdr_msg"))}; + parser {p_add_context_splunk(key("cisco_meraki")); }; + + + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_MERAKI traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index a9e8b78..234e87c 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -82,6 +82,9 @@ source s_default-ports { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf index f325b4f..322b938 100644 --- a/package/etc/context_templates/compliance_meta_by_source.conf +++ b/package/etc/context_templates/compliance_meta_by_source.conf @@ -1,5 +1,5 @@ @version: 3.24 -filter f_test_test { - host("something-*" type(glob)) or - netmask(192.168.100.1/24) -}; +#filter f_test_test { +# host("something-*" type(glob)) or +# netmask(192.168.100.1/24) +#}; diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index ec4f02c..7a418e3 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -15,6 +15,7 @@ #cisco_ios,index,netops #cisco_nx_os,index,netops #local_example,index,main +#forcepoint_webprotect,index,netproxy #fortinet_fortios_event,index,netops #fortinet_fortios_log,index,netops #fortinet_fortios_traffic,index,netfw diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 0903ca3..57c73ac 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -4,6 +4,14 @@ filter f_test_test { host("testvp-*" type(glob)) or netmask(192.168.100.1/24) }; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) +}; filter f_juniper_nsm { host("jnpnsm-*" type(glob)) or netmask(192.168.1.0/24) @@ -12,23 +20,19 @@ filter f_juniper_nsm_idp { host("jnpnsmidp-*" type(glob)) or netmask(192.168.2.0/24) }; -filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) -}; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or +filter f_cisco_meraki { + host("testcm-*" type(glob)) or netmask(192.168.4.0/24) }; filter f_cisco_nx_os { host("csconx-*" type(glob)) or netmask(192.168.5.0/24) -}; -filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) }; filter f_proofpoint_pps_filter { host("pps-*" type(glob)) or netmask(192.168.7.0/24) -}; \ No newline at end of file +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 3f90603..2f85bc4 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -1,4 +1,5 @@ f_test_test,sc4s_vendor_product,"test_test" +f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index f804816..09ea120 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -74,6 +74,9 @@ source s_dedicated_port_{{ .port_id}} { {{- else if eq .parser "cisco_parser" }} parser {cisco-parser()}; rewrite(set_cisco_ios); +{{- else if eq .parser "cisco_meraki_parser" }} + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); {{- else if eq .parser "rfc3164" }} parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); @@ -96,6 +99,9 @@ source s_dedicated_port_{{ .port_id}} { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 30537c7..ff9a112 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -2,15 +2,18 @@ source scl_source enable rh-python36 cd /opt/syslog-ng -for d in $(find /opt/syslog-ng/etc -type d) -do - echo Templating conf for $d - gomplate \ - --input-dir=$d \ - --template t=etc/go_templates/ \ - --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ - --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" -done +#The following is no longer needed but retained as a comment just in case we run into command line length issues +#for d in $(find /opt/syslog-ng/etc -type d) +#do +# echo Templating conf for $d +# gomplate \ +# --input-dir=$d \ +# --template t=etc/go_templates/ \ +# --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ +# --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" +#done +gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ + mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ diff --git a/tests/test_cisco_meraki.py b/tests/test_cisco_meraki.py new file mode 100644 index 0000000..920714a --- /dev/null +++ b/tests/test_cisco_meraki.py @@ -0,0 +1,35 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<134>1 1563249630.774247467 devicename security_event ids_alerted signature=1:28423:1 priority=1 timestamp=1468531589.810079 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection +def test_cisco_meraki_security_event(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}1 {% now 'utc', '%s' %}.123456789 testcm-{{ host }} security_event ids_alerted signature=1:28423:1 priority=1 timestamp={% now 'utc', '%s' %}.123456 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection\n") + message = mt.render(mark="<134>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"testcm-{{ host }}\" sourcetype=\"meraki\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +#<134>1 Dec 6 08:41:44 192.168.1.1 1 1386337316.207232138 MX84 events Cellular connection up diff --git a/tests/test_forcepoint_web.py b/tests/test_forcepoint_web.py new file mode 100644 index 0000000..7ecf4cb --- /dev/null +++ b/tests/test_forcepoint_web.py @@ -0,0 +1,35 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<134>Oct 16 12:13:06 sourcehost2 vendor=Websense 9f product=Security product_version=7.7.0 action=permitted severity=7 category=755 user=LDAP://user7 OU=Users,OU=Beijing,DC=com/TEST\, TEST_NAME src_host=10.0.0.4 src_port=61435 dst_host=HOST-013 dst_ip=10.0.0.19 dst_port=25404 bytes_out=4074 bytes_in=12328 http_response=200 http_method=POST http_content_type=image/gif;charset=UTF-8 http_user_agent=Mozilla/3.0 (Windows; U; Windows NT 6.1; es-def; rv:1.7.0.11) Gecko/2009060215 Firefox/8.0.11 (.NET CLR 8.5.30729) http_proxy_status_code=200 reason=- disposition=2573 policy=role-8**Default role=4 duration=63 url=http://test_web.com/contents/content1.jpg +def test_forcepoint_webprotect_kv(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} vendor=Websense 9f product=Security product_version=7.7.0 action=permitted severity=7 category=755 user=LDAP://user7 OU=Users,OU=Beijing,DC=com/TEST\, TEST_NAME src_host=10.0.0.4 src_port=61435 dst_host=HOST-013 dst_ip=10.0.0.19 dst_port=25404 bytes_out=4074 bytes_in=12328 http_response=200 http_method=POST http_content_type=image/gif;charset=UTF-8 http_user_agent=Mozilla/3.0 (Windows; U; Windows NT 6.1; es-def; rv:1.7.0.11) Gecko/2009060215 Firefox/8.0.11 (.NET CLR 8.5.30729) http_proxy_status_code=200 reason=- disposition=2573 policy=role-8**Default role=4 duration=63 url=http://test_web.com/contents/content1.jpg unknownfield=-\n") + message = mt.render(mark="<134>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"websense:cg:kv\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +#<134>1 Dec 6 08:41:44 192.168.1.1 1 1386337316.207232138 MX84 events Cellular connection up diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py new file mode 100644 index 0000000..e0f7fb1 --- /dev/null +++ b/tests/test_zscaler_proxy.py @@ -0,0 +1,55 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +#Note the long white space is a \t +#2019-10-16 15:44:36 reason=Allowed event_id=6748427317914894361 protocol=HTTPS action=Allowed transactionsize=663 responsesize=65 requestsize=598 urlcategory=UK_ALLOW_Pharmacies serverip=216.58.204.70 clienttranstime=0 requestmethod=CONNECT refererURL=None useragent=Windows Windows 10 Enterprise ZTunnel/1.0 product=NSS location=UK_Wynyard_VPN->other ClientIP=192.168.0.38 status=200 user=first.last@example.com url=4171764.fls.doubleclick.net:443 vendor=Zscaler hostname=4171764.fls.doubleclick.net clientpublicIP=213.86.221.94 threatcategory=None threatname=None filetype=None appname=DoubleClick pagerisk=0 department=Procurement, Generics urlsupercategory=User-defined appclass=Sales and Marketing dlpengine=None urlclass=Bandwidth Loss threatclass=None dlpdictionaries=None fileclass=None bwthrottle=NO servertranstime=0 md5=None +def test_zscaler_proxy(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894361\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# +def test_zscaler_proxy_pri(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894362\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# From 11c4b3ac705fa07616bc483733d704e6f2c332aa Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Fri, 18 Oct 2019 07:28:14 -0400 Subject: [PATCH 04/10] Fixes #168 (#169) --- docs/gettingstarted/docker-swarm-general.md | 8 ++++---- docs/gettingstarted/docker-swarm-rhel7.md | 8 ++++---- docs/gettingstarted/docker-systemd-general.md | 8 ++++---- docs/gettingstarted/podman-systemd-general.md | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 420065f..9a03de3 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -30,8 +30,8 @@ services: env_file: - /opt/sc4s/env_file volumes: - - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local - - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer + - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z # Uncomment the following line if custom TLS certs are provided # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` @@ -161,8 +161,8 @@ services: env_file: - /opt/sc4s/env_file volumes: - - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local - - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer + - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z #Uncomment the following line if custom TLS certs are provided # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index e8ee2ac..0d27850 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -57,8 +57,8 @@ services: env_file: - /opt/sc4s/env_file volumes: - - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local - - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer + - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z # Uncomment the following line if custom TLS certs are provided # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` @@ -190,8 +190,8 @@ services: env_file: - /opt/sc4s/env_file volumes: - - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local - - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer + - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z #Uncomment the following line if custom TLS certs are provided # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index df73c6c..51dda09 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -18,10 +18,10 @@ Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs -Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Mount point for local disk buffer (required) -Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -146,10 +146,10 @@ Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs -Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Mount point for local disk buffer (required) -Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 353bfa1..e0964a5 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -18,10 +18,10 @@ Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs -Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Mount point for local disk buffer (required) -Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -146,10 +146,10 @@ Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs -Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Mount point for local disk buffer (required) -Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" From be74c0c28c83af1802af513ae016426b769dfac9 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Thu, 24 Oct 2019 21:06:43 -0700 Subject: [PATCH 05/10] Release/1.0.3 (#176) * This change makes the location of env_file consistent however this is a procedural breaking change a deployment using the earlier docs will have the file in the wrong place. Users should move the file from opt/sc4s/default/env_file to opt/sc4s/env_file and revise the sc4s.service file to correct * Sync content from readme to RTD index page * Set proper sourcetype for checkpoint string (#175) --- README.md | 20 +++++++++++++------ docs/configuration.md | 2 +- docs/gettingstarted/byoe-rhel7.md | 4 ++-- docs/gettingstarted/docker-swarm-general.md | 2 +- docs/gettingstarted/docker-swarm-rhel7.md | 2 +- docs/gettingstarted/docker-systemd-general.md | 12 +++++------ docs/gettingstarted/podman-systemd-general.md | 12 +++++------ docs/index.md | 15 ++++++++++++++ docs/sources.md | 9 ++++++++- 9 files changed, 54 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 64043d5..3ba9077 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,15 @@ For full usage instructions, please visit the Splunk Connect for Syslog [documen ## Support -Please use the GitHub issue tracker to submit bugs or request features. - -If you have questions or need support, you can: +Splunk Connect for Syslog is an open source product developed by Splunkers with contributions from the community of partners and customers. This unique product will be enhanced, maintained and supported by the community, led by Splunkers with deep subject matter expertise. The primary reason why Splunk is taking this approach is to push product development closer to those that use and depend upon it. This direct connection will help us all be more successful and move at a rapid pace. Post a question to Splunk Answers using the tag "Splunk Connect For Syslog" -Join the #splunk-connect-for-syslog room in the splunk-usergroups Slack Workspace + +Join the #splunk-connect-for-syslog room in the splunk-usergroups Slack Workspace. If you don't yet have an account [sign up](https://docs.splunk.com/Documentation/Community/1.0/community/Chat) + +Please use the GitHub issue tracker to submit bugs or request enhancements: https://github.com/splunk/splunk-connect-for-syslog/issues + +Get involved, try it out, ask questions, contribute filters, and make new friends! ## Contributing @@ -33,6 +36,11 @@ We welcome feedback and contributions from the community! Please see our [contri ## License -Configuration and documentation licensed subject to [CC0](LICENSE-CC0) +* Configuration and documentation licensed subject to [CC0](LICENSE-CC0) + +* Code and scripts licensed subject to [BSD-2-Clause](LICENSE-BSD2) + +* Third Party Red Hat Universal Base Image see [License](https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf) + +* Third Party Syslog-NG (OSE) [License](https://github.com/balabit/syslog-ng) -Code and scripts licensed subject to [BSD-2-Clause](LICENSE-BSD2) \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md index 701fffa..02c2d18 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,7 +39,7 @@ and variables needed to properly configure SC4S for your environment. * Create a folder ``/opt/sc4s/tls`` * Save the server private key in PEM format with NO PASSWORD to ``/opt/sc4s/tls/server.key`` * Save the server certificate in PEM format to ``/opt/sc4s/tls/server.pem`` -* Add the following line to ``/opt/sc4s/default/env_file`` +* Add the following line to ``/opt/sc4s/env_file`` ```dotenv SC4S_SOURCE_TLS_ENABLE=yes diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 47d2f7c..73b7b33 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -84,7 +84,7 @@ ExecStart=/usr/sbin/syslog-ng -F $SYSLOGNG_OPTS -p /var/run/syslogd.pid ExecReload=/bin/kill -HUP $MAINPID EnvironmentFile=-/etc/default/syslog-ng EnvironmentFile=-/etc/sysconfig/syslog-ng -EnvironmentFile=/opt/sc4s/default/env_file +EnvironmentFile=/opt/sc4s/env_file StandardOutput=journal StandardError=journal Restart=on-failure @@ -125,7 +125,7 @@ mkdir -p /opt/syslog-ng/var/data/disk-buffer/ sudo bash /opt/sc4s/bin/preconfig.sh ``` -* Create the file ``/opt/sc4s/default/env_file`` and add the following environment variables: +* Create the file ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv SYSLOGNG_OPTS=-f /opt/syslog-ng/etc/syslog-ng.conf diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 9a03de3..8f536d1 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -167,7 +167,7 @@ services: # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 0d27850..ec93e82 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -196,7 +196,7 @@ services: # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 51dda09..27fa688 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -30,12 +30,12 @@ Restart=always ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE ExecStartPre=/usr/bin/docker run \ - --env-file=/opt/sc4s/default/env_file \ + --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight --rm \ $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp \ - --env-file=/opt/sc4s/default/env_file \ + --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S --rm \ @@ -68,7 +68,7 @@ unit file above. Failure to do this will cause SC4S to abort at startup. ## Configure the SC4S environment -Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: +Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -157,12 +157,12 @@ TimeoutStartSec=0 Restart=always ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE ExecStartPre=/usr/bin/docker run \ - --env-file=/opt/sc4s/default/env_file \ + --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight --rm \ $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ - --env-file=/opt/sc4s/default/env_file \ + --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S \ @@ -170,7 +170,7 @@ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 - $SC4S_IMAGE ``` -* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index e0964a5..1dea7bd 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -30,12 +30,12 @@ Restart=always ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE ExecStartPre=/usr/bin/podman run \ - --env-file=/opt/sc4s/default/env_file \ + --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight --rm \ $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp \ - --env-file=/opt/sc4s/default/env_file \ + --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S --rm \ @@ -68,7 +68,7 @@ unit file above. Failure to do this will cause SC4S to abort at startup. ## Configure the sc4s environment -Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: +Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -157,12 +157,12 @@ TimeoutStartSec=0 Restart=always ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE ExecStartPre=/usr/bin/podman run \ - --env-file=/opt/sc4s/default/env_file \ + --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight --rm \ $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ - --env-file=/opt/sc4s/default/env_file \ + --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S \ @@ -170,7 +170,7 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 - $SC4S_IMAGE ``` -* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment diff --git a/docs/index.md b/docs/index.md index 19dfcad..c7c293a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,21 @@ HTTP event Collector. * Reduce latency and improve scale by balancing event distribution across Splunk Indexers +## Support + +Splunk Connect for Syslog is an open source product developed by Splunkers with contributions from the community of partners and customers. This unique product will be enhanced, maintained and supported by the community, led by Splunkers with deep subject matter expertise. The primary reason why Splunk is taking this approach is to push product development closer to those that use and depend upon it. This direct connection will help us all be more successful and move at a rapid pace. + +Post a question to Splunk Answers using the tag "Splunk Connect For Syslog" + +Join the #splunk-connect-for-syslog room in the splunk-usergroups Slack Workspace. If you don't yet have an account [sign up](https://docs.splunk.com/Documentation/Community/1.0/community/Chat) + +Please use the GitHub issue tracker to submit bugs or request enhancements: https://github.com/splunk/splunk-connect-for-syslog/issues + +Get involved, try it out, ask questions, contribute filters, and make new friends! + +## Contributing + +We welcome feedback and contributions from the community! Please see our [contribution guidelines](CONTRIBUTING.md) for more information on how to get involved. ## License diff --git a/docs/sources.md b/docs/sources.md index f7e85cb..5b912bc 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -1,3 +1,10 @@ +# Introduction +When using Splunk Connect for Syslog to onboard a data source, the SC4S filter performs the operations that are traditionally performed at index-time by the corresponding Technical Add-on installed there. These index-time operations include linebreaking, sourcetype setting and timestamping. For this reason, if a data source is exclusively onboarded using SC4S then you will not need to install its corresponding Add-On on the indexers. You must, however, install the Add-on on the search head(s) for the user communities interested in this data source. + +SC4S "unique" filters are based either on the port upon which events arrive or the hostname/CIDR block from which they are sent. The "soup" filters run for events that arrive on port 514 (default for syslog), and contain regex and other syslog-specific parsers to identify events from a specific source, apply the correct sourcetype, and set other metadata. Data sources which generate events that are not unique enough to accurately identify with soup filters _must_ employ the "unique" filters (port/hostname/CIDR block) instead -- the soup filters are unavailable for these sources. + +If SC4S receives an event on port 514 which has no soup filter, that event will be given a "fallback" sourcetype. If you see events in Splunk with the fallback sourcetype, then you should figure out what source the events are from and determine why these events are not being sourcetyped correctly. The most common reason for events categorized as "fallback" is the lack of a SC4S filter for that source, and in some cases a misconfigured relay which alters the integrity of the message format. In most cases this means a new SC4S filter must be developed. In this situation you can either build a filter or file an issue with the community to request help. + # Vendor - Checkpoint ## Product - Log Exporter (Splunk) @@ -54,7 +61,7 @@ MSG Parse: This filter parses message content Use the following search to validate events are present ``` -index= sourcetype=cisco:asa +index= sourcetype=cp_log ``` Verify timestamp, and host values match as expected From 3f8c37076fc0eca5a74bf1f8bab469331b49f60a Mon Sep 17 00:00:00 2001 From: mkarlstrand-splunk <49571555+mkarlstrand-splunk@users.noreply.github.com> Date: Wed, 6 Nov 2019 11:50:57 -0800 Subject: [PATCH 06/10] Update configuration.md (#183) Updates to explain disk buffering --- docs/configuration.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 02c2d18..aa91a1a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -83,4 +83,26 @@ redeploy the updated service using the command: - /opt/sc4s/default/compliance_meta_by_source.csv:/opt/syslog-ng/etc/context-local/compliance_meta_by_source.csv - /opt/sc4s/default/compliance_meta_by_source.conf:/opt/syslog-ng/etc/context-local/compliance_meta_by_source.conf `` +## Data Durability - Local Disk Buffer Configuration +SC4S provides capability to minimize the number of lost events if the connection to all the Splunk Indexers goes down. This capability utilizes the disk buffering feature of Syslog-ng. SC4S receives a response from the Splunk HTTP Event Collector (HEC) when a message is received successfully. If a confirmation message from the HEC endpoint is not received (or a “server busy” reply, such as a “503” is sent), the load balancer will try the next HEC endpoint in the pool. If all pool members are exhausted (such as would occur if there were a full network outage to the HEC endpoints), events will queue to the local disk buffer on the SC4S Linux host. SC4S will continue attempting to send the failed events while it buffers all new incoming events to disk. If the disk space allocated to disk buffering fills up then SC4S will stop accepting new events and subsequent events will be lost. Once SC4S gets confirmation that events are again being received by one or more indexers, events will then stream from the buffer using FIFO queueing. The number of events in the disk buffer will reduce as long as the incoming event volume is less than the maximum SC4S (with the disk buffer in the path) can handle. When all events have been emptied from the disk buffer, SC4S will resume streaming events directly to Splunk. + +For more detail on the Syslog-ng behavior the documentation can be found here: https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.22/administration-guide/55#TOPIC-1209280 + +SC4S has disk buffering enabled by default and it is strongly recommended that you keep it on, however this feature does have a performance cost. +Without disk buffering enabled SC4S can handle up to 345K EPS (800 bytes/event avg) +With “Normal” disk buffering enabled SC4S can handle up to 60K EPS (800 bytes/event avg) -- This is still a lot of data! + +To guard against data loss it is important to configure the appropriate type and amount of storage for SC4S disk buffering. To estimate the storage allocation its best to start with your estimated maximum events per second that each SC4S server will experience. Based on the maximum throughput of SC4S with disk buffering enabled, the conservative estimate for maximum events per second is 60K (however, you should use the maximum rate in your environment for this calculation, not the max rate SC4S can handle). Next is your average estimated event size based on your data sources. It is common industry practice to estimate log events as 800 bytes on average. And the final input to the sizing estimation would be the maximum length of connectivity downtime you want disk buffering to be able to handle. This measure is very much dependent on your risk tolerance. For example, to protect against a full day of lost connectivity from SC4S to all your indexers at maximum throughput the calculation would look like the following... + +60,000 EPS * 86400 seconds * 800 bytes = 3.77186 TB of storage + +To configure storage allocation for the SC4S disk buffering, do the following... +Edit the file /opt/sc4s/default/env_file +Add the SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE variable to the file and set the value to the number of bytes based on your estimation (e.g. 4147200000000 in the example above) +Splunk does not recommend reducing the disk allocation below 500 GB +Restart SC4S + +Given that in a connectivity outage to the Indexers events will be saved and read from disk until the buffer is emptied, it is ideal to use the fastest type of storage available. For this reason, NVMe storage is recommended for SC4S disk buffering. + +It is best to design your deployment so that the disk buffer will drain after connectivity is restored to the Splunk Indexers (while incoming data continues at the same general rate). Since "your mileage may vary" with different combinations of data load, instance type, and disk subsystem performance, it is good practice to provision a box that performs twice as well as is required for your max EPS. This headroom will allow for rapid recovery after a connectivity outage. From 8d007ebb57f5a348829a2a4fcc729e14233ee5ae Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Tue, 12 Nov 2019 09:55:38 -0500 Subject: [PATCH 07/10] Release/1.1.0 (#189) * Feature/cisco ise (#178) This merge aggregates Cisco ISE messages from multiple segments into a single event in Splunk * Support for archival file destinations (#179) This merge add support for file archival in syslog-ng EWMM format * Feature/improvedmetrics (#185) * This change correct an incorrect URL issue preventing metrics flow and cleans up related documentation and filters anon metrics * Update Metrics for Splunk 8 * Resolve splunk sdk for CI unit tests move * Add disk buffer envrionment variables table (#184) * Add disk buffer envrionment variables table Add env var table to docs Adjust default disk buffer to 50 GB * Disk Buffer refinement Perform internal math to convert specified buffer sizes to values appropriate for internal (syslog-ng) destination options, which are set per worker. * Add Ubiquiti Networks Unifi product range (#188) * Resolve fallback events in msg rather than JSON mode * Add support for Ubiquiti networks unifi product line AP, Switch, USG (firewalls) --- .env.template | 4 - docker-compose-ci.yml | 1 - docker-compose-debug.yml | 1 - docker-compose-demo.yml | 1 - docker-compose-perf.yml | 1 - docker-compose.yml | 13 +- docs/configuration.md | 42 +++++- docs/gettingstarted/byoe-rhel7.md | 5 +- docs/gettingstarted/docker-swarm-general.md | 10 +- docs/gettingstarted/docker-swarm-rhel7.md | 10 +- docs/gettingstarted/docker-systemd-general.md | 10 +- docs/gettingstarted/podman-systemd-general.md | 10 +- docs/sources.md | 123 +++++++++++++++- package/Dockerfile | 4 - .../conf.d/conflib/_splunk/splunkfields.conf | 1 - .../etc/conf.d/destinations/archive_file.conf | 5 + .../conf.d/destinations/splunk_hec.conf.tmpl | 13 +- .../splunk_hec_internal.conf.tmpl | 4 +- .../destinations/splunk_hec_metrics.conf.tmpl | 2 +- .../etc/conf.d/filters/Ubiquiti/unifi.conf | 7 + package/etc/conf.d/filters/cisco/ise.conf | 4 + .../local/config/log_paths/example.conf.tmpl | 6 +- .../etc/conf.d/log_paths/internal.conf.tmpl | 28 +--- .../p_rfc3164-checkpoint_splunk.conf.tmpl | 50 ++++--- .../log_paths/p_rfc3164-cisco_asa.conf.tmpl | 30 +++- .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 30 +++- .../log_paths/p_rfc3164-cisco_ise.conf.tmpl | 106 ++++++++++++++ .../log_paths/p_rfc3164-cisco_nxos.conf.tmpl | 32 +++- .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 27 +++- .../p_rfc3164-fortinet_fortios.conf.tmpl | 38 +++-- .../log_paths/p_rfc3164-juniper_idp.conf.tmpl | 33 ++++- .../p_rfc3164-juniper_junos.conf.tmpl | 40 +++-- .../p_rfc3164-juniper_netscreen.conf.tmpl | 30 +++- .../log_paths/p_rfc3164-juniper_nsm.conf.tmpl | 32 +++- .../p_rfc3164-juniper_nsm_idp.conf.tmpl | 32 +++- .../p_rfc3164-microfocus_arcsight.conf.tmpl | 36 +++-- .../p_rfc3164-paloalto_panos.conf.tmpl | 42 ++++-- .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 29 +++- ..._rfc3164-proofpoint_pps_sendmail.conf.tmpl | 28 +++- .../p_rfc3164-ubiquiti_unifi.conf.tmpl | 137 ++++++++++++++++++ .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 48 ++++-- .../p_rfc5424-noversion_cisco_asa.conf.tmpl | 28 +++- ...rfc5424-noversion_symantec_proxy.conf.tmpl | 30 +++- .../p_rfc5424-strict_juniper_junos.conf.tmpl | 38 +++-- .../p_rfc5424_epoch-cisco_meraki.conf.tmpl | 49 +++++++ .../p_rfc5424_epoch-cisco_merkai.conf.tmpl | 42 ------ package/etc/conf.d/log_paths/zfallback.conf | 7 +- package/etc/conf.d/sources/network.conf.tmpl | 103 +------------ .../etc/context_templates/splunk_index.csv | 1 + .../vendor_product_by_source.conf | 4 + .../vendor_product_by_source.csv | 3 +- package/etc/go_templates/source_network.t | 22 ++- .../local_config/log_paths/example.conf.tmpl | 6 +- package/etc/syslog-ng.conf | 8 +- package/sbin/entrypoint.sh | 14 +- .../apps/SA-syslog-ng/default/indexes.conf | 11 ++ tests/requirements.txt | 2 +- tests/test_cisco_ise.py | 52 +++++++ tests/test_common.py | 12 ++ tests/test_ubiquiti_unifi.py | 93 ++++++++++++ 60 files changed, 1195 insertions(+), 435 deletions(-) create mode 100644 package/etc/conf.d/destinations/archive_file.conf create mode 100644 package/etc/conf.d/filters/Ubiquiti/unifi.conf create mode 100644 package/etc/conf.d/filters/cisco/ise.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl create mode 100644 tests/test_cisco_ise.py create mode 100644 tests/test_ubiquiti_unifi.py diff --git a/.env.template b/.env.template index c1f7af2..f360cd6 100644 --- a/.env.template +++ b/.env.template @@ -12,10 +12,6 @@ SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SPLUNK_PASSWORD=Changed@11 SPLUNK_START_ARGS=--accept-license SPLUNK_HEC_URL=https://splunk:8088/services/collector/event -SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download #SPLUNKBASE_USERNAME=username #SPLUNKBASE_PASSWORD=password diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 6028e6e..a699498 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -33,7 +33,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml index 9b885f3..bb22763 100644 --- a/docker-compose-debug.yml +++ b/docker-compose-debug.yml @@ -33,7 +33,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml index c4fb88f..3934fe1 100644 --- a/docker-compose-demo.yml +++ b/docker-compose-demo.yml @@ -34,7 +34,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml index 9831278..e1bd795 100644 --- a/docker-compose-perf.yml +++ b/docker-compose-perf.yml @@ -26,7 +26,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose.yml b/docker-compose.yml index a9e5e74..082040f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,8 @@ services: RH_ORG: ${RH_ORG} RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s - command: -det +#When this is enabled test_common will fail +# command: -det ports: - "514:514" - "601:601" @@ -43,14 +44,16 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - SC4S_SOURCE_TLS_ENABLE=no - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SC4S_LISTEN_DEFAULT_TCP_PORT=514 + - SC4S_LISTEN_DEFAULT_UDP_PORT=514 +# - SC4S_LISTEN_DEFAULT_TLS_PORT=6514 - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + - SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT=6000 +# - SC4S_ARCHIVE_CHECKPOINT=yes + - SC4S_ARCHIVE_GLOBAL=yes volumes: - ./tls:/opt/syslog-ng/tls splunk: diff --git a/docs/configuration.md b/docs/configuration.md index aa91a1a..379edf7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,11 +21,51 @@ and variables needed to properly configure SC4S for your environment. | SC4S_DEST_SPLUNK_HEC_SSL_VERSION | comma separated list | Open SSL version list | | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | +## SC4S Disk Buffer Configuration + +Disk buffers in SC4S are allocated _per destination_. In the future as more destinations are supported, a separate list of variables +will be used for each. This is why you see the `DEST_SPLUNK_HEC` in the variable names below. +* NOTE: "Reliable" disk buffering offeres little advantage over "normal" disk buffering, at a significant performance penalty. +For this reason, normal disk buffering is recommended. +* NOTE: If you add destinations locally in your configuration, pay attention to the _cumulative_ buffer requirements when allocating local +disk. +* NOTE: The values for the variables below represent the _total_ sizes of the buffers for the destination. These sizes are divded by the +number of workers (threads) when setting the actual syslog-ng buffer options, because the buffer options apply to each worker rather than the +entire destination. Pay careful attention to this when using the "BYOE" version of SC4S, where direct access to the syslog-ng config files +may hide this nuance. + +| Variable | Values/Default | Description | +|----------|---------------|-------------| +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE | yes(default) or no | Enable local disk buffering | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE | yes or no(default) | Enable reliable/normal disk buffering (normal recommended) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE | bytes (10241024) | Memory buffer size in bytes (used with reliable disk buffering) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH |messages (15000) | Memory buffer size in message count (used with normal disk buffering) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE | bytes (53687091200) | size of local disk buffer in bytes (default 50 GB) | + +## Archive File Configuration + +This feature is designed to support "compliance" archival of all messages. To enable this feature update the Unit file +or docker compose to mount an appropriate host folder to the container folder ``/opt/syslog-ng/var/archive``. +The files will be stored in a folder structure using the naming pattern +``${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"``. +This pattern will create one file per "vendor_product" per minute with records formatted using syslog-ng's EWMM template. + +**WARNING POTENTIAL OUTAGE CAUSING CONSEQUENCE** + +SC4S does not prune the files that are created. The administrator must provide a means of log rotation to prune files +and/or move them to an archival system to avoid disk space failures. + +| Variable | Values | Description | +|----------|---------------|-------------| +| SC4S_ARCHIVE_GLOBAL | yes or undefined | Enable archive of all vendor_products | +| SC4S_ARCHIVE_LISTEN_ | yes(default) or undefined | See sources section of documentation enables selective archival | + + ## Syslog Source Configuration | Variable | Values/Default | Description | |----------|----------------|-------------| -| SC4S_SOURCE_TLS_ENABLE | no(default) or yes | Enable a TLS listener on port 6514 | +| SC4S_LISTEN_DEFAULT_TLS_PORT | undefined or 6514 | Enable a TLS listener on port 6514 | | SC4S_SOURCE_TLS_OPTIONS | See openssl | List of SSl/TLS protocol versions to support | | SC4S_SOURCE_TLS_CIPHER_SUITE | See openssl | List of Ciphers to support | | SC4S_SOURCE_TCP_MAX_CONNECTIONS | 2000 | Max number of TCP Connections | diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 73b7b33..d970dcd 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -129,12 +129,9 @@ sudo bash /opt/sc4s/bin/preconfig.sh ```dotenv SYSLOGNG_OPTS=-f /opt/syslog-ng/etc/syslog-ng.conf -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 8f536d1..6fd894f 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -65,12 +65,9 @@ of events in the event of network failure to the Splunk infrastructure. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -179,12 +176,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index ec93e82..5ef0e66 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -92,12 +92,9 @@ again upon restart. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -208,12 +205,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 27fa688..94b164f 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -71,12 +71,9 @@ unit file above. Failure to do this will cause SC4S to abort at startup. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -182,12 +179,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 1dea7bd..1d0ddc3 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -71,12 +71,9 @@ unit file above. Failure to do this will cause SC4S to abort at startup. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -182,12 +179,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/sources.md b/docs/sources.md index 5b912bc..22c3aab 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -191,6 +191,52 @@ Use the following search to validate events are present, for NX-OS, WLC and ACI index= sourcetype=cisco:ios | stats count by host ``` +## Product - ISE + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/1915/ | +| Product Manual | https://www.cisco.com/c/en/us/td/docs/security/ise/2-6/Cisco_ISE_Syslogs/Cisco_ISE_Syslogs/Cisco_ISE_Syslogs_chapter_00.html | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cisco:ise:syslog | Aggregation used | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_ise | cisco:ise:syslog | netauth | None | + + +### Filter type + +PATTERN MATCH + +### Setup and Configuration + +* No special steps required + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_ISE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_ISE_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=cisco:ise:syslog +``` + +Verify timestamp, and host values match as expected + ## Product - Meraki Product Line MR, MS, MX, MV | Ref | Link | @@ -237,9 +283,6 @@ Use the following search to validate events are present index= sourcetype=merkai ``` -Verify timestamp, and host values match as expected - - Verify timestamp, and host values match as expected # Vendor - Forcepoint @@ -905,6 +948,80 @@ index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` +# Vendor - Ubiquiti - Unifi + +All Ubiquity Unfi firewalls, switches, and access points share a common syslog configuration via the NMS. + + +* Login to NMS +* Navigate to settings +* Navigate to Site +* Enable Remote syslog server +* Enter hostname and port +* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf `` update the host or ip mask for ``f_ubiquiti_unifi_fw`` to identify USG firewalls + +## Product - Unifi Switch and Access Points + +Unifi devices are managed using the Network Management Controller + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/4107/ | +| Product Manual | https://https://help.ubnt.com/ | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| ubnt | Used when no sub source type is required by add on | +| ubnt:fw | USG events | +| ubnt:threat | USG IDS events | +| ubnt:switch | Unifi Switches | +| ubnt:wireless | Access Point logs | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| ubiquiti_unifi | ubnt | netops | none | +| ubiquiti_unifi_fw | ubnt:fw | netfw | none | +| ubiquiti_unifi_link | ubnt:link | netops | none | +| ubiquiti_unifi_sudo | ubnt:sudo | netops | none | +| ubiquiti_unifi_switch | ubnt:switch | netops | none | +| ubiquiti_unifi_threat | ubnt:threat | netids | none | +| ubiquiti_unifi_wireless | ubnt:wireless | netops | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_ZSCALER_NSS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=zscalernss-* | stats count by host +``` + + # Vendor - Zscaler ## Product - All Products diff --git a/package/Dockerfile b/package/Dockerfile index 7008a31..a5cbb4b 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -72,8 +72,6 @@ RUN cd /tmp ;\ source scl_source enable rh-python36 ENV DEBCONF_NONINTERACTIVE_SEEN=true -ENV SPLUNK_CONNECT_METHOD=hec -ENV SYSLOGNG_HEC_WORKERS=10 RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh COPY goss.yaml /etc/goss.yaml @@ -98,8 +96,6 @@ EXPOSE 514 EXPOSE 601/tcp EXPOSE 6514/tcp -ENV SPLUNK_CONNECT_METHOD=UF - ENTRYPOINT ["/entrypoint.sh", "-F"] HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf index c195deb..bf911a6 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf @@ -17,7 +17,6 @@ block rewrite r_set_splunk_dest_default( set("`index`", value(".splunk.index")); set("`source`", value(".splunk.source")); set("`sourcetype`", value(".splunk.sourcetype")); - set("`template`", value("fields.sc4s_template")); }; diff --git a/package/etc/conf.d/destinations/archive_file.conf b/package/etc/conf.d/destinations/archive_file.conf new file mode 100644 index 0000000..efd7990 --- /dev/null +++ b/package/etc/conf.d/destinations/archive_file.conf @@ -0,0 +1,5 @@ +destination d_archive { + file("/opt/syslog-ng/var/archive/${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log" + template('$(format-ewmm)') + ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 6a31372..bc92162 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec { http( - url("{{- getenv "SPLUNK_HEC_URL"}}") + url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers({{- getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}}) @@ -18,15 +18,14 @@ destination d_hec { disk-buffer( {{- if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes"}} - mem-buf-size({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}}) + mem-buf-size({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) reliable(yes) {{- else}} - mem-buf-length({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}}) + mem-buf-length({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) reliable(no) {{- end}} - - disk-buf-size({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}}) - dir("/opt/syslog-ng/var/data/disk-buffer/") + disk-buf-size({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "53687091200") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) + dir("/opt/syslog-ng/var/data/disk-buffer/") ) {{- end}} tls(peer-verify({{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) @@ -43,7 +42,7 @@ destination d_hec { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$(template ${fields.sc4s_template} $(template "t_standard")) + event=$MSG fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl index 875d3fb..550063c 100644 --- a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec_internal { http( - url("{{- getenv "SPLUNK_HEC_URL"}}") + url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers(10) @@ -28,7 +28,7 @@ destination d_hec_internal { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$(template ${fields.sc4s_template} $(template "t_standard")) + event=$MSG fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl index a473a04..2593b8c 100644 --- a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl @@ -1,6 +1,6 @@ destination d_hecmetrics { http( - url("{{- getenv "SPLUNK_HEC_URL"}}") + url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector") method("POST") batch-lines(50) batch-bytes(1024Kb) diff --git a/package/etc/conf.d/filters/Ubiquiti/unifi.conf b/package/etc/conf.d/filters/Ubiquiti/unifi.conf new file mode 100644 index 0000000..49740eb --- /dev/null +++ b/package/etc/conf.d/filters/Ubiquiti/unifi.conf @@ -0,0 +1,7 @@ +filter f_ubiquiti_unifi { + host('^U[^,]{1,10},[a-z0-9]{9,16},v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6}') + or + program('^U[^,]{1,10},[a-z0-9]{9,16},v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6}') + or + match("ubiquiti_unifi_*", value("fields.sc4s_vendor_product") type(glob) ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco/ise.conf b/package/etc/conf.d/filters/cisco/ise.conf new file mode 100644 index 0000000..e9a2c8a --- /dev/null +++ b/package/etc/conf.d/filters/cisco/ise.conf @@ -0,0 +1,4 @@ + +filter f_cisco_ise { + program("CISE_.*"); +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl index a8ac264..6a68b45 100644 --- a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl +++ b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl @@ -27,7 +27,7 @@ log { # The first time this template is used the log_path will be linked to the default port {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); # Filters should be updated to use the simplest and most effecient logic possible to discard # the message from this path @@ -40,12 +40,12 @@ log { # In the second pass through the template a link to the dedicated port is used. This # normally does not require additional filters -source (s_dedicated_port_LOCAL_EXAMPLE); +source (s_LOCAL_EXAMPLE); {{- end}} #Set a default sourcetype and index - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; #using the key "local_example" find any cutomized index,source or sourcetype meta values diff --git a/package/etc/conf.d/log_paths/internal.conf.tmpl b/package/etc/conf.d/log_paths/internal.conf.tmpl index 7ed823e..c751b9e 100644 --- a/package/etc/conf.d/log_paths/internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/internal.conf.tmpl @@ -8,35 +8,21 @@ log { parser {p_add_context_splunk(key("sc4s_metrics")); }; rewrite { - subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', - ' - {"time": "$S_UNIXTIME.$S_MSEC", - "event": "metric", - "host": "$HOST", - "index": "${.splunk.index}", - "source": "internal", - "sourcetype": "${.splunk.sourcetype}", - "fields": { - "source_name": "${SourceName}", - "source_instance": "${SourceInstance}", - "state": "${State}", - "type": "${Type}", - "_value": ${Number}, - "metric_name": "syslogng.${SourceId}" - } - } - ', + subst('Log statistics; ', '', value("MESSAGE"), flags("utf8" "global")); + subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global")); + subst('(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27,? ?', +'{"time": "$S_UNIXTIME.$S_MSEC","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}} +', value("MESSAGE") flags("utf8" "global") - ); + ); }; destination(d_hecmetrics); #--HEC-- } else { - {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}} destination(d_stdout); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events")); }; destination(d_hec_internal); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index 0ad2806..ee81c82 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -1,64 +1,80 @@ -# Checkpoint Splunk format -{{- if (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT") "no") "no") }} +# Generate the custom port if defined {{ $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_checkpoint_splunk); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CHECKPOINT_SPLUNK); + source (s_CHECKPOINT_SPLUNK); {{- end}} parser { kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); - }; - rewrite { set("${.kv.hostname}", value("HOST")); }; + rewrite { + set("${.kv.hostname}", value("HOST")); + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cp_log"), index("netops")) + }; - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"), template("t_hdr_msg"))}; parser {p_add_context_splunk(key("checkpoint_splunk")); }; if { filter(f_checkpoint_splunk_NetworkTraffic); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; } elif { filter(f_checkpoint_splunk_Web); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; } elif { filter(f_checkpoint_splunk_NetworkSessions); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; } elif { filter(f_checkpoint_splunk_IDS_Malware); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; } elif { filter(f_checkpoint_splunk_IDS); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; } elif { filter(f_checkpoint_splunk_email); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; } elif { filter(f_checkpoint_splunk_DLP); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; }; - parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index 5e56956..e276484 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -1,25 +1,43 @@ # Cisco ASA -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common"}} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_cisco_asa); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_ASA_LEGACY); + source (s_CISCO_ASA_LEGACY); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + rewrite { + set("cisco_asa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw")) + }; parser {p_add_context_splunk(key("cisco_asa")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ASA_LEGACY") }} + destination(d_archive); +{{- end}} + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index ea17e4e..3c51cf6 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -1,26 +1,44 @@ # Cisco IOS -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_IOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_cisco_ios); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_CISCO_IOS); + source (s_CISCO_IOS); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_msg_only"))}; + rewrite { + set("cisco_ios", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops")) + }; parser { p_add_context_splunk(key("cisco_ios")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".cisco.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_IOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl new file mode 100644 index 0000000..b1d9297 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -0,0 +1,106 @@ +# Cisco ISE +{{ $context := dict "port_id" "CISCO_ISE" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + +#This filter uses a field we set to prevent the original messages before aggregation from being +#sent to Splunk +filter f_cisco_ise_complete{ + match("yes", value("ISE.COMPLETE") type(glob)); +}; + +#This parser adds messages from ISE to a context without sending them +#forward to Splunk +parser ise_grouping { + csv-parser( + columns(PID, ISE.num, ISE.seq, MESSAGE) + delimiters(chars(" ")) + flags(greedy) + ); + grouping-by( + scope(program) + key("$PID") + trigger("$(+ ${ISE.seq} 1)" == "${ISE.num}") + sort-key("${ISE.seq}") + aggregate( + value("MESSAGE" "$(implode '' $(context-values ${MESSAGE}))") + value("ISE.COMPLETE" "yes") + ) + timeout(10) + ); +}; + +#The syslog message includes a date with nano seconds and TZ which is not in the header +#So must reparse the date +parser ise_event_time { + csv-parser( + columns(ISE.DATE, ISE.TIME, ISE.TZ, MESSAGE) + delimiters(chars(" ")) + flags(greedy) + ); + + date-parser( + #YYYY- MM-DD hh:mm:ss:xxx +/-zh:zm + format("%Y-%m-%d %H:%M:%S.%f %z" ) + template("${ISE.DATE} ${ISE.TIME} ${ISE.TZ}") + ); +}; +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_is_rfc3164); + filter(f_cisco_ise); +{{- end}} +{{- if eq (.) "no"}} + source (s_CISCO_ISE); +{{- end}} + + parser(ise_grouping); + + if { + filter(f_cisco_ise_complete); + parser(ise_event_time); + rewrite { + set("cisco_ise", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ise:syslog"), index("netauth")) + }; + + parser {p_add_context_splunk(key("cisco_ise")); }; + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("PID")); + unset(value("LEGACY_MSGHDR")); + unset(value("EPOCH")); + unset(value("VERSION")); + unset(value("TIMESECFRAC")); + groupunset(values("ISE.*")); + }; + + destination(d_hec); + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ISE") }} + destination(d_archive); + {{- end}} + + flags(flow-control); + }; + + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for CISCO_ISE traffic + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_ISE traffic +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index fdb592e..f2a4478 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -1,26 +1,44 @@ # Cisco NX_OS -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_cisco_nx_os); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_CISCO_NX_OS); + source (s_CISCO_NX_OS); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))}; + rewrite { + set("cisco_nxos", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg")) + }; + parser { p_add_context_splunk(key("cisco_nx_os")); }; - parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_NXOS") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index ac0cb48..759f523 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -1,28 +1,43 @@ # Forcepoint Webprotect -{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_forcepoint_webprotect_kv); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_FORCEPOINT_WEBPROTECT); + source (s_FORCEPOINT_WEBPROTECT); {{- end}} rewrite { subst(" [^ =]+\=\-", "", value("MESSAGE"), flags("global")); + set("forcepoint_webprotect", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg")) }; - rewrite { r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg"))}; parser {p_add_context_splunk(key("forcepoint_webprotect")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORCEPOINT_WEBPROTECT") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 094075e..39d2bc9 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -1,17 +1,17 @@ # Fortinet Fortios -{{- if (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "FORTINET_FORTIOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_fortinet_fortios); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_FORTINET_FORTIOS); + source (s_FORTINET_FORTIOS); {{- end}} parser { @@ -19,25 +19,43 @@ log { date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); }; - rewrite { set("${.kv.devname}", value("HOST")); }; + rewrite { + set("${.kv.devname}", value("HOST")); + set("fortigate_fortios", value("fields.sc4s_vendor_product")); + }; if (match("traffic" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"))}; parser {p_add_context_splunk(key("fortinet_fortios_traffic")); }; } elif (match("utm" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"))}; parser {p_add_context_splunk(key("fortinet_fortios_utm")); }; } elif (match("event" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"))}; parser {p_add_context_splunk(key("fortinet_fortios_event")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"))}; parser {p_add_context_splunk(key("fortinet_fortios_log")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORTINET_FORTIOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 65dfb91..7038621 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -1,27 +1,46 @@ # Juniper IDP -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_IDP" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_strict); filter(f_juniper_idp); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_IDP); + source (s_JUNIPER_IDP); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"), template("t_hdr_sdata_msg"))}; + rewrite { + set("juniper_idp", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids")) + }; parser { - p_add_context_splunk(key("juniper_idp")); + p_add_context_splunk(key("juniper_idp")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + unset(value("PID")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_IDP") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index 5b26695..4e3eea5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -1,42 +1,60 @@ # Juniper JunOS -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_JUNOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_juniper_junos_standard); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_JUNOS); + source (s_JUNIPER_JUNOS); {{- end}} + rewrite { + set("juniper_junos", value("fields.sc4s_vendor_product")); + }; + if (program('RT_IDP')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"))}; parser {p_add_context_splunk(key("juniper_idp")); }; } elif (program('RT_FLOW') or message('PFE_FW_|DFWD_')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"))}; parser {p_add_context_splunk(key("juniper_junos_fw")); }; } elif (program('RT_IDS')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"))}; parser {p_add_context_splunk(key("juniper_junos_ids")); }; } elif (program('RT_UTM')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"))}; parser {p_add_context_splunk(key("juniper_junos_utm")); }; } elif (program('Juniper')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"))}; parser {p_add_context_splunk(key("juniper_sslvpn")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"))}; parser {p_add_context_splunk(key("juniper_legacy")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_JUNOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index f530a14..4a9952e 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -1,22 +1,21 @@ # Juniper Netscreen -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_juniper_netscreen); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_NETSCREEN); + source (s_JUNIPER_NETSCREEN); {{- end}} rewrite { - r_set_splunk_dest_default(sourcetype("netscreen:firewall"), - index("netfw"), - template("t_standard")) + set("juniper_netscreen", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("netscreen:firewall"), index("netfw")) }; parser { @@ -25,7 +24,22 @@ log { parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NETSCREEN") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index 3a84d12..b21861d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -1,28 +1,46 @@ # Juniper NSM -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_NSM" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_juniper_nsm); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_NSM); + source (s_JUNIPER_NSM); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"), template("t_standard"))}; + rewrite { + set("juniper_nsm", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"))}; parser { - p_add_context_splunk(key("juniper_nsm")); + p_add_context_splunk(key("juniper_nsm")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NSM") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index e590fd4..9d511c7 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -1,27 +1,43 @@ # Juniper NSM IDP -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_juniper_nsm_idp); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_NSM_IDP); + source (s_JUNIPER_NSM_IDP); {{- end}} - rewrite {r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"), template("t_standard"))}; + rewrite { + set("juniper_nsm_idp", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"))}; parser { - p_add_context_splunk(key("juniper_idp")); + p_add_context_splunk(key("juniper_nsm_idp")); }; - parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_standard))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NSM_IDP") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index 4c89731..ba00cc2 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -1,3 +1,7 @@ +# Microfocus ArcSight +{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + parser p_microfocus_arcsight_header { csv-parser( columns("fields.sc4s_cef_version", "fields.cef_device_vendor", "fields.cef_device_product", "fields.cef_device_version", "fields.cef_device_event_class", "fields.cef_name", "fields.cef_severity", MESSAGE) @@ -28,30 +32,25 @@ parser p_microfocus_arcsight_source { default-selector("unknown") ); }; - -# Microfocus ArcSight -{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} -{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_microfocus_arcsight); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_MICROFOCUS_ARCSIGHT); + source (s_MICROFOCUS_ARCSIGHT); {{- end}} rewrite { - r_set_splunk_dest_default(sourcetype("cef"), index("main"), template("t_msg_trim")) + set("microfocus_arcsight", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cef"), index("main")) }; parser { p_add_context_splunk(key("cef_{fields.cef_device_vendor}_${fields.cef_device_product}")); - }; parser (p_microfocus_arcsight_header); @@ -70,7 +69,22 @@ log { parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".cef.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT") }} + destination(d_archive); +{{- end}} flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 8dd5e0f..60acc74 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -1,17 +1,16 @@ # PaloAlto PanOS -{{- if (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "PALOALTO_PANOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_paloalto_panos); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_PALOALTO_PANOS); + source (s_PALOALTO_PANOS); {{- end}} # The palo message does not include a program value in the header, unfortunately. @@ -20,10 +19,10 @@ log { # While we are at it we will save the mesage type into the program field so parser can find it. rewrite { + set("paloalto_panos", value("fields.sc4s_vendor_product")); set("${LEGACY_MSGHDR}${MESSAGE}" value("MESSAGE")); unset(value("LEGACY_MSGHDR")); unset(value("PROGRAM")); - }; parser { #basic parsing @@ -46,34 +45,49 @@ log { #set the source type based on program field and lookup index from the splunk_context csv if (message(',\d+,THREAT')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"))}; parser {p_add_context_splunk(key("pan_threat")); }; } elif (message(',\d+,TRAFFIC')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"))}; parser {p_add_context_splunk(key("pan_traffic")); }; } elif (message(',\d+,SYSTEM')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"))}; parser {p_add_context_splunk(key("pan_system")); }; } elif (message(',\d+,CONFIG')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"))}; parser {p_add_context_splunk(key("pan_config")); }; } elif (message(',\d+,HIPWATCH')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"))}; parser {p_add_context_splunk(key("pan_hipwatch")); }; } elif (message(',\d+,CORRELATION')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"))}; parser {p_add_context_splunk(key("pan_correlation")); }; } elif (message(',\d+,USERID')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"))}; parser {p_add_context_splunk(key("pan_userid")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"))}; parser {p_add_context_splunk(key("pan_log")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".pan.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PALOALTO_PANOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl index e7d7087..60b6736 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl @@ -1,26 +1,43 @@ # Proofpoint -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_proofpoint_pps_filter); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_PROOFPOINT_PPS_FILTER); + source (s_PROOFPOINT_PPS_FILTER); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"), template("t_msg_only"))}; + rewrite { + set("proofpoint_pps_filter", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"))}; parser { p_add_context_splunk(key("proofpoint_pps_filter")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl index 0866ef0..11a8057 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl @@ -1,26 +1,42 @@ # Proofpoint -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "PROOFPOINT_PPS_SENDMAIL" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_proofpoint_pps_sendmail); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_PROOFPOINT_PPS_SENDMAIL); + source (s_PROOFPOINT_PPS_SENDMAIL); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"), template("t_msg_only"))}; + rewrite { + set("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"))}; parser { p_add_context_splunk(key("proofpoint_pps_sendmail")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS_SENDMAIL") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl new file mode 100644 index 0000000..0962cc6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -0,0 +1,137 @@ +#Ubiquiti unifi +{{ $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_is_rfc3164); + filter(f_ubiquiti_unifi); +{{- end}} +{{- if eq (.) "no"}} + source (s_UBIQUITI_UNIFI); +{{- end}} + + + parser {p_add_context_splunk(key("ubiquiti_unifi")); }; + + #Fiewall + if { + filter { + match("ubiquiti_unifi_fw", value("fields.sc4s_vendor_product") type(glob) ); + }; + + if (match("[^)]\s\S+\skernel:\s[^ll\sheader][^\[\d+.\d+\]]\S+\s\w+:" value("RAWMSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:threat"), index("netids")) + set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + parser {p_add_context_splunk(key("ubiquiti_unifi_threat")); }; + } elif (match("\S+\slinkcheck:" value("RAWMSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:link"), index("netops")) + set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + parser {p_add_context_splunk(key("ubiquiti_unifi_link")); }; + } elif (match("\d+:\d+:\d+\s\S+\ssudo:" value("RAWMSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:sudo"), index("netops")) + set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + parser {p_add_context_splunk(key("ubiquiti_unifi_sudo")); }; + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:fw"), index("netfw")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_fw")); }; + }; + #Switch + } elif { + filter { + host('^(?US[^,]{1,10}),(?[a-z0-9]{9,16}),(?v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches")); + }; + if (match("hostapd:\s+ath" value("RAWMSG"))) { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:hostapd"), index("netops")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${HOST_FROM}", value("HOST")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; + } elif (match("\d+:\d+:\d+\s\S+\smcad:" value("RAWMSG"))) { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:mcad"), index("netops")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${HOST_FROM}", value("HOST")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:switch"), index("netops")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${FROM_HOST}",value("HOST")); + set("${model}", value("fields.model")); + set("${serial}", value("fields.serial")); + set("${firmware}", value("fields.firmware")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_switch")); }; + + }; + + } elif { + filter { + program('^(?U\d[^,]{1,10}),(?[a-z0-9]{9,16}),(?v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches")); + }; + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:wireless"), index("netops")); + set("${FROM_HOST}",value("HOST")); + set("${model}", value("fields.model")); + set("${serial}", value("fields.serial")); + set("${firmware}", value("fields.firmware")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; + + } elif (match("traputil.c\(696\) " value("RAWMSG"))) { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:edgeswitch"), index("netops")); + set("${HOST_FROM}", value("HOST")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_edge_switch")); }; + + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt"), index("netops")); + set("${HOST_FROM}", value("HOST")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi")); }; + }; + + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_UBIQUITI_UNIFI") }} + destination(d_archive); +{{- end}} + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for UBIQUITI_UNIFI traffic + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for UBIQUITI_UNIFI traffic +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index 7bd86fc..983e13f 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -1,19 +1,19 @@ # Proofpoint -{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_zscaler_nss); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_ZSCALER_NSS); + source (s_ZSCALER_NSS); {{- end }} rewrite { + set("zscaler_nss", value("fields.sc4s_vendor_product")); subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); }; parser { @@ -22,37 +22,37 @@ log { }; if (match("alerts" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"))}; parser { p_add_context_splunk(key("zscaler_alerts")); }; } elif (match("dns" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"))}; parser { p_add_context_splunk(key("zscaler_dns")); }; } elif (match("fw" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"))}; parser { p_add_context_splunk(key("zscaler_fw")); }; } elif (match("NSS" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_web")); }; } elif (match("audit" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"))}; parser { p_add_context_splunk(key("zscaler_zia_audit")); }; } elif (match("sandbox" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"))}; parser { p_add_context_splunk(key("zscaler_zia_sandbox")); }; } elif (match("zpa" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"))}; parser { p_add_context_splunk(key("zscaler_zpa")); }; } elif (match("zpa_auth" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"))}; parser { p_add_context_splunk(key("zscaler_zpa_auth")); }; } elif (match("zpa_auth_connector" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"))}; parser { p_add_context_splunk(key("zscaler_zpa_connector")); }; } elif (match("zpa_bba" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"))}; parser { p_add_context_splunk(key("zscaler_zpa_bba")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"))}; parser { p_add_context_splunk(key("zscaler_nss")); }; @@ -61,7 +61,23 @@ log { parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_ZSCALER_NSS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index b041abb..69f4e28 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -1,25 +1,41 @@ # Cisco ASA RFC5424 -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_ASA" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_noversion); filter(f_cisco_asa); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_ASA); + source (s_CISCO_ASA); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + rewrite { + set("cisco_asa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))}; parser {p_add_context_splunk(key("cisco_asa")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ASA") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index e04290b..1ff0958 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -1,26 +1,44 @@ # Symantec Proxy (Bluecoat) -{{- if (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "SYMANTEC_PROXY" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_noversion); filter(f_symantec_bluecoat_proxy); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_SYMANTEC_PROXY); + source (s_SYMANTEC_PROXY); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy"), template("t_msg_only")) }; + rewrite { + set("bluecoat_proxy", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy")) + }; parser {p_add_context_splunk(key("bluecoat_proxy")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_PROXY") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index bd473a5..19a26c3 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -1,29 +1,32 @@ # Juniper JunOS (Structured, RFC5424-compliant) -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_strict); filter(f_juniper_junos_structured); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_JUNOS_STRUCTURED); + source (s_JUNIPER_JUNOS_STRUCTURED); {{- end}} + + rewrite { + set("juniper_junos", value("fields.sc4s_vendor_product")); + }; if (program('RT_IDP')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids")) }; parser {p_add_context_splunk(key("juniper_idp_structured")); }; } elif (program('RT_FLOW')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw")) }; parser {p_add_context_splunk(key("juniper_junos_fw_structured")); }; } elif (program('RT_IDS')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids")) }; parser {p_add_context_splunk(key("juniper_junos_ids_structured")); }; } elif (program('RT_UTM')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw")) }; parser {p_add_context_splunk(key("juniper_junos_utm_structured")); }; } # Legacy Netscreen IDP is handled in the "p_rfc3164-juniper-idp.conf" log path @@ -33,13 +36,28 @@ log { # parser {p_add_context_splunk(key("juniper_junos_idp")); }; # } else { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops")) }; parser {p_add_context_splunk(key("juniper_structured")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + unset(value("RAWMSG")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT") }} + destination(d_archive); +{{- end}} + }; {{- end}} {{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT") "no") "no") }} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl new file mode 100644 index 0000000..3ff2c86 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -0,0 +1,49 @@ +# Checkpoint Splunk format +{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_cisco_meraki); +{{- end}} +{{- if eq (.) "no"}} + source (s_CISCO_MERAKI); +{{- end}} + + rewrite { + set("cisco_meraki", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("meraki"), index("netfw")) + }; + parser {p_add_context_splunk(key("cisco_meraki")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_MERAKI") }} + destination(d_archive); +{{- end}} + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_MERAKI traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl deleted file mode 100644 index dbbb1f6..0000000 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl +++ /dev/null @@ -1,42 +0,0 @@ -# Checkpoint Splunk format -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} -{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} -{{ define "log_path" }} -log { -{{- if eq (.) "yes"}} - source(s_default-ports); - filter(f_cisco_meraki); -{{- end}} -{{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_MERAKI); -{{- end}} - - #parser { - # kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); - # date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); - # - # }; - - #rewrite { set("${.kv.hostname}", value("HOST")); }; - - rewrite { r_set_splunk_dest_default(sourcetype("meraki"), index("netfw"), template("t_hdr_msg"))}; - parser {p_add_context_splunk(key("cisco_meraki")); }; - - - - parser (compliance_meta_by_source); - - destination(d_hec); #--HEC-- - - flags(flow-control); -}; -{{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} -# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} - -# Listen on the default port (typically 514) for CISCO_MERAKI traffic -{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/zfallback.conf b/package/etc/conf.d/log_paths/zfallback.conf index 37d7a86..478ff31 100644 --- a/package/etc/conf.d/log_paths/zfallback.conf +++ b/package/etc/conf.d/log_paths/zfallback.conf @@ -1,7 +1,10 @@ log { - source(s_default-ports); + source(s_DEFAULT); - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")) }; + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); + set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); + }; parser { p_add_context_splunk(key("sc4s_fallback")); }; diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index 234e87c..be569ff 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -1,101 +1,2 @@ -# =============================================================================================== -# source definition for remote devices -# =============================================================================================== - -# =============================================================================================== -# Defaults for the default-network-drivers() source: -# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic -# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic -# 6514 TCP, for TLS-encrypted traffic -# =============================================================================================== -source s_default-ports { - - channel { - source { - syslog ( - transport("udp") - port(514) - ip-protocol(4) - so-rcvbuf({{- getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "425984"}}) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - - network ( - transport("tcp") - port(514) - ip-protocol(4) - max-connections({{- getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) - log-iw-size({{- getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) - log-fetch-limit({{- getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - - {{- if eq (getenv "SC4S_SOURCE_TLS_ENABLE") "yes"}} - network( - port(6514) - transport("tls") - ip-protocol(4) - max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) - log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) - log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - tls(allow-compress(yes) - key-file("/opt/syslog-ng/tls/server.key") - cert-file("/opt/syslog-ng/tls/server.pem") - ssl-options({{- getenv "SC4S_SOURCE_TLS_OPTIONS" "no-sslv2, no-sslv3, no-tlsv1" }}) - cipher-suite("{{- getenv "SC4S_SOURCE_TLS_CIPHER_SUITE" "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH" }}") - ) - ); - - {{- end }} - }; - #TODO: #60 Remove this function with enhancement - rewrite(set_rfcnonconformant); - - if { - filter(f_rfc5424_strict); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_strict); - } elif { - filter(f_rfc5424_noversion); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_noversion); - } elif { - parser {cisco-parser()}; - rewrite(set_cisco_ios); - } elif { - parser (p_cisco_meraki); - rewrite(set_rfc5424_epochtime); - } else { - parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); - }; - rewrite(set_rfc3164); - }; - - rewrite(r_set_splunk_default); - - parser { - vendor_product_by_source(); - }; - }; -}; \ No newline at end of file +{{ $context := dict "port_id" "DEFAULT" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 7a418e3..d4c05b9 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -13,6 +13,7 @@ #checkpoint_splunk,index,netops #cisco_asa,index,netfw #cisco_ios,index,netops +#cisco_ise,index,netauth #cisco_nx_os,index,netops #local_example,index,main #forcepoint_webprotect,index,netproxy diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 57c73ac..ec20e82 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -36,3 +36,7 @@ filter f_proofpoint_pps_sendmail { host("pps-*" type(glob)) or netmask(192.168.6.0/24) }; +filter f_ubiquiti_unifi_fw { + host("usg-*" type(glob)) or + netmask(192.168.6.0/24) +}; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 2f85bc4..698f672 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -6,4 +6,5 @@ f_juniper_idp,sc4s_vendor_product,"juniper_idp" f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" -f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" +f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" \ No newline at end of file diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 09ea120..badcbbb 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -1,10 +1,10 @@ -# The following is the dedicated source port declaration for {{ (print .port_id) }} +# The following is the source port declaration for {{ (print .port_id) }} # Two log paths will be created -- one for the dedicated port(s) and one for the default (typically port 514) - -source s_dedicated_port_{{ .port_id}} { +{{- define "T1" }} +source s_{{ .port_id}} { channel { source { -{{- if ne (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT" ) "no") "no" }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT" )) }} syslog ( transport("udp") port({{getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") }}) @@ -18,7 +18,7 @@ source s_dedicated_port_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if ne (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "no") "no" }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT" )) }} network ( transport("tcp") port({{getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") }}) @@ -34,7 +34,7 @@ source s_dedicated_port_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if ne (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "no") "no" }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT" )) }} network( port({{getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") }}) transport("tls") @@ -111,6 +111,12 @@ source s_dedicated_port_{{ .port_id}} { {{- end }} rewrite(r_set_splunk_default); + parser { + vendor_product_by_source(); + }; }; - -}; \ No newline at end of file +}; +{{- end }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) }} +{{ template "T1" (.) }} +{{- end }} diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl index a8ac264..6a68b45 100644 --- a/package/etc/local_config/log_paths/example.conf.tmpl +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -27,7 +27,7 @@ log { # The first time this template is used the log_path will be linked to the default port {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); # Filters should be updated to use the simplest and most effecient logic possible to discard # the message from this path @@ -40,12 +40,12 @@ log { # In the second pass through the template a link to the dedicated port is used. This # normally does not require additional filters -source (s_dedicated_port_LOCAL_EXAMPLE); +source (s_LOCAL_EXAMPLE); {{- end}} #Set a default sourcetype and index - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; #using the key "local_example" find any cutomized index,source or sourcetype meta values diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index e3e33e9..88897f1 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -21,7 +21,7 @@ options { chain_hostnames (off); use_dns (no); use_fqdn (no); - dns-cache(no); + dns-cache(yes); create_dirs (no); keep-hostname (yes); create_dirs(yes); @@ -39,12 +39,6 @@ options { # =============================================================================================== @define splunk-template "t_standard" -# =============================================================================================== -# Data collection parameters, buffers, and Timezone -# =============================================================================================== -#TODO: Remove once release with this PR is produced by upstream -#https://github.com/balabit/syslog-ng/pull/2932 -@define syslog-ng-sysconfdir "/opt/syslog-ng/etc" # =============================================================================================== # Global modules and includes. All device-specific filters and destinations exist in conf.d diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index ff9a112..a93736d 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,17 +1,11 @@ #!/usr/bin/env bash source scl_source enable rh-python36 +export SC4S_LISTEN_DEFAULT_TCP_PORT=514 +export SC4S_LISTEN_DEFAULT_UDP_PORT=514 + cd /opt/syslog-ng -#The following is no longer needed but retained as a comment just in case we run into command line length issues -#for d in $(find /opt/syslog-ng/etc -type d) -#do -# echo Templating conf for $d -# gomplate \ -# --input-dir=$d \ -# --template t=etc/go_templates/ \ -# --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ -# --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" -#done + gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index 612fe0b..04c2df6 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -4,6 +4,12 @@ homePath = $SPLUNK_DB/syslogng_metrics/db coldPath = $SPLUNK_DB/syslogng_metrics/colddb thawedPath = $SPLUNK_DB/syslogng_metrics/thaweddb +[em_metrics] +datatype=metric +homePath = $SPLUNK_DB/em_metrics/db +coldPath = $SPLUNK_DB/em_metrics/colddb +thawedPath = $SPLUNK_DB/em_metrics/thaweddb + [syslogng_fallback] homePath = $SPLUNK_DB/syslogng_fallback/db coldPath = $SPLUNK_DB/syslogng_fallback/colddb @@ -30,6 +36,11 @@ homePath = $SPLUNK_DB/oswinsec/db coldPath = $SPLUNK_DB/oswinsec/colddb thawedPath = $SPLUNK_DB/oswinsec/thaweddb +[netauth] +homePath = $SPLUNK_DB/netauth/db +coldPath = $SPLUNK_DB/netauth/colddb +thawedPath = $SPLUNK_DB/netauth/thaweddb + [netdlp] homePath = $SPLUNK_DB/netdlp/db coldPath = $SPLUNK_DB/netdlp/colddb diff --git a/tests/requirements.txt b/tests/requirements.txt index cc706aa..758c531 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,5 +9,5 @@ pytest jinja2 jinja2-time -http://dev.splunk.com/goto/sdk-python +splunk-sdk flake8 diff --git a/tests/test_cisco_ise.py b/tests/test_cisco_ise.py new file mode 100644 index 0000000..db1e493 --- /dev/null +++ b/tests/test_cisco_ise.py @@ -0,0 +1,52 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 0 2019-04-24 15:00:48.610 +00:00 0042009748 5200 NOTICE Passed-Authentication: Authentication succeeded, ConfigVersionId=128, Device IP Address=10.6.64.15, DestinationIPAddress=10.16.20.23, DestinationPort=1812, UserName=90-1B-0E-34-EA-92, Protocol=Radius, RequestLatency=8, NetworkDeviceName=ICPAV2-SW15, User-Name=901b0e34ea92, NAS-IP-Address=10.6.64.15, NAS-Port=50104, Service-Type=Call Check, Framed-IP-Address=10.6.226.138, Framed-MTU=1500, Called-Station-ID=B0-FA-EB-11-70-04, Calling-Station-ID=90-1B-0E-34-EA-92, NAS-Port-Type=Ethernet, NAS-Port-Id=GigabitEthernet0/4, EAP-Key-Name=, cisco-av-pair=service-type=Call Check, cisco-av-pair=audit-session-id=0A06400F000006AA6F83C371, cisco-av-pair=method=mab, OriginalUserName=901b0e34ea92, NetworkDeviceProfileName=Cisco, NetworkDeviceProfileId=b2652f13-5b3f-41ba-ada2-8385c8870809, IsThirdPartyDeviceFlow=false, RadiusFlowType=WiredMAB, SSID=B0-FA-EB-11-70-04, +#<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 1 AcsSessionID=ICDC-ISE03/341048949/1407358, AuthenticationIdentityStore=Internal Endpoints, AuthenticationMethod=Lookup, SelectedAccessService=Default Network Access, SelectedAuthorizationProfiles=WIRED_GUEST_REDIRECT, UseCase=Host Lookup, IdentityGroup=Endpoint Identity Groups:Unknown, Step=11001, Step=11017, Step=11027, Step=15049, Step=15008, Step=15048, Step=15048, Step=15041, Step=15013, Step=24209, Step=24211, Step=22037, Step=24715, Step=15036, Step=15048, Step=15048, Step=15016, Step=11002, SelectedAuthenticationIdentityStores=Internal Endpoints, AuthenticationStatus=AuthenticationPassed, NetworkDeviceGroups=Location#All Locations#Provo, NetworkDeviceGroups=Device Type#All Device Types#Switches, NetworkDeviceGroups=Migrated NDGs#All Migrated NDGs, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Guest Web Auth, UserType=Host, CPMSessionID=0A06400F000006AA6F83C371, EndPointMACAddress=90-1B-0E-34-EA-92, +#<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 2 PostureAssessmentStatus=NotApplicable, EndPointMatchedProfile=Unknown, DeviceRegistrationStatus=notRegistered, ISEPolicySetName=Wired MAB, IdentitySelectionMatchedRule=Default, StepData=5= Aruba.Aruba-Essid-Name, StepData=6= DEVICE.Device Type, StepData=8=Internal Endpoints, StepData=14= EndPoints.LogicalProfile, StepData=15= Network Access.Protocol, allowEasyWiredSession=false, DTLSSupport=Unknown, HostIdentityGroup=Endpoint Identity Groups:Unknown, Network Device Profile=Cisco, Location=Location#All Locations#Provo, Device Type=Device Type#All Device Types#Switches, Migrated NDGs=Migrated NDGs#All Migrated NDGs, Name=Endpoint Identity Groups:Unknown, +#<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 3 Response={UserName=90:1B:0E:34:EA:92; User-Name=90-1B-0E-34-EA-92; State=ReauthSession:0A06400F000006AA6F83C371; Class=CACS:0A06400F000006AA6F83C371:ICDC-ISE03/341048949/1407358; Session-Timeout=300; Idle-Timeout=240; Termination-Action=RADIUS-Request; cisco-av-pair=url-redirect-acl=ACL-WEBAUTH-REDIRECT; cisco-av-pair=url-redirect=https://ICDC-ISE03.example.com:8443/portal/gateway?sessionId=0A06400F000006AA6F83C371&portal=c5a76cb0-6150-11e5-b062-0050568d954e&action=cwa&type=drw&token=6ded1943789b345f7afdd09e91549047; cisco-av-pair=profile-name=Unknown; LicenseTypes=1; }, + +def test_cisco_ise(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 0 {% now 'utc', '%Y-%m-%d %H:%M:%S' %}.610 +00:00 0042009748 5200 NOTICE Passed-Authentication: Authentication succeeded, ConfigVersionId=128, Device IP Address=10.6.64.15, DestinationIPAddress=10.16.20.23, DestinationPort=1812, UserName=90-1B-0E-34-EA-92, Protocol=Radius, RequestLatency=8, NetworkDeviceName=ICPAV2-SW15, User-Name=901b0e34ea92, NAS-IP-Address=10.6.64.15, NAS-Port=50104, Service-Type=Call Check, Framed-IP-Address=10.6.226.138, Framed-MTU=1500, Called-Station-ID=B0-FA-EB-11-70-04, Calling-Station-ID=90-1B-0E-34-EA-92, NAS-Port-Type=Ethernet, NAS-Port-Id=GigabitEthernet0/4, EAP-Key-Name=, cisco-av-pair=service-type=Call Check, cisco-av-pair=audit-session-id=0A06400F000006AA6F83C371, cisco-av-pair=method=mab, OriginalUserName=901b0e34ea92, NetworkDeviceProfileName=Cisco, NetworkDeviceProfileId=b2652f13-5b3f-41ba-ada2-8385c8870809, IsThirdPartyDeviceFlow=false, RadiusFlowType=WiredMAB, SSID=B0-FA-EB-11-70-04,\n") + message = mt.render(mark="<165>", host=host) + sendsingle(message) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 1 AcsSessionID=ICDC-ISE03/341048949/1407358, AuthenticationIdentityStore=Internal Endpoints, AuthenticationMethod=Lookup, SelectedAccessService=Default Network Access, SelectedAuthorizationProfiles=WIRED_GUEST_REDIRECT, UseCase=Host Lookup, IdentityGroup=Endpoint Identity Groups:Unknown, Step=11001, Step=11017, Step=11027, Step=15049, Step=15008, Step=15048, Step=15048, Step=15041, Step=15013, Step=24209, Step=24211, Step=22037, Step=24715, Step=15036, Step=15048, Step=15048, Step=15016, Step=11002, SelectedAuthenticationIdentityStores=Internal Endpoints, AuthenticationStatus=AuthenticationPassed, NetworkDeviceGroups=Location#All Locations#Provo, NetworkDeviceGroups=Device Type#All Device Types#Switches, NetworkDeviceGroups=Migrated NDGs#All Migrated NDGs, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Guest Web Auth, UserType=Host, CPMSessionID=0A06400F000006AA6F83C371, EndPointMACAddress=90-1B-0E-34-EA-92,\n") + message = mt.render(mark="<111>", host=host) + sendsingle(message) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 2 PostureAssessmentStatus=NotApplicable, EndPointMatchedProfile=Unknown, DeviceRegistrationStatus=notRegistered, ISEPolicySetName=Wired MAB, IdentitySelectionMatchedRule=Default, StepData=5= Aruba.Aruba-Essid-Name, StepData=6= DEVICE.Device Type, StepData=8=Internal Endpoints, StepData=14= EndPoints.LogicalProfile, StepData=15= Network Access.Protocol, allowEasyWiredSession=false, DTLSSupport=Unknown, HostIdentityGroup=Endpoint Identity Groups:Unknown, Network Device Profile=Cisco, Location=Location#All Locations#Provo, Device Type=Device Type#All Device Types#Switches, Migrated NDGs=Migrated NDGs#All Migrated NDGs, Name=Endpoint Identity Groups:Unknown,\n") + message = mt.render(mark="<111>", host=host) + sendsingle(message) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 3 Response={UserName=90:1B:0E:34:EA:92; User-Name=90-1B-0E-34-EA-92; State=ReauthSession:0A06400F000006AA6F83C371; Class=CACS:0A06400F000006AA6F83C371:ICDC-ISE03/341048949/1407358; Session-Timeout=300; Idle-Timeout=240; Termination-Action=RADIUS-Request; cisco-av-pair=url-redirect-acl=ACL-WEBAUTH-REDIRECT; cisco-av-pair=url-redirect=https://ICDC-ISE03.example.com:8443/portal/gateway?sessionId=0A06400F000006AA6F83C371&portal=c5a76cb0-6150-11e5-b062-0050568d954e&action=cwa&type=drw&token=6ded1943789b345f7afdd09e91549047; cisco-av-pair=profile-name=Unknown; LicenseTypes=1; },\n") + message = mt.render(mark="<111>", host=host) + sendsingle(message) + + st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:ise:syslog\" | head 11") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_common.py b/tests/test_common.py index e026893..8325b52 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -67,4 +67,16 @@ def test_tag(record_property, setup_wordlist, setup_splunk): record_property("resultCount", resultCount) record_property("message", message) + assert resultCount == 1 + +# +def test_metrics(record_property, setup_wordlist, setup_splunk): + + st = env.from_string('mcatalog values(metric_name) WHERE metric_name="syslogng.d_*#0" AND ("index"="*" OR "index"="_*") BY index | fields index') + search = st.render() + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("resultCount", resultCount) + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_ubiquiti_unifi.py b/tests/test_ubiquiti_unifi.py new file mode 100644 index 0000000..4eed2b2 --- /dev/null +++ b/tests/test_ubiquiti_unifi.py @@ -0,0 +1,93 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +#<27>Nov 8 17:28:43 US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type + +def test_ubiquiti_unifi_us8p60(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<29>Nov 10 20:46:02 US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0 +def test_ubiquiti_unifi_switch_us24p250(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<30>Nov 10 11:49:46 U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514 +def test_ubiquiti_unifi_ap_u7pg2(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netops sourcetype=ubnt:wireless earliest=-2m | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<4>Nov 10 23:04:06 USG kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328 +def test_ubiquiti_unifi_usg(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} usg-{{host}} kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netfw sourcetype=ubnt:fw host=usg-{{host}} | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From eef0d5975253b0e23d5f3e3455a281d48260a6a0 Mon Sep 17 00:00:00 2001 From: mkarlstrand-splunk <49571555+mkarlstrand-splunk@users.noreply.github.com> Date: Thu, 14 Nov 2019 17:41:04 -0800 Subject: [PATCH 08/10] Update performance.md (#190) --- docs/performance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/performance.md b/docs/performance.md index ea02c72..14f1b9c 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -34,7 +34,7 @@ and forget protocol making it sensitive to performance. Given this it is highly performance with your hardware and production data samples. The syslog-ng loggen tool available in the SC4S container and the commands above can be utilized in this effort. -Deployment Size | Hardware Spec | Average EPS with average msg size 800 k +Deployment Size | Hardware Spec | Average EPS with average msg size 800 bytes -- | -- | -- Small | 2 X 3.1 ghz cores1 GB of memory | 2K msg/sec Medium | 4 X 3.1 ghz cores2 GB of memory | 4.5K msg/sec From 866092488703dce6522fa3f6af2800a0ef1c6979 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Sun, 17 Nov 2019 19:41:21 -0500 Subject: [PATCH 09/10] Release/1.2.0 (#195) * Resolve error with failure to upgrade example.conf * Automatically resolve correct time zone for near real time streaming sources * Support specific time zones by host/ip/subnet for sources with high latency --- .../p_rfc3164-checkpoint_splunk.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 1 + .../log_paths/p_rfc3164-cisco_nxos.conf.tmpl | 1 + .../p_rfc3164-fortinet_fortios.conf.tmpl | 2 +- .../p_rfc3164-paloalto_panos.conf.tmpl | 3 +- .../vendor_product_by_source.conf | 9 +++ .../vendor_product_by_source.csv | 4 +- package/etc/go_templates/source_network.t | 12 ++- package/sbin/entrypoint.sh | 2 +- tests/requirements.txt | 1 + tests/test_common.py | 73 ++++++++++++++++++- 11 files changed, 102 insertions(+), 8 deletions(-) diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index ee81c82..cd99a70 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -16,7 +16,7 @@ log { parser { kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); - date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); + date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite { diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index 3c51cf6..b9d997c 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -15,6 +15,7 @@ log { rewrite { set("cisco_ios", value("fields.sc4s_vendor_product")); + guess-time-zone(); r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops")) }; parser { diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index f2a4478..e351441 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -15,6 +15,7 @@ log { rewrite { set("cisco_nxos", value("fields.sc4s_vendor_product")); + guess-time-zone(); r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg")) }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 39d2bc9..16b35f4 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -16,7 +16,7 @@ log { parser { kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); - date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); + date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite { diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 60acc74..73dc1d0 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -38,7 +38,8 @@ log { date-parser( format("%Y/%m/%d %H:%M:%S") template("${.pan.GeneratedTime}") - time-zone("Universal") + time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) + flags(guess-timezone) ); }; diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index ec20e82..b0609de 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -39,4 +39,13 @@ filter f_proofpoint_pps_sendmail { filter f_ubiquiti_unifi_fw { host("usg-*" type(glob)) or netmask(192.168.6.0/24) +}; +filter f_tzfixhst { + host("tzfhst-*" type(glob)) or + netmask(192.168.6.0/24) +}; + +filter f_tzfixny { + host("tzfny-*" type(glob)) or + netmask(192.168.6.0/24) }; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 698f672..975d558 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -7,4 +7,6 @@ f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" -f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" \ No newline at end of file +f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" +f_tzfixhst,sc4s_time_zone,"Pacific/Honolulu" +f_tzfixny,sc4s_time_zone,"America/New_York" \ No newline at end of file diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index badcbbb..63fb6d2 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -104,7 +104,7 @@ source s_{{ .port_id}} { rewrite(set_rfc5424_epochtime); } else { parser { - syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone)); }; rewrite(set_rfc3164); }; @@ -114,6 +114,16 @@ source s_{{ .port_id}} { parser { vendor_product_by_source(); }; + + if { + filter { match("." value("fields.sc4s_time_zone") ) }; + rewrite { + fix-time-zone("${fields.sc4s_time_zone}"); + unset(value("fields.sc4s_time_zone")); + }; + }; + + }; }; {{- end }} diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index a93736d..9abad88 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -12,7 +12,7 @@ gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl -- mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ -cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ +cp --verbose -R /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index 758c531..6d908dd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,3 +11,4 @@ jinja2 jinja2-time splunk-sdk flake8 +pytz \ No newline at end of file diff --git a/tests/test_common.py b/tests/test_common.py index 8325b52..ce279a6 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -3,9 +3,11 @@ # Use of this source code is governed by a BSD-2-clause-style # license that can be found in the LICENSE-BSD2 file or at # https://opensource.org/licenses/BSD-2-Clause +import datetime import random +import pytz -from jinja2 import Environment +from jinja2 import Environment, environment from .sendmessage import * from .splunkutils import * @@ -79,4 +81,71 @@ def test_metrics(record_property, setup_wordlist, setup_splunk): record_property("resultCount", resultCount) - assert resultCount == 1 \ No newline at end of file + assert resultCount == 1 + +def test_tz_guess(record_property, setup_wordlist, setup_splunk): + + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'America/Los_Angeles', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +def test_tz_fix_hst(record_property, setup_wordlist, setup_splunk): + + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + dt = datetime.datetime.utcnow() - datetime.timedelta(hours=10, minutes=10) + mt = env.from_string( + "{{ mark }} {{ dt }} tzfhst-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") + message = mt.render(mark="<111>", host=host, dt=dt.strftime('%b %d %H:%M:%S')) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"tzfhst-{{ host }}\" sourcetype=\"cisco:asa\"") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk): + + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + tz_NY = pytz.timezone('America/New_York') + dt = datetime.datetime.now(tz_NY) - datetime.timedelta(minutes=10) + mt = env.from_string( + "{{ mark }} {{ dt }} tzfny-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") + message = mt.render(mark="<111>", host=host, dt=dt.strftime('%b %d %H:%M:%S')) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"tzfny-{{ host }}\" sourcetype=\"cisco:asa\"") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From 82bd3be435d5fca2369c9f993f1d177722ec727b Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 27 Nov 2019 23:28:00 -0500 Subject: [PATCH 10/10] Release/1.2.1 (#210) Fixes docs issues: #130 #182 #186 #207 #204 #205 #206 #209 --- .circleci/config.yml | 7 +-- CONTRIBUTING.md => docs/CONTRIBUTING.md | 0 LICENSE-BSD2 => docs/LICENSE-BSD2 | 0 LICENSE-CC0 => docs/LICENSE-CC0 | 0 docs/configuration.md | 4 +- docs/gettingstarted.md | 46 +++++++++++++++++-- docs/gettingstarted/byoe-rhel7.md | 22 +++++++-- docs/gettingstarted/docker-swarm-general.md | 21 ++++++++- docs/gettingstarted/docker-swarm-rhel7.md | 2 +- docs/gettingstarted/docker-systemd-general.md | 24 ++++++++-- docs/troubleshooting.md | 16 +++++++ .../p_rfc3164-checkpoint_splunk.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_asa.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 3 +- .../log_paths/p_rfc3164-cisco_ise.conf.tmpl | 3 +- .../log_paths/p_rfc3164-cisco_nxos.conf.tmpl | 3 +- .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 3 +- .../p_rfc3164-fortinet_fortios.conf.tmpl | 3 +- .../log_paths/p_rfc3164-juniper_idp.conf.tmpl | 3 +- .../p_rfc3164-juniper_junos.conf.tmpl | 3 +- .../p_rfc3164-juniper_netscreen.conf.tmpl | 3 +- .../log_paths/p_rfc3164-juniper_nsm.conf.tmpl | 3 +- .../p_rfc3164-juniper_nsm_idp.conf.tmpl | 3 +- .../p_rfc3164-microfocus_arcsight.conf.tmpl | 2 +- .../p_rfc3164-paloalto_panos.conf.tmpl | 3 +- .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 3 +- ..._rfc3164-proofpoint_pps_sendmail.conf.tmpl | 3 +- .../p_rfc3164-ubiquiti_unifi.conf.tmpl | 2 +- .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 3 +- .../p_rfc5424-noversion_cisco_asa.conf.tmpl | 3 +- ...rfc5424-noversion_symantec_proxy.conf.tmpl | 3 +- .../p_rfc5424-strict_juniper_junos.conf.tmpl | 5 +- .../p_rfc5424_epoch-cisco_meraki.conf.tmpl | 3 +- package/etc/go_templates/source_network.t | 2 +- .../local_config/log_paths/example.conf.tmpl | 26 +++++------ 35 files changed, 177 insertions(+), 57 deletions(-) rename CONTRIBUTING.md => docs/CONTRIBUTING.md (100%) rename LICENSE-BSD2 => docs/LICENSE-BSD2 (100%) rename LICENSE-CC0 => docs/LICENSE-CC0 (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index fda0601..8840fcf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,8 @@ jobs: path: package extra_build_args: --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION - - docker/install-goss + - docker/install-goss: + version: v0.3.7 - run: name: Test Docker command: | @@ -49,7 +50,7 @@ jobs: # Don't forget path! export PATH=$PATH:~/bin pushd package - GOSS_FMT_OPTIONS="perfdata verbose" GOSS_FILES_STRATEGY=cp GOSS_OPTS="--max-concurrent=1 --format junit" dgoss run -t $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 3>&1 1>&2 2>&3 > /tmp/goss.tmp || true ; cat /tmp/goss.tmp | grep -v "<" + GOSS_VER=v0.3.7 GOSS_FMT_OPTIONS="perfdata verbose" GOSS_FILES_STRATEGY=cp GOSS_OPTS="--max-concurrent=1 --format junit" dgoss run -t $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 3>&1 1>&2 2>&3 > /tmp/goss.tmp || true ; cat /tmp/goss.tmp | grep -v "<" popd cat /tmp/goss.tmp | grep "[<\>]" > /tmp/test-results/goss.xml @@ -413,7 +414,7 @@ workflows: requires: - build - test-unit - - test-scan-synk + #- test-scan-synk filters: tags: only: /^\d*\.\d*\.\d*.*$/ diff --git a/CONTRIBUTING.md b/docs/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to docs/CONTRIBUTING.md diff --git a/LICENSE-BSD2 b/docs/LICENSE-BSD2 similarity index 100% rename from LICENSE-BSD2 rename to docs/LICENSE-BSD2 diff --git a/LICENSE-CC0 b/docs/LICENSE-CC0 similarity index 100% rename from LICENSE-CC0 rename to docs/LICENSE-CC0 diff --git a/docs/configuration.md b/docs/configuration.md index 379edf7..5926646 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -95,8 +95,8 @@ which maps to an associated lookup of alternate indexes, sources, or other metad * Get the filter and lookup files ```bash cd /opt/sc4s/default -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/compliance_meta_by_source.conf -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/compliance_meta_by_source.csv +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context_templates/compliance_meta_by_source.conf +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context_templates/compliance_meta_by_source.csv ``` * Edit the file ``compliance_meta_by_source.conf`` to supply uniquely named filters to identify events subject to override. * Edit the file ``compliance_meta_by_source.csv`` to supply appropriate the field(s) and values. diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 887e886..e113881 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -72,12 +72,50 @@ Splunk type. | Container and Orchestration | Notes | |-----------------------------|-------| -| [Podman + systemd single node](gettingstarted/podman-systemd-general.md) | First choice for RedHat 7.x and 8.x, second choice for Debian and Ubuntu (packages provided via PPA) | -| [Docker CE + systemd single node](gettingstarted/docker-systemd-general.md) | First choice for Debian, Ubuntu, and CentOS distributions with limited existing docker experience | -| [Docker CE + Swarm single node](gettingstarted/docker-swarm-general.md) | Option for Debian, Ubuntu, and CentOS desiring swarm orchestration | -| [Docker CE + Swarm single node RHEL 7.7](gettingstarted/docker-swarm-rhel7.md) | Option for RedHat 7.7 desiring swarm orchestration | +| [Podman + systemd single node](gettingstarted/podman-systemd-general.md) | First choice for RedHat 7.x/8.x and CentOS, second choice for Debian and Ubuntu (packages provided via PPA) | +| [Docker CE + systemd single node](gettingstarted/docker-systemd-general.md) | First choice for Debian and Ubuntu; second choice for CentOS for those with limited existing Docker experience | +| [Docker CE + Swarm single node](gettingstarted/docker-swarm-general.md) | Option for Debian, Ubuntu, CentOS, and Desktop Docker desiring Docker Compose or Swarm orchestration | +| [Docker CE + Swarm single node RHEL 7.7](gettingstarted/docker-swarm-rhel7.md) | Option for RedHat 7.7 desiring Docker Compose or Swarm orchestration | | [Bring your own Envionment](gettingstarted/byoe-rhel7.md) | Option for RedHat 7.7 (centos 7) with SC4S configuration without containers | +## Offline Container Installation + +Follow these instructions to "stage" SC4S by downloading the container so that it can be loaded "out of band" on a +host machine, such as an airgapped system, without internet connectivity. + +* Download container image + +``` +sudo wget https://github.com/splunk/splunk-connect-for-syslog/releases/download/latest/oci_container.tar.gz +``` + +* Distribute the container to the airgapped host machine using an appropriate file transfer utility. +* Execute the following command, using docker or podman as appropriate + +``` + load < oci_container.tar.gz +``` + +* Note the container ID of the resultant load + +``` +Loaded image: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci:90196f77f7525bc55b3b966b5fa1ce74861c0250 +``` + +* Use the container ID to create a local label +``` + tag docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci:90196f77f7525bc55b3b966b5fa1ce74861c0250 sc4slocal:latest +``` + +* Use this local label `sc4slocal:latest` in the relevant unit or yaml file to launch SC4S (see the runtime options +above) by setting the `SC4S_IMAGE` environment variable in the unit file (example below), or the relevant `image:` tag +if using Docker Compose/Swarm. Using this label will cause the runtime to select the locally loaded image, and will not +attempt to obtain the container image via the internet. + +``` +Environment="SC4S_IMAGE=sc4slocal:latest" +``` + # Scale out Additional hosts can be deployed for syslog collection from additional network zones and locations: diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index d970dcd..881f949 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -1,13 +1,19 @@ # SC4S "Bring Your Own Environment" +* FOREWORD: The BYOE SC4S deliverable should be considered as a _secondary_ option for SC4S deployment, and should be +considered only by those with specific needs based on advanced understanding of syslog-ng architectures. The +container deliverable is the preferred deliverable of SC4S for almost all enterprises. If you are simply trying to +"get syslog working", the turnkey, container approach described in the other runtime documents will be the fastest +route to success. + The "Bring Your Own Environment" instructions that follow allow administrators to utilize the SC4S syslog-ng config files directly on the host OS running on a hardware server or virtual machine. Administrators must provide an appropriate host OS as well as an up-to-date syslog-ng installation either built from source (not documented here) or installed from community-built RPMs. Modification of the base configuration will be required for most customer -environments due to enterprise infrastructure variations. +environments due to enterprise infrastructure variations. -* NOTE: Installing or modifying system configurations can have unexpected consequences, and rudimentary linux system -administratrion and syslog-ng configuration experience is assumed. +* NOTE: Installing or modifying system configurations can have unexpected consequences, and advanced linux system +administration and syslog-ng configuration experience is assumed when using the BYOE version of SC4S. * NOTE: Do _not_ depend on the distribution-supplied version of syslog-ng, as it will likely be far too old. Read this [explanation](https://www.syslog-ng.com/community/b/blog/posts/installing-latest-syslog-ng-on-rhel-and-other-rpm-distributions) @@ -68,6 +74,10 @@ sudo chmod 755 /usr/local/bin/gomplate gomplate --help ``` +* Install the latest python + +```scl enable rh-python36 bash``` + * create the sc4s unit file drop in ``/etc/systemd/system/sc4s.service`` and add the following content ```ini @@ -93,7 +103,7 @@ Restart=on-failure WantedBy=multi-user.target ``` -* create the file ``/opt/sc4s/bin/preconfig.sh`` and add the following content +* create the file ``/opt/sc4s/bin/preconfig.sh``. This file should be made executable according to your file permission standards. Add the following content: ```bash #!/usr/bin/env bash @@ -119,7 +129,9 @@ cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/l mkdir -p /opt/syslog-ng/var/data/disk-buffer/ ``` -* Execute the preconfiguration file created above +* (Optional) Execute the preconfiguration shell script created above. You may also optionally execute it as part of the unit +file, which is recommended. If you elect _not_ to execute the script in the unit file, care must be taken to execute it manually "out of band" +when any changes are made. ```bash sudo bash /opt/sc4s/bin/preconfig.sh diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 6fd894f..dfa8b4d 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -1,7 +1,26 @@ # Install Docker CE and Swarm -Refer to [Getting Started](https://docs.docker.com/get-started/) +Refer to relevant installation guides: + +* [CentOS](https://docs.docker.com/install/linux/docker-ce/centos/) +* [Ubuntu](https://docs.docker.com/install/linux/docker-ce/ubuntu/) +* [Debian](https://docs.docker.com/install/linux/docker-ce/debian/) +* [Desktop](https://docs.docker.com/get-started/) + +NOTE: If using a CentOS image provisioned in AWS, IPV4 forwarding is _not_ enabled by default. +This needs to be enabled for container networking to function properly. The following is an example +to set this up; as usual this needs to be vetted with your enterprise security policy: + +```sudo sysctl net.ipv4.ip_forward=1``` + +Then, edit /etc/sysctl.conf, find the text below, and uncomment as shown so that the change made above will survive a +reboot: + +``` +# Uncomment the next line to enable packet forwarding for IPv4 +net.ipv4.ip_forward=1 +``` # SC4S Configuration diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 5ef0e66..7ed5afe 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -1,7 +1,7 @@ # Install Docker CE and Swarm -*Warning* this method of installing docker on RHEL does not appear to be supported: +* Warning: this method of installing docker on RHEL does not appear to be supported. Consider using podman instead. ## Enable required repositories ```bash diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 94b164f..838cdaa 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -1,7 +1,25 @@ # Install Docker CE -Refer to [Getting Started](https://docs.docker.com/get-started/) +Refer to relevant installation guides: + +* [CentOS](https://docs.docker.com/install/linux/docker-ce/centos/) +* [Ubuntu](https://docs.docker.com/install/linux/docker-ce/ubuntu/) +* [Debian](https://docs.docker.com/install/linux/docker-ce/debian/) + +NOTE: If using a CentOS image provisioned in AWS, IPV4 forwarding is _not_ enabled by default. +This needs to be enabled for container networking to function properly. The following is an example +to set this up; as usual this needs to be vetted with your enterprise security policy: + +```sudo sysctl net.ipv4.ip_forward=1``` + +Then, edit /etc/sysctl.conf, find the text below, and uncomment as shown so that the change made above will survive a +reboot: + +``` +# Uncomment the next line to enable packet forwarding for IPv4 +net.ipv4.ip_forward=1 +``` # Setup @@ -10,8 +28,8 @@ Refer to [Getting Started](https://docs.docker.com/get-started/) ```ini [Unit] Description=SC4S Container -After=network.service -Requires=network.service +Wants=network.target network-online.target +After=network.target network-online.target [Service] Environment="SC4S_IMAGE=splunk/scs:latest" diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1a78b38..aca10a7 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2,6 +2,22 @@ ## General +To test the container outside of the systemd startup environment, you can run the following to test the syntax +of the container. These commands assume the local mounted directory is set up as shown in the gettingstarted +examples (and omits the disk buffer mount): + +``` +/usr/bin/docker run --env-file=/opt/sc4s/env_file -v "/opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" --name SC4S_preflight --rm splunk/scs:latest -s +``` + +and you can run + +``` +/usr/bin/docker run --env-file=/opt/sc4s/env_file -v "/opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" --name SC4S --rm splunk/scs:latest +``` + +to test the final image. These commands can help with container errors that are hidden in the systemd process. If you +are using podman, substitute "podman" for "docker" for the container runtime command above. ### Verification of TLS Server diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index cd99a70..2fa996f 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -79,7 +79,7 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CHECKPOINT_SPLUNK traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index e276484..2506ca5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -43,7 +43,7 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_ASA_LEGACY traffic {{tmpl.Exec "log_path" "no" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index b9d997c..c47fcd9 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -44,7 +44,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_IOS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl index b1d9297..5f1bfb9 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -96,7 +96,8 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_ISE traffic {{tmpl.Exec "log_path" "no" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index e351441..458ada4 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -44,7 +44,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_NX_OS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index 759f523..5c000ba 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -42,7 +42,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT")) }} # Listen on the specified dedicated port(s) for FORCEPOINT_WEBPROTECT traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 16b35f4..6fe1189 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -60,7 +60,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for FORTINET_FORTIOS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 7038621..c1dc820 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -45,7 +45,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_IDP traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index 4e3eea5..6185a34 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -59,7 +59,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_JUNOS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index 4a9952e..9d6116f 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -43,7 +43,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NETSCREEN traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index b21861d..404781d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -44,7 +44,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NSM traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index 9d511c7..0383731 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -41,7 +41,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NSM_IDP traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index ba00cc2..549b939 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -90,8 +90,8 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT")) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT"))) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT")) }} # Listen on the specified dedicated port(s) for MICROFOCUS_ARCSIGHT traffic {{tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 73dc1d0..71335e2 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -93,7 +93,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PALOALTO_PANOS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl index 60b6736..09dd410 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl @@ -42,7 +42,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl index 11a8057..694f14e 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl @@ -41,7 +41,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PROOFPOINT_PPS_SENDMAIL traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl index 0962cc6..226b310 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -127,8 +127,8 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT")) (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT"))) (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT")) }} # Listen on the specified dedicated port(s) for UBIQUITI_UNIFI traffic {{tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index 983e13f..c9de545 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -82,7 +82,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT")) (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for ZSCALER_NSS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index 69f4e28..98e3b78 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -40,7 +40,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_ASA traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index 1ff0958..a9f881e 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -43,7 +43,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT")) (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT"))) (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TLS_PORT")) }} # Listen on the specified dedicated port(s) for SYMANTEC_PROXY traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index 19a26c3..cc2d052 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -60,9 +60,10 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_JUNOS_STRUCTURED traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_JUNOS_STRUCTURED traffic diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl index 3ff2c86..149fb4b 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -40,7 +40,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_MERAKI traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 63fb6d2..ff5be59 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -127,6 +127,6 @@ source s_{{ .port_id}} { }; }; {{- end }} -{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) }} +{{- if or (or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT"))) (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) }} {{ template "T1" (.) }} {{- end }} diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl index 6a68b45..0f12886 100644 --- a/package/etc/local_config/log_paths/example.conf.tmpl +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -7,41 +7,39 @@ # If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), # the template generator will build a custom source based on the value of one or more of the set variables. -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} # "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name # set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". # "parser" can be customized on dedicated ports only # "common" uses the same parser sequence as the default ports and is the most commonly used -{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} # The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path # The ports used are based on the values of one or more of the environment variables set above. +{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { - -# The first time this template is used the log_path will be linked to the default port - {{- if eq (.) "yes"}} source(s_DEFAULT); - -# Filters should be updated to use the simplest and most effecient logic possible to discard -# the message from this path - filter(f_is_rfc3164); filter(f_local_example); {{- end}} {{- if eq (.) "no"}} + source (s_LOCAL_EXAMPLE); +{{- end}} + + +# The first time this template is used the log_path will be linked to the default port + +# Filters should be updated to use the simplest and most effecient logic possible to discard +# the message from this path # In the second pass through the template a link to the dedicated port is used. This # normally does not require additional filters -source (s_LOCAL_EXAMPLE); -{{- end}} #Set a default sourcetype and index @@ -64,11 +62,11 @@ source (s_LOCAL_EXAMPLE); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }} # Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic - {{tmpl.Exec "log_path" "no" }} +{{tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic