diff --git a/.gitignore b/.gitignore index 59343f4..3ac4e78 100644 --- a/.gitignore +++ b/.gitignore @@ -379,4 +379,7 @@ fabric.properties !/package/scripts/ .ecs /test-results/ -/.idea/ \ No newline at end of file +/.idea/ + +tests/test_plugin_*.py +package/etc/conf.d/local/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 61760d8..2b9b2c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +1,6 @@ [submodule "package/syslog-ng"] path = package/syslog-ng url = https://github.com/balabit/syslog-ng.git -[submodule "Submod"] branch = syslog-ng-3.23.1 +# + diff --git a/docs/configuration.md b/docs/configuration.md index f314f89..949bf47 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,4 +1,9 @@ -# Global Configuration +# SC4S Configuration Variables + +Other than device filter creation, SC4S is almost entirely controlled by environment variables. Here are the categories +and variables needed to properly configure SC4S for your environment. + +## Global Configuration | Variable | Values | Description | |----------|---------------|-------------| @@ -6,16 +11,17 @@ | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | -# Splunk HEC destination Configuration +## Splunk HEC Destination Configuration | Variable | Values | Description | |----------|---------------|-------------| +| SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (threads). Set this to the number of HEC endpoints up to a max of 32. | | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE | comma separated list | Open SSL cipher suite list | | SC4S_DEST_SPLUNK_HEC_SSL_VERSION | comma separated list | Open SSL version list | | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | -# Syslog Source Configuration +## Syslog Source Configuration | Variable | Values/Default | Description | |----------|----------------|-------------| @@ -28,7 +34,7 @@ | SC4S_SOURCE_UDP_SO_RCVBUFF | 425984 | UDP server buffer size in bytes | -# Syslog Source TLS Certificate Configuration +## Syslog Source TLS Certificate Configuration * Create a folder ``/opt/sc4s/tls`` * Save the server private key in PEM format with NO PASSWORD to ``/opt/sc4s/tls/server.key`` diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index a5ea9ab..7a29b37 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -13,7 +13,7 @@ transmission of events between computer systems over UDP, TCP, or TLS. The proto overhead on the sender favoring performance over reliability. This fundamental choice means any instability or resource constraint will cause data to be lost in transmission. -* When practical and cost effective considering the importance of completeness as a requirement, place the sc4s +* When practical and cost effective (considering the importance of completeness as a requirement), place the sc4s instance in the same VLAN as the source device. * Avoid crossing a Wireless network, WAN, Firewall, Load Balancer, or inline IDS. @@ -29,9 +29,10 @@ environment. ### Create Indexes -SC4S is pre-configured to map each sourcetype to a typical index, for new installations best practice is to create the following -indexes in Splunk. The indexes can be customized easily if desired. If using defaults create the following indexes on Splunk: +SC4S is pre-configured to map each sourcetype to a typical index. For new installations, it is best practice to create them in Splunk when +using the SC4S defaults. SC4S can be easily customized to use different indexes if desired. +* email * netauth * netfw * netids @@ -54,7 +55,7 @@ Install the following: session. Alternatively, a list of HEC endpoint URLs can be configured in SC4S if no load balancer is in place. In either case, it is recommended that SC4S traffic be sent to HEC endpoints configured directly on the indexers rather than an intermediate tier of HWFs. - Create a HEC token that will be used by SC4S and ensure the token has access to place events in main, em_metrics, and all indexes used as -event destinations +event destinations. - Refer to [Splunk Cloud](http://docs.splunk.com/Documentation/Splunk/7.3.1/Data/UsetheHTTPEventCollector#Configure_HTTP_Event_Collector_on_managed_Splunk_Cloud) or [Splunk Enterprise](http://dev.splunk.com/view/event-collector/SP-CAAAE6Q) for specific HEC configuration instructions based on your Splunk type. @@ -78,6 +79,6 @@ Splunk type. # Scale out -Additional hosts can be deployed for syslog collection from additional network zones and locations +Additional hosts can be deployed for syslog collection from additional network zones and locations: ![SC4S deployment diagram](SC4S%20deployment.png) \ No newline at end of file diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 3928114..9fef61c 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -5,7 +5,7 @@ Refer to [Getting Started](https://docs.docker.com/get-started/) # SC4S Configuration -* Create a directory on the server for configuration. This should be available to all administrators, for example: +* Create a directory on the server for local configurations. This should be available to all administrators, for example: ``/opt/sc4s/`` * Create a docker-compose.yml file in the directory created above, based on the following template: @@ -33,21 +33,33 @@ services: - /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf #Uncomment the following line if custom TLS certs are provided - /opt/sc4s/tls:/opt/syslog-ng/tls - ``` +* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the yml template +above (`volumes` declarations) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. +Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. + ## Configure the SC4S environment -Create the following file ``/opt/sc4s/env_file`` +Create the following file ``/opt/sc4s/env_file`` and add the environment variables below: * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SC4S_DEST_SPLUNK_HEC_WORKERS=6 SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics +#Uncomment the following line if using untrusted SSL certificates +#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` ## Configure index destinations for Splunk @@ -77,13 +89,12 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas ``` * Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. -# Start SC4S +## Start/Restart SC4S ```bash docker stack deploy --compose-file docker-compose.yml sc4s ``` - # Scale out Additional hosts can be deployed for syslog collection from additional network zones and locations. @@ -135,16 +146,23 @@ services: - /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf #Uncomment the following line if custom TLS certs are provided - /opt/sc4s/tls:/opt/syslog-ng/tls - ``` -* Modify the following file ``/opt/sc4s/default/env_file`` after including the port-specific environment variable(s). +* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SC4S_DEST_SPLUNK_HEC_WORKERS=6 SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics @@ -153,8 +171,57 @@ SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` -* Start SC4S. +* Restart SC4S (below) + +## Start/Restart SC4S ```bash docker stack deploy --compose-file docker-compose.yml sc4s ``` + +# Stop SC4S + +Start by obtaining the stack name (ID): +```bash +docker stack ls +``` +Then, remove the stack: +```bash +docker stack rm +``` +# Verify Proper Operation + +SC4S has a number of "preflight" checks to ensure that the container starts properly and that the syntax of the underlying syslog-ng +configuration is correct. After this step completes, to verify SC4S is properly communicating with Splunk, +execute the following search in Splunk: + +```ini +index=* sourcetype=sc4s:events "starting up" +``` +This should yield the following event: +```ini +syslog-ng starting up; version='3.22.1' +``` +when the startup process proceeds normally (without syntax errors). If you do not see this, +follow the steps below before proceeding to deeper-level troubleshooting: + +* Check to see that the URL, token, and TLS/SSL settings are correct, and that the appropriate firewall ports are open (8088 or 443). + +* Check to see that the proper indexes are created in Splunk, and that the token has access to them. + +* Ensure the proper operation of the load balancer if used. + +* Lastly, execute the following command to check the internal logs of the syslog-ng process running in the container. Depending on the +traffic load, there may be quite a bit of output in the syslog-ng logs. +```bash +docker logs SC4S +``` +You should see events similar to those below in the output: +```ini +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' +Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' +``` +If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set +incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more +information. \ No newline at end of file diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index a9dbcc8..eba329c 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -33,7 +33,7 @@ sudo docker swarm init # SC4S Configuration -* Create a directory on the server for configuration. This should be available to all administrators, for example: +* Create a directory on the server for local configurations. This should be available to all administrators, for example: ``/opt/sc4s/`` * Create a docker-compose.yml file in the directory created above, based on the following template: @@ -63,21 +63,33 @@ services: - /opt/sc4s/tls:/opt/syslog-ng/tls ``` +* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the yml template +above (`volumes` declarations) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. +Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. + ## Configure the SC4S environment -Create the following file ``/opt/sc4s/env_file`` +Create the following file ``/opt/sc4s/env_file`` and add the environment variables below: * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SC4S_DEST_SPLUNK_HEC_WORKERS=6 SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics +#Uncomment the following line if using untrusted SSL certificates +#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` - ## Configure index destinations for Splunk Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow @@ -105,7 +117,7 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas ``` * Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. -# Start SC4S +## Start/Restart SC4S ```bash sudo docker stack deploy --compose-file docker-compose.yml sc4s @@ -164,13 +176,21 @@ services: - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Modify the following file ``/opt/sc4s/default/env_file`` +* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SC4S_DEST_SPLUNK_HEC_WORKERS=6 SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics @@ -178,8 +198,58 @@ SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` -* Start SC4S. + +* Restart SC4S (below) + +## Start/Restart SC4S ```bash docker stack deploy --compose-file docker-compose.yml sc4s ``` + +# Stop SC4S + +Start by obtaining the stack name (ID): +```bash +docker stack ls +``` +Then, remove the stack: +```bash +docker stack rm +``` +# Verify Proper Operation + +SC4S has a number of "preflight" checks to ensure that the container starts properly and that the syntax of the underlying syslog-ng +configuration is correct. After this step completes, to verify SC4S is properly communicating with Splunk, +execute the following search in Splunk: + +```ini +index=* sourcetype=sc4s:events "starting up" +``` +This should yield the following event: +```ini +syslog-ng starting up; version='3.22.1' +``` +when the startup process proceeds normally (without syntax errors). If you do not see this, +follow the steps below before proceeding to deeper-level troubleshooting: + +* Check to see that the URL, token, and TLS/SSL settings are correct, and that the appropriate firewall ports are open (8088 or 443). + +* Check to see that the proper indexes are created in Splunk, and that the token has access to them. + +* Ensure the proper operation of the load balancer if used. + +* Lastly, execute the following command to check the internal logs of the syslog-ng process running in the container. Depending on the +traffic load, there may be quite a bit of output in the syslog-ng logs. +```bash +docker logs SC4S +``` +You should see events similar to those below in the output: +```ini +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' +Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' +``` +If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set +incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more +information. \ No newline at end of file diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 9a683dd..80721ec 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -5,11 +5,7 @@ Refer to [Getting Started](https://docs.docker.com/get-started/) # Setup -* Create a systemd unit file use to start the container with the host os. - -``sudo vi /lib/systemd/system/sc4s.service`` - -Enter insert mode and paste the following +* Create the systemd unit file `/lib/systemd/system/sc4s.service` based on the following template: ```ini [Unit] @@ -20,10 +16,13 @@ Requires=network.service [Service] Environment="SC4S_IMAGE=splunk/sc4s:latest" -#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +#Note Uncomment the following line to use custom index names AND download the splunk_index.csv file template per getting started Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" + +#Note Uncomment the following two lines for host and ip based source type mapping AND download the two file templates per getting started Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + #Uncomment the following line if custom TLS certs are provided #Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -39,20 +38,29 @@ ExecStartPre=/usr/bin/docker run \ ExecStart=/usr/bin/docker run -p 514:514 \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \ - --name SC4S \ - --rm \ + --name SC4S --rm \ $SC4S_IMAGE ``` +* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the unit +file above (the `Environment` assignments) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. + ## Configure the SC4S environment -Create the following file ``/opt/sc4s/default/env_file`` +Create the following file ``/opt/sc4s/default/env_file`` and add the environment variables below: * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SC4S_DEST_SPLUNK_HEC_WORKERS=6 SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics @@ -88,7 +96,7 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas ``` * Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. -# Start SC4S +## Configure SC4S for systemd and start SC4S ```bash sudo systemctl daemon-reload @@ -96,7 +104,6 @@ sudo systemctl enable sc4s sudo systemctl start sc4s ``` - # Configure Dedicated Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in @@ -142,13 +149,21 @@ ExecStart=/usr/bin/docker run -p 514:514 -p 5000-5020:5000-5020 \ $SC4S_IMAGE ``` -* Modify the following file ``/opt/sc4s/default/env_file`` +* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SC4S_DEST_SPLUNK_HEC_WORKERS=6 SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics @@ -157,8 +172,67 @@ SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` -* Restart SC4S. +* Restart SC4S (below) + +# Start SC4S + +```bash +sudo systemctl start sc4s +``` + +# Restart SC4S ```bash sudo systemctl restart sc4s ``` + +If changes were made to the configuration Unit file above (e.g. to configure with dedicated ports), you must first stop SC4S and re-run +the systemd configuration commands: +```bash +sudo systemctl stop sc4s +sudo systemctl daemon-reload +sudo systemctl enable sc4s +sudo systemctl start sc4s +``` + +# Stop SC4S + +```bash +sudo systemctl stop sc4s +``` +# Verify Proper Operation + +SC4S has a number of "preflight" checks to ensure that the container starts properly and that the syntax of the underlying syslog-ng +configuration is correct. After this step completes, to verify SC4S is properly communicating with Splunk, +execute the following search in Splunk: + +```ini +index=* sourcetype=sc4s:events "starting up" +``` +This should yield the following event: +```ini +syslog-ng starting up; version='3.22.1' +``` +when the startup process proceeds normally (without syntax errors). If you do not see this, +follow the steps below before proceeding to deeper-level troubleshooting: + +* Check to see that the URL, token, and TLS/SSL settings are correct, and that the appropriate firewall ports are open (8088 or 443). + +* Check to see that the proper indexes are created in Splunk, and that the token has access to them. + +* Ensure the proper operation of the load balancer if used. + +* Lastly, execute the following command to check the internal logs of the syslog-ng process running in the container. Depending on the +traffic load, there may be quite a bit of output in the syslog-ng logs. +```bash +docker logs SC4S +``` +You should see events similar to those below in the output: +```ini +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' +Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' +``` +If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set +incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more +information. \ No newline at end of file diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 0ddb896..54d8df6 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -5,11 +5,7 @@ Refer to [Installation](https://podman.io/getting-started/installation) # Setup -* Create a systemd unit file use to start the container with the host os. - -``sudo vi /lib/systemd/system/sc4s.service`` - -Enter insert mode and paste the following +* Create the systemd unit file `/lib/systemd/system/sc4s.service` based on the following template: ```ini [Unit] @@ -20,16 +16,19 @@ Requires=network.service [Service] Environment="SC4S_IMAGE=splunk/scs:latest" -#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +#Note Uncomment the following line to use custom index names AND download the splunk_index.csv file template per getting started Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" -#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started + +#Note Uncomment the following two lines for host and ip based source type mapping AND download the two file templates per getting started #Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" #Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + #Uncomment the following line if custom TLS certs are provided #Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" TimeoutStartSec=0 Restart=always + ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE ExecStartPre=/usr/bin/podman run \ --env-file=/opt/sc4s/default/env_file \ @@ -41,18 +40,27 @@ ExecStart=/usr/bin/podman run -p 514:514 \ "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \ --name SC4S --rm \ $SC4S_IMAGE - ``` +* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the unit +file above (the `Environment` assignments) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. + ## Configure the sc4s environment -Create the following file ``/opt/sc4s/default/env_file`` +Create the following file ``/opt/sc4s/default/env_file`` and add the environment variables below: * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SC4S_DEST_SPLUNK_HEC_WORKERS=6 SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics @@ -88,7 +96,7 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas ``` * Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. -# Start SC4S +## Configure SC4S for systemd and start SC4S ```bash sudo systemctl daemon-reload @@ -139,16 +147,23 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 5000-5020:5000-5020 \ --name SC4S \ --rm \ $SC4S_IMAGE - ``` -* Modify the following file ``/opt/sc4s/default/env_file`` +* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SC4S_DEST_SPLUNK_HEC_WORKERS=6 SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics @@ -157,8 +172,67 @@ SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` -* Restart SC4S. +* Restart SC4S (below) + +# Start SC4S + +```bash +sudo systemctl start sc4s +``` + +# Restart SC4S ```bash sudo systemctl restart sc4s ``` + +If changes were made to the configuration Unit file above (e.g. to configure with dedicated ports), you must first stop SC4S and re-run +the systemd configuration commands: +```bash +sudo systemctl stop sc4s +sudo systemctl daemon-reload +sudo systemctl enable sc4s +sudo systemctl start sc4s +``` + +# Stop SC4S + +```bash +sudo systemctl stop sc4s +``` +# Verify Proper Operation + +SC4S has a number of "preflight" checks to ensure that the container starts properly and that the syntax of the underlying syslog-ng +configuration is correct. After this step completes, to verify SC4S is properly communicating with Splunk, +execute the following search in Splunk: + +```ini +index=* sourcetype=sc4s:events "starting up" +``` +This should yield the following event: +```ini +syslog-ng starting up; version='3.22.1' +``` +when the startup process proceeds normally (without syntax errors). If you do not see this, +follow the steps below before proceeding to deeper-level troubleshooting: + +* Check to see that the URL, token, and TLS/SSL settings are correct, and that the appropriate firewall ports are open (8088 or 443). + +* Check to see that the proper indexes are created in Splunk, and that the token has access to them. + +* Ensure the proper operation of the load balancer if used. + +* Lastly, execute the following command to check the internal logs of the syslog-ng process running in the container. Depending on the +traffic load, there may be quite a bit of output in the syslog-ng logs. +```bash +podman logs SC4S +``` +You should see events similar to those below in the output: +```ini +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' +Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' +``` +If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set +incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more +information. \ No newline at end of file diff --git a/docs/performance.md b/docs/performance.md index 3eba88d..966a318 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -1,27 +1,26 @@ # Performance +Performance testing against our lab configuration produces the following results and limitations. ## Tested Configuration -* SC4S instance requesting 8 cores and 4 GB of memory with K8S scheduler. +* SC4S instance requesting 8 cores and 4 GB of memory with K8S scheduler * 6 Splunk Indexers clustered in Single site -* 1 loggen test client using the following command +* 1 loggen test client using the following command: + ``` + /opt/syslog-ng/bin/loggen -i --rate=1000 --interval=180 -P -F --sdata="[test name=\"stress17\"]" -s 800 --active-connections=10 sc4s 514` + ``` * AWS instance type c5n.4xlarge -```bash -/opt/syslog-ng/bin/loggen -i --rate=1000 --interval=180 -P -F --sdata="[test name=\"stress17\"]" -s 800 --active-connections=10 sc4s 514 -``` - ## Result -The single syslog-ng container in this test is able to provided effective balancing and routing of events equivalent 632 GB per day +The single syslog-ng container in this test is able to provide effective balancing and routing of events equivalent to 632 GB per day: ``` average rate = 9717.58 msg/sec, count=1749420, time=180.026, (average) msg size=800, bandwidth=7591.86 kB/sec - ``` ## Limitations -* Splunk Enterprise's implementation of the http event collection server will respond to the client with a status code 200 and fail to commit the events to disk during a rolling restart in our testing 20-30 events per indexer may be lost +Splunk Enterprise's implementation of the http event collection server responds to the client with a status code 200 and fails to commit the events to disk during a rolling restart. In our testing, 20-30 events per indexer are lost. diff --git a/docs/sources.md b/docs/sources.md index 19f26d1..7919645 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -233,6 +233,55 @@ index= (sourcetype=fgt_log OR sourcetype=fgt_traffic OR sourcetype Verify timestamp, and host values match as expected +# Vendor - Imperva + +## Product - Incapsula + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | +| Splunk Add-on Source Specific | https://bitbucket.org/SPLServices/ta-cef-imperva-incapsula/downloads/ | +| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cef | Common sourcetype | + +### Source + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| Imperva:Incapsula | Common sourcetype | + +### Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| cef_Incapsula_SIEMintegration | Imperva:Incapsula | netwaf | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef source="Imperva:Incapsula") +``` + # Vendor - Juniper ## Product - Juniper JunOS @@ -432,6 +481,103 @@ index= sourcetype=juniper:sslvpn | stats count by host Verify timestamp, and host values match as expected +# Vendor - Microfocus ArcSight + +## Product - Internal Agent Events + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | +| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cef | Common sourcetype | + +### Source + +| source | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| ArcSight:ArcSight | Internal logs | + +### Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| cef_ArcSight_ArcSight | ArcSight:ArcSight | main | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef source="ArcSight:ArcSight") +``` + +## Product - Microsoft Windows + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-microsoft-windows-for-splunk/downloads/ | +| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cef | Common sourcetype | + +### Source + +| source | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| CEFEventLog:System or Application Event | Windows Application and System Event Logs | +| CEFEventLog:Microsoft Windows | Windows Security Event Logs | + +### Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| cef_Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | +| cef_Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef (source="CEFEventLog:Microsoft Windows" OR source="CEFEventLog:System or Application Event")) +``` + # Vendor - PaloAlto ## Product - NGFW diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index 5efd586..a08f36d 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -20,6 +20,13 @@ template t_msg_only { }; # =============================================================================================== +# Message Only trim leading or trailing white space +# =============================================================================================== + +template t_msg_trim { + template("$(strip $MSGONLY )"); + }; +# =============================================================================================== # Everything; useful for bluecoat proxy and others # =============================================================================================== diff --git a/package/etc/conf.d/filters/juniper/junos.conf b/package/etc/conf.d/filters/juniper/junos.conf index 4c259e3..f5c4256 100644 --- a/package/etc/conf.d/filters/juniper/junos.conf +++ b/package/etc/conf.d/filters/juniper/junos.conf @@ -3,5 +3,6 @@ filter f_juniper_junos_structured { }; filter f_juniper_junos_standard { - program("RT_IDP|RT_FLOW|RT_IDS|RT_UTM|Juniper"); + program("RT_IDP|RT_FLOW|RT_IDS|RT_UTM|Juniper") or + message("PFE_FW_|DFWD_") }; \ No newline at end of file diff --git a/package/etc/conf.d/filters/microfocus/arcsight.conf b/package/etc/conf.d/filters/microfocus/arcsight.conf new file mode 100644 index 0000000..287d7a4 --- /dev/null +++ b/package/etc/conf.d/filters/microfocus/arcsight.conf @@ -0,0 +1,4 @@ + +filter f_microfocus_arcsight { + program(CEF); +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/destinations/README.md b/package/etc/conf.d/local/destinations/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/destinations/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/filters/README.md b/package/etc/conf.d/local/filters/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/filters/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/filters/example.conf b/package/etc/conf.d/local/filters/example.conf new file mode 100644 index 0000000..047fdc8 --- /dev/null +++ b/package/etc/conf.d/local/filters/example.conf @@ -0,0 +1,4 @@ + +filter f_local_example { + program(sc4splugin); +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/log_paths/README.md b/package/etc/conf.d/local/log_paths/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/log_paths/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/log_paths/example.conf.tmpl b/package/etc/conf.d/local/log_paths/example.conf.tmpl new file mode 100644 index 0000000..a8ac264 --- /dev/null +++ b/package/etc/conf.d/local/log_paths/example.conf.tmpl @@ -0,0 +1,76 @@ +# LOCAL_EXAMPLE + +# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique +# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the +# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ + +# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), +# the template generator will build a custom source based on the value of one or more of the set variables. + +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} + +# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name +# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". +# "parser" can be customized on dedicated ports only +# "common" uses the same parser sequence as the default ports and is the most commonly used + +{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} + +# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path +# The ports used are based on the values of one or more of the environment variables set above. + +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { + +# The first time this template is used the log_path will be linked to the default port + +{{- if eq (.) "yes"}} + source(s_default-ports); + +# Filters should be updated to use the simplest and most effecient logic possible to discard +# the message from this path + + filter(f_is_rfc3164); + filter(f_local_example); +{{- end}} +{{- if eq (.) "no"}} + +# In the second pass through the template a link to the dedicated port is used. This +# normally does not require additional filters + +source (s_dedicated_port_LOCAL_EXAMPLE); +{{- end}} + +#Set a default sourcetype and index + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + +#using the key "local_example" find any cutomized index,source or sourcetype meta values + + parser {p_add_context_splunk(key("local_example")); }; + +# Any additional logic needed to process the event before sending to Splunk goes here + +# Send it to Splunk + + destination(d_hec); #--HEC-- + +# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will +# forward events to another system + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic + + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic + +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/local/sources/README.md b/package/etc/conf.d/local/sources/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/sources/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index abe69d4..c2ec00e 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -17,9 +17,9 @@ log { if (program('RT_IDP')) { rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; parser {p_add_context_splunk(key("juniper_idp")); }; - } elif (program('RT_FLOW')) { + } elif (program('RT_FLOW') or message('PFE_FW_|DFWD_')) { rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"), template("t_standard"))}; - parser {p_add_context_splunk(key("juniper_junos_flow")); }; + parser {p_add_context_splunk(key("juniper_junos_fw")); }; } elif (program('RT_IDS')) { rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; parser {p_add_context_splunk(key("juniper_junos_ids")); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl new file mode 100644 index 0000000..76d3e1e --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl @@ -0,0 +1,84 @@ +parser p_microfocus_arcsight_header { + csv-parser( + columns("fields.sc4s_cef_version", "fields.cef_device_vendor", "fields.cef_device_product", "fields.cef_device_version", "fields.cef_device_event_class", "fields.cef_name", "fields.cef_severity", MESSAGE) + delimiters(chars("|")) + flags(strip-whitespace, escape-none, greedy) + ); + + kv-parser(prefix(".cef.") template("t_msg_trim") + ); + +}; + +parser p_microfocus_arcsight_ts_rt { + date-parser(format("%s") template("${.cef.rt}") + ); +}; +parser p_microfocus_arcsight_ts_end { + date-parser(format("%s") template("${.cef.end}") + ); +}; + +parser p_microfocus_arcsight_source { + add-contextual-data( + selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), + database("context-local/microfocus_arcsight_source.csv") + ignore-case(yes) + prefix(".splunk.") + default-selector("unknown") + ); +}; + +# Microfocus ArcSight +{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_microfocus_arcsight); +{{- end}} +{{- if eq (.) "no"}} + source (s_dedicated_port_MICROFOCUS_ARCSIGHT); +{{- end}} + + rewrite { + r_set_splunk_dest_default(sourcetype("cef"), index("main"), template("t_msg_trim")) + }; + + parser { + p_add_context_splunk(key("cef_{fields.cef_device_vendor}_${fields.cef_device_product}")); + + }; + + parser (p_microfocus_arcsight_header); + + # We already have the syslog msg time stamp however that may not be the best one + # If we have an rt or end field that is best we use the If trick here so if this parser fails + # We don't get sent to fallback. + if { + parser (p_microfocus_arcsight_ts_rt); + } elif { + parser (p_microfocus_arcsight_ts_end); + } else {}; #Do nothing this is allows for both rt and end to be missing and still pass with the message ts + + #CEF TAs use the source as their bounds in props.conf + parser(p_microfocus_arcsight_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for MICROFOCUS_ARCSIGHT traffic + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for MICROFOCUS_ARCSIGHT traffic +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl index bf1b644..3e56a4c 100644 --- a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl @@ -18,7 +18,7 @@ log { parser {p_add_context_splunk(key("juniper_idp_structured")); }; } elif (program('RT_FLOW')) { rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) }; - parser {p_add_context_splunk(key("juniper_junos_flow_structured")); }; + parser {p_add_context_splunk(key("juniper_junos_fw_structured")); }; } elif (program('RT_IDS')) { rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; parser {p_add_context_splunk(key("juniper_junos_ids_structured")); }; diff --git a/package/etc/context-local/microfocus_arcsight_source.csv b/package/etc/context-local/microfocus_arcsight_source.csv new file mode 100644 index 0000000..6c5bd73 --- /dev/null +++ b/package/etc/context-local/microfocus_arcsight_source.csv @@ -0,0 +1,8 @@ +ArcSight_ArcSight,source,ArcSight:ArcSight +Microsoft_System or Application Event,source,CEFEventLog:System or Application Event +Microsoft_System or Application Event,index,oswin +Microsoft_Microsoft Windows,source,CEFEventLog:Microsoft Windows +Microsoft_Microsoft Windows,index,oswinsec +Incapsula_SIEMintegration,source,Imperva:Incapsula +Incapsula_SIEMintegration,index,netwaf +unknown,source,ArcSight:unknown diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv index 566c436..e93911a 100644 --- a/package/etc/context-local/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -1,7 +1,12 @@ #bluecoat_proxy,index,netproxy +#cef_ArcSight_ArcSight,index,netwaf +#cef_Incapsula_SIEMintegration,index,netwaf +#cef_Microsoft_Microsoft Windows,index,oswinsec +#cef_Microsoft_System or Application Event,index,oswin #cisco_asa,index,netfw #cisco_ios,index,netops #cisco_nx_os,index,netops +#local_example,index,main #fortinet_fortios_event,index,netops #fortinet_fortios_log,index,netops #fortinet_fortios_traffic,index,netfw @@ -9,10 +14,10 @@ #juniper_idp,index,netids #juniper_structured,index,netops #juniper_idp_structured,index,netids -#juniper_junos_flow_structured,index,netfw +#juniper_junos_fw_structured,index,netfw #juniper_junos_ids_structured,index,netids #juniper_junos_utm_structured,index,netfw -#juniper_junos_flow,index,netfw +#juniper_junos_fw,index,netfw #juniper_junos_ids,index,netids #juniper_junos_utm,index,netfw #juniper_sslvpn,index,netfw diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index f565484..e4a0a78 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -62,7 +62,8 @@ options { @include "conf.d/destinations/*.conf" @include "conf.d/log_paths/*.conf" -@include "conf.d/local/*/filters/*.conf" -@include "conf.d/local/*/sources/*.conf" -@include "conf.d/local/*/destinations/*.conf" -@include "conf.d/local/*/log_paths/*.conf" +@include "conf.d/local/filters/*.conf" +@include "conf.d/local/filters/*/*.conf" +@include "conf.d/local/sources/*.conf" +@include "conf.d/local/destinations/*.conf" +@include "conf.d/local/log_paths/*.conf" diff --git a/package/syslog-ng b/package/syslog-ng index 1742b11..26c0fe2 160000 --- a/package/syslog-ng +++ b/package/syslog-ng @@ -1 +1 @@ -Subproject commit 1742b11e5cfa6544ece38aaecde96e9b423d61c5 +Subproject commit 26c0fe20a7169a2007de561ff3c4bc7df93ca86b diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index 60d7cae..09e86e5 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -20,6 +20,15 @@ homePath = $SPLUNK_DB/test2/db coldPath = $SPLUNK_DB/test2/colddb thawedPath = $SPLUNK_DB/test2/thaweddb +[oswin] +homePath = $SPLUNK_DB/oswin/db +coldPath = $SPLUNK_DB/oswin/colddb +thawedPath = $SPLUNK_DB/oswin/thaweddb + +[oswinsec] +homePath = $SPLUNK_DB/oswinsec/db +coldPath = $SPLUNK_DB/oswinsec/colddb +thawedPath = $SPLUNK_DB/oswinsec/thaweddb [netfw] homePath = $SPLUNK_DB/netfw/db @@ -41,6 +50,11 @@ homePath = $SPLUNK_DB/netproxy/db coldPath = $SPLUNK_DB/netproxy/colddb thawedPath = $SPLUNK_DB/netproxy/thaweddb +[netwaf] +homePath = $SPLUNK_DB/netwaf/db +coldPath = $SPLUNK_DB/netwaf/colddb +thawedPath = $SPLUNK_DB/netwaf/thaweddb + [email] homePath = $SPLUNK_DB/email/db coldPath = $SPLUNK_DB/email/colddb diff --git a/tests/test_microfocus_arcsight_cef.py b/tests/test_microfocus_arcsight_cef.py new file mode 100644 index 0000000..eb3dd6d --- /dev/null +++ b/tests/test_microfocus_arcsight_cef.py @@ -0,0 +1,137 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +# Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:050|Connector Raw Event Statistics|Low| eventId=77 mrt=1539321422787 categorySignificance=/Informational categoryBehavior=/Execute/Response categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321424798 cat=/Agent/RawEvent/Statistics deviceSeverity=Warning rt=1539321422787 fileType=Agent cs1=0.23 cs2=157.72333333333333 cs3=0.23 cs4=47317 cs5=157.72333333333333 cs6=3o0OiZmYBABCACGN9CiyuGQ\=\= cn1=69 cn2=47317 cn3=69 deviceCustomDate1=1539321122775 cs1Label=Event throughput cs2Label=Raw event character throughput cs3Label=Event throughput (SLC) cs4Label=Raw event length (SLC) cs5Label=Raw event character throughput (SLC) cs6Label=Destination ID cn1Label=Total event count cn2Label=Total raw event length cn3Label=Event count (SLC) deviceCustomDate1Label=Last time ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= +# Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=30 msg=Connected to Host mrt=1539321123071 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321124967 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321123071 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= +# Mar 19 15:19:15 root CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:030|Agent [PAN1_WUC_UDP8000] type [windowsfg] started|Low| eventId=26 mrt=1539321122832 categorySignificance=/Normal categoryBehavior=/Execute/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application/Service art=1539321124967 cat=/Agent/Started deviceSeverity=Warning rt=1539321122832 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= +# Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321047341 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= +def test_microfocus_arcsight_cef_ts_rt(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=' + "{% now 'utc', '%s' %}" + ' dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\\=\\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cef\" source=ArcSight:ArcSight | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_cef_ts_end(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning end=' + "{% now 'utc', '%s' %}" + ' dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\\=\\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cef\" source=ArcSight:ArcSight| head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_cef_ts_syslog(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\\=\\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cef\" source=ArcSight:ArcSight | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Microsoft|Microsoft Windows||Microsoft-Windows-Security-Auditing:4634|An account was logged off.|Low| eventId=1 externalId=4634 msg=Network: A user or computer logged on to this computer from the network. rawEvent=EventlogType\=Security&&EventIndex\=1031&&WindowsVersion\=Windows Server 2012 R2&&WindowsKeyMapFamily\=Windows 2012 R2&&WindowsParserFamily\=Windows 2012 R2|2012|8&&DetectTime\=2018-10-12 7:25:11&&EventSource\=Microsoft-Windows-Security-Auditing&&EventID\=4634&&EventType\=Audit_success&&EventCategory\=12545&&User\=null&&ComputerName\=WIN-PAN1&&Description\=An account was logged off.&&Message\=This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer.&&Subject:Security ID\=S-1-5-21-750061412-3179291162-3140434184-500&&Subject:Account Name\=Administrator&&Subject:Account Domain\=WIN-PAN1&&Subject:Logon ID\=0x373c2&&Logon Type\=3 categorySignificance=/Informational categoryBehavior=/Access/Stop categoryDeviceGroup=/Operating System catdt=Operating System categoryOutcome=/Success categoryObject=/Host/Operating System art=1539321047369 cat=Security deviceSeverity=Audit_success rt=' + "{% now 'utc', '%s' %}" + ' dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dntdom=WIN-PAN1 duser=Administrator duid=0x373c2 cs2=Logon/Logoff:Logoff cn1=3 cs1Label=Accesses cs2Label=EventlogCategory cs4Label=Reason or Error Code cs5Label=Authentication Package Name cn1Label=LogonType cn2Label=CrashOnAuditFail cn3Label=Count ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=WIN-PAN1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 deviceNtDomain=WIN-PAN1 dtz=Asia/Riyadh _cefVer=0.1 ad.WindowsVersion=Windows Server 2012 R2 ad.WindowsParserFamily=Windows 2012 R2|2012|8 ad.WindowsKeyMapFamily=Windows 2012 R2 ad.EventIndex=1031 aid=3o0OiZmYBABCACGN9CiyuGQ\=\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=oswinsec host=\"{{ host }}\" sourcetype=\"cef\" source=\"CEFEventLog:Microsoft Windows\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Microsoft|System or Application Event||Software Protection Platform Service:902|Software Protection Platform Service|Unknown| eventId=39 externalId=902 rawEvent=EventlogType\=Application&&EventIndex\=1604&&WindowsVersion\=Windows Server 2012 R2&&WindowsKeyMapFamily\=Windows 2012 R2&&WindowsParserFamily\=Windows 2012 R2|2012|8&&DetectTime\=2018-10-12 8:12:31&&EventSource\=Software Protection Platform Service&&EventID\=902&&EventType\=null&&EventCategory\=0&&User\=null&&ComputerName\=WIN-PAN1&&Key[0]\=6.3.9600.16384 art=1539321151610 cat=Application rt=' + "{% now 'utc', '%s' %}" + ' cs2=0 cs3=Software Protection Platform Service cs2Label=EventlogCategory cs3Label=EventSource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=WIN-PAN1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dtz=Asia/Riyadh _cefVer=0.1 ad.WindowsVersion=Windows Server 2012 R2 ad.WindowsParserFamily=Windows 2012 R2|2012|8 ad.WindowsKeyMapFamily=Windows 2012 R2 ad.Key[0]=6.3.9600.16384 ad.EventIndex=1604 aid=3o0OiZmYBABCACGN9CiyuGQ\=\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=oswin host=\"{{ host }}\" sourcetype=\"cef\" source=\"CEFEventLog:System or Application Event\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_imperva_incapsula(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support src=12.12.12.12 caIP=13.13.13.13 ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsigdproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netwaf host=\"{{ host }}\" sourcetype=\"cef\" source=\"Imperva:Incapsula\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_plugin_example.py b/tests/test_plugin_example.py new file mode 100644 index 0000000..852dec5 --- /dev/null +++ b/tests/test_plugin_example.py @@ -0,0 +1,32 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +def test_plugin_local_example(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4splugin[0]: test\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"sc4s:local_example\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file