diff --git a/docker-compose.yml b/docker-compose.yml index dff7a5c..3e3bc3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,12 +44,12 @@ services: - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SC4S_SOURCE_TLS_ENABLE=no - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - - SC4S_LISTEN_DEFAULT_TCP_PORT=514 - - SC4S_LISTEN_DEFAULT_UDP_PORT=514 -# - SC4S_LISTEN_DEFAULT_TLS_PORT=6514 +# - SC4S_LISTEN_DEFAULT_TCP_PORT=514 +# - SC4S_LISTEN_DEFAULT_UDP_PORT=514 +# - SC4S_LISTEN_DEFAULT_TLS_PORT=6514 - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 - SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT=6000 -# - SC4S_ARCHIVE_CHECKPOINT=yes +# - SC4S_ARCHIVE_CHECKPOINT=yes - SC4S_ARCHIVE_GLOBAL=yes volumes: - ./tls:/opt/syslog-ng/tls diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 59baff5..e91f5b1 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -17,7 +17,7 @@ administration and syslog-ng configuration experience is assumed when using the * NOTE: Do _not_ depend on the distribution-supplied version of syslog-ng, as it will likely be far too old. Read this [explanation](https://www.syslog-ng.com/community/b/blog/posts/installing-latest-syslog-ng-on-rhel-and-other-rpm-distributions) -for the reason why syslog-ng builds are so dated in most RHEL/Debian distributions. +for the reason why syslog-ng builds are so dated in almost all RHEL/Debian distributions. # BYOE Installation Instructions @@ -157,3 +157,21 @@ sudo systemctl daemon-reload sudo systemctl enable sc4s sudo systemctl start sc4s ``` +## Configure SC4S Listening Ports + +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The standard SC4S configuration reflect these defaults. These defaults can be changed by adding the following +additional environment variables with appropriate values to the ``env_file`` above: +```dotenv +SC4S_LISTEN_DEFAULT_TCP_PORT=514 +SC4S_LISTEN_DEFAULT_UDP_PORT=514 +SC4S_LISTEN_DEFAULT_TLS_PORT=6514 +``` +### Dedicated (Unique) Listening Ports + +For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources we provide a means of dedicating a unique listening port to a specific source. + +Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology +in use. \ No newline at end of file diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 44d2255..a26a1c1 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -22,7 +22,7 @@ reboot: net.ipv4.ip_forward=1 ``` -# SC4S Configuration +# SC4S Initial Configuration * Create a directory on the server for local configurations and disk buffering. This should be available to all administrators, for example: @@ -46,6 +46,11 @@ services: protocol: udp # Comment the following line out if using docker-compose mode: host + - target: 6514 + published: 6514 + protocol: tcp +# Comment the following line out if using docker-compose + mode: host env_file: - /opt/sc4s/env_file volumes: @@ -88,9 +93,10 @@ document for details on the directory structure the archive uses. * IMPORTANT: When creating the directories above, ensure the directories created match the volume mounts specified in the `docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. -## Configure the SC4S environment +# Configure the SC4S environment -Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: +SC4S is almost entirely controlled through environment variables, which are read from a file at starteup. Create a file named +``/opt/sc4s/env_file`` and add the following environment variables and values: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 @@ -102,64 +108,46 @@ SC4S_DEST_SPLUNK_HEC_WORKERS=6 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints, up to a maxiumum of 32. +If the endpoint is a VIP, match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to uncomment the last line in the example above. -## Modify index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. - -* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your -environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in -this table that pertain to the individual data source filters that are included with SC4S. -* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further -information is covered in the "Log Path overrides" section of the Configuration document. - -## Configure source filtering by source IP or host name - -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. - -* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. -* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. -* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. - -## Configure compliance index/metadata overrides - -In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. -The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to -the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file -lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is -covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. +## Configure SC4S Listening Ports -## Start/Restart SC4S +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The docker compose file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container +port mapping can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the +``published`` port(s) in the docker compose file (which represents the actual listening ports on the host machine), like so: -```bash -docker stack deploy --compose-file docker-compose.yml sc4s ``` + ports: + - target: 514 + published: 614 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host +``` +This snippet above instructs the _host_ to listen on TCP port 614 and map that port to the default TCP 514 port on the _container_. +No changes to the underlying SC4S default configuration (environment variables) are needed. -# Scale out - -Additional hosts can be deployed for syslog collection from additional network zones and locations. - - -# Configure Dedicated Listening Ports +### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in -the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. -For collection of such sources we provide a means of dedicating a unique listening port to a specific source. +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. -Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. +The docker compose file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured +by the environment variable(s). In the following example, additional ``target`` stanzas are added for the main ``sc4s`` container, where the +``target`` and ``published`` lines provide for 21 additional technology-specific UDP and TCP ports. -In the following example the target port ranges allow for up to 21 technology-specific ports. Modify individual ports or a -range as appropriate for your network. +Follow these steps to configure unique ports: -* Modify the unit file ``/opt/sc4s/docker-compose.yml`` +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). Refer to the "Sources" +documentation to identify the specific environment variables that are mapped to each data source vendor/technology. +* Modify the compose file ``/opt/sc4s/docker-compose.yml`` and add/change port stanzas as appropriate using the example below. +* Restart SC4S using the command in the "Start/Restart SC4S" section below. ```yaml version: "3.7" services: @@ -176,6 +164,11 @@ services: protocol: udp #Comment the following line out if using docker-compose mode: host + - target: 6514 + published: 6514 + protocol: tcp +# Comment the following line out if using docker-compose + mode: host - target: 5000-5020 published: 5000-5020 protocol: tcp @@ -191,38 +184,46 @@ services: volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z -#Uncomment the following line if custom TLS certs are provided -# - /opt/sc4s/tls:/opt/syslog-ng/tls +# Uncomment the following line if local disk archiving is desired +# - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z +# Uncomment the following line if custom TLS certs are provided +# - /opt/sc4s/tls:/opt/syslog-ng/tls:z ``` -* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). +## Modify index destinations for Splunk -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. +* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further +information is covered in the "Log Path overrides" section of the Configuration document. -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +## Configure source filtering by source IP or host name -```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088 -SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 -#Uncomment the following line if using untrusted SSL certificates -#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no -``` +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* Restart SC4S (below) +* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. -## Start/Restart SC4S +## Configure compliance index/metadata overrides + +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is +covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. + +# Start/Restart SC4S ```bash docker stack deploy --compose-file docker-compose.yml sc4s ``` - # Stop SC4S Start by obtaining the stack name (ID): @@ -244,7 +245,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.22.1' +syslog-ng starting up; version='3.25.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -262,7 +263,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 0af8b6b..251c9fa 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -1,5 +1,5 @@ -# Install Docker CE and Swarm +# Install Docker CE and Swarm (RHEL 7.7) * Warning: this method of installing docker on RHEL does not appear to be supported. Consider using podman instead. @@ -31,7 +31,7 @@ systemctl start docker.service sudo docker swarm init ``` -# SC4S Configuration +# SC4S Initial Configuration * Create a directory on the server for local configurations and disk buffering. This should be available to all administrators, for example: ``/opt/sc4s/`` @@ -52,6 +52,11 @@ services: - target: 514 published: 514 protocol: udp +# Comment the following line out if using docker-compose + mode: host + - target: 6514 + published: 6514 + protocol: tcp # Comment the following line out if using docker-compose mode: host env_file: @@ -96,9 +101,10 @@ document for details on the directory structure the archive uses. * IMPORTANT: When creating the directories above, ensure the directories created match the volume mounts specified in the `docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. -## Configure the SC4S environment +# Configure the SC4S environment -Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: +SC4S is almost entirely controlled through environment variables, which are read from a file at starteup. Create a file named +``/opt/sc4s/env_file`` and add the following environment variables and values: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 @@ -110,66 +116,46 @@ SC4S_DEST_SPLUNK_HEC_WORKERS=6 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints, up to a maxiumum of 32. +If the endpoint is a VIP, match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +uncomment the last line in the example above. +## Configure SC4S Listening Ports -## Modify index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The docker compose file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container +port mapping can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the +``published`` port(s) in the docker compose file (which represents the actual listening ports on the host machine), like so: -* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your -environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in -this table that pertain to the individual data source filters that are included with SC4S. -* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further -information is covered in the "Log Path overrides" section of the Configuration document. - - -## Configure source filtering by source IP or host name - -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. - -* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. -* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. -* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. - -## Configure compliance index/metadata overrides - -In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. -The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to -the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file -lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is -covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. - -## Start/Restart SC4S - -```bash -sudo docker stack deploy --compose-file docker-compose.yml sc4s ``` + ports: + - target: 514 + published: 614 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host +``` +This snippet above instructs the _host_ to listen on TCP port 614 and map that port to the default TCP 514 port on the _container_. +No changes to the underlying SC4S default configuration (environment variables) are needed. -# Scale out - -Additional hosts can be deployed for syslog collection from additional network zones and locations. - - -# Configure Dedicated Listening Ports +### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in -the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. -For collection of such sources we provide a means of dedicating a unique listening port to a specific source. +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. -Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. +The docker compose file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured +by the environment variable(s). In the following example, additional ``target`` stanzas are added for the main ``sc4s`` container, where the +``target`` and ``published`` lines provide for 21 additional technology-specific UDP and TCP ports. -In the following example the target port ranges allow for up to 21 technology-specific ports. Modify individual ports or a -range as appropriate for your network. +Follow these steps to configure unique ports: -* Modify the unit file ``/opt/sc4s/docker-compose.yml`` +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). Refer to the "Sources" +documentation to identify the specific environment variables that are mapped to each data source vendor/technology. +* Modify the compose file ``/opt/sc4s/docker-compose.yml`` and add/change port stanzas as appropriate using the example below. +* Restart SC4S using the command in the "Start/Restart SC4S" section below. ```yaml version: "3.7" services: @@ -186,6 +172,11 @@ services: protocol: udp #Comment the following line out if using docker-compose mode: host + - target: 6514 + published: 6514 + protocol: tcp +# Comment the following line out if using docker-compose + mode: host - target: 5000-5020 published: 5000-5020 protocol: tcp @@ -201,38 +192,46 @@ services: volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z -#Uncomment the following line if custom TLS certs are provided -# - /opt/sc4s/tls:/opt/syslog-ng/tls +# Uncomment the following line if local disk archiving is desired +# - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z +# Uncomment the following line if custom TLS certs are provided +# - /opt/sc4s/tls:/opt/syslog-ng/tls:z ``` -* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). +## Modify index destinations for Splunk -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. +* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further +information is covered in the "Log Path overrides" section of the Configuration document. -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +## Configure source filtering by source IP or host name -```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088 -SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 -#Uncomment the following line if using untrusted SSL certificates -#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no -``` +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. + +* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. -* Restart SC4S (below) +## Configure compliance index/metadata overrides + +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is +covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. -## Start/Restart SC4S +# Start/Restart SC4S ```bash docker stack deploy --compose-file docker-compose.yml sc4s ``` - # Stop SC4S Start by obtaining the stack name (ID): @@ -254,7 +253,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.22.1' +syslog-ng starting up; version='3.25.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -272,7 +271,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 7eae5b0..55a4788 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -21,7 +21,7 @@ reboot: net.ipv4.ip_forward=1 ``` -# Setup +# Initial Setup * Create the systemd unit file `/lib/systemd/system/sc4s.service` based on the following template: @@ -35,13 +35,14 @@ After=network.target network-online.target Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs - Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" +# Optional mount point for local disk archive (EWMM output) files +# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Mount point for local disk buffer (required) Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" -# Uncomment the following line if local disk archiving is desired -# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls:z" @@ -52,16 +53,16 @@ ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE ExecStartPre=/usr/bin/docker run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - --name SC4S_preflight --rm \ - $SC4S_IMAGE -s -ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp \ + --name SC4S_preflight \ + --rm $SC4S_IMAGE -s +ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ "$SC4S_LOCAL_ARCHIVE_MOUNT" \ "$SC4S_TLS_DIR" \ - --name SC4S --rm \ -$SC4S_IMAGE + --name SC4S \ + --rm $SC4S_IMAGE ``` * Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. @@ -95,9 +96,10 @@ document for details on the directory structure the archive uses. * IMPORTANT: When creating the directories above, ensure the directories created match the volume mounts specified in the unit file above. Failure to do this will cause SC4S to abort at startup. -## Configure the SC4S environment +# Configure the sc4s environment -Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: +SC4S is almost entirely controlled through environment variables, which are read from a file at starteup. Create a file named +``/opt/sc4s/env_file`` and add the following environment variables and values: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 @@ -109,61 +111,41 @@ SC4S_DEST_SPLUNK_HEC_WORKERS=6 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. - -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example. - -## Modify index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. - -* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your -environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in -this table that pertain to the individual data source filters that are included with SC4S. -* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further -information is covered in the "Log Path overrides" section of the Configuration document. - -## Configure source filtering by source IP or host name - -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. - -* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. -* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. -* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints, up to a maxiumum of 32. +If the endpoint is a VIP, match this value to the total number of indexers behind the load balancer. -## Configure compliance index/metadata overrides +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example above. -In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. -The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to -the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file -lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is -covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. +## Configure SC4S Listening Ports -## Configure SC4S for systemd and start SC4S +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The unit file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container port mapping +can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the initial port in the +`ExecStart` line in the unit file for the main container (which represents the actual listening port on the host machine), like so: -```bash -sudo systemctl daemon-reload -sudo systemctl enable sc4s -sudo systemctl start sc4s ``` +-p 614:514 -p 714:514/udp -p 8514:6514 +``` +This instructs the _host_ to listen on TCP port 614, UDP 714, and TCP 8514 (for TLS) and map them to the standard UDP/TCP 514 and 6514 ports +on the _container_. No changes to the underlying SC4S default configuration (environment variables) are needed. -# Configure Dedicated Listening Ports +### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in -the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. -For collection of such sources we provide a means of dedicating a unique listening port to a specific source. +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. -Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. +The unit file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the +environment variable(s). In the example below, the `ExecStart` line for the main SC4S container is modified, where +``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. -In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify individual ports or a -range as appropriate for your network. +Follow these steps to configure unique ports: -* Modify the unit file ``/lib/systemd/system/sc4s.service`` +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). Refer to the "Sources" +documentation to identify the specific environment variables that are mapped to each data source vendor/technology. +* Modify the unit file ``/lib/systemd/system/sc4s.service`` with the appropriate ``ExecStart`` command line changes using the example below. +* Ensure that you reload the unit file as well as restarting SC4S. See the "Configure SC4S for systemd and start SC4S" section below. ```ini [Unit] Description=SC4S Container @@ -174,11 +156,14 @@ Requires=network.service Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs - Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" +# Optional mount point for local disk archive (EWMM output) files +# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Mount point for local disk buffer (required) Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" + # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -188,38 +173,54 @@ ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE ExecStartPre=/usr/bin/docker run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - --name SC4S_preflight --rm \ - $SC4S_IMAGE -s -ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ + --name SC4S_preflight \ + --rm $SC4S_IMAGE -s +ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ + "$SC4S_LOCAL_ARCHIVE_MOUNT" \ + "$SC4S_TLS_DIR" \ --name SC4S \ - --rm \ -$SC4S_IMAGE + --rm $SC4S_IMAGE ``` -* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). +## Modify index destinations for Splunk -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. +* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further +information is covered in the "Log Path overrides" section of the Configuration document. -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +## Configure source filtering by source IP or host name -```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088 -SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 -#Uncomment the following line if using untrusted SSL certificates -#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no -``` +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. + +* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. -* Restart SC4S (below) +## Configure compliance index/metadata overrides + +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is +covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. + +## Configure SC4S for systemd and start SC4S + +```bash +sudo systemctl daemon-reload +sudo systemctl enable sc4s +sudo systemctl start sc4s +``` # Start SC4S @@ -258,7 +259,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.22.1' +syslog-ng starting up; version='3.25.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -276,7 +277,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md index f686dc0..141df9c 100644 --- a/docs/gettingstarted/index.md +++ b/docs/gettingstarted/index.md @@ -76,11 +76,11 @@ Splunk type. | Container and Orchestration | Notes | |-----------------------------|-------| -| [Podman + systemd single node](gettingstarted/podman-systemd-general.md) | First choice for RedHat 7.x/8.x and CentOS, second choice for Debian and Ubuntu (packages provided via PPA) | -| [Docker CE + systemd single node](gettingstarted/docker-systemd-general.md) | First choice for Debian and Ubuntu; second choice for CentOS for those with limited existing Docker experience | -| [Docker CE + Swarm single node](gettingstarted/docker-swarm-general.md) | Option for Debian, Ubuntu, CentOS, and Desktop Docker desiring Docker Compose or Swarm orchestration | -| [Docker CE + Swarm single node RHEL 7.7](gettingstarted/docker-swarm-rhel7.md) | Option for RedHat 7.7 desiring Docker Compose or Swarm orchestration | -| [Bring your own Envionment](gettingstarted/byoe-rhel7.md) | Option for RedHat 7.7 (centos 7) with SC4S configuration without containers | +| [Podman + systemd](podman-systemd-general.md) | First choice for RedHat 7.x/8.x and CentOS, second choice for Debian and Ubuntu (packages provided via PPA) | +| [Docker CE + systemd](docker-systemd-general.md) | First choice for Debian and Ubuntu; second choice for CentOS for those with limited existing Docker experience | +| [Docker CE + Swarm](docker-swarm-general.md) | Option for Debian, Ubuntu, CentOS, and Desktop Docker desiring Docker Compose or Swarm orchestration | +| [Docker CE + Swarm RHEL 7.7](docker-swarm-rhel7.md) | Option for RedHat 7.7 desiring Docker Compose or Swarm orchestration | +| [Bring your own Envionment](byoe-rhel7.md) | Option for RedHat 7.7 (centos 7) with SC4S configuration without containers | ### Offline Container Installation @@ -120,8 +120,3 @@ attempt to obtain the container image via the internet. Environment="SC4S_IMAGE=sc4slocal:latest" ``` -## Scale out - -Additional hosts can be deployed for syslog collection from additional network zones and locations: - -![SC4S deployment diagram](SC4Sdeployment.png) diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 18f8f82..7b49762 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -3,7 +3,7 @@ Refer to [Installation](https://podman.io/getting-started/installation) -# Setup +# Initial Setup * Create the systemd unit file `/lib/systemd/system/sc4s.service` based on the following template: @@ -17,13 +17,14 @@ After=network.target network-online.target Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs - Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" +# Optional mount point for local disk archive (EWMM output) files +# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Mount point for local disk buffer (required) Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" -# Uncomment the following line if local disk archiving is desired -# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls:z" @@ -34,16 +35,16 @@ ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE ExecStartPre=/usr/bin/podman run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - --name SC4S_preflight --rm \ - $SC4S_IMAGE -s -ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp \ + --name SC4S_preflight \ + --rm $SC4S_IMAGE -s +ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ "$SC4S_LOCAL_ARCHIVE_MOUNT" \ "$SC4S_TLS_DIR" \ - --name SC4S --rm \ -$SC4S_IMAGE + --name SC4S \ + --rm $SC4S_IMAGE ``` * Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. @@ -77,9 +78,10 @@ document for details on the directory structure the archive uses. * IMPORTANT: When creating the directories above, ensure the directories created match the volume mounts specified in the unit file above. Failure to do this will cause SC4S to abort at startup. -## Configure the sc4s environment +# Configure the sc4s environment -Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: +SC4S is almost entirely controlled through environment variables, which are read from a file at starteup. Create a file named +``/opt/sc4s/env_file`` and add the following environment variables and values: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 @@ -91,61 +93,41 @@ SC4S_DEST_SPLUNK_HEC_WORKERS=6 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints, up to a maxiumum of 32. +If the endpoint is a VIP, match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example. - -## Modify index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. - -* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your -environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in -this table that pertain to the individual data source filters that are included with SC4S. -* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further -information is covered in the "Log Path overrides" section of the Configuration document. +uncomment the last line in the example above. -## Configure source filtering by source IP or host name +## Configure SC4S Listening Ports -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The unit file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container port mapping +can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the initial port in the +`ExecStart` line in the unit file for the main container (which represents the actual listening port on the host machine), like so: -* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. -* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. -* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. - -## Configure compliance index/metadata overrides - -In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. -The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to -the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file -lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is -covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. - -## Configure SC4S for systemd and start SC4S - -```bash -sudo systemctl daemon-reload -sudo systemctl enable sc4s -sudo systemctl start sc4s ``` +-p 614:514 -p 714:514/udp -p 8514:6514 +``` +This instructs the _host_ to listen on TCP port 614, UDP 714, and TCP 8514 (for TLS) and map them to the standard UDP/TCP 514 and 6514 ports +on the _container_. No changes to the underlying SC4S default configuration (environment variables) are needed. -# Configure Dedicated Listening Ports +### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in -the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. -For collection of such sources we provide a means of dedicating a unique listening port to a specific source. +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. -Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. +The unit file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the +environment variable(s). In the example below, the `ExecStart` line for the main SC4S container is modified, where +``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. -In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify individual ports or a -range as appropriate for your network. +Follow these steps to configure unique ports: -* Modify the unit file ``/lib/systemd/system/sc4s.service`` +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). Refer to the "Sources" +documentation to identify the specific environment variables that are mapped to each data source vendor/technology. +* Modify the unit file ``/lib/systemd/system/sc4s.service`` with the appropriate ``ExecStart`` command line changes using the example below. +* Ensure that you reload the unit file as well as restarting SC4S. See the "Configure SC4S for systemd and start SC4S" section below. ```ini [Unit] Description=SC4S Container @@ -156,11 +138,14 @@ Requires=network.service Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs - Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" +# Optional mount point for local disk archive (EWMM output) files +# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Mount point for local disk buffer (required) Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" + # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -170,38 +155,54 @@ ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE ExecStartPre=/usr/bin/podman run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - --name SC4S_preflight --rm \ - $SC4S_IMAGE -s -ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ + --name SC4S_preflight \ + --rm $SC4S_IMAGE -s +ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ + "$SC4S_LOCAL_ARCHIVE_MOUNT" \ + "$SC4S_TLS_DIR" \ --name SC4S \ - --rm \ -$SC4S_IMAGE + --rm $SC4S_IMAGE ``` -* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). +## Modify index destinations for Splunk -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. +* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further +information is covered in the "Log Path overrides" section of the Configuration document. -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +## Configure source filtering by source IP or host name -```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088 -SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 -#Uncomment the following line if using untrusted SSL certificates -#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no -``` +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. + +* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. -* Restart SC4S (below) +## Configure compliance index/metadata overrides + +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is +covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. + +## Configure SC4S for systemd and start SC4S + +```bash +sudo systemctl daemon-reload +sudo systemctl enable sc4s +sudo systemctl start sc4s +``` # Start SC4S @@ -240,7 +241,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.22.1' +syslog-ng starting up; version='3.25.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -258,7 +259,7 @@ podman logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/mkdocs.yml b/mkdocs.yml index 3848849..d4497cb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,9 +4,10 @@ nav: - Home: 'index.md' - Getting Started: - 'Read First': 'gettingstarted/index.md' - - 'Podman + systemd single node': 'gettingstarted/podman-systemd-general.md' - - 'Docker CE + systemd single node': 'gettingstarted/docker-systemd-general.md' - - 'Docker CE + Swarm single node': 'gettingstarted/docker-swarm-rhel7.md' + - 'Podman + systemd': 'gettingstarted/podman-systemd-general.md' + - 'Docker CE + systemd': 'gettingstarted/docker-systemd-general.md' + - 'Docker CE + Swarm': 'gettingstarted/docker-swarm-general.md' + - 'Docker CE + Swarm RHEL 7.7': 'gettingstarted/docker-swarm-rhel7.md' - 'Bring your own Envionment': 'gettingstarted/byoe-rhel7.md' - Configuration: 'configuration.md' - Sources: @@ -42,4 +43,4 @@ theme: primary: 'black' accent: 'orange' favicon: 'logo.png' - logo: 'logo.png' \ No newline at end of file + logo: 'logo.png' diff --git a/package/Dockerfile b/package/Dockerfile index 77740bb..4e5771b 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -82,4 +82,4 @@ EXPOSE 6514/tcp ENTRYPOINT ["/entrypoint.sh", "-F"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file +HEALTHCHECK --start-period=15s --interval=30s --timeout=6s CMD goss -g /etc/goss.yaml validate \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 1b2d8fe..90dc794 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec { http( - url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") + url("{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers({{- getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}}) diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl index 550063c..3bce5f0 100644 --- a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec_internal { http( - url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") + url("{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers(10) diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl index 2593b8c..7c97ce8 100644 --- a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl @@ -1,6 +1,6 @@ destination d_hecmetrics { http( - url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector") + url("{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event") method("POST") batch-lines(50) batch-bytes(1024Kb) diff --git a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl index 6eae6a3..516f954 100644 --- a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl +++ b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl @@ -1,76 +1,101 @@ # LOCAL_EXAMPLE - -# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique -# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the -# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ - -# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), -# the template generator will build a custom source based on the value of one or more of the set variables. - -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} - -# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name -# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". -# "parser" can be customized on dedicated ports only -# "common" uses the same parser sequence as the default ports and is the most commonly used - -{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} - -# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path -# The ports used are based on the values of one or more of the environment variables set above. - -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} -{{ define "log_path" }} +# DO NOT MODIFY THIS EXAMPLE DIRECTLY! It will get overwritten with the shipping example +# version each time SC4S starts. Copy this file to another name for development work. + +{{- /* To start, gomplate comments use the C++ style comment syntax you see here, enclosed by */}} +{{- /* curly braces. They will _not_ appear in the final syslog-ng config files. */}} +{{- /* Comments using this format will be specific to the templating process */}} + +# This comment, on the other hand, _will_ appear in the final syslog-ng config. +# Comments using this style will be relevant to the actual syslog-ng config files, +# independent of the templating process. + +{{- /* When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout */}} +{{- /* this file with a unique string to identify the vendor product. The string should be */}} +{{- /* of the form "VENDOR_PRODUCT" to signify the manufacturer and product type, and must */}} +{{- /* contain only characters matching this regex: [A-Z\_]+ */}} + +{{- /* If any of the "LOCAL_EXAMPLE" variables passed into the environment are set */}} +{{- /* (e.g. TLS, UDP, or TLS), the template generator will build a custom source based */}} +{{- /* on the value of one or more of the set variables. */}} + +{{- /* "port_id" is used to generate the port variable to be used. It should match the */}} +{{- /* "core" of the variable name set in the line above. For example, the "port_id" of */}} +{{- /* "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". "parser" can be customized */}} +{{- /* on dedicated ports only. "common" uses the same parser sequence as the default ports */}} +{{- /* and is the most commonly used */}} + +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { -# The first time this template is used the log_path will be linked to the default port +{{- /* The first time this template is used the log_path will be linked to the default port */}} {{- if eq (.) "yes"}} source(s_DEFAULT); - -# Filters should be updated to use the simplest and most effecient logic possible to discard -# the message from this path - filter(f_is_rfc3164); filter(f_local_example); {{- end}} -{{- if eq (.) "no"}} -# In the second pass through the template a link to the dedicated port is used. This -# normally does not require additional filters +{{- /* In the second pass through the template a link to the dedicated port is used. This */}} +{{- /* normally does not require additional filters */}} -source (s_LOCAL_EXAMPLE); +{{- if eq (.) "no"}} + source (s_LOCAL_EXAMPLE); {{- end}} -#Set a default sourcetype and index - - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; - -#using the key "local_example" find any cutomized index,source or sourcetype meta values - - parser {p_add_context_splunk(key("local_example")); }; +# Set a default sourcetype and index, as well as an appropriate value for the field +# "sc4s_vendor_product". This field is sent as an indexed field to Splunk, +# and is useful for downstream analysis. -# Any additional logic needed to process the event before sending to Splunk goes here + rewrite { + set("local_example", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main")); + }; -# Send it to Splunk +# using the key "local_example" find any customized index,source or sourcetype meta values + parser { p_add_context_splunk(key("local_example")); }; +# using any user-supplied filters, override Splunk metadata based on further hostname +# or CIDR block filters. + parser (compliance_meta_by_source); + +# Prepare the payload for sending to Splunk. This step is done here rather than in the +# destination(s) to ensure that it is performed only once. If the template value is not overridden, +# the default value (2nd argument) is used. + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + +{{- /* Check environment variables (and defaults if unset) for sending to the HEC */}} +{{- /* destination. When more destination options are offered in SC4S, this is where */}} +{{- /* output to them will be configured */}} + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_LOCAL_EXAMPLE_HEC" "no")) }} destination(d_hec); +{{- end}} -# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will -# forward events to another system +{{- /* Check environment variables (and defaults if unset) for sending to the local EWMM-format */}} +{{- /* disk archive */}} - flags(flow-control); +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_LOCAL_EXAMPLE" "no")) }} + destination(d_archive); +{{- end}} +# All passes through any matching log path will be final + flags(flow-control,final); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} -# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic +{{- /* Prepare to run two passes through this template, one for default traffic and another for */}} +{{- /* "unique ports" if they are configured. */}} - {{tmpl.Exec "log_path" "no" }} -{{- end}} +{{- if or (or (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic +{{ tmpl.Exec "log_path" "no" }} +{{- end }} # Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic - -{{tmpl.Exec "log_path" "yes" }} +{{ tmpl.Exec "log_path" "yes" }} \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index 9c64b3b..0fd2808 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -1,10 +1,10 @@ # Checkpoint -# Generate the custom port if defined -{{ $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -87,7 +87,7 @@ log { {{- end}} {{- if or (or (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CHECKPOINT_SPLUNK traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CHECKPOINT_SPLUNK traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl index 15d0963..0e3c4c1 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl @@ -1,9 +1,11 @@ # Cisco ACS -{{ $context := dict "port_id" "CISCO_ACS" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +# This filter uses a field we set to prevent the original messages before aggregation from being +# sent to Splunk -#This filter uses a field we set to prevent the original messages before aggregation from being -#sent to Splunk filter f_cisco_acs_complete{ match("yes", value("ACS.COMPLETE") type(glob)); }; @@ -29,8 +31,8 @@ parser acs_grouping { ); }; -#The syslog message includes a date with nano seconds and TZ which is not in the header -#So must reparse the date +# The syslog message includes a date with nano seconds and TZ which is not in the header +# So must reparse the date parser acs_event_time { csv-parser( columns(ACS.DATE, ACS.TIME, ACS.TZ, MESSAGE) @@ -44,7 +46,7 @@ parser acs_event_time { template("${ACS.DATE} ${ACS.TIME} ${ACS.TZ}") ); }; -# The following is an inline template; we will use this to generate the actual log path +{{- /* The following is an inline template to generate the actual log path */}} {{ define "log_path" }} log { {{- if eq (.) "yes"}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index 8b9ca9a..b794ad8 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -1,9 +1,10 @@ # Cisco ASA -{{ $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index 015f86b..2749bc8 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -1,9 +1,10 @@ # Cisco IOS -{{ $context := dict "port_id" "CISCO_IOS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_IOS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -36,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_IOS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CISCO_IOS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl index 7157f5e..b56dae5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -1,15 +1,18 @@ # Cisco ISE -{{ $context := dict "port_id" "CISCO_ISE" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +# This filter uses a field we set to prevent the original messages before aggregation from being +# sent to Splunk -#This filter uses a field we set to prevent the original messages before aggregation from being -#sent to Splunk filter f_cisco_ise_complete{ match("yes", value("ISE.COMPLETE") type(glob)); }; #This parser adds messages from ISE to a context without sending them #forward to Splunk + parser ise_grouping { csv-parser( columns(PID, ISE.num, ISE.seq, MESSAGE) @@ -31,6 +34,7 @@ parser ise_grouping { #The syslog message includes a date with nano seconds and TZ which is not in the header #So must reparse the date + parser ise_event_time { csv-parser( columns(ISE.DATE, ISE.TIME, ISE.TZ, MESSAGE) @@ -44,7 +48,7 @@ parser ise_event_time { template("${ISE.DATE} ${ISE.TIME} ${ISE.TZ}") ); }; -# The following is an inline template; we will use this to generate the actual log path +{{- /* The following is an inline template to generate the actual log path */}} {{ define "log_path" }} log { {{- if eq (.) "yes"}} @@ -81,7 +85,6 @@ log { flags(flow-control,final); }; - }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index 6c40bc0..683dc1f 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -1,9 +1,10 @@ # Cisco NX_OS -{{ $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -38,7 +39,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_NX_OS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CISCO_NX_OS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index dc65e02..a3346bd 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -1,9 +1,10 @@ # Forcepoint Webprotect -{{ $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -37,7 +38,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT")) }} # Listen on the specified dedicated port(s) for FORCEPOINT_WEBPROTECT traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for FORCEPOINT_WEBPROTECT traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index a3bfc99..837ce58 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -1,9 +1,10 @@ # Fortinet Fortios -{{ $context := dict "port_id" "FORTINET_FORTIOS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "FORTINET_FORTIOS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -56,7 +57,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for FORTINET_FORTIOS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for FORTINET_FORTIOS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl index 1c047af..a84ecaa 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl @@ -1,9 +1,10 @@ # Infoblox -{{ $context := dict "port_id" "INFOBLOX" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "INFOBLOX" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -70,7 +71,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_INFOBLOX_TCP_PORT")) (getenv (print "SC4S_LISTEN_INFOBLOX_UDP_PORT"))) (getenv (print "SC4S_LISTEN_INFOBLOX_TLS_PORT")) }} # Listen on the specified dedicated port(s) for INFOBLOX traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for INFOBLOX traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 5aecfc0..162996d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -1,9 +1,10 @@ # Juniper IDP -{{ $context := dict "port_id" "JUNIPER_IDP" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_IDP" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -36,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_IDP traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_IDP traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index 004e8c8..facaf1c 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -1,9 +1,10 @@ # Juniper JunOS -{{ $context := dict "port_id" "JUNIPER_JUNOS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_JUNOS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -55,7 +56,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_JUNOS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_JUNOS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index ece524e..6fca3d1 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -1,9 +1,10 @@ # Juniper Netscreen -{{ $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -35,8 +36,8 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NETSCREEN traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} +{{ tmpl.Exec "log_path" "no" }} +{{- end }} # Listen on the default port (typically 514) for JUNIPER_NETSCREEN traffic -{{ tmpl.Exec "log_path" "yes" }} +{{ tmpl.Exec "log_path" "yes" }} \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index 4cac2a7..668f287 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -1,9 +1,10 @@ # Juniper NSM -{{ $context := dict "port_id" "JUNIPER_NSM" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_NSM" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -36,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NSM traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_NSM traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index f33f3f6..2ac5fa0 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -1,9 +1,10 @@ # Juniper NSM IDP -{{ $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -35,7 +36,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NSM_IDP traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_NSM_IDP traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index 25ddce5..8a5a386 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -1,6 +1,7 @@ # Microfocus ArcSight -{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} parser p_microfocus_arcsight_header { csv-parser( @@ -32,7 +33,7 @@ parser p_microfocus_arcsight_source { default-selector("unknown") ); }; -# The following is an inline template; we will use this to generate the actual log path +{{- /* The following is an inline template to generate the actual log path */}} {{ define "log_path" }} log { {{- if eq (.) "yes"}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 8c6f97f..1a54790 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -1,8 +1,10 @@ # PaloAlto PanOS -{{ $context := dict "port_id" "PALOALTO_PANOS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "PALOALTO_PANOS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -34,7 +36,7 @@ log { #2012/04/10 04:39:55 #parse the date date-parser( - {{- if ((getenv "SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS") | conv.ToBool) }} + {{- if (conv.ToBool (getenv "SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS")) }} format("%Y/%m/%d %H:%M:%S.%f") {{- else}} format("%Y/%m/%d %H:%M:%S") @@ -91,7 +93,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PALOALTO_PANOS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for PALOALTO_PANOS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl index 46c5267..d17a226 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl @@ -1,8 +1,10 @@ # Proofpoint Protection Server -{{ $context := dict "port_id" "PROOFPOINT_PPS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "PROOFPOINT_PPS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -42,7 +44,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PROOFPOINT_PPS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for PROOFPOINT_PPS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl index a2b4c57..04ab7d1 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl @@ -1,5 +1,6 @@ -#Symantec Brightmail -{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} +# Symantec Brightmail + +{{- if (conv.ToBool (getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes")) }} filter f_symantec_brightmail_complete{ match("yes", value("SMG.COMPLETE") type(glob)); }; @@ -22,10 +23,12 @@ parser symantec_brightmail_grouping { }; {{- end }} -{{ $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -36,7 +39,7 @@ log { source (s_SYMANTEC_BRIGHTMAIL); {{- end }} -{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} +{{- if (conv.ToBool (getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes")) }} if { filter(f_symantec_brightmail_details); @@ -80,7 +83,7 @@ log { {{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL" "no")) }} destination(d_archive); {{- end}} -{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} +{{- if (conv.ToBool (getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes")) }} }; {{- end}} @@ -90,7 +93,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT")) (getenv (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT"))) (getenv (print "SC4S_SYMANTEC_BRIGHTMAIL_NSS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for SYMANTEC_BRIGHTMAIL traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for SYMANTEC_BRIGHTMAIL traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl index a6ab503..a8b891b 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -1,9 +1,10 @@ -#Ubiquiti unifi -{{ $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +# Ubiquiti unifi +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -14,7 +15,6 @@ log { source (s_UBIQUITI_UNIFI); {{- end}} - parser {p_add_context_splunk(key("ubiquiti_unifi")); }; #Firewall @@ -125,8 +125,8 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT")) (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT"))) (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT")) }} # Listen on the specified dedicated port(s) for UBIQUITI_UNIFI traffic - {{tmpl.Exec "log_path" "no" }} +{{tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for UBIQUITI_UNIFI traffic -{{tmpl.Exec "log_path" "yes" }} +{{tmpl.Exec "log_path" "yes" }} \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index d70f139..27f682e 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -1,8 +1,10 @@ # Zscaler -{{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -12,6 +14,7 @@ log { source (s_ZSCALER_NSS); {{- end }} + parser { date-parser(format("%Y-%m-%d %H:%M:%S") template('$(substr "$LEGACY_MSGHDR$MSG" "0" "19")')); }; rewrite { set("zscaler_nss", value("fields.sc4s_vendor_product")); subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); @@ -75,7 +78,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT")) (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for ZSCALER_NSS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for ZSCALER_NSS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index 1b93b48..59e9c50 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -1,8 +1,10 @@ # Cisco ASA RFC5424 -{{ $context := dict "port_id" "CISCO_ASA" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_ASA" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -35,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_ASA traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CISCO_ASA traffic diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index 7a40ebf..64584fd 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -1,8 +1,10 @@ # Symantec Proxy (Bluecoat) -{{ $context := dict "port_id" "SYMANTEC_PROXY" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "SYMANTEC_PROXY" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -36,7 +38,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT")) (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT"))) (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TLS_PORT")) }} # Listen on the specified dedicated port(s) for SYMANTEC_PROXY traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for SYMANTEC_PROXY traffic diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index e808f85..f5ac665 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -1,8 +1,10 @@ # Juniper JunOS (Structured, RFC5424-compliant) -{{ $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl index f080c1c..4c2ba45 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -1,8 +1,10 @@ # Cisco Meraki -{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -35,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_MERAKI traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CISCO_MERAKI traffic diff --git a/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl index 5d31966..171fccc 100644 --- a/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl @@ -1,10 +1,10 @@ -#VMware ESXi and NSX -# Generate the custom port if defined -{{ $context := dict "port_id" "VMWARE" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +# VMware ESXi and NSX +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "VMWARE" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -96,7 +96,7 @@ log { {{- end}} {{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_TLS_PORT")) }} # Listen on the specified dedicated port(s) for VMWARE traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for VMWARE traffic diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl index 91fa349..e1911f7 100644 --- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl @@ -1,8 +1,10 @@ # Linux/Unix OS system logs -{{ $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -38,7 +40,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_NIX_SYSLOG_TCP_PORT")) (getenv (print "SC4S_LISTEN_NIX_SYSLOG_UDP_PORT"))) (getenv (print "SC4S_NIX_SYSLOG_NSS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for NIX_SYSLOG traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for NIX_SYSLOG traffic diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index 44be960..d2057c2 100644 --- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -1,4 +1,5 @@ # Fallback for un-parsed sources + log { source(s_DEFAULT); diff --git a/package/etc/conf.d/log_paths/startup.conf.tmpl b/package/etc/conf.d/log_paths/startup.conf.tmpl index 3c5df61..8153c55 100644 --- a/package/etc/conf.d/log_paths/startup.conf.tmpl +++ b/package/etc/conf.d/log_paths/startup.conf.tmpl @@ -1,12 +1,13 @@ +# Startup events + {{- define "log_path"}} log { source(s_startup_out); - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events:startup:out")); }; - {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no") | conv.ToBool) }} + {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no")) }} destination(d_hec_internal); {{- end}} @@ -19,7 +20,6 @@ log { log { source(s_startup_err); - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events:startup:err"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events")); }; diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index be569ff..7f6a9e5 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -1,2 +1,3 @@ -{{ $context := dict "port_id" "DEFAULT" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +# Default "soup" syslog-ng sources, typically UDP/TCP 514; TLS 6514 +{{- $context := dict "port_id" "DEFAULT" "parser" "common" -}} +{{- tmpl.Exec "t/source_network.t" $context -}} \ No newline at end of file diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 8b43665..eef481a 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -1,13 +1,14 @@ +{{ define "T1" }} + # The following is the source port declaration for {{ (print .port_id) }} -# Two log paths will be created -- one for the dedicated port(s) and one for the default (typically port 514) -{{- define "T1" }} -source s_{{ .port_id}} { + +source s_{{ .port_id }} { channel { source { -{{- if (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT" )) }} +{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} syslog ( transport("udp") - port({{getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") }}) + port({{ getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514" }}) ip-protocol(4) so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "425984"}}) keep-hostname(yes) @@ -18,10 +19,10 @@ source s_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT" )) }} +{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} network ( transport("tcp") - port({{getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") }}) + port({{ getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514" }}) ip-protocol(4) max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) @@ -34,10 +35,10 @@ source s_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT" )) }} +{{- if (conv.ToBool (getenv "SC4S_SOURCE_TLS_ENABLE" "no")) }} network( - port({{getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") }}) transport("tls") + port({{ getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514" }}) ip-protocol(4) max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) @@ -59,7 +60,7 @@ source s_{{ .port_id}} { }; #TODO: #60 Remove this function with enhancement rewrite(set_rfcnonconformant); -{{ if eq .parser "rfc5424_strict" }} +{{- if eq .parser "rfc5424_strict" }} filter(f_rfc5424_strict); parser { syslog-parser(flags(syslog-protocol)); @@ -129,11 +130,9 @@ source s_{{ .port_id}} { unset(value("fields.sc4s_time_zone")); }; }; - - - }; + }; }; -{{- end }} -{{- if or (or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT"))) (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) }} -{{ template "T1" (.) }} -{{- end }} +{{- end -}} +{{- if or (or (or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT"))) (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT"))) (eq .port_id "DEFAULT") -}} +{{- template "T1" (.) -}} +{{- end -}} \ No newline at end of file diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl index 6f75c8e..516f954 100644 --- a/package/etc/local_config/log_paths/example.conf.tmpl +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -1,74 +1,101 @@ # LOCAL_EXAMPLE +# DO NOT MODIFY THIS EXAMPLE DIRECTLY! It will get overwritten with the shipping example +# version each time SC4S starts. Copy this file to another name for development work. + +{{- /* To start, gomplate comments use the C++ style comment syntax you see here, enclosed by */}} +{{- /* curly braces. They will _not_ appear in the final syslog-ng config files. */}} +{{- /* Comments using this format will be specific to the templating process */}} + +# This comment, on the other hand, _will_ appear in the final syslog-ng config. +# Comments using this style will be relevant to the actual syslog-ng config files, +# independent of the templating process. + +{{- /* When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout */}} +{{- /* this file with a unique string to identify the vendor product. The string should be */}} +{{- /* of the form "VENDOR_PRODUCT" to signify the manufacturer and product type, and must */}} +{{- /* contain only characters matching this regex: [A-Z\_]+ */}} + +{{- /* If any of the "LOCAL_EXAMPLE" variables passed into the environment are set */}} +{{- /* (e.g. TLS, UDP, or TLS), the template generator will build a custom source based */}} +{{- /* on the value of one or more of the set variables. */}} + +{{- /* "port_id" is used to generate the port variable to be used. It should match the */}} +{{- /* "core" of the variable name set in the line above. For example, the "port_id" of */}} +{{- /* "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". "parser" can be customized */}} +{{- /* on dedicated ports only. "common" uses the same parser sequence as the default ports */}} +{{- /* and is the most commonly used */}} + +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} +log { -# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique -# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the -# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ - -# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), -# the template generator will build a custom source based on the value of one or more of the set variables. - - -# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name -# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". -# "parser" can be customized on dedicated ports only -# "common" uses the same parser sequence as the default ports and is the most commonly used - - -# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path -# The ports used are based on the values of one or more of the environment variables set above. +{{- /* The first time this template is used the log_path will be linked to the default port */}} -{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} -log { {{- if eq (.) "yes"}} source(s_DEFAULT); filter(f_is_rfc3164); filter(f_local_example); {{- end}} + +{{- /* In the second pass through the template a link to the dedicated port is used. This */}} +{{- /* normally does not require additional filters */}} + {{- if eq (.) "no"}} source (s_LOCAL_EXAMPLE); {{- end}} +# Set a default sourcetype and index, as well as an appropriate value for the field +# "sc4s_vendor_product". This field is sent as an indexed field to Splunk, +# and is useful for downstream analysis. -# The first time this template is used the log_path will be linked to the default port + rewrite { + set("local_example", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main")); + }; -# Filters should be updated to use the simplest and most effecient logic possible to discard -# the message from this path +# using the key "local_example" find any customized index,source or sourcetype meta values + parser { p_add_context_splunk(key("local_example")); }; -# In the second pass through the template a link to the dedicated port is used. This -# normally does not require additional filters +# using any user-supplied filters, override Splunk metadata based on further hostname +# or CIDR block filters. + parser (compliance_meta_by_source); +# Prepare the payload for sending to Splunk. This step is done here rather than in the +# destination(s) to ensure that it is performed only once. If the template value is not overridden, +# the default value (2nd argument) is used. + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; -#Set a default sourcetype and index - - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; - -#using the key "local_example" find any cutomized index,source or sourcetype meta values - - parser {p_add_context_splunk(key("local_example")); }; - -# Any additional logic needed to process the event before sending to Splunk goes here - -# Send it to Splunk +{{- /* Check environment variables (and defaults if unset) for sending to the HEC */}} +{{- /* destination. When more destination options are offered in SC4S, this is where */}} +{{- /* output to them will be configured */}} +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_LOCAL_EXAMPLE_HEC" "no")) }} destination(d_hec); +{{- end}} -# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will -# forward events to another system +{{- /* Check environment variables (and defaults if unset) for sending to the local EWMM-format */}} +{{- /* disk archive */}} - flags(flow-control); +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_LOCAL_EXAMPLE" "no")) }} + destination(d_archive); +{{- end}} +# All passes through any matching log path will be final + flags(flow-control,final); }; {{- end}} +{{- /* Prepare to run two passes through this template, one for default traffic and another for */}} +{{- /* "unique ports" if they are configured. */}} + {{- if or (or (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }} # Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic - -{{tmpl.Exec "log_path" "no" }} -{{- end}} +{{ tmpl.Exec "log_path" "no" }} +{{- end }} # Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic - -{{tmpl.Exec "log_path" "yes" }} +{{ tmpl.Exec "log_path" "yes" }} \ No newline at end of file diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index aa01301..b7d5b2d 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,14 +1,10 @@ #!/usr/bin/env bash source scl_source enable rh-python36 -export SC4S_LISTEN_DEFAULT_TCP_PORT=514 -export SC4S_LISTEN_DEFAULT_UDP_PORT=514 - cd /opt/syslog-ng gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ - mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/