Merge branch 'release/0.6.0'

docker-compose-demo.yml

@@ -13,7 +13,6 @@ services:
     links:
       - splunk
       - sc4s
-      - sc4s-juniper
     volumes:
       - sc4s-tests:/work/tests
       - sc4s-results:/work/test-results

docs/gettingstarted/docker-swarm-general.md

@@ -50,12 +50,12 @@ SPLUNK_DEFAULT_INDEX=main
 SPLUNK_METRICS_INDEX=em_metrics
 ```
 
-## Configure index destinations for Splunk
+## Configure index destinations for Splunk 
 
 Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow
 customization of index destinations.
 
-* Create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above.  From this directory,
+* Create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/scs/``) created in the first step above.  From this directory,
 execute the command below to download the index context file:
 
 ```bash
@@ -66,7 +66,7 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas
 ## Configure sources by source IP or host name
 
 Legacy sources and non-standard-compliant source require configuration by source IP or hostname as included in the event. The following steps
-apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation.
+apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. 
 
 * If not already done in the step immediately above, create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``)
 created in the first step above.  From this directory, execute the commands below to download the vendor context files:
@@ -84,17 +84,16 @@ docker stack deploy --compose-file docker-compose.yml sc4s
 ```
 
 
-## Scale out
+# Scale out
 
 Additional hosts can be deployed for syslog collection from additional network zones and locations.
 
 
-# Single Source Technology instance
+# Configure Dedicated Listening Ports
 
-For certain source technologies message categorization by content is impossible to support collection
-of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using
-an alternate port.
-Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use.
+For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in
+the data.  In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise.  
+For collection of such sources we provide a means of dedicating a unique listening port to a specific source.
 
 Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use.
 
@@ -138,7 +137,7 @@ services:
 #      - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
 ```
 
-Modify the following file ``/opt/sc4s/default/env_file``
+* Modify the following file ``/opt/sc4s/default/env_file`` after including the port-specific environment variable(s).
 
 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment
 

docs/gettingstarted/docker-swarm-rhel7.md

@@ -79,7 +79,7 @@ SPLUNK_METRICS_INDEX=em_metrics
 ```
 
 
-## Configure index destinations for Splunk
+## Configure index destinations for Splunk 
 
 Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow
 customization of index destinations.
@@ -95,7 +95,7 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas
 ## Configure sources by source IP or host name
 
 Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps
-apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation.
+apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. 
 
 * If not already done in the step immediately above, create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``)
 created in the first step above.  From this directory, execute the commands below to download the vendor context files:
@@ -112,17 +112,18 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas
 sudo docker stack deploy --compose-file docker-compose.yml sc4s
 ```
 
-## Scale out
+# Scale out
 
 Additional hosts can be deployed for syslog collection from additional network zones and locations.
 
 
-# Single Source Technology instance
+# Configure Dedicated Listening Ports
 
-For certain source technologies message categorization by content is impossible to support collection
-of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using
-an alternate port.
-Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use.
+For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in
+the data.  In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise.  
+For collection of such sources we provide a means of dedicating a unique listening port to a specific source.
+
+Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use.
 
 In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports.  Modify the individual ports or a
 range as appropriate for your network.
@@ -164,7 +165,7 @@ services:
 #      - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
 ```
 
-Modify the following file ``/opt/sc4s/default/env_file``
+* Modify the following file ``/opt/sc4s/default/env_file`` 
 
 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment
 

docs/gettingstarted/docker-systemd-general.md

@@ -7,9 +7,7 @@ Refer to [Getting Started](https://docs.docker.com/get-started/)
 
 * Create a systemd unit file use to start the container with the host os. ``/lib/systemd/system/sc4s.service``
 
-*NOTE*: In a future release, the mechanism to support mounted volumes (the three -v arguments in the unit file below) will change.  For now, do not change these arguments, and ensure that the three files they reference are downloaded to the proper directory per the configuration instructions below.
-
-*NOTE-2*: Replace the URL and HEC tokens with the appropriate values for our environment
+*NOTE*: Replace the URL and HEC tokens with the appropriate values for our environment
 
 ```ini
 [Unit]
@@ -28,6 +26,7 @@ Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/s
 
 TimeoutStartSec=0
 Restart=always
+
 ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE
 ExecStartPre=/usr/bin/docker run \
         --env-file=/opt/sc4s/default/env_file \
@@ -58,15 +57,12 @@ SPLUNK_METRICS_INDEX=em_metrics
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```
 
-*NOTE*: While optional, until the final mechanism for mounted volumes in containers is released, do _not_ skip the download (wget) steps in
-the configuration steps below.  Ensure these three files are in place in the directory specified, even if there is no intent to modify them.
-
-## Configure index destinations for Splunk
+## Configure index destinations for Splunk 
 
 Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow
 customization of index destinations.
 
-* Create a directory (e.g. ``/opt/sc4s/default/`` ).  Make sure the local directory references in the unit file above (the ``-v`` arguments)
+* Create a directory (e.g. ``/opt/sc4s/default/`` ).  Make sure the local directory references in the unit file above (the ``-v`` variables)
 match the directory you created above.  From this directory, execute the following to download the latest index context file:
 
 ```bash
@@ -77,10 +73,10 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas
 ## Configure sources by source IP or host name
 
 Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps
-apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation.
+apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. 
 
-* If not already done in the step immeidately above, create a directory (e.g. ``/opt/sc4s/default/`` ).  Make sure the local directory
-references in the unit file above (the ``-v`` arguments) match the directory you created above.  From this directory, execute the following to
+* If not already done in the step immediately above, create a directory (e.g. ``/opt/sc4s/default/`` ).  Make sure the local directory
+references in the unit file above (the ``-v`` variables) match the directory you created above.  From this directory, execute the following to
 download the latest vendor context files:
 
 ```bash
@@ -98,13 +94,13 @@ sudo systemctl start sc4s
 ```
 
 
-# Single Source Technology instance
+# Configure Dedicated Listening Ports
 
-For certain source technologies message categorization by content is impossible to support collection
-of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using
-an alternate port.
+For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in
+the data.  In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise.  
+For collection of such sources we provide a means of dedicating a unique listening port to a specific source.
 
-Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use.
+Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use.
 
 In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports.  Modify the individual ports or a
 range as appropriate for your network.
@@ -139,10 +135,9 @@ ExecStart=/usr/bin/docker run -p 514:514 -p 5000-5020:5000-5020 \
         --name SC4S \
         --rm \
 $SC4S_IMAGE
-
 ```
 
-Modify the following file ``/opt/sc4s/default/env_file``
+* Modify the following file ``/opt/sc4s/default/env_file`` 
 
 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment
 

docs/gettingstarted/podman-systemd-general.md

@@ -7,9 +7,7 @@ Refer to [Installation](https://podman.io/getting-started/installation)
 
 * Create a systemd unit file use to start the container with the host os. ``/lib/systemd/system/sc4s.service``
 
-*NOTE*: In a future release, the mechanism to support mounted volumes (the three -v arguments in the unit file below) will change.  For now, do not change these arguments, and ensure that the three files they reference are downloaded to the proper directory per the configuration instructions below.
-
-*NOTE-2*: Replace the URL and HEC tokens with the appropriate values for our environment
+*NOTE*: Replace the URL and HEC tokens with the appropriate values for our environment
 
 ```ini
 [Unit]
@@ -18,7 +16,7 @@ After=network.service
 Requires=network.service
 
 [Service]
-Environment="SC4S_IMAGE=splunk/sc4s:latest"
+Environment="SC4S_IMAGE=splunk/scs:latest"
 
 #Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started
 Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv"
@@ -43,10 +41,7 @@ $SC4S_IMAGE
 
 ```
 
-*NOTE*: While optional, until the final mechanism for mounted volumes in containers is released, do _not_ skip the download (wget) steps in
-the configuration steps below.  Ensure these three files are in place in the directory specified, even if there is no intent to modify them.
-
-## Configure the SC4S environment
+## Configure the sc4s environment
 
 Create the following file ``/opt/sc4s/default/env_file``
 
@@ -62,12 +57,12 @@ SPLUNK_METRICS_INDEX=em_metrics
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```
 
-## Configure index destinations for Splunk
+## Configure index destinations for Splunk 
 
 Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow
 customization of index destinations.
 
-* Create a directory (e.g. ``/opt/sc4s/default/`` ).  Make sure the local directory references in the unit file above (the ``-v`` arguments)
+* Create a directory (e.g. ``/opt/sc4s/default/`` ).  Make sure the local directory references in the unit file above (the ``-v`` variables)
 match the directory you created above.  From this directory, execute the following to download the latest index context file:
 
 ```bash
@@ -78,10 +73,10 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas
 ## Configure sources by source IP or host name
 
 Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps
-apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation.
+apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. 
 
-* If not already done in the step immeidately above, create a directory (e.g. ``/opt/sc4s/default/`` ).  Make sure the local directory
-references in the unit file above (the ``-v`` arguments) match the directory you created above.  From this directory, execute the following to
+* If not already done in the step immediately above, create a directory (e.g. ``/opt/sc4s/default/`` ).  Make sure the local directory
+references in the unit file above (the ``-v`` variables) match the directory you created above.  From this directory, execute the following to
 download the latest vendor context files:
 
 ```bash
@@ -93,18 +88,18 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas
 # Start SC4S
 
 ```bash
-sudo systemctl daemon-reload
+sudo systemctl daemon-reload 
 sudo systemctl enable sc4s
 sudo systemctl start sc4s
 ```
 
-# Single Source Technology instance
+# Configure Dedicated Listening Ports
 
-For certain source technologies message categorization by content is impossible to support collection
-of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using
-an alternate port.
+For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in
+the data.  In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise.  
+For collection of such sources we provide a means of dedicating a unique listening port to a specific source.
 
-Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use.
+Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use.
 
 In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports.  Modify the individual ports or a
 range as appropriate for your network.
@@ -142,7 +137,7 @@ $SC4S_IMAGE
 
 ```
 
-Modify the following file ``/opt/sc4s/default/env_file``
+* Modify the following file ``/opt/sc4s/default/env_file`` 
 
 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment
 

package/etc/conf.d/conflib/_common/network.conf.tmpl

@@ -43,9 +43,7 @@
 {{ end }}
         };
         #TODO: #60 Remove this function with enhancement
-        rewrite(set_metadata_presume);
         rewrite(set_rfcnonconformant);
-        rewrite(r_set_splunk_default);
 
 {{ if eq (getenv "confgen_parser") "rfc5424_strict" }}
         filter(f_rfc5424_strict);
@@ -90,4 +88,6 @@
             rewrite(set_rfc3164);
         };
 {{ end }}
+        rewrite(r_set_splunk_default);
+
    };

package/etc/conf.d/conflib/_splunk/splunkfields.conf

@@ -1,13 +1,9 @@
 #Used to set indexed fields we will always use to global defaults
 rewrite r_set_splunk_default {
-      set("`index`", value(".splunk.index"));
-
-      set("`splunk-sourcetype`", value(".splunk.sourcetype"));
-      set("`splunk-source`:$LOGHOST", value(".splunk.source"));
-      set("`splunk-template`", value(".splunk.template"));
+      set("SC4S:$SOURCE", value(".splunk.source"));
       set($FACILITY, value("fields.sc4s_syslog_facility"));
       set($LEVEL, value("fields.sc4s_syslog_severity"));
-      set($LOGHOST, value("fields.sc4s_syslog_server"));
+      set($LOGHOST, value("fields.sc4s_log_host"));
       set($SOURCEIP, value("fields.sc4s_fromhostip"));
 };
 #used by each log-path to set index and sourcetype which may be
@@ -19,7 +15,7 @@ block rewrite r_set_splunk_dest_default(
                            ) {
       set("`index`", value(".splunk.index"));
       set("`sourcetype`", value(".splunk.sourcetype"));
-      set("`template`", value(".splunk.template"));
+      set("`template`", value("fields.sc4s_template"));
 };
 
 

package/etc/conf.d/destinations/splunk_hec.conf.tmpl

@@ -11,21 +11,23 @@
          user("sc4s")
          headers("{{getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}")
          password("{{getenv "SPLUNK_HEC_TOKEN"}}")
-         persist-name("splunk_hec")
 
-{{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE" "yes") "yes" }}
+        {{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE" "yes") "yes" }}
+         persist-name("splunk_hec")
          disk-buffer(
-{{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes" }}
-                mem-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}})
-                reliable(yes)
-{{else}}
-                mem-buf-length({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}})
-                reliable(no)
-{{ end }}
-{{ end }}
-                     disk-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}})
-                     dir("/opt/syslog-ng/var/data/disk-buffer/")
-                     )
+
+            {{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes" }}
+            mem-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}})
+            reliable(yes)
+            {{else}}
+            mem-buf-length({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}})
+            reliable(no)
+            {{ end }}
+
+             disk-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}})
+             dir("/opt/syslog-ng/var/data/disk-buffer/")
+             )
+        {{ end }}
          tls(peer-verify({{getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}})
 {{if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}}             cipher-suite("{{getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}}
 {{if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}}             ssl-version("{{getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}}

package/etc/conf.d/destinations/splunk_hec_internal.conf

@@ -0,0 +1,5 @@
+@module confgen context(destination) name(gen_splunk_hec_internal) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/destinations/splunk_hec_internal.conf.tmpl")
+
+destination d_hec_internal {
+  gen_splunk_hec_internal()
+};