Feature/don't upload p rs to releases (#126)

* update ci

.env.template

@@ -16,6 +16,6 @@ SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event
 SPLUNK_CONNECT_METHOD=hec
 SPLUNK_DEFAULT_INDEX=main
 SPLUNK_METRICS_INDEX=em_metrics
-SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download
-SPLUNKBASE_USERNAME=username
-SPLUNKBASE_PASSWORD=password
+#SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download
+#SPLUNKBASE_USERNAME=username
+#SPLUNKBASE_PASSWORD=password

.gitignore

@@ -382,4 +382,5 @@ fabric.properties
 /.idea/
 
 tests/test_plugin_*.py
-package/etc/conf.d/local/
+# package/etc/conf.d/local/
+!package/etc/conf.d/local

clair-scan.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+            set -e
+
+            REPORT_DIR=clair-reports
+            mkdir $REPORT_DIR || true
+
+            #DB=$(docker run -p 5432:5432 -d arminc/clair-db:latest)
+            docker run -p 5432:5432 -d --rm --name db arminc/clair-db:latest
+            #CLAIR=$(docker run -p 6060:6060 --link "$DB":postgres -d arminc/clair-local-scan:latest)'
+            sleep 30
+            docker run -p 6060:6060 --link db:postgres -d --rm --name clair arminc/clair-local-scan:latest
+            #CLAIR_SCANNER=$(docker run -v /var/run/docker.sock:/var/run/docker.sock --link clair:clair --name clairscanner --rm -d ovotech/clair-scanner@sha256:53fe8e8ac63af330d2dfc63498d23d8825d07f916f7d230271176de06d12acd6 tail -f /dev/null)
+
+            CLAIR_SCANNER=$(docker run --link clair:clair --name clairscanner --rm -d ovotech/clair-scanner@sha256:53fe8e8ac63af330d2dfc63498d23d8825d07f916f7d230271176de06d12acd6 tail -f /dev/null)
+
+            #clair_ip=$(docker exec -it "$CLAIR" hostname -i | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+')
+            #scanner_ip=$(docker exec -it "$CLAIR_SCANNER" hostname -i | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+')
+
+            docker cp "clair-whitelist.yml" "$CLAIR_SCANNER:/whitelist.yml"
+            WHITELIST="-w /whitelist.yml"
+
+            function scan() {
+                echo Scanning $1
+                local image=$1
+                # replace forward-slashes and colons with underscores
+                munged_image=$(echo "$image" | sed 's/\//_/g' | sed 's/:/_/g')
+                sanitised_image_filename="${munged_image}.json"
+                local ret=0
+                #--ip "$scanner_ip" \
+                #
+                local docker_cmd=(docker exec -it "$CLAIR_SCANNER" clair-scanner \
+                    --clair=http://clair:6060 \
+                    -t "high" \
+                    --report "$REPORT_DIR/$sanitised_image_filename" \
+                    --log "$REPORT_DIR/log.json" --whitelist=${WHITELIST:+"-x"}
+                    --reportAll=true \
+                    --exit-when-no-features=false \
+                    "$image")
+
+                docker pull "$image"
+
+                "${docker_cmd[@]}" 2>&1 || ret=$?
+                if [ $ret -eq 0 ]; then
+                    echo "No unapproved vulnerabilities"
+                elif [ $ret -eq 1 ]; then
+                    echo "Unapproved vulnerabilities found"
+                    EXIT_STATUS=1
+                elif [ $ret -eq 5 ]; then
+                    echo "Image was not scanned, not supported."
+                    EXIT_STATUS=1
+                else
+                    echo "Unknown clair-scanner return code $ret."
+                    EXIT_STATUS=1
+                fi
+
+                docker cp "$CLAIR_SCANNER:/$sanitised_image_filename" "$REPORT_DIR/$sanitised_image_filename" || true
+            }
+
+            EXIT_STATUS=0
+
+            scan "$IMAGE_NAME:$CIRCLE_SHA1"
+
+            docker kill clairscanner
+            docker kill clair
+            docker kill db
+
+            exit $EXIT_STATUS

docker-compose-ci.yml

@@ -19,7 +19,7 @@ services:
       - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
 
   sc4s:
-    image: rfaircloth/scs:${CIRCLE_SHA1}
+    image: ${IMAGE_NAME}:${CIRCLE_SHA1}
     hostname: sc4s
     ports:
       - "514"

docs/gettingstarted.md

@@ -21,6 +21,7 @@ instance in the same VLAN as the source device.
 environment.
 * Avoid TCP except where the source is unable to contain the event to a single UDP packet.
 * Avoid TLS except where the event may cross a untrusted network.
+* Plan for appropriately sized hardware (see)[performance.md]
 
 
 # Implementation
@@ -81,4 +82,4 @@ Splunk type.
 
 Additional hosts can be deployed for syslog collection from additional network zones and locations:
 
-![SC4S deployment diagram](SC4S%20deployment.png)
+![SC4S deployment diagram](SC4Sdeployment.png)

docs/gettingstarted/docker-swarm-general.md

@@ -35,21 +35,11 @@ services:
       - /opt/sc4s/tls:/opt/syslog-ng/tls
 ```
 
-* NOTE:  While strictly optional, it is recommended that you create and/or download all files and directories referenced in the yml template 
-above (`volumes` declarations) according to the configuration steps that follow.  The TLS options are described in the "Configuration" section.
-Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors.
+* NOTE:  If you use the default `volumes` declarations as-is from the `docker-compose.yml` file template example, you must create and/or download all files and directories referenced in the file according to the configuration steps that follow.  The TLS-specific options are described in the "Configure the sc4s Environment" section. Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors.
 
 ## Configure the SC4S environment
 
-Create the following file ``/opt/sc4s/env_file`` and add the environment variables below:
-
-* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment
-
-* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints.  If the endpoint is a VIP,
-match this value to the total number of indexers behind the load balancer.
-
-* NOTE:  Splunk Connect for Syslog defaults to secure configurations.  If you are not using trusted SSL certificates, be sure to
-uncomment the last line in the example below.
+Create a file named ``/opt/sc4s/env_file`` and add the following environment variables:
 
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
@@ -62,32 +52,38 @@ SPLUNK_METRICS_INDEX=em_metrics
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```
 
+* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment.
+
+* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints.  If the endpoint is a VIP,
+match this value to the total number of indexers behind the load balancer.
+
+* NOTE:  Splunk Connect for Syslog defaults to secure configurations.  If you are not using trusted SSL certificates, be sure to
+uncomment the last line in the example above.
+
 ## Configure index destinations for Splunk 
 
-Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow
-customization of index destinations.
+Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. 
 
-* Create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above.  From this directory,
+* Create a subdirectory called ``default`` in the directory that you created in the previous step (e.g. ``/opt/sc4s/``). Make sure the local directory volume references in the `yml` file match the directory you create here.  From this directory,
 execute the command below to download the index context file:
 
 ```bash
 sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv
 ```
-* Edit splunk_index.csv review the index configuration and revise as required for sourcetypes utilized in your environment.
+* Edit splunk_index.csv to review the index configuration and revise as required for the sourcetypes utilized in your environment.
 
 ## Configure sources by source IP or host name
 
-Legacy sources and non-standard-compliant source require configuration by source IP or hostname as included in the event. The following steps
-apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. 
+Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps
+apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. 
 
-* If not already done in the step immediately above, create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``)
-created in the first step above.  From this directory, execute the commands below to download the vendor context files:
+* If not already done, create a subdirectory called ``default`` in the ``/opt/sc4s/`` directory. Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the following commands to download the vendor context files:
 
 ```bash
 sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf
 sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv
 ```
-* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax.
+* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax.
 
 ## Start/Restart SC4S
 
@@ -224,4 +220,4 @@ Oct  1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', cl
 ```
 If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set
 incorrectly.  If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more
-information.
+information.

docs/gettingstarted/docker-swarm-rhel7.md

@@ -63,21 +63,12 @@ services:
       - /opt/sc4s/tls:/opt/syslog-ng/tls
 ```
 
-* NOTE:  While strictly optional, it is recommended that you create and/or download all files and directories referenced in the yml template 
-above (`volumes` declarations) according to the configuration steps that follow.  The TLS options are described in the "Configuration" section.
-Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors.
+* NOTE:  If you use the default `volumes` declarations as-is from the `docker-compose.yml`  file template example, do create and/or download all files and directories referenced in the file according to the configuration steps that follow.  The TLS-specific options are described in the "Configure the sc4s environment" section. Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors.
 
-## Configure the SC4S environment
-
-Create the following file ``/opt/sc4s/env_file`` and add the environment variables below:
 
-* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment
-
-* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints.  If the endpoint is a VIP,
-match this value to the total number of indexers behind the load balancer.
+## Configure the SC4S environment
 
-* NOTE:  Splunk Connect for Syslog defaults to secure configurations.  If you are not using trusted SSL certificates, be sure to
-uncomment the last line in the example below.
+Create a file named ``/opt/sc4s/env_file`` and add the following environment variables:
 
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
@@ -90,32 +81,40 @@ SPLUNK_METRICS_INDEX=em_metrics
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```
 
+* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment.
+
+* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints.  If the endpoint is a VIP,
+match this value to the total number of indexers behind the load balancer.
+
+* NOTE:  Splunk Connect for Syslog defaults to secure configurations.  If you are not using trusted SSL certificates, be sure to
+uncomment the last line in the example below.
+
+
+
 ## Configure index destinations for Splunk 
 
-Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow
-customization of index destinations.
+Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. 
 
-* Create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above.  From this directory,
+* Create a subdirectory called ``default`` in the directory that you created in the previous step (e.g. ``/opt/sc4s/``). Make sure the local directory volume references in the `yml` file match the directory you create here.  From this directory,
 execute the command below to download the index context file:
 
 ```bash
 sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv
 ```
-* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment.
+* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment.
 
 ## Configure sources by source IP or host name
 
 Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps
-apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. 
+apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. 
 
-* If not already done in the step immediately above, create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``)
-created in the first step above.  From this directory, execute the commands below to download the vendor context files:
+* If not already done, create a subdirectory called ``default`` in the ``/opt/sc4s/`` directory. Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the following commands to download the vendor context files:
 
 ```bash
 sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf
 sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv
 ```
-* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax.
+* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax.
 
 ## Start/Restart SC4S