Conf Gen support (#83)

Implement gomplate as a confgen engine to allow dynamic configuration, essentially #83 ifdefs Allow any supported filter to open a dedicated port Make options in source and destination drivers configurable Eliminate entrypoint.sh allowing command arg to syslog such as -s (verify) and -i interactive Enhance unit file templates to check config in pre steps so errors are more clear Externalize run time env variables to an env file to simplify configuration using a common structure in all methods
CSVD · Sep 20, 2019 · 1f9ac59 · 1f9ac59
1 parent d08cd20
commit 1f9ac59
Show file tree

Hide file tree

Showing 55 changed files with 1,155 additions and 536 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -168,13 +168,13 @@ jobs:
         path: /clair-reports
 
   publish-common:
+    machine:
+      image: ubuntu-1604:201903-01
+      docker_layer_caching: true    # default - false
     environment:
       IMAGE_NAME: rfaircloth/scs
-    docker:
-      - image: circleci/buildpack-deps:stretch
     steps:
-      - setup_remote_docker:
-          docker_layer_caching: true
+      - checkout
       - run:
           name: Docker Login
           command: docker login -u $DOCKER_USER -p $DOCKER_PASS
@@ -183,10 +183,14 @@ jobs:
           command: docker pull $IMAGE_NAME:$CIRCLE_SHA1
       - run:
           name: Docker tag image
-          command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$CIRCLE_BRANCH
+          command: |
+            SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch)
+            docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER
       - run:
           name: Docker push tag
-          command: docker push $IMAGE_NAME:$CIRCLE_BRANCH
+          command: |
+            SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch)
+            docker push $IMAGE_NAME:$SEMVER
 
   publish-edge:
     environment:
@@ -273,11 +277,11 @@ workflows:
           requires:
             - dgoss
             - test-unit
-          filters:
-            branches:
-              only:
-                - master
-                - develop
+#          filters:
+#            branches:
+#              only:
+#                - master
+#                - develop
       - publish-edge:
           requires:
             - dgoss

diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml
@@ -25,8 +25,8 @@ services:
       - "514"
       - "601"
       - "514/udp"
-      - "5514"
-      - "5514/udp"
+      - "5000"
+      - "5000/udp"
     stdin_open: true
     tty: true
     links:
@@ -38,23 +38,8 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-  sc4s-juniper:
-    image: rfaircloth/scs:${CIRCLE_SHA1}
-    hostname: sc4s-juniper
-    ports:
-      - "514"
-    stdin_open: true
-    tty: true
-    links:
-      - splunk
-    environment:
-      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
-      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
-      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
-      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SYSLOG_PRESUME_FILTER=juniper_netscreen
+      - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no
+      - SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
   splunk:
     image: splunk/splunk:latest
     hostname: splunk

diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml
@@ -38,6 +38,7 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no
   splunk:
     image: splunk/splunk:latest
     hostname: splunk

diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml
@@ -40,23 +40,8 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-  sc4s-juniper:
-    image: splunk/scs:latest
-    hostname: sc4s-juniper
-    ports:
-      - "514"
-    stdin_open: true
-    tty: true
-    links:
-      - splunk
-    environment:
-      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
-      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
-      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
-      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SYSLOG_PRESUME_FILTER=juniper_netscreen
+      - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no
+      - SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
   splunk:
     image: splunk/splunk:latest
     hostname: splunk

diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml
@@ -31,14 +31,9 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-#    logging:
-#      driver: splunk
-#      options:
-#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
-#        splunk-url: https://splunk:8088/services/collector/event
-#        splunk-index: main
-#        splunk-insecureskipverify: true
-#        splunk-verify-connection: false
+      - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no
+      - SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
+
   splunk:
     image: splunk/splunk:latest
     hostname: splunk
@@ -53,14 +48,6 @@ services:
       - SPLUNK_APPS_URL=${SPLUNK_APPS_URL}
       - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME}
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
-#    logging:
-#      driver: splunk
-#      options:
-#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
-#        splunk-url: https://splunk:8088/services/collector/event
-#        splunk-index: main
-#        splunk-insecureskipverify: true
-#        splunk-verify-connection: false
   egbundles:
     image: rfaircloth/scs:egb-edge
     hostname: egbundles

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -13,7 +13,6 @@ services:
     links:
       - splunk
       - sc4s
-      - sc4s-juniper
     volumes:
       - sc4s-results:/work/test-results
     environment:
@@ -34,8 +33,8 @@ services:
       - "514"
       - "601"
       - "514/udp"
-      - "5514"
-      - "5514/udp"
+      - "5000"
+      - "5000/udp"
     stdin_open: true
     tty: true
     links:
@@ -47,28 +46,8 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-  sc4s-juniper:
-    image: splunk/scs:latest
-    build:
-      context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
-    hostname: sc4s-juniper
-    ports:
-      - "514"
-    stdin_open: true
-    tty: true
-    links:
-      - splunk
-    environment:
-      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
-      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
-      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
-      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SYSLOG_PRESUME_FILTER=juniper_netscreen
+      - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no
+      - SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
   splunk:
     image: splunk/splunk:latest
     hostname: splunk

diff --git a/docs/Configuration.md b/docs/Configuration.md
@@ -0,0 +1,9 @@
+
+| Variable | Values        | Description |
+|----------|---------------|-------------|
+| SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list |
+| SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token |
+| SCS_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate |
+| SCS_DEST_SPLUNK_HEC_CIPHER_SUITE | comma separated list | Open SSL cipher suite list |
+| SCS_DEST_SPLUNK_HEC_SSL_VERSION |  comma separated list | Open SSL version list |
+| SCS_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file |
diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md
@@ -86,32 +86,3 @@ Additional hosts can be deployed for syslog collection from additional network z
 
 ![SC4S deployment diagram](SC4S%20deployment.png)
 
-## Single Source Technology instance - Alpha
-
-For certain source technologies message categorization by content is impossible to support collection 
-of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using
-an alternate port. In the following configration example a dedicated port is opened (6514) for legacy juniper netscreen devices
-
-This approach is "alpha" and subject to change
-
-```yaml
-version: "3"
-services:
-  sc4s-juniper-netscreen:
-    image: splunk/scs:latest
-    hostname: sc4s-juniper-netscreen
-    ports:
-      - "6514:514"
-      - "6514:514/udp"
-    stdin_open: true
-    tty: true
-    environment:
-      - SPLUNK_HEC_URL=https://foo:8088/services/collector/event
-      - SPLUNK_HEC_TOKEN=<token>
-      - SPLUNK_CONNECT_METHOD=hec
-      - SPLUNK_DEFAULT_INDEX=<defaultindex>
-      - SPLUNK_METRICS_INDEX=em_metrics
-      - SYSLOG_PRESUME_FILTER=f_juniper_netscreen
-    volumes:
-    - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
-```
diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md
@@ -25,12 +25,8 @@ services:
          protocol: udp
 #Comment the following line out if using docker-compose         
          mode: host
-    environment:
-      - SPLUNK_HEC_URL=https://inputs-hec.kops.spl.guru/services/collector/event
-      - SPLUNK_HEC_TOKEN=02450979-d363-4e6c-b6c9-796d8b546a6e
-      - SPLUNK_CONNECT_METHOD=hec
-      - SPLUNK_DEFAULT_INDEX=main
-      - SPLUNK_METRICS_INDEX=em_metrics
+    env_file:
+      - /opt/scs/env_file
     volumes:
 #Uncomment the following line if overriding index destinations    
 #      - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
@@ -40,6 +36,20 @@ services:
 
 ```
 
+## Configure the SCS environment
+
+Create the following file ``/opt/scs/env_file``
+
+* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment
+
+```dotenv
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
+SPLUNK_CONNECT_METHOD=hec
+SPLUNK_DEFAULT_INDEX=main
+SPLUNK_METRICS_INDEX=em_metrics
+```
+
 ## Configure index destinations for Splunk 
 
 Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow customization of index destinations.
@@ -74,38 +84,69 @@ docker stack deploy --compose-file docker-compose.yml sc4s
 Additional hosts can be deployed for syslog collection from additional network zones and locations.
 
 
-## Single Source Technology instance - Alpha
+# Single Source Technology instance
 
-For certain source technologies message categorization by content is impossible. To support collection 
-of such legacy nonstandard sources, we provide a means of dedicating a container to a specific source using
-an alternate port. In the following configration example a dedicated port is opened (6514) for legacy juniper netscreen devices.
+For certain source technologies message categorization by content is impossible to support collection 
+of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using
+an alternate port. 
+Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use.
 
-This approach is "alpha" and subject to change.
+In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology specific ports modify the range as appropriate
 
+* Modify the unit file ``/opt/scs/docker-compose.yml``
 ```yaml
-version: "3"
+version: "3.7"
 services:
-  sc4s-juniper-netscreen:
+  sc4s:
     image: splunk/scs:latest
-    hostname: sc4s-juniper-netscreen
     ports:  
        - target: 514
-         published: 6514
+         published: 514
          protocol: tcp
 #Comment the following line out if using docker-compose
          mode: host
        - target: 514
-         published: 6514
+         published: 514
+         protocol: udp
+#Comment the following line out if using docker-compose         
+         mode: host
+       - target: 5000-5021
+         published: 5000-5021
+         protocol: tcp
+#Comment the following line out if using docker-compose
+         mode: host
+       - target: 5000-5021
+         published: 5000-5021
          protocol: udp
 #Comment the following line out if using docker-compose         
          mode: host
-    environment:
-      - SPLUNK_HEC_URL=https://foo:8088/services/collector/event
-      - SPLUNK_HEC_TOKEN=<token>
-      - SPLUNK_CONNECT_METHOD=hec
-      - SPLUNK_DEFAULT_INDEX=<defaultindex>
-      - SPLUNK_METRICS_INDEX=em_metrics
-      - SYSLOG_PRESUME_FILTER=f_juniper_netscreen
+    env_file:
+      - /opt/scs/env_file
     volumes:
-    - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
+#Uncomment the following line if overriding index destinations    
+#      - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
+#Uncomment the following lines if using a host or network based filter and log_path
+#      - ./sc4s-juniper/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv
+#      - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
+```
+
+Modify the following file ``/opt/scs/default/env_file`` 
+
+* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment
+
+```dotenv
+SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event
+SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
+SPLUNK_CONNECT_METHOD=hec
+SPLUNK_DEFAULT_INDEX=main
+SPLUNK_METRICS_INDEX=em_metrics
+SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
+#Uncomment the following line if using untrusted SSL certificates
+#SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no
+```
+
+* Start SC4S.
+
+```bash
+docker stack deploy --compose-file docker-compose.yml sc4s
 ```