Release/0.5.0 (#86)

* New documentation for podman, docker and swarm! * Support for trusted certs for Splunk * Support for product specific TCP/UDP listeners
CSVD · Sep 20, 2019 · be12e52 · be12e52
1 parent 0656fb4
commit be12e52
Show file tree

Hide file tree

Showing 77 changed files with 2,223 additions and 841 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -168,13 +168,13 @@ jobs:
         path: /clair-reports
 
   publish-common:
+    machine:
+      image: ubuntu-1604:201903-01
+      docker_layer_caching: true    # default - false
     environment:
       IMAGE_NAME: rfaircloth/scs
-    docker:
-      - image: circleci/buildpack-deps:stretch
     steps:
-      - setup_remote_docker:
-          docker_layer_caching: true
+      - checkout
       - run:
           name: Docker Login
           command: docker login -u $DOCKER_USER -p $DOCKER_PASS
@@ -183,10 +183,14 @@ jobs:
           command: docker pull $IMAGE_NAME:$CIRCLE_SHA1
       - run:
           name: Docker tag image
-          command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$CIRCLE_BRANCH
+          command: |
+            SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch)
+            docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER
       - run:
           name: Docker push tag
-          command: docker push $IMAGE_NAME:$CIRCLE_BRANCH
+          command: |
+            SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch)
+            docker push $IMAGE_NAME:$SEMVER
 
   publish-edge:
     environment:
@@ -273,11 +277,11 @@ workflows:
           requires:
             - dgoss
             - test-unit
-          filters:
-            branches:
-              only:
-                - master
-                - develop
+#          filters:
+#            branches:
+#              only:
+#                - master
+#                - develop
       - publish-edge:
           requires:
             - dgoss

diff --git a/.env.template b/.env.template
@@ -15,7 +15,7 @@ SPLUNK_HEC_URL=https://splunk:8088/services/collector/event
 SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event
 SPLUNK_CONNECT_METHOD=hec
 SPLUNK_DEFAULT_INDEX=main
-SPLUNK_METRICS_INDEX=metrics
+SPLUNK_METRICS_INDEX=em_metrics
 SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download
 SPLUNKBASE_USERNAME=username
 SPLUNKBASE_PASSWORD=password
diff --git a/README.md b/README.md
@@ -25,6 +25,8 @@ cd splunk-connect-for-syslog
 cp .env.template .env
 ```
 
+- Update the splunkbase username and password in .env this allows the splunk container to install required add-ons for the demo
+
 - Start the demo environment
 
 ```bash

diff --git a/demo-with-compose.sh b/demo-with-compose.sh
@@ -4,19 +4,35 @@
 #Use of this source code is governed by a BSD-2-clause-style
 #license that can be found in the LICENSE-BSD2 file or at
 #https://opensource.org/licenses/BSD-2-Clause
-
+WAITON=${1:-test}
+compose=${2:-docker-compose-demo.yml}
+echo $WAITON $compose
 mkdir test-results
-docker volume create sc4s-tests
+docker-compose down
+docker volume rm sc4s-results
+docker volume rm splunk-etc
+
+docker volume create sc4s-results
+docker volume create splunk-etc
 
 docker container create --name dummy \
-        -v sc4s-tests:/work/tests \
+        -v sc4s-results:/work/test-results \
+        -v splunk-etc:/work/splunk-etc \
         registry.access.redhat.com/ubi7/ubi
-docker cp tests/ dummy:/work/tests/
+docker cp ./splunk/etc/* dummy:/work/splunk-etc/
 docker rm dummy
 
-docker-compose pull
-docker-compose up
-
-EXIT=$0
+docker-compose -f $compose pull
+docker-compose -f $compose up -d splunk
+docker-compose -f $compose up -d sc4s
+sleep 60
+docker-compose -f $compose build
+docker-compose -f $compose up
 
+docker container create --name dummy \
+        -v sc4s-results:/work/test-results \
+        registry.access.redhat.com/ubi7/ubi
 
+docker cp dummy:/work/test-results/functional test-results
+docker rm dummy
+EXIT=$0
diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml
@@ -14,7 +14,6 @@ services:
       - splunk
       - sc4s
     volumes:
-      - sc4s-tests:/work/tests
       - sc4s-results:/work/test-results
     environment:
       - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
@@ -26,8 +25,8 @@ services:
       - "514"
       - "601"
       - "514/udp"
-      - "5514"
-      - "5514/udp"
+      - "5000"
+      - "5000/udp"
     stdin_open: true
     tty: true
     links:
@@ -39,31 +38,8 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-  sc4s-juniper:
-    image: rfaircloth/scs:${CIRCLE_SHA1}
-    hostname: sc4s-juniper
-    ports:
-      - "514"
-    stdin_open: true
-    tty: true
-    links:
-      - splunk
-    environment:
-      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
-      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
-      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
-      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SYSLOG_PRESUME_FILTER=juniper_netscreen
-#    logging:
-#      driver: splunk
-#      options:
-#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
-#        splunk-url: https://splunk:8088/services/collector/event
-#        splunk-index: main
-#        splunk-insecureskipverify: true
-#        splunk-verify-connection: false
+      - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
+      - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
   splunk:
     image: splunk/splunk:latest
     hostname: splunk
@@ -80,17 +56,7 @@ services:
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
     volumes:
       - splunk-etc:/opt/splunk/etc
-#    logging:
-#      driver: splunk
-#      options:
-#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
-#        splunk-url: https://splunk:8088/services/collector/event
-#        splunk-index: main
-#        splunk-insecureskipverify: true
-#        splunk-verify-connection: false
 volumes:
-  sc4s-tests:
-    external: true
   sc4s-results:
     external: true
   splunk-etc:

diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml
@@ -38,6 +38,7 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
   splunk:
     image: splunk/splunk:latest
     hostname: splunk

diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml
@@ -0,0 +1,68 @@
+#Splunk Connect for Syslog (SC4S) by Splunk, Inc.
+#
+#To the extent possible under law, the person who associated CC0 with
+#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights
+#to Splunk Connect for Syslog (SC4S).
+#
+#You should have received a copy of the CC0 legalcode along with this
+#work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+version: "3.2"
+services:
+  test:
+    build: ./tests
+    links:
+      - splunk
+      - sc4s
+      - sc4s-juniper
+    volumes:
+      - sc4s-tests:/work/tests
+      - sc4s-results:/work/test-results
+    environment:
+      - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
+
+  sc4s:
+    image: splunk/scs:latest
+    hostname: sc4s
+    ports:
+      - "514"
+      - "601"
+      - "514/udp"
+      - "5514"
+      - "5514/udp"
+    stdin_open: true
+    tty: true
+    links:
+      - splunk
+    environment:
+      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
+      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
+      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
+      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
+      - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
+  splunk:
+    image: splunk/splunk:latest
+    hostname: splunk
+    ports:
+      - "8000:8000"
+      - "8088:8088"
+      - "8089:8089"
+    environment:
+      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+      - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
+      - SPLUNK_START_ARGS=${SPLUNK_START_ARGS}
+      - SPLUNK_APPS_URL=${SPLUNK_APPS_URL}
+      - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME}
+      - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
+    volumes:
+      - splunk-etc:/opt/splunk/etc
+
+volumes:
+  sc4s-tests:
+    external: true
+  sc4s-results:
+    external: true
+  splunk-etc:
+    external: true
diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml
@@ -31,14 +31,9 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-#    logging:
-#      driver: splunk
-#      options:
-#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
-#        splunk-url: https://splunk:8088/services/collector/event
-#        splunk-index: main
-#        splunk-insecureskipverify: true
-#        splunk-verify-connection: false
+      - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
+      - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
+
   splunk:
     image: splunk/splunk:latest
     hostname: splunk
@@ -53,14 +48,6 @@ services:
       - SPLUNK_APPS_URL=${SPLUNK_APPS_URL}
       - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME}
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
-#    logging:
-#      driver: splunk
-#      options:
-#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
-#        splunk-url: https://splunk:8088/services/collector/event
-#        splunk-index: main
-#        splunk-insecureskipverify: true
-#        splunk-verify-connection: false
   egbundles:
     image: rfaircloth/scs:egb-edge
     hostname: egbundles
@@ -74,4 +61,3 @@ services:
     links:
       - egbundles
       - sc4s
-
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -13,12 +13,13 @@ services:
     links:
       - splunk
       - sc4s
-      - sc4s-juniper
     volumes:
-      - sc4s-tests:/work/tests
       - sc4s-results:/work/test-results
     environment:
+      - SPLUNK_USER=admin
       - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
+      - SPLUNK_HOST=splunk
+      - SYSLOG_HOST=sc4s
 
   sc4s:
     image: splunk/scs:latest
@@ -32,8 +33,8 @@ services:
       - "514"
       - "601"
       - "514/udp"
-      - "5514"
-      - "5514/udp"
+      - "5000"
+      - "5000/udp"
     stdin_open: true
     tty: true
     links:
@@ -45,40 +46,8 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-  sc4s-juniper:
-    image: splunk/scs:latest
-    build:
-      context: ./package
-#      cache_from:
-#        - registry.access.redhat.com/ubi7/ubi
-#        - registry.access.redhat.com/rhel7/rhel
-#        - rfaircloth/scs:edge
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
-    hostname: sc4s-juniper
-    ports:
-      - "514"
-    stdin_open: true
-    tty: true
-    links:
-      - splunk
-    environment:
-      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
-      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
-      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
-      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
-      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SYSLOG_PRESUME_FILTER=juniper_netscreen
-#    logging:
-#      driver: splunk
-#      options:
-#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
-#        splunk-url: https://splunk:8088/services/collector/event
-#        splunk-index: main
-#        splunk-insecureskipverify: true
-#        splunk-verify-connection: false
+      - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
+      - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
   splunk:
     image: splunk/splunk:latest
     hostname: splunk
@@ -95,18 +64,8 @@ services:
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
     volumes:
       - splunk-etc:/opt/splunk/etc
-#    logging:
-#      driver: splunk
-#      options:
-#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
-#        splunk-url: https://splunk:8088/services/collector/event
-#        splunk-index: main
-#        splunk-insecureskipverify: true
-#        splunk-verify-connection: false
 volumes:
-  sc4s-tests:
-    external: true
   sc4s-results:
     external: true
   splunk-etc:
-    external: true
+    external: true