diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..a616137 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,101 @@ + version: 2 + jobs: + build: + environment: + IMAGE_NAME: rfaircloth/scs + docker: + - image: circleci/buildpack-deps:stretch + steps: + - checkout + - run: git submodule sync + - run: git submodule update --init --recursive + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Build Docker image + command: pushd package; docker build -f Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:build . + - run: + name: Tag Docker image + command: docker tag $IMAGE_NAME:build $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Push Docker image + command: docker push $IMAGE_NAME:$CIRCLE_SHA1 + dgoss: + environment: + IMAGE_NAME: rfaircloth/scs + GOSS_VER: v0.3.7 + GOSS_OPTS: "--max-concurrent=1 --format junit" + GOSS_FILES_STRATEGY: cp + + docker: + - image: circleci/buildpack-deps:stretch + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Install goss + command: | + # rather than give internet scripts SU rights, we install to local user bin and add to path + curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sudo sh + goss -version + - run: + name: Test Docker + command: | + mkdir -p ~/project/test-results + # Don't forget path! + export PATH=$PATH:~/bin + pushd package + GOSS_FMT_OPTIONS="perfdata verbose" GOSS_FILES_STRATEGY=cp GOSS_OPTS="--max-concurrent=1 --format junit" dgoss run -t $IMAGE_NAME:$CIRCLE_SHA1 3>&1 1>&2 2>&3 > /tmp/goss.tmp || true ; cat /tmp/goss.tmp | grep -v "<" + popd + cat /tmp/goss.tmp | grep "[<\>]" > ~/project/test-results/goss.xml + - store_test_results: + path: test-results + publish: + environment: + IMAGE_NAME: rfaircloth/scs + docker: + - image: circleci/buildpack-deps:stretch + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker tag image + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$CIRCLE_BRANCH + - run: + name: Docker push tag + command: docker push $IMAGE_NAME:$CIRCLE_BRANCH + + + + workflows: + version: 2 + build-publish: + jobs: + - build + - dgoss: + requires: + - build + - publish: + requires: + - dgoss + filters: + branches: + only: + - master + - develop diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 5f6de25..44b7535 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,8 +2,7 @@ - - + - - - - - - - - - - - - - + - - + + - - - - - - - - - - - + + - - + + @@ -66,6 +44,8 @@ pip yum java + goss.y + goss @@ -82,17 +62,20 @@ - + @@ -261,7 +266,56 @@ @@ -297,7 +351,7 @@ - @@ -309,26 +363,26 @@ - + - + - - + - + - + - + + @@ -345,7 +399,12 @@ - @@ -390,13 +449,6 @@ - - - - - - - @@ -487,20 +539,26 @@ - + - - + + - + - - + + + + + + + + @@ -508,13 +566,45 @@ - + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/LICENSE-2.0 b/LICENSE-2.0 new file mode 100644 index 0000000..913f595 --- /dev/null +++ b/LICENSE-2.0 @@ -0,0 +1,13 @@ +Copyright 2019 Splunk Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/package/Dockerfile b/package/Dockerfile index ae3583c..2c622a1 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -1,10 +1,13 @@ FROM registry.access.redhat.com/rhel7/rhel +ARG RH_ORG +ARG RH_ACTIVATION + ENV CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --enable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --enable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking" ENV DISTCHECK_CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --enable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --enable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking" COPY confluent.repo /etc/yum.repos.d/confluent.repo -RUN subscription-manager register --org=12579482 --activationkey=dev1 --force +RUN subscription-manager register --org=$RH_ORG --activationkey=$RH_ACTIVATION --force RUN subscription-manager repos --enable=rhel-7-server-extras-rpms --enable=rhel-7-server-optional-rpms --enable=rhel-server-rhscl-7-rpms RUN subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms" --enable "rhel-ha-for-rhel-*-server-rpms" RUN yum install wget -y @@ -66,8 +69,8 @@ RUN cd /tmp ;\ ENV DEBCONF_NONINTERACTIVE_SEEN=true ENV SPLUNK_CONNECT_METHOD=hec -RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | sh -COPY goss.yml /goss/goss.yml +RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh +COPY goss.yaml /etc/goss.yaml COPY --from=0 /opt/syslog-ng /opt/syslog-ng COPY scripts/splunkmetrics.sh /opt/syslog-ng/sbin/splunkmetrics.sh @@ -87,4 +90,4 @@ ENV SPLUNK_CONNECT_METHOD=UF ENTRYPOINT ["/sbin/entrypoint.sh", "--"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /goss/goss.yml validate \ No newline at end of file +HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file diff --git a/package/goss.yml b/package/goss.yaml similarity index 100% rename from package/goss.yml rename to package/goss.yaml diff --git a/package/scripts/splunkmetrics.sh b/package/scripts/splunkmetrics.sh new file mode 100755 index 0000000..d62014d --- /dev/null +++ b/package/scripts/splunkmetrics.sh @@ -0,0 +1,30 @@ +#!/bin/bash +sleep 30 +while sleep 10 +do + TS=$(date +"%s") + tmpfilecsv=$(mktemp /tmp/syslog-ng-stats-XXXXXXX-csv) + tmpfilejson=$(mktemp /tmp/syslog-ng-stats-XXXXXXX-json) + /opt/syslog-ng/sbin/syslog-ng-ctl query get "*" --reset | tail -n +2 | grep \^. | grep -v "^The selected counters" >$tmpfilecsv + IFS=';' + while read -r SourceName SourceId SourceInstance State Type Number + do + echo "{ + \"time\":\"$TS.000\", + \"event\":\"metric\", + \"source\":\"syslog-ng\", + \"host\":\"$HOSTNAME\", + \"index\":\"$SPLUNK_METRICS_INDEX\", + \"fields\":{ + \"SourceId\":\"$SourceId\", + \"SourceInstance\":\"$SourceInstance\", + \"State\":\"$State\", + \"Type\":\"$Type\", + \"_value\":$Number, + \"metric_name\":\"$SourceName\" + } + } ">>$tmpfilejson + done < $tmpfilecsv + + curl -s -S -k $SPLUNK_HEC_STATSURL -H "Authorization: Splunk $SPLUNK_HEC_TOKEN" -d "@$tmpfilejson" +done diff --git a/package/scripts/switch_transport.sh b/package/scripts/switch_transport.sh new file mode 100755 index 0000000..15863c3 --- /dev/null +++ b/package/scripts/switch_transport.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Script to change all filters to use selected transport (UF, HEC, or kafka). +# If one or more filters uses a transport different from all the others, they will need to be +# managed by hand. Unpredictable results can occur if the script is used and +# individual filters are not checked for correctness. + +case $SPLUNK_CONNECT_METHOD in + hec) + echo "Switching transport method for all filters to HEC" + sed '/^# / {/#--HEC--/ s/^#/ /}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; + sed '/^#/! {/#--KAFKA--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; + sed '/^#/! {/#--UF--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; + sed 's~__SPLUNK_HEC_URL__~"'"$SPLUNK_HEC_URL"'"~' -i /opt/syslog-ng/etc/conf.d/splunk.conf + sed 's/__SPLUNK_HEC_TOKEN__/'"$SPLUNK_HEC_TOKEN"'/' -i /opt/syslog-ng/etc/conf.d/splunk.conf + ;; + kafka) + echo "Switching transport method for all filters to Kafka" + sed '/^# / {/#--KAFKA--/ s/^#/ /}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; + sed '/^#/! {/#--HEC--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; + sed '/^#/! {/#--UF--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; + ;; + UF) + echo "Switching transport method for all filters to filesystem/UF" + sed '/^# / {/#--UF--/ s/^#/ /}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; + sed '/^#/! {/#--HEC--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; + sed '/^#/! {/#--KAFKA--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; + ;; + *) + echo "Usage: $0 hec|kafka|UF" + ;; +esac