From 5d1a7a23b0b0d646076b770d12393480f32694e1 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Mon, 7 Oct 2019 20:58:42 -0400 Subject: [PATCH] Update config.yml (#124) * CI CD Cleanup * Use GITHUB Registry for non public builds * Use docker hub for final builds --- .circleci/config.yml | 371 +++++++++++++++++++++++++++--------------- docker-compose-ci.yml | 2 +- 2 files changed, 245 insertions(+), 128 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fb30dd4..8517e64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,54 +14,55 @@ orbs: jobs: build: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false environment: - IMAGE_NAME: rfaircloth/scs - docker: - - image: circleci/buildpack-deps:stretch + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci +# docker: +# - image: circleci/buildpack-deps:stretch steps: +# - setup_remote_docker: +# docker_layer_caching: true - checkout - run: git submodule sync - run: git submodule update --init --recursive - - setup_remote_docker: - docker_layer_caching: true + - run: mkdir /tmp/artifacts + - run: docker pull gittools/gitversion:latest-linux-netcoreapp2.1 - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Build Docker image command: docker build -f package/Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:$CIRCLE_SHA1 package - run: - name: Push Docker image + name: Push Docker image by hash command: docker push $IMAGE_NAME:$CIRCLE_SHA1 - build-egb: - environment: - IMAGE_NAME: rfaircloth/scs - docker: - - image: circleci/buildpack-deps:stretch - steps: - - checkout - - run: git submodule sync - - run: git submodule update --init --recursive - - setup_remote_docker: - docker_layer_caching: true - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Build Docker image - command: docker build -f perftests/bundlesrv/Dockerfile -t $IMAGE_NAME:egb-$CIRCLE_SHA1 perftests/bundlesrv - - run: - name: Tag Docker image - command: docker tag $IMAGE_NAME:egb-$CIRCLE_SHA1 $IMAGE_NAME:egb-edge + name: Docker tag and push with version label + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + echo image $IMAGE_NAME:$SEMVER + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER + docker push $IMAGE_NAME:$SEMVER + - run: - name: Push Docker image - command: docker push $IMAGE_NAME:egb-$CIRCLE_SHA1 + name: Docker Save + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker save $IMAGE_NAME:$SEMVER | gzip -c > /tmp/artifacts/oci_container.tar.gz - run: - name: Push Docker image - command: docker push $IMAGE_NAME:egb-edge + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + + - store_artifacts: + path: /tmp/artifacts + dgoss: environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci GOSS_VER: v0.3.7 GOSS_OPTS: "--max-concurrent=1 --format junit" GOSS_FILES_STRATEGY: cp @@ -74,7 +75,7 @@ jobs: docker_layer_caching: true - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 @@ -97,6 +98,8 @@ jobs: - store_test_results: path: test-results test-unit: + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci docker: - image: circleci/buildpack-deps:stretch steps: @@ -107,7 +110,7 @@ jobs: - run: git submodule update --init --recursive - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker-compose pull @@ -153,20 +156,22 @@ jobs: docker: - image: 'circleci/buildpack-deps:stable' environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci steps: - checkout - setup_remote_docker: docker_layer_caching: true + - run: + name: Docker Login + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - snyk/scan: docker-image-name: $IMAGE_NAME:$CIRCLE_SHA1 - test-scan-clair: docker: - image: 'docker:stable' environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci steps: - checkout - setup_remote_docker: @@ -179,159 +184,225 @@ jobs: path: clair-reports - publish-common: - machine: - image: ubuntu-1604:201903-01 - docker_layer_caching: true # default - false + publish-edge: environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases + docker: + - image: circleci/buildpack-deps:stretch steps: - - checkout + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Create Directory + command: mkdir /tmp/artifacts - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - run: name: Docker tag image - command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:edge - run: name: Docker push tag - command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker push $IMAGE_NAME:$SEMVER + command: docker push $PUBLIC_IMAGE_NAME:edge + - run: name: Docker Save command: | - mkdir artifacts - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker save splunk/scs:$SEMVER | gzip -c > artifacts/oci_container.tar.gz + docker save $IMAGE_NAME:$CIRCLE_SHA1 | gzip -c > /tmp/artifacts/oci_container.tar.gz - run: name: BYOE Config command: | - tar rvf artifacts/baremetal.tar -C package etc - tar rvf artifacts/baremetal.tar -C package/sbin entrypoint.sh + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + - run: name: "Publish Release on GitHub" command: | go get -u github.com/tcnksm/ghr SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${SEMVER} ./artifacts/ - - publish-edge: - environment: - IMAGE_NAME: rfaircloth/scs - docker: - - image: circleci/buildpack-deps:stretch - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:edge - - run: - name: Docker push tag - command: docker push $IMAGE_NAME:edge + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete edge /tmp/artifacts/ publish-version: machine: image: ubuntu-1604:201903-01 docker_layer_caching: true # default - false environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases steps: + - checkout - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker tag image - command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker tag $IMAGE_NAME:$CIRCLE_SHA1 splunk/scs:$SEMVER + name: Create Directory + command: mkdir /tmp/artifacts - run: - name: Docker push tag + name: Docker Login command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker push splunk/scs:$SEMVER - release-version: - machine: - image: ubuntu-1604:201903-01 - docker_layer_caching: true # default - false - environment: - IMAGE_NAME: rfaircloth/scs - steps: - - checkout + docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - run: name: Docker tag image command: | SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker tag $IMAGE_NAME:$CIRCLE_SHA1 splunk/scs:$SEMVER + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:$SEMVER + docker push $PUBLIC_IMAGE_NAME:$SEMVER + - run: name: Docker Save command: | - mkdir artifacts SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker save splunk/scs:$SEMVER | gzip -c > artifacts/oci_container.tar.gz + docker save $PUBLIC_IMAGE_NAME:$SEMVER | gzip -c > /tmp/artifacts/oci_container.tar.gz - run: name: BYOE Config command: | - tar rvf artifacts/baremetal.tar -C package etc - tar rvf artifacts/baremetal.tar -C package/sbin entrypoint.sh + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh - run: name: "Publish Release on GitHub" command: | go get -u github.com/tcnksm/ghr SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${SEMVER} ./artifacts/ + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${SEMVER} /tmp/artifacts/ publish-latest: environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases docker: - image: circleci/buildpack-deps:stretch steps: - setup_remote_docker: docker_layer_caching: true + - run: + name: Create Directory + command: mkdir /tmp/artifacts - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - run: name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 splunk/scs:latest + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:latest - run: name: Docker push tag - command: docker push splunk/scs:latest + command: docker push $PUBLIC_IMAGE_NAME:latest + + - run: + name: Docker Save + command: | + docker save $IMAGE_NAME:$CIRCLE_SHA1 | gzip -c > /tmp/artifacts/oci_container.tar.gz + - run: + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + + - run: + name: "Publish Release on GitHub" + command: | + go get -u github.com/tcnksm/ghr + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete latest /tmp/artifacts/ + + + dockerhub-version: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: splunk/scs + steps: + + - checkout + + - run: + name: Create Directory + command: mkdir artifacts + - run: + name: Docker Login + command: | + docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker tag image + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:$SEMVER + docker push $PUBLIC_IMAGE_NAME:$SEMVER + + dockerhub-edge: + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: splunk/scs + docker: + - image: circleci/buildpack-deps:stretch + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Create Directory + command: mkdir /tmp/artifacts + - run: + name: Docker Login + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker tag image + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:edge + - run: + name: Docker push tag + command: docker push $PUBLIC_IMAGE_NAME:edge + + dockerhub-latest: + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: splunk/scs + docker: + - image: circleci/buildpack-deps:stretch + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Create Directory + command: mkdir /tmp/artifacts + - run: + name: Docker Login + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker tag image + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:latest + - run: + name: Docker push tag + command: docker push $PUBLIC_IMAGE_NAME:latest + workflows: - build-publish: + version: 2 + build-branches: jobs: - build - - build-egb - dgoss: requires: - build @@ -345,16 +416,23 @@ workflows: # - test-scan-clair: # requires: # - build - - publish-common: + - publish-edge: requires: - dgoss - test-unit filters: branches: only: - - master - develop - - publish-edge: + - publish-latest: + requires: + - dgoss + - test-unit + filters: + branches: + only: + - master + - dockerhub-edge: requires: - dgoss - test-unit @@ -362,22 +440,61 @@ workflows: branches: only: - develop - - publish-version: + - dockerhub-latest: requires: - - build - dgoss - test-unit + filters: + branches: + only: + - master + build-tags: + jobs: + - build: filters: tags: - only: /([0-9]+\.[0-9]+\.[0-9]+) - - release-version: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + - dgoss: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ requires: - - publish-version - - publish-latest: + - build + - test-unit: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - build + - test-scan-synk: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - build + - publish-version: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ requires: - dgoss - test-unit + - dockerhub-version: filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ branches: - only: - - master + ignore: /.*/ + requires: + - dgoss + - test-unit \ No newline at end of file diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index c33122c..b4bb66d 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -19,7 +19,7 @@ services: - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} sc4s: - image: rfaircloth/scs:${CIRCLE_SHA1} + image: ${IMAGE_NAME}:${CIRCLE_SHA1} hostname: sc4s ports: - "514"