From 8f21a175e7be5dcf3ab2c51b47a6e7392ed96363 Mon Sep 17 00:00:00 2001 From: badra001 Date: Thu, 18 Aug 2022 08:20:23 -0400 Subject: [PATCH] add --- copy_image.sh | 324 ++++++++++++++++++++++++++++++++++++ copy_images.tf | 56 +++++++ create-apps-ecr.tf | 28 ++++ images.json | 17 ++ settings.auto.tfvars.sample | 41 +++++ variables.ecr.tf | 57 +++++++ 6 files changed, 523 insertions(+) create mode 100755 copy_image.sh create mode 100644 copy_images.tf create mode 100644 create-apps-ecr.tf create mode 100644 images.json create mode 100644 settings.auto.tfvars.sample create mode 100644 variables.ecr.tf diff --git a/copy_image.sh b/copy_image.sh new file mode 100755 index 0000000..60e8847 --- /dev/null +++ b/copy_image.sh @@ -0,0 +1,324 @@ +#!/bin/bash + +############################################################################### +# This script uses skopeo to copy a docker image from one repository to +# another. The primary intent is to copy the image from a public repository +# to a private repository. +############################################################################### +# Expected environment variables: +# +# SOURCE_IMAGE - The image to copy to to another location. Example: +# paradyme-docker-local.jfrog.io/appetizer:dev +# SOURCE_INSECURE - Set this to 1 of the source repository is in an insecure +# docker registry. Set it to 0 or leave it unset if the +# docker registry is secure. +# +# DESTINATION_IMAGE - The image to copy to to another location. Example: +# paradyme-docker-local.jfrog.io/appetizer:dev +# DESTINATION_INSECURE - Set this to 1 of the destination repository is in +# an insecure docker registry. Set it to 0 or leave it unset +# if the docker registry is secure. +# +# When the source repository requires authentication to access, configure +# these values. Otherwise do not set them. +# +# SOURCE_USERNAME - The username to supply for credentialed access to the +# repository. `anthony-zawacki` is an example. +# SOURCE_PASSWORD - The password to supply for credentialed access to the +# repository. An artifactory API_KEY for example. +# +# When the destination repository requires authentication to access, configure +# these values. Otherwise do not set them. +# +# DESTINATION_USERNAME - The username to supply for credentialed access to the +# repository. `anthony-zawacki` is an example. +# DESTINATION_PASSWORD - The password to supply for credentialed access to the +# repository. The output of: +# `aws ecr get-login-password --region us-east-2` for example. +# +# If the destination repository does not exist, the copy_image.sh script will +# create the repository automatically. In cases where the newly created +# repository should have a mutable image (perhaps always pushing to a `latest` +# tag in a development environment), it is possible to configure the +# repository to allow mutability by configuring this environment variable. +# Otherwise, do not set it. +# +# +############################################################################### + +ensure_skopeo() { + skopeo=$(command -v skopeo) + if [[ "$skopeo" == "" ]]; then + echo "The required executable, skopeo, was not found." + echo "Please install it and ensure it is in the path." + return 1 + fi + + return 0 +} + +usage() { + local msg="${1}"; shift; + + cat < (SOURCE_IMAGE) The name of the image to copy to another + registry. + -src-username (SOURCE_USERNAME) Optional parameter in cases where + the source registry requires authentication. Use this username for the + credentials. + -src-password (SOURCE_PASSWORD) Optional parameter in cases where + the source registry requires authentication. Use this password for the + credentials. + -src-insecure (SOURCE_INSECURE=1) Optional parameter indicates that the + source registry is not a secured registry and that tls validation + should be disabled for the processing of the image. The default is + to assume that the source registry is secured. + +src-insecure (SOURCE_INSECURE=0) Optional parameter explicitly indicating + that the source registry is secure and TLS must be used to access the + registry. + + -dest-image (DESTINATION_IMAGE) The name of the image to to use in the + destination registry. + -dest-username (DESTINATION_USERNAME) Optional parameter in cases + where the destination registry requires authentication. Use this + username for the credentials. + -dest-password (DESTINATION_PASSWORD) Optional parameter in cases + where the destination registry requires authentication. Use this + password for the credentials. + -dest-insecure (DESTINATION_INSECURE=1) Optional parameter indicates that the + destination registry is not a secured registry and that tls validation + should be disabled for the processing of the image. The default is + to assume that the destination registry is secured. + +dest-insecure (DESTINATION_INSECURE=0) Optional parameter explicitly + indicating that the destination registry is secure and TLS must be + used to access the registry. + -dest-mutable (DESTINATION_MUTABLE=1) Optional parameter indicates that if + creating the ECR repository is required, create it allowing mutable + images. + +dest-mutable (DESTNATION_MUTABLE=0) Optional parameter explicitly + indicating that if creating the ECR repository is required, create it + with immutable images. + +EOF + + exit 1 +} + +parse_commandline() { + local key + local positional=() + + while [[ $# -gt 0 ]]; do + key="$1"; shift + + case "$key" in + -src-image) + SOURCE_IMAGE="$1"; shift + ;; + -src-username) + SOURCE_USERNAME="$1"; shift + ;; + -src-password) + SOURCE_PASSWORD="$1"; shift + ;; + -src-insecure) + SOURCE_INSECURE=1 + ;; + +src-insecure) + SOURCE_INSECURE=0 + ;; + -dest-image) + DESTINATION_IMAGE="$1"; shift + ;; + -dest-username) + DESTINATION_USERNAME="$1"; shift + ;; + -dest-password) + DESTINATION_PASSWORD="$1"; shift + ;; + -dest-insecure) + DESTINATION_INSECURE=1 + ;; + +dest-insecure) + DESTINATION_INSECURE=0 + ;; + -dest-mutable) + DESTINATION_MUTABLE=1 + ;; + +dest-mutable) + DESTINATION_MUTABLE=0 + ;; + *) + positional+=("$key") + ;; + esac + done + + if [[ ${#positional[@]} -gt 0 ]]; then + usage "Unrecognized parameters: ${positional[*]}" + fi +} + +ensure_parameters() { + if [[ "$SOURCE_IMAGE" == "" ]]; then + usage "Must specify SOURCE_IMAGE" + fi + + if [[ "$DESTINATION_IMAGE" == "" ]]; then + usage "Must specify DESTINATION_IMAGE" + fi + + if [[ "$SOURCE_USERNAME" != "" || "$SOURCE_PASSWORD" != "" ]]; then + if [[ "$SOURCE_USERNAME" == "" || "$SOURCE_PASSWORD" == "" ]]; then + usage "Must specify both the SOURCE_USERNAME and SOURCE_PASSWORD." + fi + fi + + if [[ "$DESTINATION_USERNAME" != "" || "$DESTINATION_PASSWORD" != "" ]]; then + if [[ "$DESTINATION_USERNAME" == "" || "$DESTINATION_PASSWORD" == "" ]]; then + usage "Must specify both the DESTINATION_USERNAME and DESTINATION_PASSWORD." + fi + fi + + return 0 +} + +image_exists() { + declare src_creds="$SOURCE_USERNAME:$SOURCE_PASSWORD" + declare command=(skopeo inspect --insecure-policy) + + if [[ "$SOURCE_USERNAME" != "" ]]; then +# command+=(--src-creds "$src_creds") + command+=(--creds "$src_creds") + else +# command+=(--src-no-creds) + command+=(--no-creds) + fi + +# if [[ "$SOURCE_INSECURE" == "1" ]]; then +# command+=(--src-tls-verify=false) +# else +# command+=(--src-tls-verify=true) +# fi + + command+=("docker://$SOURCE_IMAGE") + + ${command[@]} > /dev/null 2>&1 + status=$? + echo "* source_image_exists() status=$status" + # return 0 if it does, 1 if not + return $? +} + +destination_image_exists() { + declare dst_creds="$DESTINATION_USERNAME:$DESTINATION_PASSWORD" + declare command=(skopeo inspect --insecure-policy) + + if [[ "$DESTINATION_USERNAME" != "" ]]; then +# command+=(--dest-creds "$dst_creds") + command+=(--creds "$dst_creds") + else +# command+=(--dest-no-creds) + command+=(--no-creds) + fi + +# if [[ "$DESTINATION_INSECURE" == "1" ]]; then +# command+=(--dest-tls-verify=false) +# else +# command+=(--dest-tls-verify=true) +# fi + + command+=("docker://$DESTINATION_IMAGE") + + ${command[@]} > /dev/null 2>&1 + status=$? + echo "* destination_image_exists() status=$status" + # return 0 if it does, 1 if not + return $? +} + +copy_image() { + declare src_creds="$SOURCE_USERNAME:$SOURCE_PASSWORD" + declare dest_creds="$DESTINATION_USERNAME:$DESTINATION_PASSWORD" + declare command=(skopeo copy --insecure-policy) + + if [[ "$SOURCE_USERNAME" != "" ]]; then + command+=(--src-creds "$src_creds") + else + command+=(--src-no-creds) + fi + + if [[ "$SOURCE_INSECURE" == "1" ]]; then + command+=(--src-tls-verify=false) + else + command+=(--src-tls-verify=true) + fi + + if [[ "$DESTINATION_USERNAME" != "" ]]; then + command+=(--dest-creds "$dest_creds") + else + command+=(--dest-no-creds) + fi + + if [[ "$DESTINATION_INSECURE" == "1" ]]; then + command+=(--dest-tls-verify=false) + else + command+=(--dest-tls-verify=true) + fi + + command+=("docker://$SOURCE_IMAGE" "docker://$DESTINATION_IMAGE") + + if [[ "$DESTINATION_IMAGE" == *.dkr.ecr.*.amazonaws.com/* ]]; then + echo "ECR registry detected, ensuring repository." + declare repository="${DESTINATION_IMAGE##*.amazonaws.com/}" + repository="${repository%%:*}" + declare region="${DESTINATION_IMAGE%%.amazonaws.com/*}" + region="${region##*.}" + export AWS_PAGER="" + if ! aws ecr describe-repositories \ + --region "$region" \ + --output "json" \ + --repository-names "$repository" \ + > /dev/null 2>&1; then + local mutability="IMMUTABLE" + if [ "$DESTINATION_MUTABLE" == "1" ]; then + mutability="MUTABLE" + fi + echo "creating repository $repository." + aws ecr create-repository \ + --image-tag-mutability "$mutability" \ + --image-scanning-configuration "scanOnPush=true" \ + --encryption-configuration "encryptionType=KMS" \ + --repository-name "$repository" \ + --region "$region" \ + > /dev/null 2>&1 || return $? + else + echo "repository $repository exists." + fi + fi + + echo "Copying $SOURCE_IMAGE" + echo "to $DESTINATION_IMAGE" + + ${command[@]} +} + + +ensure_image() { + ( image_exists && ! destination_image_exists ) || copy_image +} + +main() { + ensure_skopeo && \ + parse_commandline "$@" && \ + ensure_parameters && \ + ensure_image && \ + echo "Done" +} + +return 0 > /dev/null 2>&1 || main "$@" + diff --git a/copy_images.tf b/copy_images.tf new file mode 100644 index 0000000..5db8ee1 --- /dev/null +++ b/copy_images.tf @@ -0,0 +1,56 @@ +data "aws_ecr_authorization_token" "token" {} + +# ECR format +# {application_name}/{image}:{ag} +# application_name = {org}-{program} +# adsd-cumulus +# dice-mojo +# dice-centurion + +locals { + repo_parent_name = format("%v", var.app_name) + ecr_region = var.ecr_region == null ? local.region : var.ecr_region + + account_ecr_registry = format("%v.dkr.ecr.%v.amazonaws.com", local.account_id, local.ecr_region) + account_ecr = format("%v/%v", local.account_ecr_registry, local.repo_parent_name) + + images = { for i in var.image_config : format("%v#%v", i.name, i.tag) => + merge(i, tomap({ + key = format("%v#%v", i.name, i.tag), + source_full_path = format("%v/%v:%v", i.source_registry, i.source_image, element(compact(concat([lookup(i,"source_tag",null)],[i.tag])),0)), + dest_registry = local.account_ecr_registry, + dest_full_path = i.repo_path != null ? format("%v/%v/%v/%v:%v", local.account_ecr_registry, local.repo_parent_name, i.repo_path, i.name, i.tag) : format("%v/%v/%v:%v", local.account_ecr_registry, local.repo_parent_name, i.name, i.tag), + dest_repository = i.repo_path != null ? format("%v/%v/%v", local.repo_parent_name, i.repo_path, i.name) : format("%v/%v", local.repo_parent_name, i.name), + })) } + + image_repos = { for k, v in local.images : k => format("%v/%v", local.account_ecr, v.name) } +} + +resource "null_resource" "copy_images" { + triggers = { + ecr_region = local.ecr_region + } + for_each = { for image in local.images : image.key => image if image.enabled } + + provisioner "local-exec" { + command = "${path.module}/copy_image.sh" + environment = { + AWS_PROFILE = var.profile + AWS_REGION = local.ecr_region + # SOURCE_IMAGE = format("%v:%v", each.value.image, each.value.tag) + SOURCE_IMAGE = each.value.source_full_path + # DESTINATION_IMAGE = format("%v/%v:%v", local.account_ecr, each.value.name, each.value.tag) + DESTINATION_IMAGE = each.value.dest_full_path + SOURCE_USERNAME = var.source_username == null ? "" : var.source_username + SOURCE_PASSWORD = var.source_password == null ? "" : var.source_password + DESTINATION_USERNAME = var.destination_username == null ? data.aws_ecr_authorization_token.token.user_name : var.destination_username + DESTINATION_PASSWORD = var.destination_password == null ? data.aws_ecr_authorization_token.token.password : var.destination_password + } + } +} + + +output "images" { + description = "Final full merge of images with extra details" + value = local.images +} diff --git a/create-apps-ecr.tf b/create-apps-ecr.tf new file mode 100644 index 0000000..e0129c7 --- /dev/null +++ b/create-apps-ecr.tf @@ -0,0 +1,28 @@ +locals { + application_list = var.application_list + ecr_repo_list = { for app in local.application_list : app => format("%v/%v", var.app_name, app) } +} + +resource "aws_ecr_repository" "apps_repos" { + for_each = local.ecr_repo_list + name = each.value + + image_tag_mutability = "IMMUTABLE" + image_scanning_configuration { + scan_on_push = true + } + + encryption_configuration { + encryption_type = "KMS" + } + + tags = merge( + local.common_tags, + local.base_tags, + var.application_tags, + tomap({ + "Name" = format("ecr_%v/%v", var.app_name, each.key) + "Environment" = "application" + }), + ) +} diff --git a/images.json b/images.json new file mode 100644 index 0000000..9cafef0 --- /dev/null +++ b/images.json @@ -0,0 +1,17 @@ +[ + { + "name": "istio/pilot", + "tag": "1.10.1", + "source_image": "istio/pilot", + "source_registry": "docker.io", + "enabled": "true", + }, + { + "name": "amazonlinux", + "tag": "latest"/ + "source_image": "amazonlinux/amazonlinux", + "source_registry": "public.ecr.aws", + "enabled": "true", + }, +] + diff --git a/settings.auto.tfvars.sample b/settings.auto.tfvars.sample new file mode 100644 index 0000000..4d37526 --- /dev/null +++ b/settings.auto.tfvars.sample @@ -0,0 +1,41 @@ +app_name = "adsd-cumulus" +ecr_region = "us-gov-east-1" +#destination_password = "" +#destination_username = "" +source_password = "" +source_username = "" + +application_list = [ + "mft-batch-job", + "cumulus-monitor-api", +] + +image_config = [ + { + enabled = true + dest_path = null + name = "openjdk-8" + source_image = "ubi8/openjdk-8" + source_registry = "registry.access.redhat.com" + source_tag = null + tag = "latest" + }, + { + enabled = true + name = "nginx-118" + dest_path = null + source_image = "ubi8/nginx-118" + source_registry = "registry.access.redhat.com" + source_tag = null + tag = "latest" + }, + { + enabled = true + name = "nodejs-14" + dest_path = null + source_image = "ubi8/nodejs-14" + source_registry = "registry.access.redhat.com" + source_tag = null + tag = "latest" + }, +] diff --git a/variables.ecr.tf b/variables.ecr.tf new file mode 100644 index 0000000..ae9dd86 --- /dev/null +++ b/variables.ecr.tf @@ -0,0 +1,57 @@ +variable "app_name" { + description = "Appliication name, usually {org}-{project}, which is likely a prefix to the EKS cluster name" + type = string +} + +variable "application_list" { + description = "List of application repositories to create for /{app_name}/{image_name} for those not in image_config" + type = list(string) + default = [] +} + +variable "ecr_region" { + description = "Region in which to create the ECR repositories (default of current region)" + type = string + default = null +} + +variable "image_config" { + description = "List of image configuration objects to copy from NRC to DST" + type = list(object({ + name = string, + tag = string, + dest_path = string, + source_registry = string, + source_image = string, + source_tag = string, + enabled = bool, + })) + default = [] +} + +variable "source_username" { + description = "OCI source repository username" + type = string + default = null +} + +variable "source_password" { + description = "OCI source repository password" + type = string + # sensitive = true + default = null +} + +variable "destination_username" { + description = "OCI destination repository username" + type = string + default = null +} + +variable "destination_password" { + description = "OCI destination repository password" + type = string + # sensitive = true + default = null +} +