diff --git a/bin/README.md b/bin/README.md index df3a35a..3cead49 100644 --- a/bin/README.md +++ b/bin/README.md @@ -15,8 +15,8 @@ to one of them. Don't include this in a `git commit`. For example: -``` -ln -s .terraform/modules/s3_thing/bin/upgrade-s3-provider.sh +```console +% ln -s .terraform/modules/s3_thing/bin/upgrade-s3-provider.sh ``` First, change the source to use `?ref=3`. @@ -31,7 +31,7 @@ module "s3_thing" { Next, run `tf-init -upgrade` to grab the new code. -```c +```console % tf-init -ugprade ``` @@ -54,7 +54,7 @@ Then, run the script on the module resource `module.s3_thing`. Here is an examp If you do a whole directory full of s3 modules, you can run this through in a loop. This assumes all the S3 module calls are named `s3_`: -```script +```console # change source as listed above % tf-init -upgrade % for f in $(grep module.*s3_ s3.tf | awk '{print "module." $2}' | sed -e 's/"//g'); do ./upgrade-s3-provider.sh $f; done @@ -64,9 +64,42 @@ This creates import logs in `logs/upgrade-s3-provider.*.log`. Once you have converted them, if you've made the link to the current directory, please remove it. +```console +% rm ./upgrade-s3-provider.sh ``` -rm ./upgrade-s3-provider.sh + +This script takes a second argument, the resource name (defaults to `this`). For example, for the `module.logs`, which has this `module.logs.aws_s3_bucket.logs` +S3 resource, you would use: + +```console +% ./upgrade-s3-provider.sh module.logs log +# same output as above, but with the proper resource name ``` +## upgrade-s3-provider-objects.sh + +When an S3 module creates objects, they were in `aws_s3_bucket_object` resources. These are now `aws_s3_object` in the new provider. We cannot rename +them because they re a differnt object, so we must import and remove the old one from state. -# + +```console +% ./upgrade-s3-provider-objects.sh module.logs log +* getting tf-plan for module.logs resource_name logs to /tmp/tfplan.sVplq (logfile logs/upgrade-s3-provider-objects.20220718.1658155713.log) +* checking that a bucket exists in module.logs +* getting bucket ID from module.logs +* found bucket inf-logs-252903981224-us-gov-west-1 +* checking for bucket_objects in module.logs +* importing s3_object resources to be created +. resource: tf-import module.logs.aws_s3_object.logs["inventory"] s3://inf-logs-252903981224-us-gov-west-1/inventory/ +. resource: tf-state rm module.logs.aws_s3_bucket_object.logs["inventory"] +. resource: tf-import module.logs.aws_s3_object.logs["nlb-logs"] s3://inf-logs-252903981224-us-gov-west-1/nlb-logs/ +. resource: tf-state rm module.logs.aws_s3_bucket_object.logs["nlb-logs"] +. resource: tf-import module.logs.aws_s3_object.logs["alb-logs"] s3://inf-logs-252903981224-us-gov-west-1/alb-logs/ +. resource: tf-state rm module.logs.aws_s3_bucket_object.logs["alb-logs"] +. resource: tf-import module.logs.aws_s3_object.logs["s3"] s3://inf-logs-252903981224-us-gov-west-1/s3/ +. resource: tf-state rm module.logs.aws_s3_bucket_object.logs["s3"] +. resource: tf-import module.logs.aws_s3_object.logs["elasticmapreduce"] s3://inf-logs-252903981224-us-gov-west-1/elasticmapreduce/ +. resource: tf-state rm module.logs.aws_s3_bucket_object.logs["elasticmapreduce"] +* imported 5 resources +* removed 5 resources +``` diff --git a/bin/upgrade-s3-provider-objects.sh b/bin/upgrade-s3-provider-objects.sh new file mode 100755 index 0000000..0748504 --- /dev/null +++ b/bin/upgrade-s3-provider-objects.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +VERSION="1.0.0" +THIS=$(basename $0 .sh) +STATUS=0 +MODULE=$1 +RNAME=$2 +if [ -z $MODULE ] +then + echo "* missing module, expecting 'module.s3_name'" + exit 1 +fi +if [ -z $RNAME ] +then + RNAME="this" +fi + +LOGDIR="logs" +test -d $LOGDIR || mkdir -p $LOGDIR +YMDSTAMP=$(date +%Y%m%d) +start=$(date +%s) +STAMP="$YMDSTAMP.$start" +LOGFILE="$LOGDIR/$THIS.$STAMP.log" +if [ -z $TFCOMMAND ] +then + TFCOMMAND=terraform +fi + +ERRORS="" +FILE=$(mktemp -t tfplan.XXXXX) +echo "* getting tf-plan for $MODULE resource_name $RNAME to $FILE (logfile $LOGFILE)" +$TFCOMMAND plan -no-color -target=$MODULE > $FILE + +echo "* checking that a bucket exists in $MODULE" +EXISTS=$(grep -c ^$MODULE.aws_s3_bucket.$RNAME: $FILE) +if [ $EXISTS == 0 ] +then + echo "* no S3 bucket at module $MODULE aws_s3_bucket.$RNAME" + exit 1 +fi + +echo "* getting bucket ID from $MODULE" +BUCKETID=$($TFCOMMAND state show -no-color $MODULE.aws_s3_bucket.$RNAME|grep -E 'id.* *='|awk '{print $1,$3}' |grep ^id|awk '{print $2}'|sed -e 's/"//g') +if [ -z $BUCKETID ] +then + echo "* cannot determine bucket id for $MODULE" + exit 1 +else + echo "* found bucket $BUCKETID" +fi + +COUNT=0 +RMCOUNT=0 + +echo "* checking for bucket_objects in $MODULE" +EXISTS=$(grep -c ^$MODULE.aws_s3_bucket_object.$RNAME $FILE) +if [ $EXISTS == 0 ] +then + echo "* no S3 bucket objects at module $MODULE aws_s3_bucket_object.$RNAME" + exit 1 +fi + +echo "* importing s3_object resources to be created" +for resource in $(grep ^$MODULE.aws_s3_bucket_object.$RNAME $FILE | awk '{print $1}' | sed -e's/:$//') +do + nresource=$(echo $resource | sed -e 's/aws_s3_bucket_object/aws_s3_object/') + oentry=$($TFCOMMAND state show -no-color $resource | grep -iE " key " | awk '{print $3}' | sed -e 's/"//g') + echo ". resource: tf-import $nresource s3://${BUCKETID}/${oentry}" + $TFCOMMAND import -no-color $nresource s3://${BUCKETID}/${oentry} >> $LOGFILE + if [ $? != 0 ] + then + echo "* error importing resource $nresource" + STATUS=$(( $STATUS + 1 )) + ERRORS+=" iomport:$nresource" + else + COUNT=$(( $COUNT + 1 )) + fi + echo ". resource: tf-state rm $resource" + $TFCOMMAND state rm "$resource" >> $LOGFILE + if [ $? != 0 ] + then + echo "* error removing resource $resource" + STATUS=$(( $STATUS + 1 )) + ERRORS+=" rm:$resource" + else + RMCOUNT=$(( $RMCOUNT + 1 )) + fi +done + +if [ $COUNT == 0 ] +then + echo "* no resources to import" +else + echo "* imported $COUNT resources" +fi + +if [ $RMCOUNT == 0 ] +then + echo "* no resources to remove" +else + echo "* removed $RMCOUNT resources" +fi + +rm -f $FILE +if [ $STATUS == 0 ] +then + echo "* import complete" + exit 0 +else + echo "* some portion of import failed: $ERRORS" + exit 1 +fi