From 89c6c700a9c50160322139199e92e2255a88b05f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 14 Mar 2025 14:46:29 -0400 Subject: [PATCH] start of gliffy --- .../gfl-resource-actions/README.md | 99 ++- .../gfl-resource-actions/gliffy.py | 822 ++++++++++++++++++ 2 files changed, 920 insertions(+), 1 deletion(-) create mode 100644 local-app/python-tools/gfl-resource-actions/gliffy.py diff --git a/local-app/python-tools/gfl-resource-actions/README.md b/local-app/python-tools/gfl-resource-actions/README.md index ed6376c6..57690e38 100644 --- a/local-app/python-tools/gfl-resource-actions/README.md +++ b/local-app/python-tools/gfl-resource-actions/README.md @@ -1,3 +1,100 @@ -# gfl-resource-actions +# AWS Organization Resource Management Tool - Gliffy +This script discovers and manages (start/stop) AWS resources across multiple accounts in an AWS organization. +## Prerequisites + +- Python 3.6+ +- AWS SDK for Python (boto3) +- Proper IAM permissions: + - Access to AWS Organizations API (in the management account) + - Ability to assume `OrganizationAccountAccessRole` in member accounts + - Permissions to manage EC2, RDS, EKS, and EMR resources + +## Installation + +```bash +# Install dependencies +pip install boto3 +``` + +## IAM Permissions + +Ensure your IAM user/role has the following permissions: +- `organizations:ListAccounts` +- `sts:AssumeRole` +- Permission to assume a role in each target account + +The role being assumed in each account needs: +- `ec2:DescribeInstances`, `ec2:StopInstances`, `ec2:StartInstances` +- `rds:DescribeDBInstances`, `rds:StopDBInstance`, `rds:StartDBInstance` +- `eks:ListClusters`, `eks:DescribeCluster`, `eks:ListNodegroups`, `eks:DescribeNodegroup` +- `autoscaling:DescribeAutoScalingGroups`, `autoscaling:UpdateAutoScalingGroup` +- `emr:ListClusters`, `emr:DescribeCluster`, `emr:StopCluster`, `emr:StartCluster`, `emr:TerminateJobFlows` + +## Usage + +```bash +# Stop resources (default action) +python aws_org_resource_shutdown.py + +# Start resources +python aws_org_resource_shutdown.py --action start + +# List all resources without modifying them (dry run) +python aws_org_resource_shutdown.py --dry-run + +# Specify regions to scan +python aws_org_resource_shutdown.py --regions us-east-1,us-west-2 + +# Exclude specific accounts +python aws_org_resource_shutdown.py --exclude-accounts 123456789012,098765432109 + +# Exclude specific resource types +python aws_org_resource_shutdown.py --exclude-resources ec2 eks emr + +# Combine options +python aws_org_resource_shutdown.py --action start --regions us-east-1 --exclude-resources rds --dry-run +``` + +## How It Works + +1. Connects to AWS Organizations API to list all member accounts +2. For each account: + - Assumes the `OrganizationAccountAccessRole` role + - Queries for EC2 instances, RDS instances, EKS clusters, and EMR clusters + - Performs shutdown or startup operations based on the selected action (if not in dry-run mode) + +### Tag-Based Exclusion + +Resources with the tag `gfl_exclude` will be automatically excluded from stop/start actions. The value of the tag is not checked, only its presence. This allows you to mark critical resources that should not be automatically managed. + +Additionally, EC2 instances with the following tags are automatically excluded from direct EC2 management: +- `eks:cluster-name` - These instances are managed by EKS and should only be scaled through the EKS API +- `aws:elasticmapreduce:job-flow-id` - These instances are managed by EMR and should only be controlled through the EMR API + +### Stop Action (Default) +- EC2: Stops running instances (unless they have exclusion tags or are service-managed) +- RDS: Stops available database instances (unless they have the exclusion tag) +- EKS: Scales down node groups to 0 (unless the cluster has the exclusion tag) +- EMR: Stops clusters if in RUNNING or WAITING state, terminates if in STARTING or BOOTSTRAPPING state + +### Start Action +- EC2: Starts stopped instances (unless they have exclusion tags or are service-managed) +- RDS: Starts stopped database instances (unless they have the exclusion tag) +- EKS: Scales up node groups using original sizes from tags (unless the cluster has the exclusion tag) +- EMR: Starts stopped clusters + +## Logging + +The script logs all actions to both the console and a file named `aws_resource_management.log`. + +## Safety Features + +- Dry-run mode to list resources without modifying them +- Ability to exclude specific accounts or resource types +- Detailed logging of all actions + +## Note + +This script requires proper IAM permissions and should be tested in a non-production environment first. diff --git a/local-app/python-tools/gfl-resource-actions/gliffy.py b/local-app/python-tools/gfl-resource-actions/gliffy.py new file mode 100644 index 00000000..191c55b5 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/gliffy.py @@ -0,0 +1,822 @@ +#!/usr/bin/env python3 +import boto3 +import logging +import argparse +import sys +import time +from botocore.exceptions import ClientError + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler("aws_resource_management.log"), + logging.StreamHandler(sys.stdout) + ] +) +logger = logging.getLogger("AWS-Resource-Management") + +# Constants +ASSUME_ROLE_NAME = "OrganizationAccountAccessRole" # Typically available in member accounts +REGIONS = [] # Empty list means all regions will be discovered +DEFAULT_NG_MIN_SIZE = 1 # Default minimum size when starting EKS node groups +DEFAULT_NG_DESIRED_SIZE = 1 # Default desired capacity when starting EKS node groups +# Tag keys for storing node group sizes +TAG_KEY_ORIGINAL_MIN_SIZE = "managed-by:resource-management:original-min-size" +TAG_KEY_ORIGINAL_DESIRED_SIZE = "managed-by:resource-management:original-desired-size" +# Exclusion tag - resources with this tag will be excluded from management +EXCLUSION_TAG = "gfl_exclude" +# Tag for tracking stop action +STOP_TAG = "gfl_stop" +# Service-specific tags that indicate managed EC2 instances +EKS_TAG = "eks:cluster-name" +EMR_TAG = "aws:elasticmapreduce:job-flow-id" + +def parse_arguments(): + """Parse command-line arguments""" + parser = argparse.ArgumentParser(description="AWS Organization Resource Management Tool") + parser.add_argument("--action", choices=["stop", "start"], default="stop", + help="Action to perform: stop or start resources (default: stop)") + parser.add_argument("--dry-run", action="store_true", help="List resources without modifying them") + parser.add_argument("--regions", type=str, help="Comma-separated list of regions to scan") + parser.add_argument("--exclude-accounts", type=str, help="Comma-separated list of account IDs to exclude") + parser.add_argument("--exclude-resources", choices=["ec2", "rds", "eks", "emr"], nargs="+", + help="Resource types to exclude from management") + return parser.parse_args() + +def get_available_regions(): + """Get all available AWS regions""" + ec2_client = boto3.client('ec2') + regions = [region['RegionName'] for region in ec2_client.describe_regions()['Regions']] + return regions + +def assume_role(account_id, role_name): + """Assume role in the specified account""" + try: + sts_client = boto3.client('sts') + assumed_role = sts_client.assume_role( + RoleArn=f"arn:aws:iam::{account_id}:role/{role_name}", + RoleSessionName="ResourceManagementSession" + ) + credentials = assumed_role['Credentials'] + return credentials + except Exception as e: + logger.error(f"Failed to assume role in account {account_id}: {e}") + return None + +def get_account_resources(credentials, regions, exclude_resources): + """Get resources from an account across specified regions""" + resources = { + "ec2_instances": [], + "rds_instances": [], + "eks_clusters": [], + "emr_clusters": [] + } + + for region in regions: + session_kwargs = { + "aws_access_key_id": credentials['AccessKeyId'], + "aws_secret_access_key": credentials['SecretAccessKey'], + "aws_session_token": credentials['SessionToken'], + "region_name": region + } + + # Get EC2 instances + if "ec2" not in exclude_resources: + try: + ec2_client = boto3.client('ec2', **session_kwargs) + response = ec2_client.describe_instances() + for reservation in response.get('Reservations', []): + for instance in reservation.get('Instances', []): + # Capture tags for exclusion check + tags = {tag['Key']: tag['Value'] for tag in instance.get('Tags', [])} + + resources["ec2_instances"].append({ + "id": instance['InstanceId'], + "state": instance['State']['Name'], + "region": region, + "tags": tags + }) + + # Handle pagination + while 'NextToken' in response: + response = ec2_client.describe_instances(NextToken=response['NextToken']) + for reservation in response.get('Reservations', []): + for instance in reservation.get('Instances', []): + # Capture tags for exclusion check + tags = {tag['Key']: tag['Value'] for tag in instance.get('Tags', [])} + + resources["ec2_instances"].append({ + "id": instance['InstanceId'], + "state": instance['State']['Name'], + "region": region, + "tags": tags + }) + except Exception as e: + logger.error(f"Error getting EC2 instances in region {region}: {e}") + + # Get RDS instances + if "rds" not in exclude_resources: + try: + rds_client = boto3.client('rds', **session_kwargs) + response = rds_client.describe_db_instances() + for instance in response.get('DBInstances', []): + # Capture tags for exclusion check + tags = {} + try: + tag_list = rds_client.list_tags_for_resource( + ResourceName=instance['DBInstanceArn'] + ).get('TagList', []) + tags = {tag['Key']: tag['Value'] for tag in tag_list} + except Exception as tag_e: + logger.warning(f"Error getting tags for RDS instance {instance['DBInstanceIdentifier']}: {tag_e}") + + resources["rds_instances"].append({ + "id": instance['DBInstanceIdentifier'], + "status": instance['DBInstanceStatus'], + "region": region, + "tags": tags + }) + + # Handle pagination + while 'Marker' in response: + response = rds_client.describe_db_instances(Marker=response['Marker']) + for instance in response.get('DBInstances', []): + # Capture tags for exclusion check + tags = {} + try: + tag_list = rds_client.list_tags_for_resource( + ResourceName=instance['DBInstanceArn'] + ).get('TagList', []) + tags = {tag['Key']: tag['Value'] for tag in tag_list} + except Exception as tag_e: + logger.warning(f"Error getting tags for RDS instance {instance['DBInstanceIdentifier']}: {tag_e}") + + resources["rds_instances"].append({ + "id": instance['DBInstanceIdentifier'], + "status": instance['DBInstanceStatus'], + "region": region, + "tags": tags + }) + except Exception as e: + logger.error(f"Error getting RDS instances in region {region}: {e}") + + # Get EKS clusters + if "eks" not in exclude_resources: + try: + eks_client = boto3.client('eks', **session_kwargs) + response = eks_client.list_clusters() + clusters = response.get('clusters', []) + + # Handle pagination + while 'nextToken' in response: + response = eks_client.list_clusters(nextToken=response['nextToken']) + clusters.extend(response.get('clusters', [])) + + for cluster_name in clusters: + try: + cluster_info = eks_client.describe_cluster(name=cluster_name) + + # Capture tags for exclusion check + cluster_tags = cluster_info['cluster'].get('tags', {}) + + resources["eks_clusters"].append({ + "name": cluster_name, + "status": cluster_info['cluster']['status'], + "region": region, + "tags": cluster_tags + }) + except Exception as e: + logger.error(f"Error getting details for EKS cluster {cluster_name} in region {region}: {e}") + except Exception as e: + logger.error(f"Error getting EKS clusters in region {region}: {e}") + + # Get EMR clusters + if "emr" not in exclude_resources: + try: + emr_client = boto3.client('emr', **session_kwargs) + # For stopping: get active clusters + active_states = ['STARTING', 'BOOTSTRAPPING', 'RUNNING', 'WAITING'] + response = emr_client.list_clusters(ClusterStates=active_states) + clusters = response.get('Clusters', []) + + # Handle pagination for active clusters + while 'Marker' in response: + response = emr_client.list_clusters( + ClusterStates=active_states, + Marker=response['Marker'] + ) + clusters.extend(response.get('Clusters', [])) + + # For starting: also get stopped clusters + response = emr_client.list_clusters(ClusterStates=['TERMINATED_WITH_ERRORS', 'STOPPED']) + stopped_clusters = response.get('Clusters', []) + + # Handle pagination for stopped clusters + while 'Marker' in response: + response = emr_client.list_clusters( + ClusterStates=['TERMINATED_WITH_ERRORS', 'STOPPED'], + Marker=response['Marker'] + ) + stopped_clusters.extend(response.get('Clusters', [])) + + # Add all clusters to the resources list + for cluster in clusters + stopped_clusters: + resources["emr_clusters"].append({ + "id": cluster['Id'], + "name": cluster['Name'], + "status": cluster['Status']['State'], + "region": region + }) + except Exception as e: + logger.error(f"Error getting EMR clusters in region {region}: {e}") + + return resources + +def shutdown_ec2_instances(credentials, instances, account_id): + """Gracefully stop EC2 instances""" + for instance in instances: + # Skip instances with the exclusion tag + if EXCLUSION_TAG in instance.get("tags", {}): + logger.info(f"Skipping EC2 instance {instance['id']} with exclusion tag {EXCLUSION_TAG}") + continue + + # Skip instances that are part of EKS clusters + if EKS_TAG in instance.get("tags", {}): + logger.info(f"Skipping EC2 instance {instance['id']} with tag {EKS_TAG}={instance['tags'][EKS_TAG]} (EKS managed node)") + continue + + # Skip instances that are part of EMR clusters + if EMR_TAG in instance.get("tags", {}): + logger.info(f"Skipping EC2 instance {instance['id']} with tag {EMR_TAG}={instance['tags'][EMR_TAG]} (EMR managed node)") + continue + + if instance["state"] == "running": + try: + region = instance["region"] + ec2_client = boto3.client( + 'ec2', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + # Create ISO 8601 timestamp + timestamp = datetime.datetime.now().isoformat() + + # Tag the instance before stopping + logger.info(f"Tagging EC2 instance {instance['id']} with {STOP_TAG}={timestamp}") + ec2_client.create_tags( + Resources=[instance['id']], + Tags=[ + { + 'Key': STOP_TAG, + 'Value': timestamp + } + ] + ) + + # Stop the instance + logger.info(f"Stopping EC2 instance {instance['id']} in region {region}") + ec2_client.stop_instances(InstanceIds=[instance['id']]) + + # Log with detailed information + logger.info(f"ACTION: instance={instance['id']}, region={region}, account_id={account_id}, action=stop, timestamp={timestamp}") + + except Exception as e: + logger.error(f"Failed to stop EC2 instance {instance['id']}: {e}") + +def start_ec2_instances(credentials, instances, account_id): + """Start stopped EC2 instances""" + for instance in instances: + # Skip instances with the exclusion tag + if EXCLUSION_TAG in instance.get("tags", {}): + logger.info(f"Skipping EC2 instance {instance['id']} with exclusion tag {EXCLUSION_TAG}") + continue + + # Skip instances that are part of EKS clusters + if EKS_TAG in instance.get("tags", {}): + logger.info(f"Skipping EC2 instance {instance['id']} with tag {EKS_TAG}={instance['tags'][EKS_TAG]} (EKS managed node)") + continue + + # Skip instances that are part of EMR clusters + if EMR_TAG in instance.get("tags", {}): + logger.info(f"Skipping EC2 instance {instance['id']} with tag {EMR_TAG}={instance['tags'][EMR_TAG]} (EMR managed node)") + continue + + if instance["state"] == "stopped": + try: + region = instance["region"] + ec2_client = boto3.client( + 'ec2', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + timestamp = datetime.datetime.now().isoformat() + + logger.info(f"Starting EC2 instance {instance['id']} in region {region}") + ec2_client.start_instances(InstanceIds=[instance['id']]) + + # Log with detailed information + logger.info(f"ACTION: instance={instance['id']}, region={region}, account_id={account_id}, action=start, timestamp={timestamp}") + + except Exception as e: + logger.error(f"Failed to start EC2 instance {instance['id']}: {e}") + +def shutdown_rds_instances(credentials, instances): + """Stop RDS instances""" + for instance in instances: + # Skip instances with the exclusion tag + if EXCLUSION_TAG in instance.get("tags", {}): + logger.info(f"Skipping RDS instance {instance['id']} with exclusion tag {EXCLUSION_TAG}") + continue + + if instance["status"] == "available": + try: + region = instance["region"] + rds_client = boto3.client( + 'rds', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + logger.info(f"Stopping RDS instance {instance['id']} in region {region}") + rds_client.stop_db_instance(DBInstanceIdentifier=instance['id']) + except Exception as e: + logger.error(f"Failed to stop RDS instance {instance['id']}: {e}") + +def start_rds_instances(credentials, instances): + """Start stopped RDS instances""" + for instance in instances: + # Skip instances with the exclusion tag + if EXCLUSION_TAG in instance.get("tags", {}): + logger.info(f"Skipping RDS instance {instance['id']} with exclusion tag {EXCLUSION_TAG}") + continue + + if instance["status"] == "stopped": + try: + region = instance["region"] + rds_client = boto3.client( + 'rds', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + logger.info(f"Starting RDS instance {instance['id']} in region {region}") + rds_client.start_db_instance(DBInstanceIdentifier=instance['id']) + except Exception as e: + logger.error(f"Failed to start RDS instance {instance['id']}: {e}") + +def shutdown_eks_clusters(credentials, clusters): + """ + Shutdown EKS clusters gracefully + Note: This doesn't actually delete clusters, just scales down node groups + """ + for cluster in clusters: + # Skip clusters with the exclusion tag + if EXCLUSION_TAG in cluster.get("tags", {}): + logger.info(f"Skipping EKS cluster {cluster['name']} with exclusion tag {EXCLUSION_TAG}") + continue + + if cluster["status"] == "ACTIVE": + try: + region = cluster["region"] + eks_client = boto3.client( + 'eks', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + autoscaling_client = boto3.client( + 'autoscaling', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + # Get nodegroups + nodegroups = eks_client.list_nodegroups(clusterName=cluster['name']).get('nodegroups', []) + + logger.info(f"Scaling down EKS cluster {cluster['name']} in region {region}") + + # Scale down each nodegroup + for nodegroup in nodegroups: + ng_info = eks_client.describe_nodegroup( + clusterName=cluster['name'], + nodegroupName=nodegroup + ) + + if 'autoScalingGroups' in ng_info['nodegroup']: + for asg in ng_info['nodegroup']['autoScalingGroups']: + asg_name = asg['name'] + + # Get current ASG configuration + asg_info = autoscaling_client.describe_auto_scaling_groups( + AutoScalingGroupNames=[asg_name] + ) + + if len(asg_info['AutoScalingGroups']) > 0: + current_asg = asg_info['AutoScalingGroups'][0] + + # Save the current min size and desired capacity as tags + current_min_size = current_asg['MinSize'] + current_desired_size = current_asg['DesiredCapacity'] + + # Only save if greater than 0 + if current_min_size > 0 or current_desired_size > 0: + logger.info(f"Saving original ASG {asg_name} sizes: min={current_min_size}, desired={current_desired_size}") + autoscaling_client.create_or_update_tags( + Tags=[ + { + 'ResourceId': asg_name, + 'ResourceType': 'auto-scaling-group', + 'Key': TAG_KEY_ORIGINAL_MIN_SIZE, + 'Value': str(current_min_size), + 'PropagateAtLaunch': False + }, + { + 'ResourceId': asg_name, + 'ResourceType': 'auto-scaling-group', + 'Key': TAG_KEY_ORIGINAL_DESIRED_SIZE, + 'Value': str(current_desired_size), + 'PropagateAtLaunch': False + } + ] + ) + + # Now scale down the ASG + logger.info(f"Scaling down ASG {asg_name} for nodegroup {nodegroup}") + autoscaling_client.update_auto_scaling_group( + AutoScalingGroupName=asg_name, + MinSize=0, + DesiredCapacity=0 + ) + except Exception as e: + logger.error(f"Failed to scale down EKS cluster {cluster['name']}: {e}") + +def shutdown_eks_clusters(credentials, clusters): + """ + Shutdown EKS clusters gracefully + Note: This doesn't actually delete clusters, just scales down node groups + """ + for cluster in clusters: + if cluster["status"] == "ACTIVE": + try: + region = cluster["region"] + eks_client = boto3.client( + 'eks', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + autoscaling_client = boto3.client( + 'autoscaling', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + # Get nodegroups + nodegroups = eks_client.list_nodegroups(clusterName=cluster['name']).get('nodegroups', []) + + logger.info(f"Scaling down EKS cluster {cluster['name']} in region {region}") + + # Scale down each nodegroup + for nodegroup in nodegroups: + ng_info = eks_client.describe_nodegroup( + clusterName=cluster['name'], + nodegroupName=nodegroup + ) + + if 'autoScalingGroups' in ng_info['nodegroup']: + for asg in ng_info['nodegroup']['autoScalingGroups']: + asg_name = asg['name'] + + # Get current ASG configuration + asg_info = autoscaling_client.describe_auto_scaling_groups( + AutoScalingGroupNames=[asg_name] + ) + + if len(asg_info['AutoScalingGroups']) > 0: + current_asg = asg_info['AutoScalingGroups'][0] + + # Save the current min size and desired capacity as tags + current_min_size = current_asg['MinSize'] + current_desired_size = current_asg['DesiredCapacity'] + + # Only save if greater than 0 + if current_min_size > 0 or current_desired_size > 0: + logger.info(f"Saving original ASG {asg_name} sizes: min={current_min_size}, desired={current_desired_size}") + autoscaling_client.create_or_update_tags( + Tags=[ + { + 'ResourceId': asg_name, + 'ResourceType': 'auto-scaling-group', + 'Key': TAG_KEY_ORIGINAL_MIN_SIZE, + 'Value': str(current_min_size), + 'PropagateAtLaunch': False + }, + { + 'ResourceId': asg_name, + 'ResourceType': 'auto-scaling-group', + 'Key': TAG_KEY_ORIGINAL_DESIRED_SIZE, + 'Value': str(current_desired_size), + 'PropagateAtLaunch': False + } + ] + ) + + # Now scale down the ASG + logger.info(f"Scaling down ASG {asg_name} for nodegroup {nodegroup}") + autoscaling_client.update_auto_scaling_group( + AutoScalingGroupName=asg_name, + MinSize=0, + DesiredCapacity=0 + ) + except Exception as e: + logger.error(f"Failed to scale down EKS cluster {cluster['name']}: {e}") + +def start_eks_clusters(credentials, clusters): + """ + Start EKS clusters by scaling up node groups + Uses saved tags to restore original capacity if available + """ + for cluster in clusters: + if EXCLUSION_TAG in cluster.get("tags", {}): + logger.info(f"Skipping EKS cluster {cluster['name']} with exclusion tag {EXCLUSION_TAG}") + continue + + if cluster["status"] == "ACTIVE": + try: + region = cluster["region"] + eks_client = boto3.client( + 'eks', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + autoscaling_client = boto3.client( + 'autoscaling', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + # Get nodegroups + nodegroups = eks_client.list_nodegroups(clusterName=cluster['name']).get('nodegroups', []) + + logger.info(f"Scaling up EKS cluster {cluster['name']} in region {region}") + + # Scale up each nodegroup + for nodegroup in nodegroups: + ng_info = eks_client.describe_nodegroup( + clusterName=cluster['name'], + nodegroupName=nodegroup + ) + + if 'autoScalingGroups' in ng_info['nodegroup']: + for asg in ng_info['nodegroup']['autoScalingGroups']: + asg_name = asg['name'] + + # Check current capacity + asg_info = autoscaling_client.describe_auto_scaling_groups( + AutoScalingGroupNames=[asg_name] + ) + + if len(asg_info['AutoScalingGroups']) > 0: + current_asg = asg_info['AutoScalingGroups'][0] + + # Only scale up if currently at zero + if current_asg['MinSize'] == 0 and current_asg['DesiredCapacity'] == 0: + # Check if we have saved sizes in tags + min_size = DEFAULT_NG_MIN_SIZE + desired_size = DEFAULT_NG_DESIRED_SIZE + + # Look for original size tags + for tag in current_asg.get('Tags', []): + if tag['Key'] == TAG_KEY_ORIGINAL_MIN_SIZE: + try: + min_size = int(tag['Value']) + except (ValueError, TypeError): + logger.warning(f"Invalid tag value for {TAG_KEY_ORIGINAL_MIN_SIZE} on ASG {asg_name}: {tag['Value']}") + + if tag['Key'] == TAG_KEY_ORIGINAL_DESIRED_SIZE: + try: + desired_size = int(tag['Value']) + except (ValueError, TypeError): + logger.warning(f"Invalid tag value for {TAG_KEY_ORIGINAL_DESIRED_SIZE} on ASG {asg_name}: {tag['Value']}") + + logger.info(f"Scaling up ASG {asg_name} for nodegroup {nodegroup} to minimum={min_size}, desired={desired_size}") + autoscaling_client.update_auto_scaling_group( + AutoScalingGroupName=asg_name, + MinSize=min_size, + DesiredCapacity=desired_size + ) + + # Clean up the tags after restoring + try: + autoscaling_client.delete_tags( + Tags=[ + { + 'ResourceId': asg_name, + 'ResourceType': 'auto-scaling-group', + 'Key': TAG_KEY_ORIGINAL_MIN_SIZE + }, + { + 'ResourceId': asg_name, + 'ResourceType': 'auto-scaling-group', + 'Key': TAG_KEY_ORIGINAL_DESIRED_SIZE + } + ] + ) + except Exception as tag_e: + logger.warning(f"Failed to clean up size tags on ASG {asg_name}: {tag_e}") + else: + logger.info(f"ASG {asg_name} already has capacity. Skipping scale up.") + except Exception as e: + logger.error(f"Failed to scale up EKS cluster {cluster['name']}: {e}") + +def shutdown_emr_clusters(credentials, clusters): + """ + Stop EMR clusters gracefully + Note: This preserves the cluster configuration and data + """ + for cluster in clusters: + # Only RUNNING and WAITING clusters can be stopped + if cluster["status"] in ["RUNNING", "WAITING"]: + try: + region = cluster["region"] + emr_client = boto3.client( + 'emr', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + logger.info(f"Stopping EMR cluster {cluster['name']} ({cluster['id']}) in region {region}") + emr_client.stop_cluster(ClusterId=cluster['id']) + except Exception as e: + logger.error(f"Failed to stop EMR cluster {cluster['id']}: {e}") + # For STARTING and BOOTSTRAPPING clusters, we need to terminate them as they can't be stopped + elif cluster["status"] in ["STARTING", "BOOTSTRAPPING"]: + try: + region = cluster["region"] + emr_client = boto3.client( + 'emr', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + logger.info(f"Terminating EMR cluster {cluster['name']} ({cluster['id']}) in region {region} (cannot stop a cluster in {cluster['status']} state)") + emr_client.terminate_job_flows(JobFlowIds=[cluster['id']]) + except Exception as e: + logger.error(f"Failed to terminate EMR cluster {cluster['id']}: {e}") + +def start_emr_clusters(credentials, clusters): + """ + Start stopped EMR clusters + """ + for cluster in clusters: + if cluster["status"] == "STOPPED": + try: + region = cluster["region"] + emr_client = boto3.client( + 'emr', + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + + logger.info(f"Starting EMR cluster {cluster['name']} ({cluster['id']}) in region {region}") + emr_client.start_cluster(ClusterId=cluster['id']) + except Exception as e: + logger.error(f"Failed to start EMR cluster {cluster['id']}: {e}") + +def main(): + args = parse_arguments() + + # Process arguments + action = args.action + dry_run = args.dry_run + regions = args.regions.split(',') if args.regions else get_available_regions() + exclude_accounts = args.exclude_accounts.split(',') if args.exclude_accounts else [] + exclude_resources = args.exclude_resources or [] + + logger.info(f"Starting AWS organization resource {'discovery' if dry_run else action} {'(DRY RUN)' if dry_run else ''}") + + # Get organization accounts + try: + org_client = boto3.client('organizations') + accounts = [] + + paginator = org_client.get_paginator('list_accounts') + for page in paginator.paginate(): + for account in page['Accounts']: + if account['Status'] == 'ACTIVE' and account['Id'] not in exclude_accounts: + accounts.append({ + 'id': account['Id'], + 'name': account['Name'] + }) + except Exception as e: + logger.error(f"Failed to list organization accounts: {e}") + logger.info("Falling back to using current account only") + + # Fallback to current account if can't access organization + sts_client = boto3.client('sts') + identity = sts_client.get_caller_identity() + accounts = [{ + 'id': identity['Account'], + 'name': 'Current Account' + }] + + # Process each account +for account in accounts: + logger.info(f"Processing account: {account['name']} ({account['id']})") + + # Assume role in the account + credentials = assume_role(account['id'], ASSUME_ROLE_NAME) + if not credentials: + logger.warning(f"Skipping account {account['id']} due to role assumption failure") + continue + + # Get resources + resources = get_account_resources(credentials, regions, exclude_resources) + + # Count service-managed EC2 instances + eks_managed = sum(1 for i in resources['ec2_instances'] if EKS_TAG in i.get('tags', {})) + emr_managed = sum(1 for i in resources['ec2_instances'] if EMR_TAG in i.get('tags', {})) + + # Log discovered resources with service-managed counts + + + # Log discovered resources + total_ec2 = len(resources['ec2_instances']) + total_rds = len(resources['rds_instances']) + total_eks = len(resources['eks_clusters']) + total_emr = len(resources['emr_instances']) + + # Count excluded resources + excluded_ec2 = sum(1 for i in resources['ec2_instances'] if EXCLUSION_TAG in i.get('tags', {})) + excluded_rds = sum(1 for i in resources['rds_instances'] if EXCLUSION_TAG in i.get('tags', {})) + excluded_eks = sum(1 for i in resources['eks_clusters'] if EXCLUSION_TAG in i.get('tags', {})) + + logger.info(f"Account {account['id']} - Found {total_ec2} EC2 instances ({excluded_ec2} explicitly excluded, {eks_managed} EKS-managed, {emr_managed} EMR-managed)") + logger.info(f"Account {account['id']} - Found {total_rds} RDS instances ({excluded_rds} excluded)") + logger.info(f"Account {account['id']} - Found {total_eks} EKS clusters ({excluded_eks} excluded)") + logger.info(f"Account {account['id']} - Found {total_emr} EMR clusters") + + if not dry_run: + # Perform actions based on selected mode + if action == "stop": + # Shutdown resources + if "ec2" not in exclude_resources: + shutdown_ec2_instances(credentials, resources['ec2_instances'], account['id']) + + if "rds" not in exclude_resources: + shutdown_rds_instances(credentials, resources['rds_instances']) + + if "eks" not in exclude_resources: + shutdown_eks_clusters(credentials, resources['eks_clusters']) + + if "emr" not in exclude_resources: + shutdown_emr_clusters(credentials, resources['emr_clusters']) + else: # action == "start" + # Start resources + if "ec2" not in exclude_resources: + start_ec2_instances(credentials, resources['ec2_instances'], account['id']) + + if "rds" not in exclude_resources: + start_rds_instances(credentials, resources['rds_instances']) + + if "eks" not in exclude_resources: + start_eks_clusters(credentials, resources['eks_clusters']) + + if "emr" not in exclude_resources: + start_emr_clusters(credentials, resources['emr_clusters']) + + logger.info(f"Resource {'discovery' if dry_run else action} completed") + +if __name__ == "__main__": + main()