From 355fb169cfca62500958eb45117ef185b1c68d23 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 14 Mar 2025 16:58:03 -0400 Subject: [PATCH] functional 0.0.1 --- .../gfl-resource-actions/.gitignore | 55 +++++ .../gfl-resource-actions/README.md | 188 ++++++++++++------ .../gfl-resource-actions/aws_utils.py | 141 ++++++++++++- .../gfl-resource-actions/config.py | 2 +- 4 files changed, 313 insertions(+), 73 deletions(-) create mode 100644 local-app/python-tools/gfl-resource-actions/.gitignore diff --git a/local-app/python-tools/gfl-resource-actions/.gitignore b/local-app/python-tools/gfl-resource-actions/.gitignore new file mode 100644 index 00000000..db01cfc1 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/.gitignore @@ -0,0 +1,55 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +dist/ +build/ +*.egg-info/ +*.egg + +# Virtual environments +venv/ +env/ +ENV/ +.env/ +.venv/ + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ +nosetests.xml + +# IDE specific files +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store + +# Project specific +config.local.py +credentials.json +*.log +logs/ +temp/ + +# AWS +.aws-sam/ +samconfig.toml +.chalice/ +.aws_credentials + +# Terraform +.terraform/ +*.tfstate +*.tfstate.backup +*.tfplan +.terraform.lock.hcl diff --git a/local-app/python-tools/gfl-resource-actions/README.md b/local-app/python-tools/gfl-resource-actions/README.md index 57690e38..41167bed 100644 --- a/local-app/python-tools/gfl-resource-actions/README.md +++ b/local-app/python-tools/gfl-resource-actions/README.md @@ -1,100 +1,156 @@ -# AWS Organization Resource Management Tool - Gliffy +# AWS Resource Management Utilities -This script discovers and manages (start/stop) AWS resources across multiple accounts in an AWS organization. +A collection of Python utilities for managing and interacting with AWS resources across multiple accounts, with special support for GovCloud environments and AWS SSO. -## Prerequisites +## Features + +- Cross-account resource management +- Dynamic AWS partition detection (supports commercial AWS, GovCloud, AWS China) +- AWS SSO profile integration +- Fallback mechanisms for authentication +- Support for AWS Organizations + +## Requirements - Python 3.6+ -- AWS SDK for Python (boto3) -- Proper IAM permissions: - - Access to AWS Organizations API (in the management account) - - Ability to assume `OrganizationAccountAccessRole` in member accounts - - Permissions to manage EC2, RDS, EKS, and EMR resources +- boto3 +- AWS credentials configured through AWS SSO or with standard credentials ## Installation -```bash -# Install dependencies +1. Clone this repository or copy the files to your desired location +2. Install required dependencies: +``` pip install boto3 ``` -## IAM Permissions +## Usage -Ensure your IAM user/role has the following permissions: -- `organizations:ListAccounts` -- `sts:AssumeRole` -- Permission to assume a role in each target account +### Basic Usage -The role being assumed in each account needs: -- `ec2:DescribeInstances`, `ec2:StopInstances`, `ec2:StartInstances` -- `rds:DescribeDBInstances`, `rds:StopDBInstance`, `rds:StartDBInstance` -- `eks:ListClusters`, `eks:DescribeCluster`, `eks:ListNodegroups`, `eks:DescribeNodegroup` -- `autoscaling:DescribeAutoScalingGroups`, `autoscaling:UpdateAutoScalingGroup` -- `emr:ListClusters`, `emr:DescribeCluster`, `emr:StopCluster`, `emr:StartCluster`, `emr:TerminateJobFlows` +```python +import aws_utils -## Usage +# Create a client using SSO profiles or role assumption +ec2_client = aws_utils.create_boto3_client_for_account( + '123456789012', # AWS account ID + 'ec2', # AWS service + 'us-gov-west-1' # AWS region +) + +# Use the client +instances = ec2_client.describe_instances() +``` + +### Using AWS SSO Profiles + +The library will automatically find and use appropriate AWS SSO profiles from your AWS config file (`~/.aws/config`). It will: + +1. Look for profiles matching the requested account ID +2. If a role_name is specified, it will find a profile with that role +3. Otherwise, it will prefer profiles with admin permissions +4. Fall back to role assumption if no profile is found or profile authentication fails + +### Key Functions + +#### `create_boto3_client_for_account(account_id, service, region=None, role_name=None)` + +Creates a boto3 client for the specified AWS service in the target account. + +- `account_id`: Target AWS account ID +- `service`: AWS service name (e.g., 'ec2', 'rds') +- `region`: AWS region (defaults to config.DEFAULT_REGION) +- `role_name`: Specific role to assume or look for in SSO profiles + +#### `get_partition_for_region(region)` + +Determines the correct AWS partition for a given region. -```bash -# Stop resources (default action) -python aws_org_resource_shutdown.py +- `region`: AWS region name +- Returns: 'aws', 'aws-us-gov', or 'aws-cn' -# Start resources -python aws_org_resource_shutdown.py --action start +#### `find_profile_for_account(account_id, role_name=None)` -# List all resources without modifying them (dry run) -python aws_org_resource_shutdown.py --dry-run +Finds an SSO profile in the AWS config file for the specified account. -# Specify regions to scan -python aws_org_resource_shutdown.py --regions us-east-1,us-west-2 +- `account_id`: AWS account ID to find profile for +- `role_name`: Optional preferred role name +- Returns: Profile name if found, None otherwise -# Exclude specific accounts -python aws_org_resource_shutdown.py --exclude-accounts 123456789012,098765432109 +#### `assume_role(account_id, region=None, role_name=None)` -# Exclude specific resource types -python aws_org_resource_shutdown.py --exclude-resources ec2 eks emr +Gets credentials for the specified account, trying SSO profiles first. + +- `account_id`: AWS account ID +- `region`: AWS region for determining partition +- `role_name`: Role name to assume +- Returns: Credentials dictionary + +#### `get_organization_accounts(exclude_accounts=None)` + +Gets a list of accounts in the AWS organization. + +- `exclude_accounts`: List of account IDs to exclude +- Returns: List of dictionaries with account ID and name + +## Configuration + +Create a `config.py` file with the following variables: + +```python +# Default AWS region to use +DEFAULT_REGION = 'us-gov-east-1' + +# Default role name to assume if one is not provided +ASSUME_ROLE_NAME = 'OrganizationAccountAccessRole' +``` + +## Makefile Usage + +The project includes a Makefile to simplify common operations: -# Combine options -python aws_org_resource_shutdown.py --action start --regions us-east-1 --exclude-resources rds --dry-run ``` +# Install dependencies +make install -## How It Works +# Run tests +make test -1. Connects to AWS Organizations API to list all member accounts -2. For each account: - - Assumes the `OrganizationAccountAccessRole` role - - Queries for EC2 instances, RDS instances, EKS clusters, and EMR clusters - - Performs shutdown or startup operations based on the selected action (if not in dry-run mode) +# Run linting +make lint -### Tag-Based Exclusion +# Format code +make format -Resources with the tag `gfl_exclude` will be automatically excluded from stop/start actions. The value of the tag is not checked, only its presence. This allows you to mark critical resources that should not be automatically managed. +# Clean up temporary files +make clean -Additionally, EC2 instances with the following tags are automatically excluded from direct EC2 management: -- `eks:cluster-name` - These instances are managed by EKS and should only be scaled through the EKS API -- `aws:elasticmapreduce:job-flow-id` - These instances are managed by EMR and should only be controlled through the EMR API +# Build distribution package +make dist -### Stop Action (Default) -- EC2: Stops running instances (unless they have exclusion tags or are service-managed) -- RDS: Stops available database instances (unless they have the exclusion tag) -- EKS: Scales down node groups to 0 (unless the cluster has the exclusion tag) -- EMR: Stops clusters if in RUNNING or WAITING state, terminates if in STARTING or BOOTSTRAPPING state +# Deploy to development environment +make deploy-dev -### Start Action -- EC2: Starts stopped instances (unless they have exclusion tags or are service-managed) -- RDS: Starts stopped database instances (unless they have the exclusion tag) -- EKS: Scales up node groups using original sizes from tags (unless the cluster has the exclusion tag) -- EMR: Starts stopped clusters +# Deploy to production environment +make deploy-prod +``` + +The Makefile helps standardize development workflows and ensures consistent environment setup across different machines. To view all available commands: -## Logging +``` +make help +``` -The script logs all actions to both the console and a file named `aws_resource_management.log`. +## Tips for GovCloud Usage -## Safety Features +When working with GovCloud: -- Dry-run mode to list resources without modifying them -- Ability to exclude specific accounts or resource types -- Detailed logging of all actions +1. Ensure your AWS SSO is properly configured +2. AWS SSO profiles should include the GovCloud regions in their configuration +3. The library will automatically detect GovCloud regions and use the correct partition ('aws-us-gov') -## Note +## Troubleshooting -This script requires proper IAM permissions and should be tested in a non-production environment first. +- **Authentication failures**: Check that your SSO session is active (`aws sso login --profile your-profile`) +- **Profile not found**: Verify your `~/.aws/config` file contains the correct account IDs +- **Permission issues**: Ensure the roles in your SSO profiles have the necessary permissions diff --git a/local-app/python-tools/gfl-resource-actions/aws_utils.py b/local-app/python-tools/gfl-resource-actions/aws_utils.py index 2ffdead0..9510d06f 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_utils.py @@ -3,6 +3,8 @@ """ import boto3 import config +import os +import re from logging_utils import setup_logging logger = setup_logging() @@ -60,18 +62,111 @@ def get_partition_for_region(region): else: return 'aws' -def assume_role(account_id, region=None): +def find_profile_for_account(account_id, role_name=None): """ - Assume the OrganizationAccountAccessRole in the specified account. + Find an appropriate SSO profile for the given account ID and role name. Args: - account_id (str): AWS account ID to assume role in + account_id (str): AWS account ID + role_name (str, optional): Preferred role name (e.g., 'AdministratorAccess', 'inf-admin-t2') + + Returns: + str: Profile name if found, None otherwise + """ + try: + # Try to read profiles from AWS config + import configparser + config_file = os.path.expanduser("~/.aws/config") + if not os.path.exists(config_file): + return None + + aws_config = configparser.ConfigParser() + aws_config.read(config_file) + + # Look for profiles matching the account ID + matching_profiles = [] + for section in aws_config.sections(): + # Extract the actual profile name from section (removing 'profile ' prefix if present) + profile_name = section + if section.startswith('profile '): + profile_name = section[8:] + + # Check if this profile is for the target account ID + if 'sso_account_id' in aws_config[section] and aws_config[section]['sso_account_id'] == account_id: + matching_profiles.append((profile_name, aws_config[section].get('sso_role_name', ''))) + + # If role_name is specified, try to find a profile with that role + if role_name and matching_profiles: + for profile, profile_role in matching_profiles: + if profile_role.lower() == role_name.lower(): + logger.info(f"Found matching profile {profile} for account {account_id} with role {role_name}") + return profile + + # If we have any matching profiles, return the first one + if matching_profiles: + # Prefer profiles with admin roles if available + admin_profiles = [p for p, r in matching_profiles if 'admin' in r.lower()] + if admin_profiles: + logger.info(f"Using profile {admin_profiles[0]} for account {account_id}") + return admin_profiles[0] + + logger.info(f"Using profile {matching_profiles[0][0]} for account {account_id}") + return matching_profiles[0][0] + + return None + except Exception as e: + logger.warning(f"Error finding profile for account {account_id}: {e}") + return None + +def create_boto3_session_from_profile(profile_name): + """ + Create a boto3 session using the specified profile. + + Args: + profile_name (str): AWS profile name + + Returns: + boto3.Session: Configured boto3 session or None if failed + """ + try: + logger.info(f"Creating session using profile: {profile_name}") + session = boto3.Session(profile_name=profile_name) + # Test if the session works by getting the caller identity + sts = session.client('sts') + sts.get_caller_identity() + return session + except Exception as e: + logger.warning(f"Failed to create session with profile {profile_name}: {e}") + return None + +def assume_role(account_id, region=None, role_name=None): + """ + Get credentials for the specified account, preferring SSO profiles when available. + + Args: + account_id (str): AWS account ID to access region (str, optional): AWS region, used to determine partition + role_name (str, optional): Role name to assume (defaults to config.ASSUME_ROLE_NAME) Returns: dict: Credentials dictionary or None if failed """ try: + # First try using SSO profiles if available + profile_name = find_profile_for_account(account_id, role_name) + if profile_name: + session = create_boto3_session_from_profile(profile_name) + if session: + # Get credentials from the session + credentials = session.get_credentials() + frozen_credentials = credentials.get_frozen_credentials() + return { + 'AccessKeyId': frozen_credentials.access_key, + 'SecretAccessKey': frozen_credentials.secret_key, + 'SessionToken': frozen_credentials.token + } + + # Fall back to assuming role directly if no profile found or profile didn't work # Use default region from config if not provided if region is None: region = config.DEFAULT_REGION @@ -79,19 +174,53 @@ def assume_role(account_id, region=None): # Determine the correct partition for the ARN partition = get_partition_for_region(region) - role_arn = f"arn:{partition}:iam::{account_id}:role/{config.ASSUME_ROLE_NAME}" + # Use the specified role name or fall back to the default + actual_role_name = role_name if role_name else config.ASSUME_ROLE_NAME + + role_arn = f"arn:{partition}:iam::{account_id}:role/{actual_role_name}" sts_client = boto3.client('sts', region_name=region) + logger.info(f"Assuming role {role_arn}") response = sts_client.assume_role( RoleArn=role_arn, - RoleSessionName="Gliffy" + RoleSessionName="ResourceManagementSession" ) return response['Credentials'] except Exception as e: - logger.error(f"Error assuming role in account {account_id}: {e}") + logger.error(f"Error accessing account {account_id}: {e}") return None +def create_boto3_client_for_account(account_id, service, region=None, role_name=None): + """ + Create a boto3 client for a specific service in the specified account. + + Args: + account_id (str): AWS account ID + service (str): AWS service name (e.g., 'ec2', 'rds') + region (str, optional): AWS region name + role_name (str, optional): Role name to use + + Returns: + boto3.client: Configured boto3 client or None if failed + """ + # First try using a profile directly + profile_name = find_profile_for_account(account_id, role_name) + if profile_name: + session = create_boto3_session_from_profile(profile_name) + if session: + if region: + return session.client(service, region_name=region) + else: + return session.client(service) + + # Fall back to assume_role method + credentials = assume_role(account_id, region, role_name) + if not credentials: + return None + + return create_boto3_client(service, credentials, region) + def get_organization_accounts(exclude_accounts=None): """ Get a list of accounts in the AWS organization. diff --git a/local-app/python-tools/gfl-resource-actions/config.py b/local-app/python-tools/gfl-resource-actions/config.py index 53f6a5ae..a3640eba 100644 --- a/local-app/python-tools/gfl-resource-actions/config.py +++ b/local-app/python-tools/gfl-resource-actions/config.py @@ -8,7 +8,7 @@ # Default node group sizes for EKS when starting DEFAULT_NG_MIN_SIZE = 1 DEFAULT_NG_DESIRED_SIZE = 1 - +DEFAULT_REGION = "us-gov-east-1" # Tag keys for storing node group sizes TAG_KEY_ORIGINAL_MIN_SIZE = "managed-by:resource-management:original-min-size" TAG_KEY_ORIGINAL_DESIRED_SIZE = "managed-by:resource-management:original-desired-size"