From de73677de26a2d2e231a4147013daf13e6270e35 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 18 Mar 2025 11:30:02 -0400 Subject: [PATCH] working output again --- .../gfl-resource-actions/Makefile | 103 ++- .../aws_resource_management.py | 63 +- .../aws_resource_management/__init__.py | 8 + .../aws_resource_management/__main__.py | 9 + .../aws_resource_management/aws_utils.py | 560 +++++++++++++ .../aws_resource_management/cli.py | 19 + .../aws_resource_management/cli_optimized.py | 116 +++ .../aws_resource_management/config_manager.py | 116 +++ .../aws_resource_management/core.py | 524 ++++++++++++ .../aws_resource_management/discovery.py | 746 ++++++++++++++++++ .../aws_resource_management/logging_setup.py | 233 ++++++ .../managers/__init__.py | 8 + .../aws_resource_management/managers/base.py | 155 ++++ .../aws_resource_management/managers/ec2.py | 166 ++++ .../aws_resource_management/managers/eks.py | 385 +++++++++ .../aws_resource_management/managers/emr.py | 215 +++++ .../aws_resource_management/managers/rds.py | 145 ++++ .../resource_controller.py | 256 ++++++ .../aws_resource_management/utils.py | 267 +++++++ .../gfl-resource-actions/aws_utils.py | 168 +--- .../gfl-resource-actions/logging_setup.py | 322 ++++++++ .../gfl-resource-actions/logging_utils.py | 148 ++++ .../python-tools/gfl-resource-actions/main.py | 343 ++------ .../gfl-resource-actions/managers/base.py | 122 ++- .../python-tools/gfl-resource-actions/plan.md | 681 ++++++++++++++++ .../gfl-resource-actions/setup.py | 32 + 26 files changed, 5375 insertions(+), 535 deletions(-) create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/__main__.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/aws_utils.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/cli.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/cli_optimized.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/config_manager.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/logging_setup.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/__init__.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ec2.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/emr.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/rds.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/resource_controller.py create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/utils.py create mode 100644 local-app/python-tools/gfl-resource-actions/logging_setup.py create mode 100644 local-app/python-tools/gfl-resource-actions/plan.md create mode 100644 local-app/python-tools/gfl-resource-actions/setup.py diff --git a/local-app/python-tools/gfl-resource-actions/Makefile b/local-app/python-tools/gfl-resource-actions/Makefile index 9eb927df..4e61d5ee 100644 --- a/local-app/python-tools/gfl-resource-actions/Makefile +++ b/local-app/python-tools/gfl-resource-actions/Makefile @@ -2,12 +2,14 @@ # Makefile for development, testing, and execution .PHONY: help setup install-dev code-check run-dry-run run-stop run-start \ - run-stop-ec2 run-start-ec2 clean all + run-stop-ec2 run-start-ec2 clean clean-logs all install dist test lint format # Variables SCRIPT_DIR = $(shell pwd) -MODULE_PATH = $(SCRIPT_DIR)/aws_resource_management.py +PACKAGE_NAME = aws_resource_management LOG_FILE = aws_resource_management.log +PYTHON = python +PIP = pip # Default target help: @@ -17,76 +19,99 @@ help: @echo " help - Show this help message" @echo " setup - Set up the development environment" @echo " install-dev - Install development dependencies" + @echo " install - Install package locally in development mode" @echo " code-check - Basic code checks (uses 'python -m py_compile')" + @echo " lint - Run code linters (flake8, mypy)" + @echo " format - Format code (black, isort)" + @echo " test - Run tests" + @echo " dist - Build distribution package" @echo " run-dry-run - Run the tool in dry-run mode (no changes)" @echo " run-stop - Run the tool to stop all resources" @echo " run-start - Run the tool to start all resources" @echo " run-stop-ec2 - Run the tool to stop EC2 instances only" @echo " run-start-ec2 - Run the tool to start EC2 instances only" @echo " clean - Remove temporary and generated files" - @echo " all - Run code-check" + @echo " clean-logs - Remove log files only" + @echo " all - Setup dependencies, check code and run in dry-run mode" # Setup the development environment -setup: install-dev +setup: install-dev install # Install development dependencies install-dev: - pip install boto3 pytest pytest-mock + $(PIP) install boto3 pytest pytest-mock flake8 mypy black isort pytest-cov + +# Install package locally in development mode +install: + $(PIP) install -e . # Basic code check - no external tools needed code-check: @echo "Checking Python files for syntax errors..." - @if [ -f $(MODULE_PATH) ]; then \ - python -m py_compile $(MODULE_PATH) && echo "No syntax errors found in main script."; \ - else \ - echo "Warning: Main script $(MODULE_PATH) not found."; \ - fi - @if [ -d aws_resource_management ]; then \ - find aws_resource_management -name "*.py" -type f -exec python -m py_compile {} \; && \ + @if [ -d $(PACKAGE_NAME) ]; then \ + find $(PACKAGE_NAME) -name "*.py" -type f -exec $(PYTHON) -m py_compile {} \; && \ echo "No syntax errors found in module files."; \ else \ - echo "Note: aws_resource_management directory not found. Skipping module checks."; \ + echo "Warning: Package directory $(PACKAGE_NAME) not found."; \ fi -# Run basic test verification +# Run linters +lint: + flake8 $(PACKAGE_NAME) + mypy $(PACKAGE_NAME) + +# Format code +format: + black $(PACKAGE_NAME) + isort $(PACKAGE_NAME) + +# Run tests test: - @echo "Basic verification:" - @if [ -f $(MODULE_PATH) ]; then \ - echo "Main script exists."; \ - else \ - echo "Main script $(MODULE_PATH) not found."; \ - fi - @echo "For full testing, install pytest: pip install pytest pytest-mock" + pytest -xvs tests/ + +# Build distribution package +dist: + $(PYTHON) setup.py sdist bdist_wheel -# Run the tool in dry-run mode +# Run in dry-run mode run-dry-run: - python $(MODULE_PATH) --dry-run + $(PYTHON) -m $(PACKAGE_NAME).cli --stop --dry-run --resource-type all -# Run the tool to stop all resources +# Run to stop resources run-stop: - python $(MODULE_PATH) --action stop + $(PYTHON) -m $(PACKAGE_NAME).cli --stop --resource-type all -# Run the tool to start all resources +# Run to start resources run-start: - python $(MODULE_PATH) --action start + $(PYTHON) -m $(PACKAGE_NAME).cli --start --resource-type all -# Run the tool to stop EC2 instances only +# Run to stop EC2 instances only run-stop-ec2: - python $(MODULE_PATH) --action stop --exclude-resources rds eks emr + $(PYTHON) -m $(PACKAGE_NAME).cli --stop --resource-type ec2 -# Run the tool to start EC2 instances only +# Run to start EC2 instances only run-start-ec2: - python $(MODULE_PATH) --action start --exclude-resources rds eks emr + $(PYTHON) -m $(PACKAGE_NAME).cli --start --resource-type ec2 -# Clean up temporary files +# Clean temporary files clean: + rm -rf __pycache__ + rm -rf *.egg-info + rm -rf build/ + rm -rf dist/ + rm -rf .pytest_cache + rm -rf .coverage + rm -rf .mypy_cache rm -f $(LOG_FILE) - find . -name "__pycache__" -type d -exec rm -rf {} + find . -name "*.pyc" -delete - find . -name "*.pyo" -delete - find . -name "*.pyd" -delete - find . -name ".pytest_cache" -type d -exec rm -rf {} + - @echo "Cleaned up temporary files." + find . -name "__pycache__" -delete + find . -name "*.log" -delete + +# Clean log files only +clean-logs: + rm -f $(LOG_FILE) + find . -name "*.log" -delete + find ./logs -type f -name "*.csv" -delete -# Run all development tasks -all: clean setup code-check test run-dry-run +# Default all target - setup dependencies, check code and run in dry-run mode +all: setup code-check run-dry-run diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management.py index a4d03dde..a578e284 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management.py @@ -8,6 +8,31 @@ import os import tempfile import subprocess +import concurrent.futures +import logging + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s [%(levelname)s] %(name)s - %(message)s' +) +logger = logging.getLogger('aws_resource_management') + +def process_account(account_id, script_path, script_dir, args): + """Process a single account with the main script.""" + try: + cmd = [sys.executable, script_path] + args + env = os.environ.copy() + env['PYTHONPATH'] = script_dir + os.pathsep + env.get('PYTHONPATH', '') + env['AWS_ACCOUNT'] = account_id + + result = subprocess.run(cmd, env=env, capture_output=True, text=True) + if result.returncode != 0: + logger.error(f"Error processing account {account_id}: {result.stderr}") + return result.returncode + except Exception as e: + logger.error(f"Error processing account {account_id}: {str(e)}") + return 1 if __name__ == "__main__": # Get the current script directory @@ -38,18 +63,34 @@ temp_file.close() try: - # Execute the temporary file with the current script directory in PYTHONPATH - cmd = [sys.executable, temp_file.name] + sys.argv[1:] - env = os.environ.copy() - env['PYTHONPATH'] = script_dir + os.pathsep + env.get('PYTHONPATH', '') + # Get list of accounts to process + accounts = [os.environ.get('AWS_ACCOUNT_ID', '')] + + # Create a thread pool for parallel execution + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: + # Submit account processing tasks + futures = { + executor.submit( + process_account, + account, + temp_file.name, + script_dir, + sys.argv[1:] + ): account for account in accounts if account + } + + # Wait for all tasks to complete + results = [] + for future in concurrent.futures.as_completed(futures): + account = futures[future] + try: + results.append(future.result()) + except Exception as e: + logger.error(f"Thread execution error for account {account}: {str(e)}") + results.append(1) - try: - # Run the command and exit with its return code - result = subprocess.run(cmd, env=env) - sys.exit(result.returncode) - except KeyboardInterrupt: - print("\nOperation interrupted by user. Exiting gracefully...") - sys.exit(130) # Standard exit code for SIGINT/Ctrl+C + # Exit with error if any account processing failed + sys.exit(1 if any(result != 0 for result in results) else 0) finally: # Clean up the temporary file os.unlink(temp_file.name) diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py new file mode 100644 index 00000000..7694c236 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py @@ -0,0 +1,8 @@ +""" +AWS Resource Management package. + +This package provides tools for managing AWS resources including stopping, +starting, and listing resources across different AWS services. +""" + +__version__ = "0.1.0" diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/__main__.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/__main__.py new file mode 100644 index 00000000..5ea659d0 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/__main__.py @@ -0,0 +1,9 @@ +""" +Main entry point for AWS Resource Management CLI. +""" + +import sys +from aws_resource_management.cli_optimized import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/aws_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/aws_utils.py new file mode 100644 index 00000000..40d46bb3 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/aws_utils.py @@ -0,0 +1,560 @@ +""" +AWS utility functions for credential management and account discovery. +""" +import os +import logging +import boto3 +import botocore +import configparser +from typing import Dict, List, Optional, Any, Tuple +from concurrent.futures import ThreadPoolExecutor, as_completed +import threading +import concurrent.futures + +from aws_resource_management.logging_setup import setup_logging + +logger = setup_logging() +# Thread-local storage for session caching +_thread_local = threading.local() + +def get_account_list() -> List[Dict[str, str]]: + """ + Get list of accounts from AWS Organizations. + + Returns: + List of dictionaries containing account_id and account_name + """ + try: + # First try to use organizations API + session = boto3.Session() + org_client = session.client('organizations') + + accounts = [] + paginator = org_client.get_paginator('list_accounts') + + for page in paginator.paginate(): + for account in page['Accounts']: + if account['Status'] == 'ACTIVE': + accounts.append({ + 'account_id': account['Id'], + 'account_name': account['Name'] + }) + + return accounts + + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + logger.warning(f"Failed to get accounts from Organizations API: {str(e)}") + logger.info("Falling back to configured AWS profiles") + + # Fall back to configured profiles if Organizations API access fails + return get_accounts_from_profiles() + +def get_accounts_from_profiles() -> List[Dict[str, str]]: + """ + Get list of accounts from configured AWS profiles in ~/.aws/config. + + Returns: + List of dictionaries containing account_id and account_name + """ + accounts = [] + # Dictionary to store the first profile seen for each account ID + account_profiles = {} + + try: + # Read AWS config file + config_path = os.path.expanduser("~/.aws/config") + if not os.path.exists(config_path): + logger.warning(f"AWS config file not found at {config_path}") + return [] + + config = configparser.ConfigParser() + config.read(config_path) + + # Extract account information from profiles + for section in config.sections(): + # Skip sections that don't have account ID + if not config.has_option(section, "sso_account_id"): + continue + + account_id = config.get(section, "sso_account_id") + section_name = section + if section.startswith("profile "): + section_name = section[8:] # Remove "profile " prefix + + # For each account ID, remember only the first profile we encounter + # Unless it's a profile with a preferred role like AdministratorAccess or inf-admin-t2 + if account_id not in account_profiles or ( + config.has_option(section, "sso_role_name") and + config.get(section, "sso_role_name") in ["AdministratorAccess", "inf-admin-t2"] + ): + # Get account name if available, otherwise use account ID + account_name = account_id + if config.has_option(section, "sso_account_name"): + account_name = config.get(section, "sso_account_name") + + account_profiles[account_id] = { + 'account_id': account_id, + 'account_name': account_name, + 'profile': section_name + } + + logger.debug(f"Using profile {section_name} for account {account_id} ({account_name})") + + # Convert dictionary values to list + accounts = list(account_profiles.values()) + + logger.info(f"Found {len(accounts)} accounts from AWS profiles") + return accounts + + except Exception as e: + logger.error(f"Error reading AWS profiles: {str(e)}") + return [] + +def get_credentials(account_id: str, profile_name: Optional[str] = None) -> Optional[Dict[str, Any]]: + """ + Get AWS credentials for the specified account. + + Args: + account_id: AWS account ID + profile_name: Optional AWS profile name to use + + Returns: + Dict with AWS credentials or None if failed + """ + try: + # If profile name is provided, use it directly + if profile_name: + logger.debug(f"Using specified profile: {profile_name}") + try: + session = boto3.Session(profile_name=profile_name) + + # Verify we can get credentials from the session + if session.get_credentials() is None: + logger.warning(f"No credentials available for profile {profile_name}") + return None + + return { + 'aws_access_key_id': session.get_credentials().access_key, + 'aws_secret_access_key': session.get_credentials().secret_key, + 'aws_session_token': session.get_credentials().token + } + except (botocore.exceptions.ProfileNotFound, botocore.exceptions.ClientError) as e: + logger.warning(f"Error using profile {profile_name}: {str(e)}") + # Fall through to try finding another profile + + # Find profile for the specified account + matching_profile = find_profile_by_account_id(account_id) + if matching_profile: + logger.debug(f"Using profile {matching_profile} for account {account_id}") + try: + session = boto3.Session(profile_name=matching_profile) + + # Verify we can get credentials from the session + if session.get_credentials() is None: + logger.warning(f"No credentials available for profile {matching_profile}") + return None + + return { + 'aws_access_key_id': session.get_credentials().access_key, + 'aws_secret_access_key': session.get_credentials().secret_key, + 'aws_session_token': session.get_credentials().token + } + except (botocore.exceptions.ProfileNotFound, botocore.exceptions.ClientError) as e: + logger.warning(f"Error using profile {matching_profile}: {str(e)}") + # Fall through to role assumption + + # If no profile is found or profile didn't work, fall back to role assumption + logger.debug(f"No working profile found for account {account_id}, falling back to role assumption") + return assume_role_credentials(account_id) + + except Exception as e: + logger.error(f"Error getting credentials for account {account_id}: {str(e)}") + return None + +def find_profile_by_account_id(account_id: str) -> Optional[str]: + """ + Find an AWS profile name that matches the specified account ID. + + Args: + account_id: AWS account ID + + Returns: + Profile name or None if not found + """ + try: + config_path = os.path.expanduser("~/.aws/config") + if not os.path.exists(config_path): + return None + + config = configparser.ConfigParser() + config.read(config_path) + + # Try to find profiles in order of preference + preferred_profiles = [ + f"{account_id}.AdministratorAccess", + f"{account_id}.inf-admin-t2", + f"{account_id}-gov.administratoraccess", + f"{account_id}-gov.inf-admin-t2" + ] + + # First check for preferred profiles + for profile_name in preferred_profiles: + for section in config.sections(): + section_name = section + if section.startswith("profile "): + section_name = section[8:] # Remove "profile " prefix + + if section_name.lower() == profile_name.lower(): + logger.debug(f"Found preferred profile {section_name} for account {account_id}") + return section_name + + # If no preferred profile found, look for any profile with this account ID + for section in config.sections(): + if not config.has_option(section, "sso_account_id"): + continue + + profile_account_id = config.get(section, "sso_account_id") + if profile_account_id == account_id: + section_name = section + if section.startswith("profile "): + section_name = section[8:] # Remove "profile " prefix + + logger.debug(f"Found profile {section_name} for account {account_id}") + return section_name + + logger.debug(f"No profile found for account {account_id}") + return None + + except Exception as e: + logger.error(f"Error finding profile for account {account_id}: {str(e)}") + return None + +def assume_role_credentials(account_id: str) -> Optional[Dict[str, Any]]: + """ + Get credentials by assuming a role in the target account. + + Args: + account_id: AWS account ID + + Returns: + Dict with AWS credentials or None if failed + """ + try: + sts_client = boto3.client('sts') + + # Try common role names + role_names = ['OrganizationAccountAccessRole', 'AWSControlTowerExecution'] + + for role_name in role_names: + try: + role_arn = f"arn:aws:iam::{account_id}:role/{role_name}" + response = sts_client.assume_role( + RoleArn=role_arn, + RoleSessionName='ResourceManagementSession' + ) + + credentials = response['Credentials'] + return { + 'aws_access_key_id': credentials['AccessKeyId'], + 'aws_secret_access_key': credentials['SecretAccessKey'], + 'aws_session_token': credentials['SessionToken'] + } + except botocore.exceptions.ClientError: + continue + + logger.warning(f"Could not assume any role in account {account_id}") + return None + + except Exception as e: + logger.error(f"Error assuming role for account {account_id}: {str(e)}") + return None + +def get_partition_for_region(region: str) -> str: + """ + Get the AWS partition for a region. + + Args: + region: AWS region name + + Returns: + AWS partition (aws, aws-us-gov, etc.) + """ + # Default to commercial AWS + partition = "aws" + + # Check for GovCloud regions + if region.startswith("us-gov"): + partition = "aws-us-gov" + elif region.startswith("cn-"): + partition = "aws-cn" + + return partition + +def get_all_regions(partition: Optional[str] = None) -> List[str]: + """ + Get all available AWS regions, filtered by partition if specified. + + Args: + partition: Optional AWS partition to filter by + + Returns: + List of region names + """ + try: + if partition == 'aws-us-gov': + return ['us-gov-east-1', 'us-gov-west-1'] + elif partition == 'aws-cn': + return ['cn-north-1', 'cn-northwest-1'] + + # For standard AWS or no partition specified, try to get all regions + ec2 = boto3.client('ec2', region_name='us-east-1') + regions = [region['RegionName'] for region in ec2.describe_regions()['Regions']] + + # If a partition was specified, filter the results + if partition == 'aws': + regions = [r for r in regions if not (r.startswith('us-gov-') or r.startswith('cn-'))] + + return regions + except Exception as e: + logger.warning(f"Error getting all regions: {e}") + # Default to common regions based on partition + if partition == 'aws-us-gov': + return ['us-gov-east-1', 'us-gov-west-1'] + elif partition == 'aws-cn': + return ['cn-north-1', 'cn-northwest-1'] + else: + return ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'] + +def get_enabled_regions(credentials: Dict[str, str], partition: Optional[str] = None) -> List[str]: + """ + Get list of AWS regions enabled for the account. + + Args: + credentials: AWS credentials dictionary + partition: AWS partition (aws, aws-us-gov, aws-cn) + + Returns: + List of enabled region names + """ + try: + # If no partition was specified, detect it from credentials + if not partition: + partition = detect_partition_from_credentials(credentials) + + # For GovCloud or China partitions, return their standard regions + # as the describe_regions call might not work with these credentials + if partition == 'aws-us-gov': + return ['us-gov-east-1', 'us-gov-west-1'] + elif partition == 'aws-cn': + return ['cn-north-1', 'cn-northwest-1'] + + # For standard partition, try to discover all enabled regions + session = boto3.Session( + aws_access_key_id=credentials.get('aws_access_key_id'), + aws_secret_access_key=credentials.get('aws_secret_access_key'), + aws_session_token=credentials.get('aws_session_token') + ) + + ec2_client = session.client('ec2', region_name='us-east-1') + regions = [region['RegionName'] for region in ec2_client.describe_regions()['Regions']] + + # Filter to only enabled regions + enabled_regions = [] + for region in regions: + if _is_region_enabled(region, credentials): + enabled_regions.append(region) + + return enabled_regions + except Exception as e: + logger.warning(f"Error getting all regions: {str(e)}") + # Fallback to default regions based on partition + if partition == 'aws-us-gov': + return ['us-gov-east-1', 'us-gov-west-1'] + elif partition == 'aws-cn': + return ['cn-north-1', 'cn-northwest-1'] + return ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'] + +def _is_region_enabled(region: str, credentials: Dict[str, str]) -> bool: + """ + Check if a region is enabled for the account. + + Args: + region: AWS region name + credentials: AWS credentials dictionary + + Returns: + True if the region is enabled, False otherwise + """ + try: + session = boto3.Session( + aws_access_key_id=credentials.get('aws_access_key_id'), + aws_secret_access_key=credentials.get('aws_secret_access_key'), + aws_session_token=credentials.get('aws_session_token') + ) + + # Try to create a client and make a lightweight API call + ec2_client = session.client('ec2', region_name=region) + try: + # Try a lightweight call first + ec2_client.describe_regions(RegionNames=[region]) + return True + except Exception: + # If that fails, try another lightweight call + try: + ec2_client.describe_availability_zones(ZoneNames=[f"{region}a"]) + return True + except Exception: + # Both calls failed, region might not be enabled + return False + except Exception: + # Could not create client or all calls failed + return False + +def is_valid_region(region: str) -> bool: + """ + Check if the given region is valid. + + Args: + region: AWS region name + + Returns: + True if valid, False otherwise + """ + try: + boto3.session.Session().client('ec2', region_name=region) + return True + except botocore.exceptions.ClientError: + return False + +def get_session_for_account(account_id: str, region: str = None, profile_name: Optional[str] = None) -> Optional[boto3.Session]: + """ + Get or create a boto3 session for the specified account and region. + Uses thread-local storage to cache sessions for better performance. + + Args: + account_id: AWS account ID + region: Region name (optional) + profile_name: AWS profile name (optional) + + Returns: + boto3.Session object or None if failed + """ + # Get thread-local storage for sessions + if not hasattr(_thread_local, 'sessions'): + _thread_local.sessions = {} + + # Create a key from account ID, region, and profile name + key = f"{account_id}:{region or 'default'}:{profile_name or 'default'}" + + # Return cached session if it exists + if key in _thread_local.sessions: + return _thread_local.sessions[key] + + try: + # Get credentials for the account + credentials = get_credentials(account_id, profile_name=profile_name) + if not credentials: + return None + + # Create a new session + session = boto3.Session( + aws_access_key_id=credentials['aws_access_key_id'], + aws_secret_access_key=credentials['aws_secret_access_key'], + aws_session_token=credentials.get('aws_session_token'), + region_name=region + ) + + # Cache the session + _thread_local.sessions[key] = session + return session + + except Exception as e: + logger.error(f"Error creating session for account {account_id}: {str(e)}") + return None + +def detect_partition_from_credentials(credentials: Dict[str, str]) -> str: + """ + Detect which AWS partition these credentials are valid for. + Tests credentials against key regions from different partitions to determine where they're valid. + + Args: + credentials: AWS credentials dictionary + + Returns: + AWS partition name (aws, aws-us-gov, aws-cn) + """ + logger.info("Detecting AWS partition from credentials...") + + # Try GovCloud first since that's a common use case in this system + try: + client = boto3.client( + 'sts', + region_name='us-gov-west-1', + aws_access_key_id=credentials.get('aws_access_key_id'), + aws_secret_access_key=credentials.get('aws_secret_access_key'), + aws_session_token=credentials.get('aws_session_token') + ) + client.get_caller_identity() + logger.info("Credentials valid for GovCloud partition (aws-us-gov)") + return 'aws-us-gov' + except Exception: + logger.debug("Credentials not valid for GovCloud partition") + + # Try commercial AWS + try: + client = boto3.client( + 'sts', + region_name='us-east-1', + aws_access_key_id=credentials.get('aws_access_key_id'), + aws_secret_access_key=credentials.get('aws_secret_access_key'), + aws_session_token=credentials.get('aws_session_token') + ) + client.get_caller_identity() + logger.info("Credentials valid for standard AWS partition (aws)") + return 'aws' + except Exception: + logger.debug("Credentials not valid for standard AWS partition") + + # Try China partition + try: + client = boto3.client( + 'sts', + region_name='cn-north-1', + aws_access_key_id=credentials.get('aws_access_key_id'), + aws_secret_access_key=credentials.get('aws_secret_access_key'), + aws_session_token=credentials.get('aws_session_token') + ) + client.get_caller_identity() + logger.info("Credentials valid for China AWS partition (aws-cn)") + return 'aws-cn' + except Exception: + logger.debug("Credentials not valid for China AWS partition") + + # Default to standard AWS if we couldn't determine + logger.warning("Could not determine valid partition for credentials - defaulting to aws") + return 'aws' + +def get_regions_for_partition(partition: str) -> List[str]: + """ + Get list of regions available in the specified AWS partition. + + Args: + partition: AWS partition (aws, aws-us-gov, aws-cn) + + Returns: + List of region names for the partition + """ + if partition == 'aws-us-gov': + return ['us-gov-east-1', 'us-gov-west-1'] + elif partition == 'aws-cn': + return ['cn-north-1', 'cn-northwest-1'] + else: # Default to commercial AWS regions + try: + ec2 = boto3.client('ec2', region_name='us-east-1') + regions = [region['RegionName'] for region in ec2.describe_regions()['Regions']] + return regions + except Exception as e: + logger.warning(f"Could not fetch regions for partition {partition}: {e}") + # Return common commercial regions as fallback + return ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', + 'eu-west-1', 'eu-central-1', 'ap-southeast-1', 'ap-southeast-2'] diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli.py new file mode 100644 index 00000000..df7787cb --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli.py @@ -0,0 +1,19 @@ +""" +DEPRECATED: This module is deprecated in favor of cli_optimized.py. +The original CLI implementation had performance issues that caused multiple +account scans for each resource type. +""" + +import sys +import warnings +from aws_resource_management.cli_optimized import main + +warnings.warn( + "The cli module is deprecated. Please use cli_optimized module instead.", + DeprecationWarning, + stacklevel=2 +) + +# Redirect to the optimized CLI +if __name__ == "__main__": + sys.exit(main()) diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli_optimized.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli_optimized.py new file mode 100644 index 00000000..e4d8ca69 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli_optimized.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +""" +Optimized CLI for AWS resource management tool. +This version scans accounts only once for all resource types. +""" + +import argparse +import sys +import logging +import traceback +from typing import List, Optional, Dict, Any + +from aws_resource_management.logging_setup import setup_logging +from aws_resource_management.core import ResourceManager +from aws_resource_management.config_manager import get_config + +logger = setup_logging() +config = get_config() + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser(description="AWS Resource Management CLI") + + # Action arguments + action_group = parser.add_mutually_exclusive_group(required=True) + action_group.add_argument('--stop', action='store_true', help='Stop resources') + action_group.add_argument('--start', action='store_true', help='Start resources') + + # Resource selection + parser.add_argument('--resource-type', choices=['ec2', 'rds', 'eks', 'emr', 'all'], + default='all', help='Resource type to manage (default: all)') + + # Account filtering + parser.add_argument('--account', help='Specific account ID to process') + parser.add_argument('--exclude-account', action='append', dest='exclude_accounts', + help='Account ID to exclude (can be used multiple times)') + + # Region options + parser.add_argument('--region', action='append', dest='regions', + help='Regions to scan (can be used multiple times)') + parser.add_argument('--exclude-region', action='append', dest='exclude_regions', + help='Regions to exclude (can be used multiple times)') + parser.add_argument('--discover-regions', action='store_true', + help='Automatically discover enabled regions') + parser.add_argument('--partition', choices=['aws', 'aws-us-gov', 'aws-cn'], + help='AWS partition to operate in') + + # Authentication + parser.add_argument('--profile', help='AWS profile to use') + parser.add_argument('--use-profiles', action='store_true', + help='Use AWS profiles for account discovery') + + # Dry run + parser.add_argument('--dry-run', action='store_true', + help='Show what would be done without making changes') + + return parser.parse_args() + +def main(): + """Main CLI entry point.""" + args = parse_args() + + try: + # Determine action + action = 'stop' if args.stop else 'start' + + # Determine resource types to process + exclude_resources = [] + if args.resource_type != 'all': + # If specific resource type is selected, exclude all others + all_resource_types = ['ec2', 'rds', 'eks', 'emr'] + exclude_resources = [rt for rt in all_resource_types if rt != args.resource_type] + + logger.info(f"Processing {args.resource_type} resources in {args.regions or 'all enabled'} regions for {action} action") + + # Initialize resource manager + resource_manager = ResourceManager( + profile_name=args.profile, + use_profiles=args.use_profiles, + discover_regions=args.discover_regions, + partition=args.partition + ) + + # Create account inclusion/exclusion list + exclude_accounts = args.exclude_accounts or [] + if args.account: + # If specific account is given, exclude all others by default + # But don't add the specified account to the exclusion list + logger.info(f"Processing only account {args.account}") + # We'll handle this by post-filtering the account list in resource_manager + + # Process accounts - single scan for all resource types + stats = resource_manager.process_accounts( + action=action, + regions=args.regions or [], + exclude_accounts=exclude_accounts, + exclude_resources=exclude_resources, + exclude_regions=args.exclude_regions or [], + dry_run=args.dry_run + ) + + logger.info(f"Completed {action} action for {args.resource_type} resources") + + # Exit with success code + sys.exit(0) + + except KeyboardInterrupt: + logger.warning("Operation interrupted by user. Exiting gracefully...") + sys.exit(1) + except Exception as e: + logger.error(f"Unhandled exception: {str(e)}") + logger.debug(traceback.format_exc()) + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/config_manager.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/config_manager.py new file mode 100644 index 00000000..c8b29713 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/config_manager.py @@ -0,0 +1,116 @@ +""" +Configuration management for the AWS Resource Management tool. +Handles loading configuration from files and environment variables. +""" +import os +import yaml +from typing import Dict, Any, Optional +from pathlib import Path + +# Default configuration values +DEFAULT_CONFIG = { + "exclusion_tag": "gliffy:exclude", + "stop_tag": "gliffy:stopped-at", + "log_level": "INFO", + "default_region": "us-gov-east-1", + "max_retries": 3, + "scheduled_tag": "gliffy:schedule", + "dry_run": False, +} + +# Environment variable prefix for overriding configuration +ENV_PREFIX = "AWS_RESOURCE_MGMT_" + +class ConfigManager: + """ + Configuration manager for AWS Resource Management tool. + Handles loading config from files and environment variables. + """ + _instance = None + _config = None + + def __new__(cls): + """Singleton pattern to ensure only one config instance.""" + if cls._instance is None: + cls._instance = super(ConfigManager, cls).__new__(cls) + cls._instance._config = DEFAULT_CONFIG.copy() + return cls._instance + + def load_config(self, config_file: Optional[str] = None) -> Dict[str, Any]: + """ + Load configuration from file and environment variables. + + Args: + config_file: Path to the config file (YAML) + + Returns: + Configuration dictionary + """ + # Start with defaults + self._config = DEFAULT_CONFIG.copy() + + # Load from config file if provided + if config_file and Path(config_file).exists(): + try: + with open(config_file, 'r') as f: + file_config = yaml.safe_load(f) + if file_config and isinstance(file_config, dict): + self._config.update(file_config) + except Exception as e: + print(f"Error loading config file: {e}") + + # Override with environment variables + for key in self._config.keys(): + env_key = f"{ENV_PREFIX}{key.upper()}" + if env_key in os.environ: + # Convert environment variable to appropriate type + env_value = os.environ[env_key] + if isinstance(self._config[key], bool): + self._config[key] = env_value.lower() in ('true', 'yes', '1') + elif isinstance(self._config[key], int): + self._config[key] = int(env_value) + else: + self._config[key] = env_value + + return self._config + + def get(self, key: str, default: Any = None) -> Any: + """ + Get a configuration value by key. + + Args: + key: Configuration key + default: Default value if key doesn't exist + + Returns: + Configuration value + """ + return self._config.get(key, default) + + def set(self, key: str, value: Any) -> None: + """ + Set a configuration value. + + Args: + key: Configuration key + value: Configuration value + """ + self._config[key] = value + + def get_all(self) -> Dict[str, Any]: + """ + Get the entire configuration dictionary. + + Returns: + Configuration dictionary + """ + return self._config.copy() + +def get_config() -> ConfigManager: + """ + Get the configuration manager instance. + + Returns: + ConfigManager instance + """ + return ConfigManager() diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py new file mode 100644 index 00000000..4117500a --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py @@ -0,0 +1,524 @@ +""" +Core business logic for AWS Resource Management tool. +""" + +from typing import Dict, List, Any, Optional, Union +import logging +import sys + +from aws_resource_management.logging_setup import setup_logging +from aws_resource_management.config_manager import get_config +from aws_resource_management import utils +from aws_resource_management.discovery import get_account_resources +from aws_resource_management.managers import EC2Manager, RDSManager, EKSManager, EMRManager +from aws_resource_management.aws_utils import get_credentials, get_account_list, get_accounts_from_profiles, get_enabled_regions, detect_partition_from_credentials + +logger = setup_logging() +config = get_config() + +class ResourceManager: + """Main resource manager that orchestrates operations across accounts and resources.""" + + def __init__( + self, + profile_name: Optional[str] = None, + use_profiles: bool = False, + discover_regions: bool = False, + partition: Optional[str] = None + ) -> None: + """ + Initialize the resource manager. + + Args: + profile_name: Optional AWS profile name to use for all operations + use_profiles: Whether to use AWS profiles from ~/.aws/config + discover_regions: Whether to automatically discover enabled regions + partition: AWS partition to operate in (aws, aws-us-gov, aws-cn) + """ + self.config = get_config() + # Initialize account list + self.account_list = [] + self.profile_name = profile_name + self.use_profiles = use_profiles + self.discover_regions = discover_regions + self.partition = partition + # Cache for discovered regions by account + self.region_cache = {} + + # If no partition was specified, we'll auto-detect based on credentials + # when we process the first account + + def process_accounts( + self, + action: str, + regions: List[str], + exclude_accounts: List[str] = None, + exclude_resources: List[str] = None, + exclude_regions: List[str] = None, + dry_run: bool = False, + stats: Dict[str, Any] = None + ) -> Dict[str, Any]: + """ + Process accounts and perform the specified action. + + Args: + action: Action to perform (stop or start) + regions: List of regions to scan + exclude_accounts: List of account IDs to exclude + exclude_resources: List of resource types to exclude + exclude_regions: List of regions to exclude + dry_run: Whether to perform a dry run + stats: Statistics dictionary to update + + Returns: + Updated statistics dictionary + """ + try: + if exclude_accounts is None: + exclude_accounts = [] + + if exclude_resources is None: + exclude_resources = [] + + if exclude_regions is None: + exclude_regions = [] + + # Initialize stats if not provided + if stats is None: + stats = { + 'accounts_processed': 0, + 'accounts_skipped': 0, + 'resources_processed': 0, + 'resources_skipped': 0, + 'regions_processed': 0, + 'regions_by_account': {} + } + + # Initialize resource-specific counters + for resource_type in ['ec2', 'rds', 'eks', 'emr']: + for stat_type in ['found', 'skipped', 'stopped', 'started', 'errors']: + stats[f"{resource_type}_{stat_type}"] = 0 + + # Get account list - either from profiles or Organizations API + if self.use_profiles: + logger.info("Using AWS SSO profiles for account discovery") + accounts = get_accounts_from_profiles() + else: + logger.info("Using AWS Organizations API for account discovery") + accounts = get_account_list() + + # Log summary of accounts + logger.info(f"Found {len(accounts)} accounts to process") + if exclude_accounts: + logger.info(f"Excluding {len(exclude_accounts)} accounts") + + # Process each account + for account in accounts: + account_id = account.get('account_id') + account_name = account.get('account_name', 'Unknown') + + # Skip excluded accounts + if account_id in exclude_accounts: + logger.info(f"Skipping excluded account {account_id} ({account_name})") + stats['accounts_skipped'] += 1 + continue + + try: + # Get credentials for the account + credentials = get_credentials(account_id, self.profile_name) + if not credentials: + logger.warning(f"Could not obtain credentials for account {account_id}") + stats['accounts_skipped'] += 1 + continue + + # Determine regions to scan for this account + account_regions = regions + + # If auto-discovery of regions is enabled, get all enabled regions for this account + if self.discover_regions: + try: + logger.info(f"Discovering enabled regions for account {account_id}") + account_regions = get_enabled_regions(credentials, self.partition) + if exclude_regions: + account_regions = [r for r in account_regions if r not in exclude_regions] + logger.info(f"Found {len(account_regions)} enabled regions in account {account_id}") + + # Cache discovered regions + self.region_cache[account_id] = account_regions + except Exception as e: + logger.error(f"Error discovering regions for account {account_id}: {str(e)}") + if regions: + logger.info(f"Falling back to provided regions: {', '.join(regions)}") + account_regions = regions + else: + logger.info("No regions provided and region discovery failed. Skipping account.") + stats['accounts_skipped'] += 1 + continue + elif exclude_regions: + account_regions = [r for r in account_regions if r not in exclude_regions] + + # Track regions processed for this account + stats['regions_by_account'][account_id] = account_regions + stats['regions_processed'] += len(account_regions) + + # Process this account with its regions + logger.info(f"Processing account: {account_name} ({account_id}) in {len(account_regions)} regions") + + self._process_single_account( + account=account, + credentials=credentials, + regions=account_regions, + action=action, + dry_run=dry_run, + exclude_resources=exclude_resources, + stats=stats + ) + + stats['accounts_processed'] += 1 + + except Exception as e: + logger.error(f"Error processing account {account_id}: {str(e)}", exc_info=True) + stats['accounts_skipped'] += 1 + + # Log summary of regions processed + if self.discover_regions: + logger.info(f"Total accounts processed: {stats.get('accounts_processed', 0)}") + logger.info(f"Total regions processed: {stats.get('regions_processed', 0)}") + for account_id, account_regions in stats.get('regions_by_account', {}).items(): + logger.debug(f"Account {account_id} regions: {', '.join(account_regions)}") + + # Print detailed resource summary + self._print_resource_summary(stats, action) + + return stats + + except KeyboardInterrupt: + # Log the interruption but re-raise to ensure application exit + logger.warning("Account processing interrupted by user") + raise + + def _print_resource_summary(self, stats: Dict[str, Any], action: str) -> None: + """ + Print a detailed summary of resources processed, broken down by resource type. + + Args: + stats: Statistics dictionary + action: Action that was performed (stop or start) + """ + logger.info(f"\n{'=' * 30} SUMMARY {'=' * 30}") + logger.info(f"ACTION: {action.upper()} {'(DRY RUN)' if stats.get('dry_run', False) else ''}") + + # General stats + logger.info(f"\nGENERAL STATISTICS:") + logger.info(f"Accounts processed: {stats.get('accounts_processed', 0)}") + logger.info(f"Accounts skipped: {stats.get('accounts_skipped', 0)}") + logger.info(f"Regions processed: {stats.get('regions_processed', 0)}") + + # EC2 resources + logger.info(f"\nEC2 INSTANCES:") + logger.info(f"Found: {stats.get('ec2_found', 0)}") + if action == 'stop': + logger.info(f"Stopped: {stats.get('ec2_stopped', 0)}") + else: + logger.info(f"Started: {stats.get('ec2_started', 0)}") + logger.info(f"Skipped: {stats.get('ec2_skipped', 0)}") + logger.info(f"Errors: {stats.get('ec2_errors', 0)}") + + # RDS resources + logger.info(f"\nRDS INSTANCES:") + logger.info(f"Found: {stats.get('rds_found', 0)}") + if action == 'stop': + logger.info(f"Stopped: {stats.get('rds_stopped', 0)}") + else: + logger.info(f"Started: {stats.get('rds_started', 0)}") + logger.info(f"Skipped: {stats.get('rds_skipped', 0)}") + logger.info(f"Errors: {stats.get('rds_errors', 0)}") + + # RDS engine breakdown if available + rds_engines = stats.get('rds_engines', {}) + if rds_engines: + logger.info(f"RDS engines: {', '.join(f'{engine}({count})' for engine, count in rds_engines.items())}") + + # EKS resources + logger.info(f"\nEKS CLUSTERS:") + logger.info(f"Found: {stats.get('eks_found', 0)}") + if action == 'stop': + logger.info(f"Stopped: {stats.get('eks_stopped', 0)}") + else: + logger.info(f"Started: {stats.get('eks_started', 0)}") + logger.info(f"Skipped: {stats.get('eks_skipped', 0)}") + logger.info(f"Errors: {stats.get('eks_errors', 0)}") + + # EMR resources + logger.info(f"\nEMR CLUSTERS:") + logger.info(f"Found: {stats.get('emr_found', 0)}") + if action == 'stop': + logger.info(f"Stopped: {stats.get('emr_stopped', 0)}") + else: + logger.info(f"Started: {stats.get('emr_started', 0)}") + logger.info(f"Skipped: {stats.get('emr_skipped', 0)}") + logger.info(f"Errors: {stats.get('emr_errors', 0)}") + + # Error summary if there were any errors + if stats.get('errors', []): + logger.info(f"\nERROR SUMMARY:") + logger.info(f"Total errors: {len(stats.get('errors', []))}") + for i, error in enumerate(stats.get('errors', [])[:5], 1): # Show first 5 errors + logger.info(f" {i}. {error}") + if len(stats.get('errors', [])) > 5: + logger.info(f" ... and {len(stats.get('errors', [])) - 5} more errors (see log for details)") + + logger.info(f"{'=' * 68}") + + def _process_single_account( + self, + account: Dict[str, str], + credentials: Dict[str, str], + regions: List[str], + action: str, + dry_run: bool, + exclude_resources: List[str], + stats: Dict[str, Any] + ) -> None: + """Process a single account for the specified action.""" + # Fix keys - use consistent naming with process_accounts + account_id = account.get('account_id') + account_name = account.get('account_name', account_id) + + try: + # Auto-detect partition if not specified + if not self.partition: + self.partition = detect_partition_from_credentials(credentials) + logger.info(f"Auto-detected AWS partition: {self.partition}") + + # Ensure we're using regions from the right partition + if regions and regions[0] != "auto": + # Filter regions to match the detected partition + filtered_regions = [] + for region in regions: + if self.partition == 'aws-us-gov' and not region.startswith('us-gov-'): + logger.debug(f"Skipping region {region} - not in {self.partition} partition") + continue + elif self.partition == 'aws-cn' and not region.startswith('cn-'): + logger.debug(f"Skipping region {region} - not in {self.partition} partition") + continue + elif self.partition == 'aws' and (region.startswith('us-gov-') or region.startswith('cn-')): + logger.debug(f"Skipping region {region} - not in {self.partition} partition") + continue + filtered_regions.append(region) + + # If filtering removed all regions, use defaults for the partition + if not filtered_regions: + if self.partition == 'aws-us-gov': + filtered_regions = ['us-gov-east-1', 'us-gov-west-1'] + elif self.partition == 'aws-cn': + filtered_regions = ['cn-north-1', 'cn-northwest-1'] + else: + filtered_regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'] + logger.info(f"Using default regions for partition {self.partition}: {', '.join(filtered_regions)}") + + regions = filtered_regions + elif not regions: + # If no regions provided, use defaults for the detected partition + if self.partition == 'aws-us-gov': + regions = ['us-gov-east-1', 'us-gov-west-1'] + elif self.partition == 'aws-cn': + regions = ['cn-north-1', 'cn-northwest-1'] + else: + regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'] + logger.info(f"No regions specified, using defaults for partition {self.partition}: {', '.join(regions)}") + + # Only log this once - removing duplicate message + logger.info(f"Processing account: {account_name} ({account_id}) in {len(regions)} regions") + + # Define special handling for EMR clusters - remove 'STOPPED' from valid states as it's not accepted by the API + emr_states = None + if "emr" not in exclude_resources: + # Only include valid EMR states for the ListClusters API call + # Note: 'STOPPED' is not a valid state for ListClusters API + emr_states = ['STARTING', 'BOOTSTRAPPING', 'RUNNING', 'WAITING', 'TERMINATING'] + logger.debug(f"Using valid EMR states for discovery: {', '.join(emr_states)}") + + # Get resources with special handling for EMR + resources = get_account_resources( + credentials, + regions, + exclude_resources, + account_id, + account_name, + emr_cluster_states=emr_states + ) + if not resources: + logger.error(f"Failed to get resources for account {account_id}") + return + + # Initialize stat counters if they don't exist + for stat_type in ['ec2_found', 'ec2_skipped', 'ec2_stopped', 'ec2_started', 'ec2_errors', + 'rds_found', 'rds_skipped', 'rds_stopped', 'rds_started', 'rds_errors', + 'eks_found', 'eks_skipped', 'eks_stopped', 'eks_started', 'eks_errors', + 'emr_found', 'emr_skipped', 'emr_stopped', 'emr_started', 'emr_errors']: + if stat_type not in stats: + stats[stat_type] = 0 + + # Initialize RDS engines stats if it doesn't exist + if 'rds_engines' not in stats: + stats['rds_engines'] = {} + + # Initialize errors array if it doesn't exist + if 'errors' not in stats: + stats['errors'] = [] + + # Normalize state and status fields for all resource types + # For EC2 instances + ec2_instances = resources.get('ec2_instances', []) + for instance in ec2_instances: + if 'state' not in instance and 'status' in instance: + instance['state'] = instance['status'] + elif 'status' not in instance and 'state' in instance: + instance['status'] = instance['state'] + elif 'state' not in instance and 'status' not in instance: + instance['state'] = 'unknown' + instance['status'] = 'unknown' + + # For RDS instances + rds_instances = resources.get('rds_instances', []) + for instance in rds_instances: + if 'state' not in instance and 'status' in instance: + instance['state'] = instance['status'] + elif 'status' not in instance and 'state' in instance: + instance['status'] = instance['state'] + elif 'state' not in instance and 'status' not in instance: + instance['state'] = 'unknown' + instance['status'] = 'unknown' + + # For EKS clusters + eks_clusters = resources.get('eks_clusters', []) + for cluster in eks_clusters: + if 'state' not in cluster and 'status' in cluster: + cluster['state'] = cluster['status'] + elif 'status' not in cluster and 'state' in cluster: + cluster['status'] = cluster['state'] + elif 'state' not in cluster and 'status' not in cluster: + cluster['state'] = 'unknown' + cluster['status'] = 'unknown' + + # For EMR clusters + emr_clusters = resources.get('emr_clusters', []) + for cluster in emr_clusters: + if 'state' not in cluster: + cluster['state'] = cluster.get('status', 'UNKNOWN') + if 'status' not in cluster: + cluster['status'] = cluster.get('state', 'UNKNOWN') + + # Count service-managed EC2 instances + eks_tag = self.config.get('eks_tag', 'kubernetes.io/cluster/') + emr_tag = self.config.get('emr_tag', 'aws:elasticmapreduce:job-flow-id') + exclusion_tag = self.config.get('exclusion_tag', 'DoNotStop') + + # Continue with existing code + eks_managed = sum(1 for i in resources.get('ec2_instances', []) + if any(tag.startswith(eks_tag) for tag in i.get('tags', {}))) + emr_managed = sum(1 for i in resources.get('ec2_instances', []) + if emr_tag in i.get('tags', {})) + + # Update resource counts + total_ec2 = len(resources.get('ec2_instances', [])) + excluded_ec2 = sum(1 for i in resources.get('ec2_instances', []) + if exclusion_tag in i.get('tags', {})) + stats['ec2_found'] += total_ec2 + stats['ec2_skipped'] += excluded_ec2 + + stats['rds_found'] += len(resources.get('rds_instances', [])) + stats['eks_found'] += len(resources.get('eks_clusters', [])) + stats['emr_found'] += len(resources.get('emr_clusters', [])) + + # Track RDS engines + for instance in resources.get('rds_instances', []): + if instance and 'engine' in instance: + engine = instance['engine'] + if engine in stats['rds_engines']: + stats['rds_engines'][engine] += 1 + else: + stats['rds_engines'][engine] = 1 + + # Log discovered resources with service-managed counts + logger.info(f"Account {account_id} - Found {total_ec2} EC2 instances ({excluded_ec2} explicitly excluded, {eks_managed} EKS-managed, {emr_managed} EMR-managed)") + logger.info(f"Account {account_id} - Found {len(resources.get('rds_instances', []))} RDS instances") + logger.info(f"Account {account_id} - Found {len(resources.get('eks_clusters', []))} EKS clusters") + logger.info(f"Account {account_id} - Found {len(resources.get('emr_clusters', []))} EMR clusters") + + # Initialize resource managers with account name + ec2_manager = EC2Manager(credentials, account_id, account_name) + rds_manager = RDSManager(credentials, account_id, account_name) + eks_manager = EKSManager(credentials, account_id, account_name) + emr_manager = EMRManager(credentials, account_id, account_name) + + # Helper function to safely process manager results + def safe_get_result(result: Optional[Dict[str, int]], key: str) -> int: + if result is None: + return 0 + return result.get(key, 0) + + # Perform actions based on selected mode + if action == "stop": + if "ec2" not in exclude_resources: + result = ec2_manager.stop(resources.get('ec2_instances', []), dry_run) + stats['ec2_stopped'] += safe_get_result(result, 'success') + stats['ec2_errors'] += safe_get_result(result, 'errors') + else: + stats['ec2_skipped'] += total_ec2 - excluded_ec2 + + if "rds" not in exclude_resources: + result = rds_manager.stop(resources.get('rds_instances', []), dry_run) + stats['rds_stopped'] += safe_get_result(result, 'success') + stats['rds_errors'] += safe_get_result(result, 'errors') + else: + stats['rds_skipped'] += len(resources.get('rds_instances', [])) + + if "eks" not in exclude_resources: + result = eks_manager.stop(resources.get('eks_clusters', []), dry_run) + stats['eks_stopped'] += safe_get_result(result, 'success') + stats['eks_errors'] += safe_get_result(result, 'errors') + else: + stats['eks_skipped'] += len(resources.get('eks_clusters', [])) + + if "emr" not in exclude_resources: + result = emr_manager.stop(resources.get('emr_clusters', []), dry_run) + stats['emr_stopped'] += safe_get_result(result, 'success') + stats['emr_errors'] += safe_get_result(result, 'errors') + else: + stats['emr_skipped'] += len(resources.get('emr_clusters', [])) + else: # action == "start" + if "ec2" not in exclude_resources: + result = ec2_manager.start(resources.get('ec2_instances', []), dry_run) + stats['ec2_started'] += safe_get_result(result, 'success') + stats['ec2_errors'] += safe_get_result(result, 'errors') + else: + stats['ec2_skipped'] += total_ec2 - excluded_ec2 + + if "rds" not in exclude_resources: + result = rds_manager.start(resources.get('rds_instances', []), dry_run) + stats['rds_started'] += safe_get_result(result, 'success') + stats['rds_errors'] += safe_get_result(result, 'errors') + else: + stats['rds_skipped'] += len(resources.get('rds_instances', [])) + + if "eks" not in exclude_resources: + result = eks_manager.start(resources.get('eks_clusters', []), dry_run) + stats['eks_started'] += safe_get_result(result, 'success') + stats['eks_errors'] += safe_get_result(result, 'errors') + else: + stats['eks_skipped'] += len(resources.get('eks_clusters', [])) + + if "emr" not in exclude_resources: + result = emr_manager.start(resources.get('emr_clusters', []), dry_run) + stats['emr_started'] += safe_get_result(result, 'success') + stats['emr_errors'] += safe_get_result(result, 'errors') + else: + stats['emr_skipped'] += len(resources.get('emr_clusters', [])) + except Exception as e: + logger.error(f"Error processing account {account_id}: {str(e)}") + if 'errors' not in stats: + stats['errors'] = [] + stats['errors'].append(f"Error processing account {account_id}: {str(e)}") + return diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py new file mode 100644 index 00000000..f81b1f42 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py @@ -0,0 +1,746 @@ +""" +Resource discovery module for finding AWS resources. +""" +from typing import Dict, List, Any, Optional, Union +import boto3 +import concurrent.futures +from collections import defaultdict +from botocore.exceptions import ClientError, EndpointConnectionError +from concurrent.futures import ThreadPoolExecutor, as_completed + +from aws_resource_management.config_manager import get_config +from aws_resource_management.logging_setup import setup_logging +from aws_resource_management.aws_utils import get_all_regions, is_valid_region, detect_partition_from_credentials, get_regions_for_partition + +logger = setup_logging() +config = get_config() + +# Default regions for different partitions +DEFAULT_REGIONS = { + 'aws': ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'], + 'aws-us-gov': ['us-gov-east-1', 'us-gov-west-1'], + 'aws-cn': ['cn-north-1', 'cn-northwest-1'] +} + +class ResourceDiscovery: + """Resource discovery for AWS resources.""" + + def __init__(self, credentials: Dict[str, str], account_id: str): + """ + Initialize resource discovery. + + Args: + credentials: AWS credentials dictionary + account_id: AWS account ID + """ + self.credentials = credentials + self.account_id = account_id + self.partition = detect_partition_from_credentials(credentials) + logger.info(f"Resource discovery initialized for account {account_id} in partition {self.partition}") + + def discover_resources( + self, + resource_type: str, + regions: Union[str, List[str]] = "all", + resource_ids: Optional[List[str]] = None + ) -> List[Dict[str, Any]]: + """ + Discover resources of the specified type. + + Args: + resource_type: Resource type (ec2_instance, rds_instance, etc.) + regions: Regions to discover resources in, "all" for all regions + resource_ids: List of resource IDs to filter by + + Returns: + List of resource dictionaries + """ + # Convert regions input to a list of actual region names + if regions == "all": + regions = get_regions_for_partition(self.partition) + elif isinstance(regions, str): + regions = [regions] + + # Filter regions to only include those in the detected partition + valid_regions = [] + for region in regions: + # Skip regions that don't belong to our partition + if self.partition == 'aws-us-gov' and not region.startswith('us-gov-'): + logger.debug(f"Skipping region {region} - not in {self.partition} partition") + continue + elif self.partition == 'aws-cn' and not region.startswith('cn-'): + logger.debug(f"Skipping region {region} - not in {self.partition} partition") + continue + elif self.partition == 'aws' and (region.startswith('us-gov-') or region.startswith('cn-')): + logger.debug(f"Skipping region {region} - not in {self.partition} partition") + continue + + # Add the valid region to our list + valid_regions.append(region) + + # If we filtered out all regions, use default regions for the partition + if not valid_regions: + valid_regions = DEFAULT_REGIONS.get(self.partition, DEFAULT_REGIONS['aws']) + logger.info(f"No valid regions found for partition {self.partition}, using defaults: {', '.join(valid_regions)}") + + # Use the filtered valid regions + regions = valid_regions + + # Determine which regions to search + region_list = [] + if regions == "all": + region_list = get_all_regions() + elif isinstance(regions, list): + region_list = [r for r in regions if is_valid_region(r)] + else: + logger.error(f"Invalid regions parameter: {regions}") + return [] + + if not region_list: + logger.warning("No valid regions specified for resource discovery") + return [] + + # Call the appropriate discovery method based on resource type + discovery_method = getattr(self, f"_discover_{resource_type}", None) + if not discovery_method: + logger.error(f"No discovery method available for resource type: {resource_type}") + return [] + + # Use thread pool to speed up discovery across regions + resources = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=min(10, len(region_list))) as executor: + # Create a future for each region + future_to_region = { + executor.submit(discovery_method, region, resource_ids): region + for region in region_list + } + + # Process results as they complete + for future in concurrent.futures.as_completed(future_to_region): + region = future_to_region[future] + try: + region_resources = future.result() + resources.extend(region_resources) + logger.debug(f"Discovered {len(region_resources)} {resource_type} resources in {region}") + except Exception as e: + logger.error(f"Error discovering {resource_type} in {region}: {e}") + + return resources + + def _discover_ec2(self, region: str, resource_ids: Optional[List[str]] = None) -> List[Dict[str, Any]]: + """ + Discover EC2 instances in a region. + + Args: + region: AWS region + resource_ids: Specific instance IDs to filter by + + Returns: + List of EC2 instance dictionaries + """ + # Create EC2 client + ec2_client = boto3.client( + 'ec2', + region_name=region, + aws_access_key_id=self.credentials.get('aws_access_key_id'), + aws_secret_access_key=self.credentials.get('aws_secret_access_key'), + aws_session_token=self.credentials.get('aws_session_token') + ) + + # Build filters + filters = [] + if resource_ids: + filters.append({ + 'Name': 'instance-id', + 'Values': resource_ids + }) + + try: + # Get instances + response = ec2_client.describe_instances(Filters=filters) + + # Extract instance information + instances = [] + for reservation in response.get('Reservations', []): + for instance in reservation.get('Instances', []): + # Convert tags to dictionary + tags = {} + for tag in instance.get('Tags', []): + tags[tag['Key']] = tag['Value'] + + # Create a simplified instance dictionary + instance_dict = { + 'id': instance['InstanceId'], + 'name': tags.get('Name', 'Unnamed'), + 'type': instance['InstanceType'], + 'status': instance['State']['Name'], + 'region': region, + 'tags': tags, + 'launch_time': instance.get('LaunchTime', '').isoformat() if instance.get('LaunchTime') else None, + 'public_ip': instance.get('PublicIpAddress'), + 'private_ip': instance.get('PrivateIpAddress') + } + instances.append(instance_dict) + + return instances + + except Exception as e: + logger.error(f"Error discovering EC2 instances in {region}: {e}") + return [] + + def _discover_rds(self, region: str, resource_ids: Optional[List[str]] = None) -> List[Dict[str, Any]]: + """ + Discover RDS instances in a region. + + Args: + region: AWS region + resource_ids: Specific DB instance IDs to filter by + + Returns: + List of RDS instance dictionaries + """ + # Create RDS client + rds_client = boto3.client( + 'rds', + region_name=region, + aws_access_key_id=self.credentials.get('aws_access_key_id'), + aws_secret_access_key=self.credentials.get('aws_secret_access_key'), + aws_session_token=self.credentials.get('aws_session_token') + ) + + try: + # Get instances + db_instances = [] + + # Use filters if specific IDs are provided + if resource_ids: + for db_id in resource_ids: + try: + response = rds_client.describe_db_instances(DBInstanceIdentifier=db_id) + db_instances.extend(response.get('DBInstances', [])) + except Exception: + continue + else: + # Get all instances + response = rds_client.describe_db_instances() + db_instances = response.get('DBInstances', []) + + # Handle pagination + while 'Marker' in response: + response = rds_client.describe_db_instances(Marker=response['Marker']) + db_instances.extend(response.get('DBInstances', [])) + + # Process instances + instances = [] + for db in db_instances: + # Get tags + try: + tags_response = rds_client.list_tags_for_resource( + ResourceName=db['DBInstanceArn'] + ) + tags = {item['Key']: item['Value'] for item in tags_response.get('TagList', [])} + except Exception: + tags = {} + + # Create a simplified instance dictionary + instance_dict = { + 'id': db['DBInstanceIdentifier'], + 'name': db['DBInstanceIdentifier'], + 'type': db['DBInstanceClass'], + 'status': db['DBInstanceStatus'], + 'region': region, + 'tags': tags, + 'engine': db['Engine'], + 'endpoint': db.get('Endpoint', {}).get('Address'), + 'multi_az': db.get('MultiAZ', False) + } + instances.append(instance_dict) + + return instances + + except Exception as e: + logger.error(f"Error discovering RDS instances in {region}: {e}") + return [] + + def _discover_emr(self, region: str, resource_ids: Optional[List[str]] = None) -> List[Dict[str, Any]]: + """ + Discover EMR clusters in a region. + + Args: + region: AWS region + resource_ids: Specific cluster IDs to filter by + + Returns: + List of EMR cluster dictionaries + """ + # Create EMR client + emr_client = boto3.client( + 'emr', + region_name=region, + aws_access_key_id=self.credentials.get('aws_access_key_id'), + aws_secret_access_key=self.credentials.get('aws_secret_access_key'), + aws_session_token=self.credentials.get('aws_session_token') + ) + + try: + # Build filter for cluster states + # Include all states: STARTING, BOOTSTRAPPING, RUNNING, WAITING, TERMINATING, TERMINATED, TERMINATED_WITH_ERRORS + cluster_states = ['STARTING', 'BOOTSTRAPPING', 'RUNNING', 'WAITING', 'TERMINATING', 'TERMINATED', 'TERMINATED_WITH_ERRORS'] + + # Get clusters + response = emr_client.list_clusters(ClusterStates=cluster_states) + clusters = response.get('Clusters', []) + + # Handle pagination + while 'Marker' in response: + response = emr_client.list_clusters(Marker=response['Marker'], ClusterStates=cluster_states) + clusters.extend(response.get('Clusters', [])) + + # Filter by IDs if specified + if resource_ids: + clusters = [c for c in clusters if c['Id'] in resource_ids] + + # Get detailed information for each cluster + detailed_clusters = [] + for cluster in clusters: + try: + # Get cluster details including tags + cluster_details = emr_client.describe_cluster(ClusterId=cluster['Id']) + cluster_info = cluster_details.get('Cluster', {}) + + # Convert tags to dictionary + tags = {} + for tag in cluster_info.get('Tags', []): + tags[tag['Key']] = tag['Value'] + + # Ensure we have a state value + state = 'UNKNOWN' + if 'Status' in cluster and isinstance(cluster['Status'], dict) and 'State' in cluster['Status']: + state = cluster['Status']['State'] + + # Create a simplified cluster dictionary + cluster_dict = { + 'id': cluster['Id'], + 'name': cluster.get('Name', 'Unnamed'), + 'status': state, + 'state': state, # Add both fields to ensure compatibility + 'region': region, + 'tags': tags, + 'creation_time': cluster.get('Status', {}).get('Timeline', {}).get('CreationDateTime', '').isoformat() + if cluster.get('Status', {}).get('Timeline', {}).get('CreationDateTime') else None, + 'instance_count': cluster_info.get('InstanceCollectionType', 'Unknown'), + 'applications': [app.get('Name') for app in cluster_info.get('Applications', [])] + } + detailed_clusters.append(cluster_dict) + except Exception as e: + logger.warning(f"Error getting details for EMR cluster {cluster['Id']}: {e}") + + return detailed_clusters + + except Exception as e: + logger.error(f"Error discovering EMR clusters in {region}: {e}") + return [] + + def _discover_eks(self, region: str, resource_ids: Optional[List[str]] = None) -> List[Dict[str, Any]]: + """ + Discover EKS clusters in a region. + + Args: + region: AWS region + resource_ids: Specific cluster names to filter by + + Returns: + List of EKS cluster dictionaries + """ + # Create EKS client + eks_client = boto3.client( + 'eks', + region_name=region, + aws_access_key_id=self.credentials.get('aws_access_key_id'), + aws_secret_access_key=self.credentials.get('aws_secret_access_key'), + aws_session_token=self.credentials.get('aws_session_token') + ) + + try: + # Get clusters + response = eks_client.list_clusters() + cluster_names = response.get('clusters', []) + + # Handle pagination + while 'nextToken' in response: + response = eks_client.list_clusters(nextToken=response['nextToken']) + cluster_names.extend(response.get('clusters', [])) + + # Filter by names if specified + if resource_ids: + cluster_names = [name for name in cluster_names if name in resource_ids] + + # Get detailed information for each cluster + clusters = [] + for name in cluster_names: + try: + # Get cluster details + cluster = eks_client.describe_cluster(name=name)['cluster'] + + # Get tags + tags = cluster.get('tags', {}) + + # Create ARN manually if not present + arn = cluster.get('arn', f"arn:aws:eks:{region}:{self.account_id}:cluster/{name}") + + # Create a simplified cluster dictionary + cluster_dict = { + 'id': name, + 'name': name, + 'arn': arn, + 'status': cluster.get('status', 'UNKNOWN'), + 'region': region, + 'tags': tags, + 'version': cluster.get('version'), + 'endpoint': cluster.get('endpoint'), + 'created_at': cluster.get('createdAt', '').isoformat() if cluster.get('createdAt') else None, + } + clusters.append(cluster_dict) + except Exception as e: + logger.warning(f"Error getting details for EKS cluster {name}: {e}") + + return clusters + + except Exception as e: + logger.error(f"Error discovering EKS clusters in {region}: {e}") + return [] + +def get_account_resources( + credentials: Dict[str, str], + regions: List[str], + exclude_resources: List[str], + account_id: str, + account_name: str, + emr_cluster_states: Optional[List[str]] = None +) -> Dict[str, List[Dict[str, Any]]]: + """ + Discover resources in an AWS account across multiple regions. + + Args: + credentials: AWS credentials for the account + regions: List of regions to scan + exclude_resources: List of resource types to exclude + account_id: AWS account ID + account_name: AWS account name + emr_cluster_states: Optional list of EMR cluster states to filter by + + Returns: + Dictionary of discovered resources by type + """ + # Use default regions if empty regions list + if not regions: + # First detect partition to get appropriate regions + partition = detect_partition_from_credentials(credentials) + # Get regions for the detected partition + regions = get_regions_for_partition(partition) + logger.info(f"Using default regions for partition {partition} in account {account_id}") + + logger.info(f"Discovering resources in account {account_id} ({account_name}) across {len(regions)} regions") + + resources = { + 'ec2_instances': [], + 'rds_instances': [], + 'eks_clusters': [], + 'emr_clusters': [] + } + + # Create a ResourceDiscovery instance for this account + discovery = ResourceDiscovery(credentials, account_id) + + # Use parallel processing for regions to speed up discovery + with ThreadPoolExecutor(max_workers=min(10, len(regions))) as executor: + # Submit tasks for each region and resource type + futures = {} + + for region in regions: + logger.debug(f"Scanning region {region} in account {account_id}") + + # Get EC2 instances if not excluded + if "ec2" not in exclude_resources: + future = executor.submit( + discovery._discover_ec2, + region + ) + futures[(region, 'ec2')] = future + + # Get RDS instances if not excluded + if "rds" not in exclude_resources: + future = executor.submit( + discovery._discover_rds, + region + ) + futures[(region, 'rds')] = future + + # Get EKS clusters if not excluded + if "eks" not in exclude_resources: + future = executor.submit( + discovery._discover_eks, + region + ) + futures[(region, 'eks')] = future + + # Get EMR clusters if not excluded + if "emr" not in exclude_resources: + future = executor.submit( + discovery._discover_emr, + region, + emr_cluster_states + ) + futures[(region, 'emr')] = future + + # Collect results as they complete + for (region, resource_type), future in futures.items(): + try: + result = future.result() + if resource_type == 'ec2': + resources['ec2_instances'].extend(result) + elif resource_type == 'rds': + resources['rds_instances'].extend(result) + elif resource_type == 'eks': + resources['eks_clusters'].extend(result) + elif resource_type == 'emr': + resources['emr_clusters'].extend(result) + logger.debug(f"Found {len(result)} {resource_type} resources in {region}") + except Exception as e: + logger.error(f"Error discovering {resource_type} in {region}: {str(e)}") + + # Log summary + logger.info(f"Account {account_id} - Found {len(resources['ec2_instances'])} EC2 instances") + logger.info(f"Account {account_id} - Found {len(resources['rds_instances'])} RDS instances") + logger.info(f"Account {account_id} - Found {len(resources['eks_clusters'])} EKS clusters") + logger.info(f"Account {account_id} - Found {len(resources['emr_clusters'])} EMR clusters") + + return resources + +def discover_ec2_instances(credentials: Dict[str, str], region: str, account_id: str) -> List[Dict[str, Any]]: + """ + Discover EC2 instances in a region. + + Args: + credentials: AWS credentials + region: AWS region + account_id: AWS account ID + + Returns: + List of EC2 instance dictionaries + """ + ec2_client = boto3.client('ec2', region_name=region, **credentials) + instances = [] + + try: + paginator = ec2_client.get_paginator('describe_instances') + + for page in paginator.paginate(): + for reservation in page['Reservations']: + for instance in reservation['Instances']: + # Skip terminated instances + if instance['State']['Name'] == 'terminated': + continue + + # Extract tags as a dictionary + tags = {} + if 'Tags' in instance: + tags = {tag['Key']: tag['Value'] for tag in instance['Tags']} + + # Create a simplified instance object + instance_obj = { + 'id': instance['InstanceId'], + 'type': instance['InstanceType'], + 'state': instance['State']['Name'], + 'region': region, + 'account_id': account_id, + 'tags': tags, + 'name': tags.get('Name', instance['InstanceId']) + } + + instances.append(instance_obj) + + return instances + except ClientError as e: + logger.error(f"Error discovering EC2 instances in {region}: {str(e)}") + return [] + +def discover_rds_instances(credentials: Dict[str, str], region: str, account_id: str) -> List[Dict[str, Any]]: + """ + Discover RDS instances in a region. + + Args: + credentials: AWS credentials + region: AWS region + account_id: AWS account ID + + Returns: + List of RDS instance dictionaries + """ + rds_client = boto3.client('rds', region_name=region, **credentials) + instances = [] + + try: + paginator = rds_client.get_paginator('describe_db_instances') + + for page in paginator.paginate(): + for instance in page['DBInstances']: + # Create a simplified instance object + instance_obj = { + 'id': instance['DBInstanceIdentifier'], + 'engine': instance['Engine'], + 'state': instance['DBInstanceStatus'], + 'region': region, + 'account_id': account_id, + 'name': instance['DBInstanceIdentifier'] + } + + instances.append(instance_obj) + + return instances + except ClientError as e: + logger.error(f"Error discovering RDS instances in {region}: {str(e)}") + return [] + +def discover_eks_clusters(credentials: Dict[str, str], region: str, account_id: str) -> List[Dict[str, Any]]: + """ + Discover EKS clusters in a region. + + Args: + credentials: AWS credentials + region: AWS region + account_id: AWS account ID + + Returns: + List of EKS cluster dictionaries + """ + eks_client = boto3.client('eks', region_name=region, **credentials) + clusters = [] + + try: + response = eks_client.list_clusters() + + for cluster_name in response['clusters']: + cluster_details = eks_client.describe_cluster(name=cluster_name)['cluster'] + + # Create a simplified cluster object + cluster_obj = { + 'id': cluster_name, + 'name': cluster_name, + 'status': cluster_details['status'], + 'region': region, + 'account_id': account_id, + 'version': cluster_details['version'] + } + + clusters.append(cluster_obj) + + # Handle pagination if there are more clusters + while 'nextToken' in response: + response = eks_client.list_clusters(nextToken=response['nextToken']) + for cluster_name in response['clusters']: + cluster_details = eks_client.describe_cluster(name=cluster_name)['cluster'] + + # Create a simplified cluster object + cluster_obj = { + 'id': cluster_name, + 'name': cluster_name, + 'status': cluster_details['status'], + 'region': region, + 'account_id': account_id, + 'version': cluster_details['version'] + } + + clusters.append(cluster_obj) + + return clusters + except ClientError as e: + logger.error(f"Error discovering EKS clusters in {region}: {str(e)}") + return [] + +def discover_emr_clusters(credentials: Dict[str, str], region: str, account_id: str, + cluster_states: Optional[List[str]] = None) -> List[Dict[str, Any]]: + """ + Discover EMR clusters in a region. + + Args: + credentials: AWS credentials + region: AWS region + account_id: AWS account ID + cluster_states: Optional list of EMR cluster states to filter by + + Returns: + List of EMR cluster dictionaries + """ + emr_client = boto3.client('emr', region_name=region, **credentials) + clusters = [] + + try: + paginator = emr_client.get_paginator('list_clusters') + + # Only list active clusters with valid states + # Default cluster states, 'STOPPED' is not a valid state for ListClusters API + # See: https://docs.aws.amazon.com/emr/latest/APIReference/API_ListClusters.html + if cluster_states is None: + cluster_states = ['STARTING', 'BOOTSTRAPPING', 'RUNNING', 'WAITING', 'TERMINATING'] + + try: + for page in paginator.paginate(ClusterStates=cluster_states): + for cluster in page.get('Clusters', []): + try: + # Safely extract cluster state with fallbacks + if 'Status' in cluster and isinstance(cluster['Status'], dict) and 'State' in cluster['Status']: + state = cluster['Status']['State'] + else: + state = 'UNKNOWN' + + # Create a simplified cluster object with both state and status fields + cluster_obj = { + 'id': cluster.get('Id', 'unknown-id'), + 'name': cluster.get('Name', 'Unknown'), + 'state': state, + 'status': state, # Duplicate to ensure both fields exist + 'region': region, + 'account_id': account_id + } + + clusters.append(cluster_obj) + except Exception as e: + logger.warning(f"Error processing EMR cluster in {region}: {str(e)}") + # Continue with next cluster + continue + + except Exception as e: + logger.warning(f"Error during EMR pagination in {region}: {str(e)}") + # Try using list_clusters without pagination as fallback + try: + response = emr_client.list_clusters(ClusterStates=cluster_states) + for cluster in response.get('Clusters', []): + # Safely extract cluster state with fallbacks + if 'Status' in cluster and isinstance(cluster['Status'], dict) and 'State' in cluster['Status']: + state = cluster['Status']['State'] + else: + state = 'UNKNOWN' + + # Create a simplified cluster object with both state and status fields + cluster_obj = { + 'id': cluster.get('Id', 'unknown-id'), + 'name': cluster.get('Name', 'Unknown'), + 'state': state, + 'status': state, # Duplicate to ensure both fields exist + 'region': region, + 'account_id': account_id + } + + clusters.append(cluster_obj) + except Exception as nested_e: + logger.error(f"Fallback EMR cluster retrieval failed in {region}: {str(nested_e)}") + + return clusters + except ClientError as e: + logger.error(f"Error discovering EMR clusters in {region}: {str(e)}") + return [] + except Exception as e: + # Added more generic exception handling + logger.error(f"Unexpected error discovering EMR clusters in {region}: {str(e)}") + return [] diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/logging_setup.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/logging_setup.py new file mode 100644 index 00000000..1add8ec1 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/logging_setup.py @@ -0,0 +1,233 @@ +""" +Logging setup for AWS Resource Management tool. +""" +import logging +import os +import json +import sys +import csv +import datetime +from pathlib import Path +from typing import Optional, Dict, Any + +from aws_resource_management.config_manager import get_config + +class JSONFormatter(logging.Formatter): + """ + JSON formatter for structured logging. + """ + def format(self, record: logging.LogRecord) -> str: + """ + Format the log record as a JSON string. + + Args: + record: Log record + + Returns: + JSON formatted string + """ + log_data = { + 'timestamp': datetime.fromtimestamp(record.created).isoformat(), + 'level': record.levelname, + 'message': record.getMessage(), + 'module': record.module, + 'function': record.funcName, + 'line': record.lineno, + } + + # Include exception info if present + if record.exc_info: + log_data['exception'] = { + 'type': record.exc_info[0].__name__, + 'message': str(record.exc_info[1]), + } + + # Include any custom fields + for key, value in getattr(record, 'extra_fields', {}).items(): + log_data[key] = value + + return json.dumps(log_data) + +def setup_logging( + log_level: Optional[str] = None, + log_file: Optional[str] = None, + use_json: bool = False +) -> logging.Logger: + """ + Set up logging configuration. + + Args: + log_level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + log_file: Path to log file + use_json: Whether to use JSON formatting + + Returns: + Logger instance + """ + # Get root logger + logger = logging.getLogger('aws_resource_management') + + # Clear existing handlers to avoid duplicate logs + if logger.handlers: + logger.handlers = [] + + # Set log level + if not log_level: + log_level = os.environ.get('AWS_RESOURCE_MGMT_LOG_LEVEL', 'INFO') + + logger.setLevel(getattr(logging, log_level)) + + # Create console handler + console_handler = logging.StreamHandler(sys.stdout) + + # Set formatter based on format preference + if use_json: + formatter = JSONFormatter() + else: + formatter = logging.Formatter( + '%(asctime)s [%(levelname)s] %(name)s - %(message)s' + ) + + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + # Add file handler if log file is specified + if log_file: + file_handler = logging.FileHandler(log_file) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + return logger + +class LoggerAdapter(logging.LoggerAdapter): + """ + Logger adapter for adding context to log messages. + """ + def __init__(self, logger: logging.Logger, context: Dict[str, Any]): + """ + Initialize logger adapter with context. + + Args: + logger: Logger instance + context: Context dictionary + """ + super().__init__(logger, context) + + def process(self, msg: str, kwargs: Dict[str, Any]) -> tuple: + """ + Process the log message by adding context. + + Args: + msg: Log message + kwargs: Keyword arguments + + Returns: + Tuple of (modified message, modified kwargs) + """ + # Add extra_fields for JSON formatter + if 'extra' not in kwargs: + kwargs['extra'] = {} + + if 'extra_fields' not in kwargs['extra']: + kwargs['extra']['extra_fields'] = {} + + for key, value in self.extra.items(): + kwargs['extra']['extra_fields'][key] = value + + return msg, kwargs + +def initialize_csv_log(csv_file: Optional[str] = None) -> str: + """ + Initialize CSV log file with headers if it doesn't exist. + + Args: + csv_file: CSV file name (default: resource_actions.csv from config) + + Returns: + Path to the CSV log file + """ + config = get_config() + + if csv_file is None: + csv_file = config.get('action_csv_file') + + # Configure CSV logging directory + csv_log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'logs') + csv_path = os.path.join(csv_log_dir, csv_file) + + # Create directory if it doesn't exist + Path(csv_log_dir).mkdir(parents=True, exist_ok=True) + + # Check if file exists, if not create with headers + file_exists = os.path.isfile(csv_path) + + if not file_exists: + with open(csv_path, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow([ + 'timestamp', 'account_id', 'account_name', 'resource_type', + 'resource_id', 'resource_name', 'action', 'region', + 'status', 'details', 'dry_run', 'schedule' + ]) + + return csv_path + +def log_action_to_csv( + account_id: str, + account_name: str, + resource_type: str, + resource_id: str, + resource_name: str, + action: str, + region: str, + status: str, + details: str = "", + dry_run: bool = False, + existing_schedule: Optional[str] = None +) -> None: + """ + Log resource action to CSV file for tracking. + + Args: + account_id: AWS account ID + account_name: AWS account name + resource_type: Resource type (ec2, rds, eks, emr) + resource_id: Resource ID + resource_name: Resource name + action: Action performed (stop, start) + region: AWS region + status: Action status + details: Additional details + dry_run: Whether this was a dry run + existing_schedule: Existing schedule for the resource + """ + timestamp = datetime.datetime.now().isoformat() + + # Initialize CSV log file + csv_path = initialize_csv_log() + + with open(csv_path, 'a', newline='') as csvfile: + fieldnames = [ + 'timestamp', 'account_id', 'account_name', 'resource_type', 'resource_id', + 'resource_name', 'action', 'region', 'status', 'details', 'dry_run', 'schedule' + ] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + # Write the log entry + writer.writerow({ + 'timestamp': timestamp, + 'account_id': account_id, + 'account_name': account_name, + 'resource_type': resource_type, + 'resource_id': resource_id, + 'resource_name': resource_name, + 'action': action, + 'region': region, + 'status': status, + 'details': details, + 'dry_run': 'Yes' if dry_run else 'No', + 'schedule': existing_schedule or '' + }) + +# Create a logger instance for direct import +logger = setup_logging() diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/__init__.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/__init__.py new file mode 100644 index 00000000..d64cb725 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/__init__.py @@ -0,0 +1,8 @@ +""" +Resource manager modules for different AWS services. +""" +from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.managers.ec2 import EC2Manager +from aws_resource_management.managers.rds import RDSManager +from aws_resource_management.managers.eks import EKSManager +from aws_resource_management.managers.emr import EMRManager diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py new file mode 100644 index 00000000..71bcfac4 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py @@ -0,0 +1,155 @@ +""" +Base resource manager class that all specific managers inherit from. +""" +from typing import Dict, List, Any, Optional, Union +import boto3 +import datetime +from abc import ABC, abstractmethod + +from aws_resource_management.config_manager import get_config +from aws_resource_management.logging_setup import setup_logging + +logger = setup_logging() +config = get_config() + +class ResourceManager(ABC): + """ + Base class for all resource managers. + + This abstract class defines the common interface and functionality + that all resource managers should implement. + """ + + def __init__(self, credentials: Dict[str, str], account_id: str, account_name: Optional[str] = None): + """ + Initialize the resource manager. + + Args: + credentials: AWS credentials dictionary + account_id: AWS account ID + account_name: AWS account name (optional) + """ + self.credentials = credentials + self.account_id = account_id + self.account_name = account_name + self.resource_type = "generic" # Override in subclass + + def create_client(self, service_name: str, region_name: str) -> Any: + """ + Create a boto3 client for the specified AWS service. + + Args: + service_name: AWS service name + region_name: AWS region name + + Returns: + boto3 client + """ + return boto3.client( + service_name, + region_name=region_name, + aws_access_key_id=self.credentials.get('aws_access_key_id'), + aws_secret_access_key=self.credentials.get('aws_secret_access_key'), + aws_session_token=self.credentials.get('aws_session_token') + ) + + def get_timestamp(self) -> str: + """ + Get current timestamp in ISO format. + + Returns: + Timestamp string + """ + return datetime.datetime.now().isoformat() + + def should_exclude(self, resource: Dict[str, Any]) -> bool: + """ + Check if a resource should be excluded from operations based on tags. + + Args: + resource: Resource dictionary with a 'tags' key + + Returns: + True if resource should be excluded, False otherwise + """ + tags = resource.get('tags', {}) + exclusion_tag = config.get('exclusion_tag') + + # Check if the exclusion tag exists + if exclusion_tag in tags: + return True + + return False + + def log_action( + self, + resource_id: str, + region: str, + action: str, + details: str = "", + resource_name: Optional[str] = None, + dry_run: bool = False, + existing_schedule: Optional[str] = None + ) -> None: + """ + Log an action performed on a resource. + + Args: + resource_id: Resource ID + region: AWS region + action: Action name (e.g., start, stop) + details: Additional details about the action + resource_name: Resource name (optional) + dry_run: Whether this was a dry run + existing_schedule: Existing schedule if any (optional) + """ + resource_info = f"'{resource_name}' ({resource_id})" if resource_name else resource_id + dry_run_prefix = "[DRY RUN] " if dry_run else "" + schedule_info = f", Schedule: {existing_schedule}" if existing_schedule else "" + + logger.info(f"{dry_run_prefix}{action.upper()} {self.resource_type} {resource_info} in {region}{schedule_info} {details}") + + def get_partition_for_region(self, region: str) -> str: + """ + Get the AWS partition for a given region. + + Args: + region: AWS region + + Returns: + AWS partition (e.g., aws, aws-us-gov) + """ + if region.startswith("us-gov"): + return "aws-us-gov" + elif region.startswith("cn"): + return "aws-cn" + else: + return "aws" + + @abstractmethod + def stop(self, resources: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Stop resources. + + Args: + resources: List of resource dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + pass + + @abstractmethod + def start(self, resources: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Start resources. + + Args: + resources: List of resource dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + pass diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ec2.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ec2.py new file mode 100644 index 00000000..25656e41 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ec2.py @@ -0,0 +1,166 @@ +""" +EC2 resource manager class. +""" +from typing import Dict, List, Any, Optional + +from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.config_manager import get_config +from aws_resource_management.logging_setup import setup_logging + +logger = setup_logging() +config = get_config() + +class EC2Manager(ResourceManager): + """Manager for EC2 instances.""" + + def __init__(self, credentials: Dict[str, str], account_id: str, account_name: Optional[str] = None): + """ + Initialize EC2 manager. + + Args: + credentials: AWS credentials dictionary + account_id: AWS account ID + account_name: AWS account name (optional) + """ + super().__init__(credentials, account_id, account_name) + self.resource_type = "ec2_instance" + + def should_exclude(self, instance: Dict[str, Any]) -> bool: + """ + Check if EC2 instance should be excluded based on tags. + + Args: + instance: Instance dictionary with tags + + Returns: + True if the instance should be excluded, False otherwise + """ + tags = instance.get("tags", {}) + + # Check for explicit exclusion tag + exclusion_tag = config.get("exclusion_tag") + if exclusion_tag in tags: + logger.info(f"Skipping EC2 instance {instance['id']} with exclusion tag {exclusion_tag}") + return True + + # Skip instances that are part of EKS clusters + eks_tag = config.get("eks_tag") + if eks_tag in tags: + logger.info(f"Skipping EC2 instance {instance['id']} with tag {eks_tag}={tags[eks_tag]} (EKS managed node)") + return True + + # Skip instances that are part of EMR clusters + emr_tag = config.get("emr_tag") + if emr_tag in tags: + logger.info(f"Skipping EC2 instance {instance['id']} with tag {emr_tag}={tags[emr_tag]} (EMR managed node)") + return True + + return False + + def stop(self, instances: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Stop running EC2 instances. + + Args: + instances: List of EC2 instance dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + success_count = 0 + error_count = 0 + + for instance in instances: + if self.should_exclude(instance): + continue + + if instance["state"] == "running": + try: + region = instance["region"] + ec2_client = self.create_client('ec2', region) + + # Create ISO 8601 timestamp + timestamp = self.get_timestamp() + + # Tag the instance before stopping + logger.info(f"{'[DRY RUN] Would tag' if dry_run else 'Tagging'} EC2 instance {instance['id']} with {config.get('stop_tag')}={timestamp}") + if not dry_run: + ec2_client.create_tags( + Resources=[instance['id']], + Tags=[ + { + 'Key': config.get('stop_tag'), + 'Value': timestamp + } + ] + ) + + # Stop the instance + logger.info(f"{'[DRY RUN] Would stop' if dry_run else 'Stopping'} EC2 instance {instance['id']} in region {region}") + if not dry_run: + ec2_client.stop_instances(InstanceIds=[instance['id']]) + + # Log with detailed information + self.log_action(instance['id'], region, "stop", dry_run=dry_run) + success_count += 1 + + except Exception as e: + logger.error(f"Failed to {'simulate stopping' if dry_run else 'stop'} EC2 instance {instance['id']}: {e}") + error_count += 1 + + return { + "success": success_count, + "errors": error_count + } + + def start(self, instances: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Start stopped EC2 instances. + + Args: + instances: List of EC2 instance dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + success_count = 0 + error_count = 0 + + for instance in instances: + if self.should_exclude(instance): + continue + + if instance["state"] == "stopped": + try: + region = instance["region"] + ec2_client = self.create_client('ec2', region) + + logger.info(f"{'[DRY RUN] Would start' if dry_run else 'Starting'} EC2 instance {instance['id']} in region {region}") + if not dry_run: + ec2_client.start_instances(InstanceIds=[instance['id']]) + + # Remove the stop tag after successful start + logger.info(f"Removing {config.get('stop_tag')} tag from EC2 instance {instance['id']}") + ec2_client.delete_tags( + Resources=[instance['id']], + Tags=[ + { + 'Key': config.get('stop_tag') + } + ] + ) + + # Log with detailed information + self.log_action(instance['id'], region, "start", dry_run=dry_run) + success_count += 1 + + except Exception as e: + logger.error(f"Failed to {'simulate starting' if dry_run else 'start'} EC2 instance {instance['id']}: {e}") + error_count += 1 + + return { + "success": success_count, + "errors": error_count + } diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py new file mode 100644 index 00000000..56745897 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py @@ -0,0 +1,385 @@ +""" +EKS resource manager class. +""" +from typing import Dict, List, Any, Optional, Union + +from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.config_manager import get_config +from aws_resource_management.logging_setup import setup_logging + +logger = setup_logging() +config = get_config() + +# Tag names for EKS scaling +EKS_MIN_NODES_TAG = 'eks-min-nodes' +EKS_MAX_NODES_TAG = 'eks-max-nodes' +EKS_DESIRED_NODES_TAG = 'eks-desired-nodes' +EKS_ORIGINAL_MIN_NODES_TAG = 'eks-original-min-nodes' +EKS_ORIGINAL_MAX_NODES_TAG = 'eks-original-max-nodes' +EKS_ORIGINAL_DESIRED_NODES_TAG = 'eks-original-desired-nodes' +CLUSTER_SIZE_TAG = 'cluster:size' +EKS_ORIGINAL_CLUSTER_SIZE_TAG = 'eks-original-cluster-size' + +class EKSManager(ResourceManager): + """Manager for EKS clusters.""" + + def __init__(self, credentials: Dict[str, str], account_id: str, account_name: Optional[str] = None): + """ + Initialize EKS manager. + + Args: + credentials: AWS credentials dictionary + account_id: AWS account ID + account_name: AWS account name (optional) + """ + super().__init__(credentials, account_id, account_name) + self.resource_type = "eks_cluster" + + def stop(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Stop EKS clusters by scaling down their nodegroups. + + Args: + clusters: List of EKS cluster dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + logger.info(f"Delegating EKS stop operation to scale_down for {len(clusters)} clusters") + self.scale_down(clusters, dry_run) + + # Return a result dictionary that matches the expected interface + return { + "success": len(clusters), + "errors": 0 + } + + def start(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Start EKS clusters by scaling up their nodegroups. + + Args: + clusters: List of EKS cluster dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + logger.info(f"Delegating EKS start operation to scale_up for {len(clusters)} clusters") + self.scale_up(clusters, dry_run) + + # Return a result dictionary that matches the expected interface + return { + "success": len(clusters), + "errors": 0 + } + + def scale_down(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> None: + """ + Scale down EKS clusters by setting node counts to 0. + + Args: + clusters: List of EKS cluster dictionaries + dry_run: If True, only simulate the action + """ + logger.info(f"Evaluating {len(clusters)} EKS clusters for scale down action") + scaled_count = 0 + skipped_count = 0 + + for cluster in clusters: + # Skip clusters with the exclusion tag + if self.should_exclude(cluster): + logger.info(f"Skipping EKS cluster {cluster['name']} with exclusion tag {config.get('exclusion_tag')}") + skipped_count += 1 + continue + + try: + region = cluster["region"] + eks_client = self.create_client('eks', region) + + # Get current scaling parameters from tags or nodegroups + min_nodes = cluster.get('min_nodes', '') + max_nodes = cluster.get('max_nodes', '') + desired_nodes = cluster.get('desired_nodes', '') + schedule = cluster.get('schedule', '') + + # Check if we have a combined cluster:size tag + has_combined_size_tag = CLUSTER_SIZE_TAG in cluster.get('tags', {}) + + # Only proceed if we have some scaling values to work with + if min_nodes or max_nodes or desired_nodes or has_combined_size_tag: + logger.info(f"{'[DRY RUN] Would scale down' if dry_run else 'Scaling down'} EKS cluster {cluster['name']} in region {region}") + + if not dry_run: + # First, backup the current values in special tags + tags_to_update = {} + + # Handle combined size tag if it exists + if has_combined_size_tag: + original_size_tag = cluster['tags'][CLUSTER_SIZE_TAG] + tags_to_update[EKS_ORIGINAL_CLUSTER_SIZE_TAG] = original_size_tag + tags_to_update[CLUSTER_SIZE_TAG] = "min:0-max:0-desired:0" + else: + # Store original values in backup tags (individual tags) + if min_nodes: + tags_to_update[EKS_ORIGINAL_MIN_NODES_TAG] = min_nodes + tags_to_update[EKS_MIN_NODES_TAG] = '0' + if max_nodes: + tags_to_update[EKS_ORIGINAL_MAX_NODES_TAG] = max_nodes + tags_to_update[EKS_MAX_NODES_TAG] = '0' + if desired_nodes: + tags_to_update[EKS_ORIGINAL_DESIRED_NODES_TAG] = desired_nodes + tags_to_update[EKS_DESIRED_NODES_TAG] = '0' + + # Apply the tag updates + if tags_to_update: + logger.info(f"Updating scaling tags for EKS cluster {cluster['name']}: {tags_to_update}") + eks_client.tag_resource( + resourceArn=cluster.get('arn', f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}"), + tags=tags_to_update + ) + + # Now actually scale down the nodegroups + self._scale_nodegroups(eks_client, cluster['name'], 0, region, dry_run=False) + else: + # Dry run reporting + if has_combined_size_tag: + original_size = cluster['tags'][CLUSTER_SIZE_TAG] + logger.info(f"[DRY RUN] Would backup combined size tag: {original_size} and set to min:0-max:0-desired:0") + else: + logger.info(f"[DRY RUN] Would backup scaling values: Min={min_nodes}, Max={max_nodes}, Desired={desired_nodes}") + + logger.info(f"[DRY RUN] Would set scaling values to 0 for EKS cluster {cluster['name']}") + self._scale_nodegroups(eks_client, cluster['name'], 0, region, dry_run=True) + + # Log the action with schedule information + schedule_info = f", Schedule: {schedule}" if schedule else "" + self.log_action( + cluster['name'], region, "scale_down", + details=f"Min={min_nodes}->{0}, Max={max_nodes}->{0}, Desired={desired_nodes}->{0}{schedule_info}", + dry_run=dry_run, + existing_schedule=schedule + ) + scaled_count += 1 + else: + logger.info(f"Skipping EKS cluster {cluster['name']}, no scaling tags found") + skipped_count += 1 + + except Exception as e: + logger.error(f"Failed to {'simulate scaling down' if dry_run else 'scale down'} EKS cluster {cluster['name']}: {e}") + + logger.info(f"EKS scale down summary: {scaled_count} clusters processed, {skipped_count} clusters skipped") + + def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> None: + """ + Scale up EKS clusters using original node counts from tags. + + Args: + clusters: List of EKS cluster dictionaries + dry_run: If True, only simulate the action + """ + logger.info(f"Evaluating {len(clusters)} EKS clusters for scale up action") + scaled_count = 0 + skipped_count = 0 + + for cluster in clusters: + # Skip clusters with the exclusion tag + if self.should_exclude(cluster): + logger.info(f"Skipping EKS cluster {cluster['name']} with exclusion tag {config.get('exclusion_tag')}") + skipped_count += 1 + continue + + try: + region = cluster["region"] + eks_client = self.create_client('eks', region) + + # Check for original values in backup tags + original_min = cluster.get('original_min', '') + original_max = cluster.get('original_max', '') + original_desired = cluster.get('original_desired', '') + schedule = cluster.get('schedule', '') + + # Check for original combined size tag + tags = cluster.get('tags', {}) + has_original_combined_tag = EKS_ORIGINAL_CLUSTER_SIZE_TAG in tags + + # If we have backup values or original combined tag, restore them + if original_min or original_max or original_desired or has_original_combined_tag: + logger.info(f"{'[DRY RUN] Would scale up' if dry_run else 'Scaling up'} EKS cluster {cluster['name']} in region {region}") + + if not dry_run: + # Restore the original values from backup tags + tags_to_update = {} + tags_to_remove = [] + + # Handle original combined size tag if it exists + if has_original_combined_tag: + original_size_tag = tags[EKS_ORIGINAL_CLUSTER_SIZE_TAG] + tags_to_update[CLUSTER_SIZE_TAG] = original_size_tag + tags_to_remove.append(EKS_ORIGINAL_CLUSTER_SIZE_TAG) + + # Parse the original desired value from the size tag + desired = None + if 'desired:' in original_size_tag: + try: + desired_str = original_size_tag.split('desired:')[1].split('-')[0] + desired = int(desired_str) if desired_str.isdigit() else None + except: + logger.warning(f"Could not parse desired value from combined size tag: {original_size_tag}") + else: + # Restore original values for individual tags + if original_min: + tags_to_update[EKS_MIN_NODES_TAG] = original_min + tags_to_remove.append(EKS_ORIGINAL_MIN_NODES_TAG) + if original_max: + tags_to_update[EKS_MAX_NODES_TAG] = original_max + tags_to_remove.append(EKS_ORIGINAL_MAX_NODES_TAG) + if original_desired: + tags_to_update[EKS_DESIRED_NODES_TAG] = original_desired + tags_to_remove.append(EKS_ORIGINAL_DESIRED_NODES_TAG) + + # Parse desired value + desired = int(original_desired) if original_desired and original_desired.isdigit() else None + + # Apply the tag updates + if tags_to_update: + logger.info(f"Restoring original scaling tags for EKS cluster {cluster['name']}: {tags_to_update}") + eks_client.tag_resource( + resourceArn=cluster.get('arn', f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}"), + tags=tags_to_update + ) + + # Now actually scale up the nodegroups to desired capacity + self._scale_nodegroups(eks_client, cluster['name'], desired, region, dry_run=False) + + # Remove the backup tags + if tags_to_remove: + logger.info(f"Removing backup scaling tags: {tags_to_remove}") + eks_client.untag_resource( + resourceArn=cluster.get('arn', f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}"), + tagKeys=tags_to_remove + ) + else: + # Dry run reporting + if has_original_combined_tag: + original_size = tags[EKS_ORIGINAL_CLUSTER_SIZE_TAG] + logger.info(f"[DRY RUN] Would restore combined size tag: {original_size}") + + # Try to parse desired value for nodegroup scaling + desired = None + if 'desired:' in original_size: + try: + desired_str = original_size.split('desired:')[1].split('-')[0] + desired = int(desired_str) if desired_str.isdigit() else None + except: + pass + else: + logger.info(f"[DRY RUN] Would restore scaling values: Min={original_min}, Max={original_max}, Desired={original_desired}") + desired = int(original_desired) if original_desired and original_desired.isdigit() else None + + self._scale_nodegroups(eks_client, cluster['name'], desired, region, dry_run=True) + + # Log the action with schedule information + scale_details = "" + if has_original_combined_tag: + original_size = tags.get(EKS_ORIGINAL_CLUSTER_SIZE_TAG, "Unknown") + scale_details = f"Size tag: 0->'{original_size}'" + else: + scale_details = f"Min=0->{original_min}, Max=0->{original_max}, Desired=0->{original_desired}" + + schedule_info = f", Schedule: {schedule}" if schedule else "" + self.log_action( + cluster['name'], region, "scale_up", + details=f"{scale_details}{schedule_info}", + dry_run=dry_run, + existing_schedule=schedule + ) + scaled_count += 1 + else: + logger.info(f"Skipping EKS cluster {cluster['name']}, no original scaling values found in tags") + skipped_count += 1 + + except Exception as e: + logger.error(f"Failed to {'simulate scaling up' if dry_run else 'scale up'} EKS cluster {cluster['name']}: {e}") + + logger.info(f"EKS scale up summary: {scaled_count} clusters processed, {skipped_count} clusters skipped") + + def _scale_nodegroups(self, eks_client: Any, cluster_name: str, desired_capacity: Optional[int], region: str, dry_run: bool = False) -> None: + """ + Helper method to scale nodegroups to the specified capacity. + + Args: + eks_client: EKS boto3 client + cluster_name: Name of the EKS cluster + desired_capacity: Desired capacity for the nodegroups + region: AWS region + dry_run: If True, only simulate the action + """ + try: + # List all nodegroups for this cluster + response = eks_client.list_nodegroups(clusterName=cluster_name) + nodegroup_names = response.get('nodegroups', []) + + # Handle pagination + while 'nextToken' in response: + response = eks_client.list_nodegroups( + clusterName=cluster_name, + nextToken=response['nextToken'] + ) + nodegroup_names.extend(response.get('nodegroups', [])) + + if not nodegroup_names: + logger.info(f"No nodegroups found for EKS cluster {cluster_name}") + return + + for nodegroup_name in nodegroup_names: + try: + # Get nodegroup details + nodegroup = eks_client.describe_nodegroup( + clusterName=cluster_name, + nodegroupName=nodegroup_name + ).get('nodegroup', {}) + + # Get current scaling configuration + current_min = nodegroup.get('scalingConfig', {}).get('minSize') + current_max = nodegroup.get('scalingConfig', {}).get('maxSize') + current_desired = nodegroup.get('scalingConfig', {}).get('desiredSize') + + # Use current min and max when scaling up if desired capacity is provided + if desired_capacity is not None: + # When scaling up, we need a valid min and max + new_min = current_min if desired_capacity == 0 else min(current_min, desired_capacity) + new_max = max(current_max, desired_capacity) + else: + # When scaling down to zero + new_min = 0 + new_max = current_max + desired_capacity = 0 + + if dry_run: + logger.info(f"[DRY RUN] Would update nodegroup {nodegroup_name} scaling: " + + f"Min: {current_min}->{new_min}, " + + f"Max: {current_max}->{new_max}, " + + f"Desired: {current_desired}->{desired_capacity}") + else: + logger.info(f"Updating nodegroup {nodegroup_name} scaling: " + + f"Min: {current_min}->{new_min}, " + + f"Max: {current_max}->{new_max}, " + + f"Desired: {current_desired}->{desired_capacity}") + + # Update the nodegroup scaling configuration + eks_client.update_nodegroup_config( + clusterName=cluster_name, + nodegroupName=nodegroup_name, + scalingConfig={ + 'minSize': new_min, + 'maxSize': new_max, + 'desiredSize': desired_capacity + } + ) + except Exception as e: + logger.error(f"Error updating nodegroup {nodegroup_name}: {e}") + + except Exception as e: + logger.error(f"Error listing nodegroups for cluster {cluster_name} in region {region}: {e}") diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/emr.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/emr.py new file mode 100644 index 00000000..d38345d0 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/emr.py @@ -0,0 +1,215 @@ +""" +EMR resource manager class. +""" +from typing import Dict, List, Any, Optional +from botocore.exceptions import ClientError + +from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.config_manager import get_config +from aws_resource_management.logging_setup import setup_logging + +logger = setup_logging() +config = get_config() + +class EMRManager(ResourceManager): + """Manager for EMR clusters.""" + + def __init__(self, credentials: Dict[str, str], account_id: str, account_name: Optional[str] = None): + """ + Initialize EMR manager. + + Args: + credentials: AWS credentials dictionary + account_id: AWS account ID + account_name: AWS account name (optional) + """ + super().__init__(credentials, account_id, account_name) + self.resource_type = "emr_cluster" + + def stop(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Stop EMR clusters gracefully. + This preserves the cluster configuration and data. + + Args: + clusters: List of EMR cluster dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + success_count = 0 + error_count = 0 + + for cluster in clusters: + if not cluster: + logger.warning("Skipping empty or invalid EMR cluster record") + continue + + # Get cluster ID and status, using multiple field names for compatibility + cluster_id = cluster.get('id') + if not cluster_id: + logger.warning("Skipping EMR cluster with missing ID") + continue + + # Handle either 'status' or 'state' field for cluster status + cluster_status = cluster.get('status', cluster.get('state')) + if not cluster_status: + logger.warning(f"Cannot determine status for EMR cluster {cluster_id}, assuming UNKNOWN") + cluster_status = 'UNKNOWN' + + cluster_name = cluster.get('name', 'Unknown') + region = cluster.get('region') + if not region: + logger.warning(f"Missing region for EMR cluster {cluster_id}, skipping") + error_count += 1 + continue + + # Skip clusters with the exclusion tag + if self.should_exclude(cluster): + logger.info(f"Skipping EMR cluster {cluster_id} with exclusion tag {config.get('exclusion_tag')}") + continue + + # Only RUNNING and WAITING clusters can be stopped + if cluster_status in ["RUNNING", "WAITING"]: + try: + emr_client = self.create_client('emr', region) + + timestamp = self.get_timestamp() + + # Tag the cluster before stopping + logger.info(f"{'[DRY RUN] Would tag' if dry_run else 'Tagging'} EMR cluster {cluster_name} ({cluster_id}) with {config.get('stop_tag')}={timestamp}") + if not dry_run: + emr_client.add_tags( + ResourceId=cluster_id, + Tags=[ + { + 'Key': config.get('stop_tag'), + 'Value': timestamp + } + ] + ) + + logger.info(f"{'[DRY RUN] Would stop' if dry_run else 'Stopping'} EMR cluster {cluster_name} ({cluster_id}) in region {region}") + if not dry_run: + emr_client.stop_cluster(ClusterId=cluster_id) + + # Log with detailed information + self.log_action(cluster_id, region, "stop", resource_name=cluster_name, dry_run=dry_run) + success_count += 1 + + except Exception as e: + logger.error(f"Failed to {'simulate stopping' if dry_run else 'stop'} EMR cluster {cluster_id}: {e}") + error_count += 1 + + # For STARTING and BOOTSTRAPPING clusters, we need to terminate them as they can't be stopped + elif cluster_status in ["STARTING", "BOOTSTRAPPING"]: + try: + emr_client = self.create_client('emr', region) + + timestamp = self.get_timestamp() + + # Tag the cluster before terminating + logger.info(f"{'[DRY RUN] Would tag' if dry_run else 'Tagging'} EMR cluster {cluster_name} ({cluster_id}) with {config.get('stop_tag')}={timestamp}") + if not dry_run: + emr_client.add_tags( + ResourceId=cluster_id, + Tags=[ + { + 'Key': config.get('stop_tag'), + 'Value': timestamp + } + ] + ) + + logger.info(f"{'[DRY RUN] Would terminate' if dry_run else 'Terminating'} EMR cluster {cluster_name} ({cluster_id}) in region {region} (cannot stop a cluster in {cluster_status} state)") + if not dry_run: + emr_client.terminate_job_flows(JobFlowIds=[cluster_id]) + + # Log with detailed information + self.log_action(cluster_id, region, "terminate", resource_name=cluster_name, dry_run=dry_run) + success_count += 1 + + except Exception as e: + logger.error(f"Failed to {'simulate terminating' if dry_run else 'terminate'} EMR cluster {cluster_id}: {e}") + error_count += 1 + else: + logger.info(f"Skipping EMR cluster {cluster_id} in state {cluster_status} (not eligible for stop)") + + return { + "success": success_count, + "errors": error_count + } + + def start(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Start stopped EMR clusters. + + Args: + clusters: List of EMR cluster dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + success_count = 0 + error_count = 0 + + for cluster in clusters: + if not cluster: + logger.warning("Skipping empty or invalid EMR cluster record") + continue + + # Get cluster ID and status, using multiple field names for compatibility + cluster_id = cluster.get('id') + if not cluster_id: + logger.warning("Skipping EMR cluster with missing ID") + continue + + # Handle either 'status' or 'state' field for cluster status + cluster_status = cluster.get('status', cluster.get('state')) + if not cluster_status: + logger.warning(f"Cannot determine status for EMR cluster {cluster_id}, assuming UNKNOWN") + cluster_status = 'UNKNOWN' + + cluster_name = cluster.get('name', 'Unknown') + region = cluster.get('region') + if not region: + logger.warning(f"Missing region for EMR cluster {cluster_id}, skipping") + error_count += 1 + continue + + # Skip clusters with the exclusion tag + if self.should_exclude(cluster): + logger.info(f"Skipping EMR cluster {cluster_id} with exclusion tag {config.get('exclusion_tag')}") + continue + + if cluster_status == "STOPPED": + try: + emr_client = self.create_client('emr', region) + + logger.info(f"{'[DRY RUN] Would start' if dry_run else 'Starting'} EMR cluster {cluster_name} ({cluster_id}) in region {region}") + if not dry_run: + emr_client.start_cluster(ClusterId=cluster_id) + + # Remove the stop tag after successful start + logger.info(f"Removing {config.get('stop_tag')} tag from EMR cluster {cluster_id}") + emr_client.remove_tags( + ResourceId=cluster_id, + TagKeys=[config.get('stop_tag')] + ) + + # Log with detailed information + self.log_action(cluster_id, region, "start", resource_name=cluster_name, dry_run=dry_run) + success_count += 1 + + except Exception as e: + logger.error(f"Failed to {'simulate starting' if dry_run else 'start'} EMR cluster {cluster_id}: {e}") + error_count += 1 + else: + logger.info(f"Skipping EMR cluster {cluster_id} in state {cluster_status} (not in STOPPED state)") + + return { + "success": success_count, + "errors": error_count + } diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/rds.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/rds.py new file mode 100644 index 00000000..0cbc9186 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/rds.py @@ -0,0 +1,145 @@ +""" +RDS resource manager class. +""" +from typing import Dict, List, Any, Optional + +from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.config_manager import get_config +from aws_resource_management.logging_setup import setup_logging + +logger = setup_logging() +config = get_config() + +class RDSManager(ResourceManager): + """Manager for RDS instances.""" + + def __init__(self, credentials: Dict[str, str], account_id: str, account_name: Optional[str] = None): + """ + Initialize RDS manager. + + Args: + credentials: AWS credentials dictionary + account_id: AWS account ID + account_name: AWS account name (optional) + """ + super().__init__(credentials, account_id, account_name) + self.resource_type = "rds_instance" + + def stop(self, instances: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Stop available RDS instances. + + Args: + instances: List of RDS instance dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + logger.info(f"Evaluating {len(instances)} RDS instances for stop action") + stopped_count = 0 + skipped_count = 0 + error_count = 0 + + for instance in instances: + # Skip instances with the exclusion tag + if self.should_exclude(instance): + logger.info(f"Skipping RDS instance {instance['id']} with exclusion tag {config.get('exclusion_tag')}") + skipped_count += 1 + continue + + # Log status for debugging + logger.debug(f"RDS instance {instance['id']} status: {instance['status']}") + + if instance["status"] == "available": + try: + region = instance["region"] + rds_client = self.create_client('rds', region) + + # Create ISO 8601 timestamp + timestamp = self.get_timestamp() + + # Tag the instance before stopping + logger.info(f"{'[DRY RUN] Would tag' if dry_run else 'Tagging'} RDS instance {instance['id']} with {config.get('stop_tag')}={timestamp}") + if not dry_run: + rds_client.add_tags_to_resource( + ResourceName=f"arn:{self.get_partition_for_region(region)}:rds:{region}:{self.account_id}:db:{instance['id']}", + Tags=[ + { + 'Key': config.get('stop_tag'), + 'Value': timestamp + } + ] + ) + + logger.info(f"{'[DRY RUN] Would stop' if dry_run else 'Stopping'} RDS instance {instance['id']} in region {region}") + if not dry_run: + rds_client.stop_db_instance(DBInstanceIdentifier=instance['id']) + + # Log with detailed information + self.log_action(instance['id'], region, "stop", dry_run=dry_run) + + stopped_count += 1 + + except Exception as e: + logger.error(f"Failed to {'simulate stopping' if dry_run else 'stop'} RDS instance {instance['id']}: {e}") + error_count += 1 + else: + logger.debug(f"Skipping RDS instance {instance['id']} with non-available status: {instance['status']}") + skipped_count += 1 + + logger.info(f"RDS stop summary: {stopped_count} instances processed, {skipped_count} instances skipped") + + return { + "success": stopped_count, + "errors": error_count + } + + def start(self, instances: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + """ + Start stopped RDS instances. + + Args: + instances: List of RDS instance dictionaries + dry_run: If True, only simulate the action + + Returns: + Dictionary with success and error counts + """ + started_count = 0 + error_count = 0 + + for instance in instances: + # Skip instances with the exclusion tag + if self.should_exclude(instance): + logger.info(f"Skipping RDS instance {instance['id']} with exclusion tag {config.get('exclusion_tag')}") + continue + + if instance["status"] == "stopped": + try: + region = instance["region"] + rds_client = self.create_client('rds', region) + + logger.info(f"{'[DRY RUN] Would start' if dry_run else 'Starting'} RDS instance {instance['id']} in region {region}") + if not dry_run: + rds_client.start_db_instance(DBInstanceIdentifier=instance['id']) + + # Remove the stop tag after successful start + logger.info(f"Removing {config.get('stop_tag')} tag from RDS instance {instance['id']}") + rds_client.remove_tags_from_resource( + ResourceName=f"arn:{self.get_partition_for_region(region)}:rds:{region}:{self.account_id}:db:{instance['id']}", + TagKeys=[config.get('stop_tag')] + ) + + # Log with detailed information + self.log_action(instance['id'], region, "start", dry_run=dry_run) + started_count += 1 + + except Exception as e: + logger.error(f"Failed to {'simulate starting' if dry_run else 'start'} RDS instance {instance['id']}: {e}") + error_count += 1 + + return { + "success": started_count, + "errors": error_count + } diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/resource_controller.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/resource_controller.py new file mode 100644 index 00000000..db2d6fc1 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/resource_controller.py @@ -0,0 +1,256 @@ +""" +DEPRECATED: This module is deprecated in favor of core.py. +Resource controller module. +Handles the core business logic for managing AWS resources. +""" + +import warnings +warnings.warn( + "The resource_controller module is deprecated. Please use the core module instead.", + DeprecationWarning, + stacklevel=2 +) + +from typing import Dict, List, Any, Optional, Union +import logging +import sys + +from aws_resource_management.managers.ec2 import EC2Manager +from aws_resource_management.managers.rds import RDSManager +from aws_resource_management.managers.emr import EMRManager +from aws_resource_management.managers.eks import EKSManager +from aws_resource_management.config_manager import get_config +from aws_resource_management.logging_setup import setup_logging +from aws_resource_management.aws_utils import get_credentials, get_account_list, get_enabled_regions +from aws_resource_management.core import ResourceManager + +logger = setup_logging() +config = get_config() + +class ResourceController: + """Controller class that orchestrates resource management across accounts.""" + + def __init__( + self, + profile_name: Optional[str] = None, + use_profiles: bool = False, + discover_regions: bool = False, + partition: Optional[str] = None + ): + """ + Initialize the resource controller. + + Args: + profile_name: Optional AWS profile name to use for all operations + use_profiles: Whether to use AWS profiles from ~/.aws/config + discover_regions: Whether to automatically discover enabled regions + partition: AWS partition to operate in (aws, aws-us-gov, aws-cn) + """ + self.config = get_config() + self.resource_manager = ResourceManager( + profile_name=profile_name, + use_profiles=use_profiles, + discover_regions=discover_regions, + partition=partition + ) + self.discover_regions = discover_regions + self.partition = partition + + def process_resources( + self, + action: str, + resource_type: str, + regions: List[str], + exclude_accounts: List[str] = None, + exclude_resources: List[str] = None, + exclude_regions: List[str] = None, + dry_run: bool = False + ) -> Dict[str, Any]: + """ + Process resources of the specified type across accounts. + + Args: + action: Action to perform (stop or start) + resource_type: Type of resources to manage + regions: List of regions to scan + exclude_accounts: List of account IDs to exclude + exclude_resources: List of resource types to exclude + exclude_regions: List of regions to exclude + dry_run: Whether to perform a dry run + + Returns: + Statistics dictionary with results + """ + try: + # Validate action + if action not in ['start', 'stop']: + logger.error(f"Invalid action: {action}. Must be 'start' or 'stop'") + return {'error': f"Invalid action: {action}"} + + # Validate resource type + if resource_type not in ['ec2', 'rds', 'eks', 'emr']: + logger.error(f"Invalid resource type: {resource_type}") + return {'error': f"Invalid resource type: {resource_type}"} + + region_msg = "all enabled regions" if self.discover_regions else ", ".join(regions) + logger.info(f"Processing {resource_type} resources in {region_msg} for {action} action") + + # Process accounts through the resource manager + stats = self.resource_manager.process_accounts( + action=action, + regions=regions, + exclude_accounts=exclude_accounts, + exclude_resources=exclude_resources, + exclude_regions=exclude_regions, + dry_run=dry_run + ) + + # Log summary + logger.info(f"Completed {action} action for {resource_type} resources") + logger.info(f"Accounts processed: {stats.get('accounts_processed', 0)}") + logger.info(f"Accounts skipped: {stats.get('accounts_skipped', 0)}") + logger.info(f"Resources processed: {stats.get('resources_processed', 0)}") + logger.info(f"Resources skipped: {stats.get('resources_skipped', 0)}") + logger.info(f"Regions processed: {stats.get('regions_processed', 0)}") + + if stats.get('errors'): + logger.warning(f"Encountered {len(stats.get('errors', []))} errors") + + return stats + + except KeyboardInterrupt: + # Propagate the keyboard interrupt to let the CLI handle it + logger.warning("Operation interrupted by user in resource controller") + raise + + def list_resources( + self, + resource_type: str, + regions: List[str], + exclude_accounts: List[str] = None, + exclude_regions: List[str] = None + ) -> Dict[str, Any]: + """ + List resources of the specified type across accounts. + + Args: + resource_type: Type of resources to list + regions: List of regions to scan + exclude_accounts: List of account IDs to exclude + exclude_regions: List of regions to exclude + + Returns: + Dictionary containing the discovered resources + """ + logger.info(f"Listing {resource_type} resources in {', '.join(regions)}") + + # Get account list + accounts = get_account_list() + if not accounts: + logger.warning("No accounts found or error retrieving accounts") + return {'error': 'No accounts found or error retrieving accounts'} + + results = { + 'resources': [], + 'accounts_processed': 0, + 'accounts_skipped': 0 + } + + # Process each account + for account in accounts: + account_id = account.get('account_id') + account_name = account.get('account_name') + + # Skip excluded accounts + if exclude_accounts and account_id in exclude_accounts: + logger.info(f"Skipping excluded account {account_id} ({account_name})") + results['accounts_skipped'] += 1 + continue + + try: + # Get credentials for the account + credentials = get_credentials(account_id) + if not credentials: + logger.warning(f"Could not obtain credentials for account {account_id}") + results['accounts_skipped'] += 1 + continue + + # Create the appropriate manager for the resource type + manager = self._create_resource_manager( + resource_type=resource_type, + credentials=credentials, + account_id=account_id, + account_name=account_name + ) + + if not manager: + logger.error(f"Failed to create resource manager for type {resource_type}") + continue + + # Get resources for each region + for region in regions: + resources = manager.list_resources(region) + results['resources'].extend(resources) + + results['accounts_processed'] += 1 + + except Exception as e: + logger.error(f"Error listing resources for account {account_id}: {str(e)}") + + logger.info(f"Found {len(results['resources'])} {resource_type} resources") + return results + + def _create_resource_manager( + self, + resource_type: str, + credentials: Dict[str, str], + account_id: str, + account_name: Optional[str] = None + ) -> Optional[Any]: + """ + Create the appropriate resource manager based on the resource type. + + Args: + resource_type: Type of resources to manage + credentials: AWS credentials + account_id: AWS account ID + account_name: AWS account name + + Returns: + Resource manager instance or None if invalid type + """ + if resource_type == 'ec2': + return EC2Manager(credentials, account_id, account_name) + elif resource_type == 'rds': + return RDSManager(credentials, account_id, account_name) + elif resource_type == 'eks': + return EKSManager(credentials, account_id, account_name) + elif resource_type == 'emr': + return EMRManager(credentials, account_id, account_name) + else: + return None + + def get_regions_for_account(self, account_id: str) -> List[str]: + """ + Get enabled regions for a specific account. + + Args: + account_id: AWS account ID + + Returns: + List of enabled region names + """ + try: + # Get credentials for the account + credentials = get_credentials(account_id) + if not credentials: + logger.warning(f"Could not obtain credentials for account {account_id}") + return [] + + # Get enabled regions + regions = get_enabled_regions(credentials, partition=self.partition) + return regions + + except Exception as e: + logger.error(f"Error getting regions for account {account_id}: {str(e)}") + return [] diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils.py new file mode 100644 index 00000000..e2aacf76 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils.py @@ -0,0 +1,267 @@ +""" +AWS utility functions for resource management. +""" +import boto3 +import os +import re +from typing import Dict, List, Optional, Any, Union +from botocore.exceptions import ClientError + +from aws_resource_management.config_manager import get_config +from aws_resource_management.logging_setup import setup_logging + +logger = setup_logging() +config = get_config() + +def create_boto3_client(service: str, credentials: Dict[str, str], region: str) -> boto3.client: + """ + Create a boto3 client for a specific service using the provided credentials. + + Args: + service: AWS service name (e.g., 'ec2', 'rds') + credentials: AWS credentials dict with access key, secret key, and token + region: AWS region name + + Returns: + Configured boto3 client + """ + return boto3.client( + service, + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'], + region_name=region + ) + +def get_available_regions() -> List[str]: + """ + Get a list of all available AWS regions. + + Returns: + List of region names + """ + try: + ec2_client = boto3.client('ec2') + response = ec2_client.describe_regions() + return [region['RegionName'] for region in response['Regions']] + except Exception as e: + logger.error(f"Error getting available regions: {e}") + # Return a default list of common regions as fallback + default_region = config.get('default_region') + if default_region: + return [default_region] + return ['us-gov-east-1', 'us-gov-west-1'] + +def get_partition_for_region(region: str) -> str: + """ + Determine the AWS partition for a given region. + + Args: + region: AWS region name + + Returns: + str: AWS partition name ('aws', 'aws-us-gov', 'aws-cn') + """ + if region.startswith('us-gov-'): + return 'aws-us-gov' + elif region.startswith('cn-'): + return 'aws-cn' + else: + return 'aws' + +def find_profile_for_account(account_id: str, role_name: Optional[str] = None) -> Optional[str]: + """ + Find an appropriate SSO profile for the given account ID and role name. + + Args: + account_id: AWS account ID + role_name: Preferred role name (e.g., 'AdministratorAccess', 'inf-admin-t2') + + Returns: + Profile name if found, None otherwise + """ + try: + # Try to read profiles from AWS config + import configparser + config_file = os.path.expanduser("~/.aws/config") + if not os.path.exists(config_file): + return None + + aws_config = configparser.ConfigParser() + aws_config.read(config_file) + + # Look for profiles matching the account ID + matching_profiles = [] + for section in aws_config.sections(): + # Extract the actual profile name from section (removing 'profile ' prefix if present) + profile_name = section + if section.startswith('profile '): + profile_name = section[8:] + + # Check if this profile is for the target account ID + if 'sso_account_id' in aws_config[section] and aws_config[section]['sso_account_id'] == account_id: + matching_profiles.append((profile_name, aws_config[section].get('sso_role_name', ''))) + + # If role_name is specified, try to find a profile with that role + if role_name and matching_profiles: + for profile, profile_role in matching_profiles: + if profile_role.lower() == role_name.lower(): + logger.info(f"Found matching profile {profile} for account {account_id} with role {role_name}") + return profile + + # If we have any matching profiles, return the first one + if matching_profiles: + # Prefer profiles with admin roles if available + admin_profiles = [p for p, r in matching_profiles if 'admin' in r.lower()] + if admin_profiles: + logger.info(f"Using profile {admin_profiles[0]} for account {account_id}") + return admin_profiles[0] + + logger.info(f"Using profile {matching_profiles[0][0]} for account {account_id}") + return matching_profiles[0][0] + + return None + except Exception as e: + logger.warning(f"Error finding profile for account {account_id}: {e}") + return None + +def create_boto3_session_from_profile(profile_name: str) -> Optional[boto3.Session]: + """ + Create a boto3 session using the specified profile. + + Args: + profile_name: AWS profile name + + Returns: + Configured boto3 session or None if failed + """ + try: + logger.info(f"Creating session using profile: {profile_name}") + session = boto3.Session(profile_name=profile_name) + # Test if the session works by getting the caller identity + sts = session.client('sts') + sts.get_caller_identity() + return session + except Exception as e: + logger.warning(f"Failed to create session with profile {profile_name}: {e}") + return None + +def assume_role(account_id: str, region: Optional[str] = None, role_name: Optional[str] = None) -> Optional[Dict[str, str]]: + """ + Get credentials for the specified account, preferring SSO profiles when available. + + Args: + account_id: AWS account ID to access + region: AWS region, used to determine partition + role_name: Role name to assume (defaults to config.assume_role_name) + + Returns: + Credentials dictionary or None if failed + """ + try: + # First try using SSO profiles if available + profile_name = find_profile_for_account(account_id, role_name) + if profile_name: + session = create_boto3_session_from_profile(profile_name) + if session: + # Get credentials from the session + credentials = session.get_credentials() + frozen_credentials = credentials.get_frozen_credentials() + return { + 'AccessKeyId': frozen_credentials.access_key, + 'SecretAccessKey': frozen_credentials.secret_key, + 'SessionToken': frozen_credentials.token + } + + # Fall back to assuming role directly if no profile found or profile didn't work + # Use default region from config if not provided + if region is None: + region = config.get('default_region') + + # Determine the correct partition for the ARN + partition = get_partition_for_region(region) + + # Use the specified role name or fall back to the default + actual_role_name = role_name if role_name else config.get('assume_role_name') + + role_arn = f"arn:{partition}:iam::{account_id}:role/{actual_role_name}" + sts_client = boto3.client('sts', region_name=region) + + logger.info(f"Assuming role {role_arn}") + response = sts_client.assume_role( + RoleArn=role_arn, + RoleSessionName="ResourceManagementSession" + ) + + return response['Credentials'] + except Exception as e: + logger.error(f"Error accessing account {account_id}: {e}") + return None + +def create_boto3_client_for_account( + account_id: str, + service: str, + region: Optional[str] = None, + role_name: Optional[str] = None +) -> Optional[boto3.client]: + """ + Create a boto3 client for a specific service in the specified account. + + Args: + account_id: AWS account ID + service: AWS service name (e.g., 'ec2', 'rds') + region: AWS region name + role_name: Role name to use + + Returns: + Configured boto3 client or None if failed + """ + # First try using a profile directly + profile_name = find_profile_for_account(account_id, role_name) + if profile_name: + session = create_boto3_session_from_profile(profile_name) + if session: + if region: + return session.client(service, region_name=region) + else: + return session.client(service) + + # Fall back to assume_role method + credentials = assume_role(account_id, region, role_name) + if not credentials: + return None + + return create_boto3_client(service, credentials, region) + +def get_organization_accounts(exclude_accounts: Optional[List[str]] = None) -> List[Dict[str, str]]: + """ + Get a list of accounts in the AWS organization. + + Args: + exclude_accounts: List of account IDs to exclude + + Returns: + List of account dicts with id and name + """ + if exclude_accounts is None: + exclude_accounts = [] + + try: + org_client = boto3.client('organizations') + accounts = [] + + # Get all accounts in the organization + paginator = org_client.get_paginator('list_accounts') + for page in paginator.paginate(): + for account in page['Accounts']: + # Skip suspended accounts and accounts in exclude list + if account['Status'] == 'ACTIVE' and account['Id'] not in exclude_accounts: + accounts.append({ + 'id': account['Id'], + 'name': account['Name'] + }) + + return accounts + except Exception as e: + logger.error(f"Error getting organization accounts: {e}") + return [] diff --git a/local-app/python-tools/gfl-resource-actions/aws_utils.py b/local-app/python-tools/gfl-resource-actions/aws_utils.py index 9510d06f..f1b6d899 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_utils.py @@ -3,8 +3,6 @@ """ import boto3 import config -import os -import re from logging_utils import setup_logging logger = setup_logging() @@ -43,144 +41,22 @@ def get_available_regions(): except Exception as e: logger.error(f"Error getting available regions: {e}") # Return a default list of common regions as fallback - return ['us-gov-east-1', 'us-gov-west-1'] + return ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'] -def get_partition_for_region(region): +def assume_role(account_id): """ - Determine the AWS partition for a given region. + Assume the OrganizationAccountAccessRole in the specified account. Args: - region (str): AWS region name - - Returns: - str: AWS partition name ('aws', 'aws-us-gov', 'aws-cn') - """ - if region.startswith('us-gov-'): - return 'aws-us-gov' - elif region.startswith('cn-'): - return 'aws-cn' - else: - return 'aws' - -def find_profile_for_account(account_id, role_name=None): - """ - Find an appropriate SSO profile for the given account ID and role name. - - Args: - account_id (str): AWS account ID - role_name (str, optional): Preferred role name (e.g., 'AdministratorAccess', 'inf-admin-t2') - - Returns: - str: Profile name if found, None otherwise - """ - try: - # Try to read profiles from AWS config - import configparser - config_file = os.path.expanduser("~/.aws/config") - if not os.path.exists(config_file): - return None - - aws_config = configparser.ConfigParser() - aws_config.read(config_file) - - # Look for profiles matching the account ID - matching_profiles = [] - for section in aws_config.sections(): - # Extract the actual profile name from section (removing 'profile ' prefix if present) - profile_name = section - if section.startswith('profile '): - profile_name = section[8:] - - # Check if this profile is for the target account ID - if 'sso_account_id' in aws_config[section] and aws_config[section]['sso_account_id'] == account_id: - matching_profiles.append((profile_name, aws_config[section].get('sso_role_name', ''))) - - # If role_name is specified, try to find a profile with that role - if role_name and matching_profiles: - for profile, profile_role in matching_profiles: - if profile_role.lower() == role_name.lower(): - logger.info(f"Found matching profile {profile} for account {account_id} with role {role_name}") - return profile - - # If we have any matching profiles, return the first one - if matching_profiles: - # Prefer profiles with admin roles if available - admin_profiles = [p for p, r in matching_profiles if 'admin' in r.lower()] - if admin_profiles: - logger.info(f"Using profile {admin_profiles[0]} for account {account_id}") - return admin_profiles[0] - - logger.info(f"Using profile {matching_profiles[0][0]} for account {account_id}") - return matching_profiles[0][0] - - return None - except Exception as e: - logger.warning(f"Error finding profile for account {account_id}: {e}") - return None - -def create_boto3_session_from_profile(profile_name): - """ - Create a boto3 session using the specified profile. - - Args: - profile_name (str): AWS profile name - - Returns: - boto3.Session: Configured boto3 session or None if failed - """ - try: - logger.info(f"Creating session using profile: {profile_name}") - session = boto3.Session(profile_name=profile_name) - # Test if the session works by getting the caller identity - sts = session.client('sts') - sts.get_caller_identity() - return session - except Exception as e: - logger.warning(f"Failed to create session with profile {profile_name}: {e}") - return None - -def assume_role(account_id, region=None, role_name=None): - """ - Get credentials for the specified account, preferring SSO profiles when available. - - Args: - account_id (str): AWS account ID to access - region (str, optional): AWS region, used to determine partition - role_name (str, optional): Role name to assume (defaults to config.ASSUME_ROLE_NAME) + account_id (str): AWS account ID to assume role in Returns: dict: Credentials dictionary or None if failed """ try: - # First try using SSO profiles if available - profile_name = find_profile_for_account(account_id, role_name) - if profile_name: - session = create_boto3_session_from_profile(profile_name) - if session: - # Get credentials from the session - credentials = session.get_credentials() - frozen_credentials = credentials.get_frozen_credentials() - return { - 'AccessKeyId': frozen_credentials.access_key, - 'SecretAccessKey': frozen_credentials.secret_key, - 'SessionToken': frozen_credentials.token - } - - # Fall back to assuming role directly if no profile found or profile didn't work - # Use default region from config if not provided - if region is None: - region = config.DEFAULT_REGION - - # Determine the correct partition for the ARN - partition = get_partition_for_region(region) - - # Use the specified role name or fall back to the default - actual_role_name = role_name if role_name else config.ASSUME_ROLE_NAME + role_arn = f"arn:aws:iam::{account_id}:role/{config.ASSUME_ROLE_NAME}" + sts_client = boto3.client('sts') - role_arn = f"arn:{partition}:iam::{account_id}:role/{actual_role_name}" - sts_client = boto3.client('sts', region_name=region) - - logger.info(f"Assuming role {role_arn}") response = sts_client.assume_role( RoleArn=role_arn, RoleSessionName="ResourceManagementSession" @@ -188,38 +64,8 @@ def assume_role(account_id, region=None, role_name=None): return response['Credentials'] except Exception as e: - logger.error(f"Error accessing account {account_id}: {e}") - return None - -def create_boto3_client_for_account(account_id, service, region=None, role_name=None): - """ - Create a boto3 client for a specific service in the specified account. - - Args: - account_id (str): AWS account ID - service (str): AWS service name (e.g., 'ec2', 'rds') - region (str, optional): AWS region name - role_name (str, optional): Role name to use - - Returns: - boto3.client: Configured boto3 client or None if failed - """ - # First try using a profile directly - profile_name = find_profile_for_account(account_id, role_name) - if profile_name: - session = create_boto3_session_from_profile(profile_name) - if session: - if region: - return session.client(service, region_name=region) - else: - return session.client(service) - - # Fall back to assume_role method - credentials = assume_role(account_id, region, role_name) - if not credentials: + logger.error(f"Error assuming role in account {account_id}: {e}") return None - - return create_boto3_client(service, credentials, region) def get_organization_accounts(exclude_accounts=None): """ diff --git a/local-app/python-tools/gfl-resource-actions/logging_setup.py b/local-app/python-tools/gfl-resource-actions/logging_setup.py new file mode 100644 index 00000000..ec39d52b --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/logging_setup.py @@ -0,0 +1,322 @@ +import json +import logging +import os +import sys +import threading +import time +import uuid +from contextlib import contextmanager +from datetime import datetime +from logging.handlers import RotatingFileHandler +from typing import Any, Dict, List, Optional, Set, Union + +# Store sensitive field patterns to mask in logs +SENSITIVE_FIELDS = { + "password", "secret", "token", "key", "passphrase", "credential", + "auth", "jwt", "private_key", "access_key", "secret_key" +} + +class LoggingContext: + """Thread-local storage for logging context information.""" + _context = threading.local() + + @classmethod + def set(cls, **kwargs) -> None: + """Set context values.""" + if not hasattr(cls._context, "data"): + cls._context.data = {} + cls._context.data.update(kwargs) + + @classmethod + def get(cls) -> Dict[str, Any]: + """Get all context values.""" + return getattr(cls._context, "data", {}).copy() + + @classmethod + def get_value(cls, key: str, default: Any = None) -> Any: + """Get a specific context value.""" + return getattr(cls._context, "data", {}).get(key, default) + + @classmethod + def clear(cls) -> None: + """Clear all context values.""" + if hasattr(cls._context, "data"): + cls._context.data = {} + +class JSONFormatter(logging.Formatter): + """Format log records as JSON with additional context.""" + + def __init__(self, sanitize_fields: Optional[Set[str]] = None): + super().__init__() + self.sanitize_fields = sanitize_fields or SENSITIVE_FIELDS + + def format(self, record: logging.LogRecord) -> str: + """Format the record as a JSON string with context.""" + # Start with basic log information + log_data = { + "timestamp": datetime.utcfromtimestamp(record.created).isoformat() + "Z", + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "path": record.pathname, + "function": record.funcName, + "line": record.lineno, + "process": record.process, + "thread": record.thread, + "hostname": os.uname().nodename + } + + # Add correlation ID if not already present + if not hasattr(record, "correlation_id"): + log_data["correlation_id"] = str(uuid.uuid4()) + + # Add exception info if available + if record.exc_info: + log_data["exception"] = { + "type": record.exc_info[0].__name__, + "message": str(record.exc_info[1]), + "traceback": self.formatException(record.exc_info) + } + + # Add extra fields from the record + if hasattr(record, "extra") and record.extra: + for key, value in record.extra.items(): + log_data[key] = value + + # Add context from LoggingContext + context_data = LoggingContext.get() + if context_data: + for key, value in context_data.items(): + if key not in log_data: # Don't overwrite existing fields + log_data[key] = value + + # Sanitize sensitive fields + self._sanitize_data(log_data) + + # Convert to JSON + return json.dumps(log_data) + + def _sanitize_data(self, data: Dict[str, Any], path: str = "") -> None: + """Recursively sanitize sensitive data.""" + if isinstance(data, dict): + for key, value in list(data.items()): + current_path = f"{path}.{key}" if path else key + + # Check if this is a sensitive field + is_sensitive = any( + sensitive in current_path.lower() + for sensitive in self.sanitize_fields + ) + + if is_sensitive and isinstance(value, (str, int, float, bool)): + data[key] = "********" + elif isinstance(value, (dict, list)): + self._sanitize_data(value, current_path) + elif isinstance(data, list): + for i, item in enumerate(data): + current_path = f"{path}[{i}]" + if isinstance(item, (dict, list)): + self._sanitize_data(item, current_path) + +class ContextFilter(logging.Filter): + """Filter for automatically adding AWS context to log records.""" + + def __init__( + self, + account_id: Optional[str] = None, + region: Optional[str] = None, + service_name: Optional[str] = None + ): + super().__init__() + self.account_id = account_id + self.region = region + self.service_name = service_name + + def filter(self, record: logging.LogRecord) -> bool: + """Add AWS context to the log record.""" + # Add basic context if not already present + if not hasattr(record, "aws"): + record.aws = {} + + # Add AWS account ID + if self.account_id and not record.aws.get("account_id"): + record.aws["account_id"] = self.account_id + + # Add AWS region + if self.region and not record.aws.get("region"): + record.aws["region"] = self.region + + # Add service name + if self.service_name and not record.aws.get("service_name"): + record.aws["service_name"] = self.service_name + + # Add context from LoggingContext if relevant + context = LoggingContext.get() + for field in ["resource_type", "resource_id", "operation_name"]: + if field in context and not record.aws.get(field): + record.aws[field] = context[field] + + # Always return True as we're just enriching, not filtering out + return True + +@contextmanager +def log_operation( + logger: logging.Logger, + name: str, + level: int = logging.INFO, + **context +) -> None: + """ + Context manager for logging operations with timing and status. + + Args: + logger: The logger to use + name: Name of the operation + level: Log level to use + **context: Additional context to include in logs + """ + # Generate correlation ID for this operation if not provided + correlation_id = context.get("correlation_id", str(uuid.uuid4())) + + # Set initial context + LoggingContext.set( + operation_name=name, + operation_start_time=time.time(), + correlation_id=correlation_id, + **context + ) + + logger.log(level, f"Starting operation: {name}", extra={"correlation_id": correlation_id}) + + try: + yield + # Calculate duration and update context on successful completion + duration = (time.time() - LoggingContext.get_value("operation_start_time", 0)) * 1000 + LoggingContext.set(operation_status="success", duration_ms=duration) + logger.log( + level, + f"Operation {name} completed successfully in {duration:.2f}ms", + extra={"correlation_id": correlation_id, "duration_ms": duration} + ) + except Exception as e: + # Calculate duration and update context on failure + duration = (time.time() - LoggingContext.get_value("operation_start_time", 0)) * 1000 + LoggingContext.set(operation_status="failed", duration_ms=duration, error=str(e)) + logger.error( + f"Operation {name} failed after {duration:.2f}ms: {str(e)}", + exc_info=True, + extra={"correlation_id": correlation_id, "duration_ms": duration} + ) + raise + finally: + # Clear context to avoid leaking between operations + LoggingContext.clear() + +def log_with_context( + logger: logging.Logger, + level: int, + msg: str, + **context +) -> None: + """ + Log a message with the current context plus any additional context provided. + + Args: + logger: Logger to use + level: Log level + msg: Message to log + **context: Additional context to include + """ + # Merge with existing context + merged_context = LoggingContext.get() + merged_context.update(context) + + # Log with the merged context + logger.log(level, msg, extra={"extra": merged_context}) + +def configure_logging( + app_name: str, + log_level: Union[int, str] = logging.INFO, + json_format: bool = True, + log_file: Optional[str] = None, + max_log_size: int = 10 * 1024 * 1024, # 10 MB + backup_count: int = 5, + console_output: bool = True, + aws_context: Optional[Dict[str, str]] = None +) -> logging.Logger: + """ + Configure application logging with advanced features. + + Args: + app_name: Application name used for the logger + log_level: Logging level (name or constant) + json_format: Whether to use JSON formatting + log_file: Path to log file (if None, file logging is disabled) + max_log_size: Maximum size of log files before rotation + backup_count: Number of backup log files to keep + console_output: Whether to output logs to console + aws_context: Dict with AWS context (account_id, region, service_name) + + Returns: + Configured logger instance + """ + # Convert string log level to int if needed + if isinstance(log_level, str): + log_level = getattr(logging, log_level.upper(), logging.INFO) + + # Create logger + logger = logging.getLogger(app_name) + logger.setLevel(log_level) + logger.handlers = [] # Remove existing handlers + + # Create formatters + if json_format: + json_formatter = JSONFormatter() + console_formatter = json_formatter + else: + # Human-readable format for console + console_formatter = logging.Formatter( + '%(asctime)s [%(levelname)s] [%(name)s] [%(correlation_id)s] %(message)s' + ) + # JSON for file even if console is human-readable + json_formatter = JSONFormatter() + + # Add AWS context filter if provided + if aws_context: + context_filter = ContextFilter( + account_id=aws_context.get("account_id"), + region=aws_context.get("region"), + service_name=aws_context.get("service_name") + ) + logger.addFilter(context_filter) + + # Add console handler if requested + if console_output: + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(console_formatter) + logger.addHandler(console_handler) + + # Add file handler if requested + if log_file: + os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True) + file_handler = RotatingFileHandler( + log_file, + maxBytes=max_log_size, + backupCount=backup_count + ) + file_handler.setFormatter(json_formatter) + logger.addHandler(file_handler) + + return logger + +def get_logger(name: str) -> logging.Logger: + """ + Get a logger with the given name, inheriting configuration from the root logger. + + Args: + name: Logger name + + Returns: + Logger instance + """ + return logging.getLogger(name) diff --git a/local-app/python-tools/gfl-resource-actions/logging_utils.py b/local-app/python-tools/gfl-resource-actions/logging_utils.py index 8f22f3b0..b0ac0083 100644 --- a/local-app/python-tools/gfl-resource-actions/logging_utils.py +++ b/local-app/python-tools/gfl-resource-actions/logging_utils.py @@ -107,3 +107,151 @@ def log_action_to_csv(account_id, account_name, resource_type, resource_id, reso # Create a logger instance for direct import logger = setup_logging() + +import logging +from typing import Any, Dict, Optional + +from logging_setup import LoggingContext, log_operation, log_with_context + +logger = logging.getLogger('aws_resource_management') + +def log_resource_operation( + level: int, + operation: str, + resource_type: str, + resource_id: str, + region: str, + details: Optional[Dict[str, Any]] = None, + exception: Optional[Exception] = None +) -> None: + """ + Log resource operations with consistent structure. + + Args: + level: Log level + operation: Operation being performed (e.g., 'start', 'stop') + resource_type: AWS resource type (e.g., 'ec2_instance', 'rds_instance') + resource_id: Resource identifier + region: AWS region + details: Optional additional details + exception: Optional exception if the operation failed + """ + msg = f"{operation} {resource_type} {resource_id} in {region}" + context = { + "resource_type": resource_type, + "resource_id": resource_id, + "operation": operation, + "region": region + } + + if details: + context["details"] = details + msg += f": {details}" + + if exception: + context["exception"] = str(exception) + context["exception_type"] = exception.__class__.__name__ + msg += f" (failed: {exception})" + + log_with_context(logger, level, msg, **context) + +def initialize_aws_logging( + app_name: str, + log_level: str = "INFO", + account_id: Optional[str] = None, + region: Optional[str] = None, + log_to_file: bool = False, + log_file_path: Optional[str] = None +) -> logging.Logger: + """ + Initialize AWS-aware logging for the application. + + Args: + app_name: Application name + log_level: Log level name + account_id: AWS account ID + region: AWS default region + log_to_file: Whether to log to a file + log_file_path: Path to log file (if log_to_file is True) + + Returns: + Configured logger instance + """ + from logging_setup import configure_logging + + # Set up default log file path if needed + if log_to_file and not log_file_path: + import os + log_dir = os.path.join(os.path.expanduser("~"), ".aws-resource-management", "logs") + os.makedirs(log_dir, exist_ok=True) + log_file_path = os.path.join(log_dir, f"{app_name}.log") + + # Configure AWS context + aws_context = {} + if account_id: + aws_context["account_id"] = account_id + if region: + aws_context["region"] = region + + # Initialize logging + return configure_logging( + app_name=app_name, + log_level=log_level, + json_format=True, + log_file=log_file_path if log_to_file else None, + console_output=True, + aws_context=aws_context + ) + +# Example usage function to demonstrate the structured logging features +def example_usage(): + # Initialize logging + logger = initialize_aws_logging( + app_name="resource-manager", + log_level="INFO", + account_id="123456789012", + region="us-west-2", + log_to_file=True + ) + + # Simple logging with context + log_with_context( + logger, + logging.INFO, + "Starting application", + app_version="1.0.0", + environment="development" + ) + + # Log resource operation + log_resource_operation( + logging.INFO, + "start", + "ec2_instance", + "i-1234567890abcdef0", + "us-west-2", + details={"instance_type": "t2.micro", "target_state": "running"} + ) + + # Use operation context manager + try: + with log_operation(logger, "resize_rds_instance", logging.INFO, + resource_type="rds_instance", + resource_id="mydb-instance-1", + region="us-west-2"): + # Simulated operation that takes time + import time + time.sleep(1.5) + + # Set additional context during operation + LoggingContext.set(instance_class="db.t3.large") + + # Log with the updated context + log_with_context(logger, logging.INFO, "Changed instance class") + + # Simulate successful completion + pass + except Exception as e: + # The log_operation context manager will log the exception + # and re-raise it + pass diff --git a/local-app/python-tools/gfl-resource-actions/main.py b/local-app/python-tools/gfl-resource-actions/main.py index 36d4c291..37624fd0 100644 --- a/local-app/python-tools/gfl-resource-actions/main.py +++ b/local-app/python-tools/gfl-resource-actions/main.py @@ -4,317 +4,64 @@ """ import argparse +from concurrent.futures import ThreadPoolExecutor import config -import sys -from collections import defaultdict from logging_utils import setup_logging from aws_utils import get_available_regions, assume_role, get_organization_accounts from discovery import get_account_resources from managers import EC2Manager, RDSManager, EKSManager, EMRManager -# Application version information -APP_NAME = "AWS Organization Resource Management Tool" -APP_VERSION = "1.0.0" # Semantic versioning: MAJOR.MINOR.PATCH - logger = setup_logging() -def parse_arguments(): - """Parse command-line arguments""" - parser = argparse.ArgumentParser(description="AWS Organization Resource Management Tool") - parser.add_argument("--action", choices=["stop", "start"], default="stop", - help="Action to perform: stop or start resources (default: stop)") - parser.add_argument("--dry-run", action="store_true", help="List resources without modifying them") - parser.add_argument("--regions", type=str, help="Comma-separated list of regions to scan") - parser.add_argument("--exclude-accounts", type=str, help="Comma-separated list of account IDs to exclude") - parser.add_argument("--exclude-resources", choices=["ec2", "rds", "eks", "emr"], nargs="+", - help="Resource types to exclude from management") - parser.add_argument("--help-detail", action="store_true", help="Show detailed help information and exit") - parser.add_argument("--version", action="store_true", help="Show version information and exit") - return parser.parse_args() - -def show_detailed_help(): - """Show detailed help information about the tool""" - help_text = """ -AWS Organization Resource Management Tool -======================================== - -This tool helps manage AWS resources across multiple accounts in an organization. - -ACTIONS: - stop - Stop running resources (EC2, RDS, EKS, EMR) - start - Start stopped resources - -OPTIONS: - --dry-run - Only show resources that would be affected without making changes - --regions - Specify specific AWS regions to scan (comma separated) - --exclude-accounts - Accounts to skip (comma separated account IDs) - --exclude-resources - Resource types to skip (ec2, rds, eks, emr) - --help-detail - Display this detailed help message - -EXAMPLES: - # Stop all resources in all accounts (dry run mode) - python main.py --action stop --dry-run - - # Start all resources in specific regions - python main.py --action start --regions us-east-1,us-west-2 - - # Stop only EC2 and RDS instances - python main.py --exclude-resources eks emr - """ - print(help_text) +def process_region(credentials, region, exclude_resources): + """Process resources in a single region.""" + try: + return get_account_resources(credentials, [region], exclude_resources) + except Exception as e: + logger.error(f"Error processing region {region}: {e}") + return { + "ec2_instances": [], + "rds_instances": [], + "eks_clusters": [], + "emr_clusters": [] + } -def display_summary(stats, action, was_interrupted=False): - """Display a summary of the actions taken""" - print("\n" + "="*50) - title = f"SUMMARY OF {action.upper()} OPERATION" - if was_interrupted: - title += " (INTERRUPTED)" - print(title) - print("="*50) - - print(f"\nAccounts processed: {stats['accounts_processed']}") - print(f"Accounts skipped: {stats['accounts_skipped']}") - - # Handle irregular verb conjugation - action_past = "stopped" if action == "stop" else "started" - - # EC2 summary - print("\nEC2 Instances:") - print(f" Found: {stats['ec2_found']}") - print(f" Skipped: {stats['ec2_skipped']}") - print(f" {action_past.capitalize()}: {stats['ec2_' + action_past]}") - print(f" Errors: {stats['ec2_errors']}") - - # RDS summary - print("\nRDS Instances:") - print(f" Found: {stats['rds_found']}") - print(f" Skipped: {stats['rds_skipped']}") - print(f" {action_past.capitalize()}: {stats['rds_' + action_past]}") - print(f" Errors: {stats['rds_errors']}") - - # RDS by engine type - if stats['rds_engines'] and stats['rds_found'] > 0: - print("\n RDS by Engine Type:") - for engine, count in stats['rds_engines'].items(): - print(f" {engine}: {count}") - - # EKS summary - print("\nEKS Clusters:") - print(f" Found: {stats['eks_found']}") - print(f" Skipped: {stats['eks_skipped']}") - print(f" {action_past.capitalize()}: {stats['eks_' + action_past]}") - print(f" Errors: {stats['eks_errors']}") - - # EMR summary - print("\nEMR Clusters:") - print(f" Found: {stats['emr_found']}") - print(f" Skipped: {stats['emr_skipped']}") - print(f" {action_past.capitalize()}: {stats['emr_' + action_past]}") - print(f" Errors: {stats['emr_errors']}") - - print("\n" + "="*50) +def main(): + # ...existing code until after args parsing... -def process_account(account, credentials, regions, action, dry_run, exclude_resources, stats): - """Process a single account and update stats""" - logger.info(f"Processing account: {account['name']} ({account['id']})") - - # Get resources - resources = get_account_resources(credentials, regions, exclude_resources) - if not resources: - logger.error(f"Failed to get resources for account {account['id']}") - return - - # Count service-managed EC2 instances - eks_managed = sum(1 for i in resources.get('ec2_instances', []) if config.EKS_TAG in i.get('tags', {})) - emr_managed = sum(1 for i in resources.get('ec2_instances', []) if config.EMR_TAG in i.get('tags', {})) - - # Update resource counts - total_ec2 = len(resources.get('ec2_instances', [])) - excluded_ec2 = sum(1 for i in resources.get('ec2_instances', []) if config.EXCLUSION_TAG in i.get('tags', {})) - stats['ec2_found'] += total_ec2 - stats['ec2_skipped'] += excluded_ec2 - - stats['rds_found'] += len(resources.get('rds_instances', [])) - stats['eks_found'] += len(resources.get('eks_clusters', [])) - stats['emr_found'] += len(resources.get('emr_clusters', [])) - - # Track RDS engines - for instance in resources.get('rds_instances', []): - if instance and 'engine' in instance: - engine = instance['engine'] - stats['rds_engines'][engine] += 1 - - # Log discovered resources with service-managed counts - logger.info(f"Account {account['id']} - Found {total_ec2} EC2 instances ({excluded_ec2} explicitly excluded, {eks_managed} EKS-managed, {emr_managed} EMR-managed)") - logger.info(f"Account {account['id']} - Found {len(resources.get('rds_instances', []))} RDS instances") - logger.info(f"Account {account['id']} - Found {len(resources.get('eks_clusters', []))} EKS clusters") - logger.info(f"Account {account['id']} - Found {len(resources.get('emr_clusters', []))} EMR clusters") - - # Initialize resource managers with account name - ec2_manager = EC2Manager(credentials, account['id'], account['name']) - rds_manager = RDSManager(credentials, account['id'], account['name']) - eks_manager = EKSManager(credentials, account['id'], account['name']) - emr_manager = EMRManager(credentials, account['id'], account['name']) - - # Helper function to safely process manager results - def safe_get_result(result, key): - if result is None: - return 0 - return result.get(key, 0) - - # Perform actions based on selected mode - if action == "stop": - if "ec2" not in exclude_resources: - result = ec2_manager.stop(resources.get('ec2_instances', []), dry_run) - stats['ec2_stopped'] += safe_get_result(result, 'success') - stats['ec2_errors'] += safe_get_result(result, 'errors') - else: - stats['ec2_skipped'] += total_ec2 - excluded_ec2 + # Process each account + for account in accounts: + logger.info(f"Processing account: {account['name']} ({account['id']}) in {len(regions)} regions") - if "rds" not in exclude_resources: - result = rds_manager.stop(resources.get('rds_instances', []), dry_run) - stats['rds_stopped'] += safe_get_result(result, 'success') - stats['rds_errors'] += safe_get_result(result, 'errors') - else: - stats['rds_skipped'] += len(resources.get('rds_instances', [])) - - if "eks" not in exclude_resources: - result = eks_manager.stop(resources.get('eks_clusters', []), dry_run) - stats['eks_stopped'] += safe_get_result(result, 'success') - stats['eks_errors'] += safe_get_result(result, 'errors') - else: - stats['eks_skipped'] += len(resources.get('eks_clusters', [])) - - if "emr" not in exclude_resources: - result = emr_manager.stop(resources.get('emr_clusters', []), dry_run) - stats['emr_stopped'] += safe_get_result(result, 'success') - stats['emr_errors'] += safe_get_result(result, 'errors') - else: - stats['emr_skipped'] += len(resources.get('emr_clusters', [])) - else: # action == "start" - if "ec2" not in exclude_resources: - result = ec2_manager.start(resources.get('ec2_instances', []), dry_run) - stats['ec2_started'] += safe_get_result(result, 'success') - stats['ec2_errors'] += safe_get_result(result, 'errors') - else: - stats['ec2_skipped'] += total_ec2 - excluded_ec2 - - if "rds" not in exclude_resources: - result = rds_manager.start(resources.get('rds_instances', []), dry_run) - stats['rds_started'] += safe_get_result(result, 'success') - stats['rds_errors'] += safe_get_result(result, 'errors') - else: - stats['rds_skipped'] += len(resources.get('rds_instances', [])) + credentials = assume_role(account['id']) + if not credentials: + logger.warning(f"Skipping account {account['id']} due to role assumption failure") + continue + + # Process regions in parallel + all_resources = { + "ec2_instances": [], + "rds_instances": [], + "eks_clusters": [], + "emr_clusters": [] + } - if "eks" not in exclude_resources: - result = eks_manager.start(resources.get('eks_clusters', []), dry_run) - stats['eks_started'] += safe_get_result(result, 'success') - stats['eks_errors'] += safe_get_result(result, 'errors') - else: - stats['eks_skipped'] += len(resources.get('eks_clusters', [])) + with ThreadPoolExecutor(max_workers=4) as executor: + future_to_region = { + executor.submit(process_region, credentials, region, exclude_resources): region + for region in regions + } - if "emr" not in exclude_resources: - result = emr_manager.start(resources.get('emr_clusters', []), dry_run) - stats['emr_started'] += safe_get_result(result, 'success') - stats['emr_errors'] += safe_get_result(result, 'errors') - else: - stats['emr_skipped'] += len(resources.get('emr_clusters', [])) + for future in concurrent.futures.as_completed(future_to_region): + region = future_to_region[future] + try: + region_resources = future.result() + for resource_type in all_resources: + all_resources[resource_type].extend(region_resources[resource_type]) + except Exception as e: + logger.error(f"Error processing region {region}: {e}") -def initialize_stats(): - """Initialize the statistics tracking dictionary""" - stats = { - 'accounts_processed': 0, - 'accounts_skipped': 0, - 'ec2_found': 0, - 'ec2_skipped': 0, - 'ec2_stopped': 0, - 'ec2_started': 0, - 'ec2_errors': 0, - 'rds_found': 0, - 'rds_skipped': 0, - 'rds_stopped': 0, - 'rds_started': 0, - 'rds_errors': 0, - 'eks_found': 0, - 'eks_skipped': 0, - 'eks_stopped': 0, - 'eks_started': 0, - 'eks_errors': 0, - 'emr_found': 0, - 'emr_skipped': 0, - 'emr_stopped': 0, - 'emr_started': 0, - 'emr_errors': 0, - } - # Add RDS engine tracking using defaultdict - stats['rds_engines'] = defaultdict(int) - return stats - -def main(): - """Main execution function.""" - args = parse_arguments() - - # Check if version information was requested - if args.version: - print(f"{APP_NAME} v{APP_VERSION}") - sys.exit(0) - - # Check if detailed help was requested - if args.help_detail: - show_detailed_help() - sys.exit(0) - - # Process arguments - action = args.action - dry_run = args.dry_run - regions = args.regions.split(',') if args.regions else get_available_regions() - exclude_accounts = args.exclude_accounts.split(',') if args.exclude_accounts else [] - exclude_resources = args.exclude_resources or [] - - # Initialize statistics dictionary - stats = initialize_stats() - - logger.info(f"Starting AWS organization resource {action} {'(DRY RUN)' if dry_run else ''}") - - # Get organization accounts - accounts = get_organization_accounts(exclude_accounts) - - # Flag to track interruptions - was_interrupted = False - - try: - # Process each account - for account in accounts: - try: - # Assume role in the account - credentials = assume_role(account['id']) - if not credentials: - logger.warning(f"Skipping account {account['id']} due to role assumption failure") - stats['accounts_skipped'] += 1 - continue - - stats['accounts_processed'] += 1 - - # Process this account - process_account(account, credentials, regions, action, dry_run, exclude_resources, stats) - - except Exception as e: - logger.error(f"Error processing account {account['id']}: {str(e)}") - stats['accounts_skipped'] += 1 - - logger.info(f"Resource {action} completed {'(DRY RUN)' if dry_run else ''}") - - except KeyboardInterrupt: - was_interrupted = True - print("\n\nProcess interrupted by user (Ctrl+C)") - logger.warning("Process was interrupted by user (Ctrl+C)") - - # Display summary of actions (will run even if interrupted) - display_summary(stats, action, was_interrupted) - - # Exit with appropriate code - if was_interrupted: - sys.exit(130) # Standard exit code for Ctrl+C/SIGINT + # ...rest of existing code... if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/local-app/python-tools/gfl-resource-actions/managers/base.py b/local-app/python-tools/gfl-resource-actions/managers/base.py index 6842eb4b..f518ff30 100644 --- a/local-app/python-tools/gfl-resource-actions/managers/base.py +++ b/local-app/python-tools/gfl-resource-actions/managers/base.py @@ -1,59 +1,118 @@ """ -Base resource manager class that provides common functionality. +Base resource manager class that provides common functionality for AWS resource management. + +This module contains the ResourceManager base class which handles common operations +such as client creation, action logging, and resource filtering. """ import datetime +from typing import Dict, Optional, Any, Union import boto3 +from botocore.exceptions import ClientError import config from aws_utils import get_partition_for_region from logging_utils import setup_logging, log_action_to_csv logger = setup_logging() + class ResourceManager: - """Base class for all resource managers.""" + """Base class for all resource managers providing common AWS resource functionality.""" - def __init__(self, credentials, account_id, account_name=None): + def __init__(self, credentials: Dict[str, str], account_id: str, account_name: Optional[str] = None): """ - Initialize resource manager with credentials. + Initialize resource manager with AWS credentials. Args: - credentials: AWS credentials dictionary + credentials: AWS credentials dictionary containing AccessKeyId, SecretAccessKey, and SessionToken account_id: AWS account ID - account_name: AWS account name (optional) + account_name: AWS account name (optional, defaults to account_id if not provided) """ self.credentials = credentials self.account_id = account_id self.account_name = account_name or account_id self.resource_type = "generic" + self._clients = {} # Cache for boto3 clients - def create_client(self, service, region): - """Create a boto3 client for the specified service and region.""" + def create_client(self, service: str, region: str) -> Optional[boto3.client]: + """ + Create a boto3 client for the specified service and region. + + Args: + service: AWS service name (e.g., 'ec2', 's3') + region: AWS region name (e.g., 'us-east-1') + + Returns: + Configured boto3 client or None if credentials are not available + + Raises: + ClientError: If there's an error creating the client + """ if not self.credentials: + logger.warning(f"No credentials available to create {service} client in {region}") return None + + # Use client caching to avoid creating redundant clients + cache_key = f"{service}:{region}" + if cache_key in self._clients: + return self._clients[cache_key] - return boto3.client( - service, - region_name=region, - aws_access_key_id=self.credentials['AccessKeyId'], - aws_secret_access_key=self.credentials['SecretAccessKey'], - aws_session_token=self.credentials['SessionToken'] - ) + try: + client = boto3.client( + service, + region_name=region, + aws_access_key_id=self.credentials['AccessKeyId'], + aws_secret_access_key=self.credentials['SecretAccessKey'], + aws_session_token=self.credentials['SessionToken'] + ) + self._clients[cache_key] = client + return client + except ClientError as e: + logger.error(f"Failed to create {service} client in {region}: {str(e)}") + raise - # Use aws_utils function directly rather than duplicating logic - def get_partition_for_region(self, region): - """Get AWS partition for the specified region.""" + def get_partition_for_region(self, region: str) -> str: + """ + Get AWS partition for the specified region. + + Args: + region: AWS region name + + Returns: + AWS partition (e.g., 'aws', 'aws-cn', 'aws-us-gov') + """ return get_partition_for_region(region) - def get_timestamp(self): - """Get ISO 8601 timestamp for tagging.""" + def get_timestamp(self) -> str: + """ + Get ISO 8601 timestamp for tagging and logging purposes. + + Returns: + Current UTC timestamp in ISO 8601 format + """ return datetime.datetime.utcnow().isoformat() - def should_exclude(self, resource): - """Check if resource should be excluded based on tags.""" + def should_exclude(self, resource: Dict[str, Any]) -> bool: + """ + Check if resource should be excluded based on tags. + + Args: + resource: Resource dictionary containing tags + + Returns: + True if the resource should be excluded, False otherwise + """ tags = resource.get("tags", {}) return config.EXCLUSION_TAG in tags - def log_action(self, resource_id, region, action, resource_name=None, details=None, dry_run=False, existing_schedule=None): + def log_action(self, + resource_id: str, + region: str, + action: str, + resource_name: Optional[str] = None, + details: Optional[str] = None, + dry_run: bool = False, + existing_schedule: Optional[str] = None, + status: str = "success") -> None: """ Log an action performed on a resource. @@ -65,6 +124,7 @@ def log_action(self, resource_id, region, action, resource_name=None, details=No details: Optional additional details dry_run: Whether this was a dry run existing_schedule: Existing schedule for the resource + status: Action status (default: 'success') """ dry_run_prefix = "[DRY RUN] " if dry_run else "" @@ -77,7 +137,8 @@ def log_action(self, resource_id, region, action, resource_name=None, details=No if self.account_name and self.account_name != self.account_id: account_desc = f"{self.account_name} ({self.account_id})" - action_desc = f"{action}ed" if action[-1] != 'e' else f"{action}d" + # Convert action to past tense for logging + action_desc = f"{action}ed" if not action.endswith('e') else f"{action}d" message = f"{dry_run_prefix}{self.resource_type.upper()} {resource_desc} {action_desc} in {account_desc} region {region}" @@ -88,7 +149,8 @@ def log_action(self, resource_id, region, action, resource_name=None, details=No if existing_schedule: message += f" (Schedule: {existing_schedule})" - # Use logging_utils.log_action_to_csv for consistent CSV logging across the application + # Use logging_utils.log_action_to_csv for consistent CSV logging + status_value = "simulated" if dry_run else status log_action_to_csv( account_id=self.account_id, account_name=self.account_name, @@ -97,9 +159,17 @@ def log_action(self, resource_id, region, action, resource_name=None, details=No resource_name=resource_name or resource_id, # Use ID as name if not provided action=action, region=region, - status="simulated" if dry_run else "success", + status=status_value, details=details, dry_run=dry_run, existing_schedule=existing_schedule ) logger.info(message) + + def cleanup(self) -> None: + """ + Perform cleanup operations when the manager is no longer needed. + This method should be called when finished with the resource manager. + """ + # Clear client cache to allow garbage collection + self._clients.clear() diff --git a/local-app/python-tools/gfl-resource-actions/plan.md b/local-app/python-tools/gfl-resource-actions/plan.md new file mode 100644 index 00000000..5209f1f6 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/plan.md @@ -0,0 +1,681 @@ +Step-by-Step Improvement Plan for AWS Resource Management Tool +1. ✅ Code Organization and Structure (COMPLETED) +Currently, the codebase is well-organized with separate modules for different resource managers: + +✅ Implemented proper Python packaging with a setup.py file for easier installation +✅ Added type hints throughout the codebase for better IDE support and code quality +✅ Created a dedicated config management module with environment variable support +✅ Separated CLI functionality from core business logic for better testability +✅ Added AWS SSO profile support for credential management +✅ Added multi-region discovery and support for checking all regions per account + +2. Error Handling and Logging +The logging implementation could be enhanced: + +2.1 Structured Logging: +- Expand the existing JSONFormatter class in logging_setup.py to include additional context: + - Add AWS-specific fields: account_id, region, resource_type, resource_id, request_id + - Include operation context: operation_name, operation_status, duration_ms + - Add correlation_id field to track related log entries across a single operation + - Support custom context through thread-local storage or LogRecord extra parameter + - Include hostname, process ID, and thread ID for distributed debugging + +- Implementation approach: + - Create a LoggingContext class to store and manage contextual information + - Add a log_with_context() utility function that includes current context + - Update the JSONFormatter.format() method to incorporate context + - Add context manager for tracking operation timing and results + - Create logging filter to automatically enrich logs with AWS context + +- Configuration improvements: + - Add support for different log formats per handler (JSON for files, human-readable for console) + - Include log rotation by size and time with configurable retention + - Support configurable log destinations (file, console, syslog, CloudWatch Logs) + - Add option to disable certain context fields for privacy/security + +- Sample implementation structure: + ```python + # Context management + class LoggingContext: + _context = threading.local() + @classmethod + def set(cls, **kwargs): ... + @classmethod + def get(cls): ... + @classmethod + def clear(cls): ... + + # Context manager for operations + @contextmanager + def log_operation(name, **context): + start = time.time() + LoggingContext.set(operation_name=name, **context) + try: + yield + duration = (time.time() - start) * 1000 + LoggingContext.set(operation_status="success", duration_ms=duration) + logger.info(f"Operation {name} completed successfully") + except Exception as e: + duration = (time.time() - start) * 1000 + LoggingContext.set(operation_status="failed", duration_ms=duration) + logger.error(f"Operation {name} failed: {e}") + raise + finally: + LoggingContext.clear() + ``` + +- Handling sensitive information: + - Create a SanitizingFormatter that masks sensitive fields (access keys, passwords) + - Define a list of sensitive field patterns to detect and mask + - Use regex patterns to identify potential sensitive data in unstructured log messages + - Add configuration options to control sanitization levels + +2.2 Granular Log Levels: +- Implementation details for each log level: + - DEBUG: + - AWS API request/response bodies and headers + - Parameter values for all functions + - Resource discovery detailed progress + - Authentication token acquisition + - Configuration loading and resolution steps + - Connection attempts and retries + + - INFO: + - Resource operations start/completion (e.g., "Starting EC2 instance i-123456") + - Resource state changes (e.g., "RDS instance changed to 'stopping'") + - Important configuration values loaded + - Number of resources discovered/affected + - Operation timing information + - Profile and region information in use + + - WARNING: + - Resources excluded due to tags + - Missing optional configuration + - Slow API responses + - Resources in unexpected states + - Deprecated API usage + - Retrying operations after recoverable errors + - Missing tags or metadata + + - ERROR: + - Failed operations on specific resources + - Authentication or permission issues + - Invalid parameters or configurations + - API rate limiting or throttling + - Network connectivity issues to specific regions + - Resource not found + + - CRITICAL: + - Complete failure to authenticate + - Fatal configuration errors + - Inability to access any AWS services + - Unrecoverable program state + +- Implementation approach: + - Create logging helper functions for consistent usage: + ```python + def log_resource_operation(logger, level, operation, resource_type, resource_id, + region, details=None, exception=None): + """Log resource operations with consistent structure.""" + msg = f"{operation} {resource_type} {resource_id} in {region}" + extra = { + "resource_type": resource_type, + "resource_id": resource_id, + "operation": operation, + "region": region + } + if details: + extra["details"] = details + msg += f": {details}" + if exception: + extra["exception"] = str(exception) + msg += f" (failed: {exception})" + logger.log(level, msg, extra=extra) + ``` + + - Add log level configuration: + - Command line option to override log level + - Environment variable for log level + - Configuration file setting + - Different log levels for console vs file output + + - Create module-level logger constants with appropriate levels: + ```python + # Base module logger + logger = logging.getLogger('aws_resource_management') + + # Component-specific loggers with potential different levels + api_logger = logger.getChild('api') # For AWS API calls + discovery_logger = logger.getChild('discovery') # For resource discovery + operation_logger = logger.getChild('operation') # For resource operations + ``` + + - Add utility function to adjust verbosity: + ```python + def set_verbosity(level_name: str) -> None: + """ + Set the verbosity level of the application. + + Arguments: + level_name: One of 'debug', 'info', 'warning', 'error', 'critical' + """ + level = getattr(logging, level_name.upper()) + logger.setLevel(level) + + # Adjust component loggers as needed + if level <= logging.DEBUG: + api_logger.setLevel(logging.DEBUG) # Always show API details in debug mode + else: + api_logger.setLevel(logging.INFO) # Otherwise just show API operation summaries + ``` + +2.3 Error Handling Module: +- Create aws_errors.py module with custom exception hierarchy: + - BaseAWSResourceError (base exception): + - Common attributes for all errors: error_code, message, details + - Methods for serializing errors to JSON + - Support for contextual information (account_id, region, etc.) + + - ConfigurationError (config issues): + - Missing required configuration + - Invalid configuration values + - Configuration file parsing errors + - Environment variable conflicts + + - AuthenticationError (credential issues): + - Invalid credentials + - Expired credentials + - Missing credentials + - Insufficient permissions + + - ResourceOperationError (resource actions): + - Failed state transitions + - Resource in incompatible state + - Resource already in target state + - Dependencies preventing operation + - Resource-specific error subclasses (EC2Error, RDSError, etc.) + + - NetworkError (connectivity issues): + - Connection timeouts + - Rate limiting/throttling + - DNS resolution failures + - Endpoint unavailability + + - ValidationError (input validation): + - Invalid parameter types + - Value range violations + - Missing required parameters + - Incompatible parameter combinations + +- Implementation approach: + - Define error codes for each exception type: + ```python + class ErrorCode(Enum): + # General errors (1-99) + UNKNOWN_ERROR = 1 + INTERNAL_ERROR = 2 + + # Configuration errors (100-199) + CONFIG_NOT_FOUND = 100 + CONFIG_PARSE_ERROR = 101 + CONFIG_VALIDATION_ERROR = 102 + + # Authentication errors (200-299) + AUTH_INVALID_CREDENTIALS = 200 + AUTH_EXPIRED_CREDENTIALS = 201 + AUTH_INSUFFICIENT_PERMISSIONS = 202 + + # Resource operation errors (300-399) + RESOURCE_NOT_FOUND = 300 + RESOURCE_STATE_CONFLICT = 301 + RESOURCE_DEPENDENCY_ERROR = 302 + + # Service-specific error ranges + # EC2 errors (1000-1099) + EC2_INVALID_STATE = 1000 + EC2_INSTANCE_NOT_FOUND = 1001 + + # RDS errors (1100-1199) + RDS_INVALID_STATE = 1100 + RDS_INSTANCE_NOT_FOUND = 1101 + ``` + + - Add contextual information to exceptions: + ```python + class BaseAWSResourceError(Exception): + """Base exception for all AWS Resource Management errors.""" + + def __init__( + self, + message: str, + error_code: ErrorCode = ErrorCode.UNKNOWN_ERROR, + account_id: Optional[str] = None, + region: Optional[str] = None, + resource_type: Optional[str] = None, + resource_id: Optional[str] = None, + details: Optional[Dict[str, Any]] = None, + original_exception: Optional[Exception] = None + ): + self.message = message + self.error_code = error_code + self.account_id = account_id + self.region = region + self.resource_type = resource_type + self.resource_id = resource_id + self.details = details or {} + self.original_exception = original_exception + + # Build detailed error message + detailed_msg = f"[{error_code.name}] {message}" + if resource_type and resource_id: + detailed_msg += f" (Resource: {resource_type}/{resource_id})" + if region: + detailed_msg += f" in region {region}" + + super().__init__(detailed_msg) + + def to_dict(self) -> Dict[str, Any]: + """Convert exception to a dictionary for serialization.""" + result = { + "error_code": self.error_code.value, + "error_name": self.error_code.name, + "message": self.message, + } + + # Add contextual information if available + for field in ["account_id", "region", "resource_type", "resource_id"]: + if getattr(self, field): + result[field] = getattr(self, field) + + # Add any additional details + if self.details: + result["details"] = self.details + + return result + ``` + + - Add utility functions for error handling: + ```python + def handle_boto3_error( + boto3_error: botocore.exceptions.ClientError, + resource_type: Optional[str] = None, + resource_id: Optional[str] = None, + region: Optional[str] = None, + operation: Optional[str] = None + ) -> BaseAWSResourceError: + """Convert boto3 errors to our custom exceptions with proper context.""" + error_code = boto3_error.response.get("Error", {}).get("Code", "Unknown") + error_message = boto3_error.response.get("Error", {}).get("Message", str(boto3_error)) + + # Map boto3 error codes to our error codes + if error_code == "AuthFailure": + return AuthenticationError( + f"Authentication failed: {error_message}", + error_code=ErrorCode.AUTH_INVALID_CREDENTIALS, + resource_type=resource_type, + resource_id=resource_id, + region=region, + details={"operation": operation}, + original_exception=boto3_error + ) + + # Handle throttling/rate limiting + if error_code == "Throttling" or error_code == "ThrottlingException": + return NetworkError( + f"API rate limit exceeded: {error_message}", + error_code=ErrorCode.API_THROTTLING, + resource_type=resource_type, + resource_id=resource_id, + region=region, + details={"operation": operation}, + original_exception=boto3_error + ) + + # Default to ResourceOperationError for other cases + return ResourceOperationError( + f"AWS operation failed: {error_message}", + error_code=ErrorCode.from_boto3_code(error_code), + resource_type=resource_type, + resource_id=resource_id, + region=region, + details={ + "operation": operation, + "boto3_error_code": error_code + }, + original_exception=boto3_error + ) + ``` + +- Error code mapping: + - Map AWS/boto3 error codes to internal error codes + - Create utility to convert between different error representations + - Support HTTP status codes for API responses + +- Integration with logging: + - Log exceptions with appropriate levels based on severity + - Include stack traces for unexpected errors + - Provide consistent error reporting format + +2.4 Stack Trace and Debug Information: +- Comprehensive exception handling: + - Use `traceback` module to capture and format stack traces + - Include stack trace context in log messages at DEBUG level + - Create wrapper functions for common try/except patterns + - Record exception chain for nested exceptions + +- Implementation approach for stack trace logging: + ```python + def log_exception( + logger, + exc: Exception, + msg: str = "An exception occurred", + level: int = logging.ERROR, + include_traceback: bool = True, + include_cause_chain: bool = True, + context: Optional[Dict[str, Any]] = None + ) -> None: + """ + Log an exception with stack trace at the specified level. + + Args: + logger: Logger to use + exc: The exception to log + msg: Message to log with the exception + level: Log level to use + include_traceback: Whether to include the stack trace + include_cause_chain: Whether to include cause chain for chained exceptions + context: Additional context to include in the log + """ + exc_info = sys.exc_info() if include_traceback else None + + # Extract info from exception + extra = context or {} + extra['exception_type'] = exc.__class__.__name__ + extra['exception_msg'] = str(exc) + + # Format cause chain if requested and available + if include_cause_chain and exc.__cause__: + causes = [] + current = exc.__cause__ + while current: + causes.append(f"{current.__class__.__name__}: {str(current)}") + current = current.__cause__ + + extra['exception_causes'] = causes + + # Log the exception + logger.log(level, msg, exc_info=exc_info, extra=extra) + ``` + +- AWS API request/response logging: + - Create a boto3 event hook for logging API calls + - Log request parameters and response data + - Sanitize sensitive information in requests/responses + - Track AWS request IDs for correlation + - Calculate and log API latency metrics + +- Implementation approach for boto3 request logging: + ```python + def add_request_logging_to_session(session: boto3.Session, log_level: int = logging.DEBUG) -> None: + """ + Add request/response logging hooks to a boto3 session. + + Args: + session: boto3 Session to instrument + log_level: Log level to use for API requests/responses + """ + api_logger = logging.getLogger('aws_resource_management.api') + + def log_request(request, **kwargs): + # Create a unique ID for this request for correlation + request_id = str(uuid.uuid4()) + request.context['request_log_id'] = request_id + + # Get sanitized parameters (remove sensitive info) + params = _sanitize_params(request.context.get('params', {})) + + # Log the request details + api_logger.log( + log_level, + f"AWS API Request: {request.context.get('operation_name')}", + extra={ + 'request_log_id': request_id, + 'service': request.context.get('client_meta').service_model.service_name, + 'operation': request.context.get('operation_name'), + 'params': params, + 'region': request.context.get('client_region'), + 'endpoint_url': request.context.get('client_meta').endpoint_url, + } + ) + + def log_response(request, response, **kwargs): + # Get the request ID we set earlier + request_id = getattr(request.context, 'request_log_id', 'unknown') + + # Get AWS request ID from response + aws_request_id = response.get('ResponseMetadata', {}).get('RequestId', 'unknown') + + # Calculate latency + latency_ms = response.get('ResponseMetadata', {}).get('HTTPHeaders', {}).get('x-amz-request-latency', -1) + + # Log the response + api_logger.log( + log_level, + f"AWS API Response: {request.context.get('operation_name')}", + extra={ + 'request_log_id': request_id, + 'aws_request_id': aws_request_id, + 'status_code': response.get('ResponseMetadata', {}).get('HTTPStatusCode'), + 'latency_ms': latency_ms, + 'service': request.context.get('client_meta').service_model.service_name, + 'operation': request.context.get('operation_name'), + } + ) + + # Register the event handlers with boto3 + session.events.register('before-send.*.*.', log_request) + session.events.register('after-call.*.*.', log_response) + ``` + +- Utility functions for error handling and reporting: + - Create consistent error message formatting + - Extract useful error information from boto3 exceptions + - Format error messages for CLI output vs. logs + - Handle known error patterns with custom messages + - Retrieve error documentation links for common errors + +- Implementation approach for error reporting: + ```python + def format_error_for_display( + error: Exception, + verbose: bool = False, + include_help: bool = True + ) -> str: + """ + Format an error for display to users. + + Args: + error: The exception to format + verbose: Whether to include detailed information + include_help: Whether to include help text for known errors + + Returns: + Formatted error message + """ + # Handle our custom exceptions + if isinstance(error, BaseAWSResourceError): + message = f"Error ({error.error_code.name}): {error.message}" + + # Add context for resource errors if available + if error.resource_type and error.resource_id: + message += f"\nResource: {error.resource_type}/{error.resource_id}" + if error.region: + message += f"\nRegion: {error.region}" + + # Add details for verbose mode + if verbose and error.details: + message += "\nDetails:" + for k, v in error.details.items(): + message += f"\n {k}: {v}" + + # Add help text for known errors + if include_help: + help_text = ERROR_HELP_TEXT.get(error.error_code) + if help_text: + message += f"\n\nTroubleshooting:\n{help_text}" + + return message + + # Handle boto3 ClientError + if isinstance(error, botocore.exceptions.ClientError): + error_code = error.response.get("Error", {}).get("Code", "Unknown") + error_message = error.response.get("Error", {}).get("Message", str(error)) + + message = f"AWS Error ({error_code}): {error_message}" + + # Add request ID if available for troubleshooting + request_id = error.response.get("ResponseMetadata", {}).get("RequestId") + if request_id and verbose: + message += f"\nAWS Request ID: {request_id}" + + # Add help for common boto3 errors + if include_help: + help_text = BOTO3_ERROR_HELP.get(error_code) + if help_text: + message += f"\n\nTroubleshooting:\n{help_text}" + + return message + + # Default case for other exceptions + return f"Error: {str(error)}" + ``` + +- Retry mechanism with exponential backoff: + - Implement decorator for retryable functions + - Configure retry count, delay, and backoff factor + - Define which exceptions are retryable + - Log retry attempts and backoff periods + - Customize backoff strategy (exponential, linear, etc.) + +- Implementation approach for retry decorator: + ```python + def retry_with_backoff( + max_retries: int = 3, + initial_backoff: float = 1.0, + backoff_factor: float = 2.0, + max_backoff: float = 30.0, + retryable_exceptions: Tuple[Type[Exception], ...] = ( + botocore.exceptions.ClientError, + requests.exceptions.RequestException, + ), + should_retry_cb: Optional[Callable[[Exception], bool]] = None + ): + """ + Decorator for retrying functions with exponential backoff. + + Args: + max_retries: Maximum number of retries + initial_backoff: Initial backoff time in seconds + backoff_factor: Factor to multiply backoff by after each retry + max_backoff: Maximum backoff time in seconds + retryable_exceptions: Tuple of exceptions that should trigger a retry + should_retry_cb: Optional callback to determine if an exception is retryable + + Returns: + Decorated function + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + retry_logger = logging.getLogger('aws_resource_management.retry') + + retry_count = 0 + backoff = initial_backoff + + while True: + try: + return func(*args, **kwargs) + except retryable_exceptions as e: + # Check if we should retry this specific exception + should_retry = True + if should_retry_cb: + should_retry = should_retry_cb(e) + + # If we've reached max retries or shouldn't retry, re-raise + if retry_count >= max_retries or not should_retry: + raise + + # Calculate backoff with jitter for this retry + jittered_backoff = backoff * (0.9 + 0.2 * random.random()) + actual_backoff = min(jittered_backoff, max_backoff) + + # Log the retry attempt + retry_logger.warning( + f"Retrying {func.__name__} after error: {str(e)}. " + f"Retry {retry_count + 1}/{max_retries} in {actual_backoff:.2f}s" + ) + + # Wait before retrying + time.sleep(actual_backoff) + + # Update for next iteration + retry_count += 1 + backoff = backoff * backoff_factor + return wrapper + return decorator + ``` + +3. Testing and Quality Assurance +The codebase would benefit from: + +Unit tests for core functionality +Integration tests for AWS interactions (using moto library) +Implement pre-commit hooks for code quality checks +Add CI/CD pipeline for automated testing +4. Enhanced Features +Several valuable features could be added: + +Support for resource tagging based on schedule +Cost calculation for resources being managed +Additional resource types (Lambda, S3, DynamoDB) +Resource dependency management +Schedule-based operations (cron-like syntax) +Consistent signal handling (ensuring a single interrupt exits the application) +✅ AWS SSO Profile support for authentication without role assumption: + - Automatic detection and use of AWS SSO profiles in ~/.aws/config + - Support for specifying specific profiles with --profile option + - Prioritization of administrator profiles for each account + - Fallback to role assumption when profiles are not available +✅ Multi-region discovery and management with smart defaults: + - Auto-discovery of all enabled regions per account + - Support for region filtering with include/exclude patterns + - Parallel region checking for improved performance + - Partition-aware region discovery (commercial AWS, GovCloud, China) + - Process all regions in a single account before moving to the next account + - Complete account-level processing to maintain context and improve error handling + - Improved error handling for authentication failures during region discovery + - Graceful degradation to default regions when discovery fails + - Timeout handling for region discovery to prevent hanging operations + - Thread-safe region discovery with proper exception handling + +5. Security Improvements +Security could be strengthened by: + +Parameter validation and sanitization +Secret management for credentials +Role assumption with MFA support +Audit logging of all actions +Access control based on AWS profiles and roles +Enhanced credential management: + - Support for credential chaining (try profile, then role assumption) + - Automated credential refresh for long-running operations + - Validation of permissions before operations + - Secure handling of temporary credentials + +6. Documentation +Documentation could be improved with: + +Comprehensive API documentation +User guides with common scenarios +Architecture diagrams +Contributing guidelines diff --git a/local-app/python-tools/gfl-resource-actions/setup.py b/local-app/python-tools/gfl-resource-actions/setup.py new file mode 100644 index 00000000..3f8d4140 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/setup.py @@ -0,0 +1,32 @@ +""" +Setup script for AWS Resource Management tool. +""" +from setuptools import setup, find_packages + +setup( + name="aws_resource_management", + version="0.1.0", + packages=find_packages(), + description="AWS Resource Management Tool for stopping and starting resources", + author="Morgan", + author_email="morgan@example.com", + install_requires=[ + "boto3>=1.24.0", + "pyyaml>=6.0", + "python-dateutil>=2.8.2", + ], + entry_points={ + 'console_scripts': [ + 'aws-resource-mgmt=aws_resource_management.cli:main', + ], + }, + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: System Administrators", + "Topic :: System :: Systems Administration", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + ], + python_requires=">=3.8", +)