From 81e398f83e7f6bcc348c4690f0ff89a57c5a5d2f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 4 Apr 2025 01:02:10 -0400 Subject: [PATCH] fix lints --- .../gfl-resource-actions/README.md | 187 ++++-- .../aws_resource_management/__init__.py | 9 - .../aws_resource_management/cli.py | 214 +++--- .../aws_resource_management/core.py | 620 +++++++++--------- .../aws_resource_management/discovery.py | 368 +++++++---- .../aws_resource_management/managers/base.py | 137 ++-- .../aws_resource_management/managers/ec2.py | 109 +-- .../aws_resource_management/managers/ecr.py | 477 +++++++------- .../aws_resource_management/managers/eks.py | 103 ++- .../aws_resource_management/managers/emr.py | 84 ++- .../aws_resource_management/managers/rds.py | 30 +- .../aws_resource_management/reporting.py | 123 ++-- .../resource_registry.py | 224 +++++-- .../aws_resource_management/utils/__init__.py | 40 +- .../utils/api_utils.py | 149 +++-- .../utils/aws_core_utils.py | 267 +++++--- .../utils/config_utils.py | 60 +- .../utils/datetime_utils.py | 62 ++ .../utils/file_utils.py | 129 ++-- .../utils/logging_utils.py | 84 +-- .../utils/region_utils.py | 204 ++++-- 21 files changed, 2220 insertions(+), 1460 deletions(-) create mode 100644 local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/datetime_utils.py diff --git a/local-app/python-tools/gfl-resource-actions/README.md b/local-app/python-tools/gfl-resource-actions/README.md index 9ccf10d6..d5bfe457 100644 --- a/local-app/python-tools/gfl-resource-actions/README.md +++ b/local-app/python-tools/gfl-resource-actions/README.md @@ -1,15 +1,25 @@ # AWS Resource Management Tool +A Python-based tool for discovering, starting, stopping, and managing AWS resources across multiple accounts, primarily designed for GovCloud environments to help with cost management. + ## Overview -The AWS Resource Management Tool is a Python-based utility designed to discover, start, and stop AWS resources across multiple accounts, primarily in GovCloud environments. Its primary purpose is cost management by automating resource state management. -## Features -- Multi-account and multi-region support -- Resource discovery and management for EC2, RDS, EKS, EMR, and ECR -- AWS SSO and role-based authentication -- CLI for easy interaction -- Dry-run mode for safe testing -- CSV reporting for resource actions +This tool helps you manage AWS resources across multiple accounts and regions by allowing you to: + +1. Discover resources of different types (EC2, RDS, EKS, EMR, ECR) +2. Start or stop resources on demand +3. Export resource information to CSV files +4. Automatically delete old ECR images + +The tool is designed to work efficiently with large AWS organizations by implementing parallel processing of accounts and intelligent caching to minimize API calls. + +## Supported Resource Types + +- **EC2**: EC2 instances can be started/stopped +- **RDS**: RDS database instances can be started/stopped +- **EKS**: EKS clusters can be started/stopped +- **EMR**: EMR clusters can be started/stopped +- **ECR**: ECR images can be discovered and old images deleted (based on age) ## Package Structure ``` @@ -32,70 +42,109 @@ aws_resource_management/ # Main package ## Installation -1. Clone the repository: - ```bash - git clone - cd - ``` +### Prerequisites + +- Python 3.8 or newer +- AWS credentials with appropriate permissions +- For organization management: OrganizationsReadOnlyAccess permission +- For resource management: appropriate permissions to start/stop resources -2. Install dependencies: - ```bash - make install-dev - ``` +### Setup + +1. Clone the repository +2. Install the package: + +```bash +cd /path/to/gfl-resource-actions +pip install -e . +``` ## Usage -### CLI Commands -The tool exposes a CLI through `aws-resource-mgmt` with the following options: - -- `--start`/`--stop`: Specify the action to perform. -- `--resource-type`: Select resource type (ec2, rds, eks, emr, ecr, all). -- `--region`/`--exclude-region`: Specify regions to include or exclude. -- `--account`/`--exclude-account`: Specify accounts to include or exclude. -- `--dry-run`: Simulate actions without making changes. - -#### Examples -- Run in dry-run mode: - ```bash - python -m aws_resource_management.cli --stop --dry-run --resource-type ec2 - ``` - -- Stop all resources: - ```bash - python -m aws_resource_management.cli --stop --resource-type all - ``` - -- Start EC2 instances only: - ```bash - python -m aws_resource_management.cli --start --resource-type ec2 - ``` - -## Development - -### Code Quality -- Run linters: - ```bash - make lint - ``` -- Format code: - ```bash - make format - ``` - -### Testing -- Run tests: - ```bash - make test - ``` - -### Build Distribution -- Build the package: - ```bash - make dist - ``` - -## Contributing -Contributions are welcome! Please see the `CONTRIBUTING.md` file for guidelines. +### Command Line Interface + +The tool provides a command-line interface with the following main options: + +```bash +# Stop resources with dry run +aws-resource-mgmt --stop --dry-run --resource-type ec2 + +# Start resources in specific regions +aws-resource-mgmt --start --resource-type rds --region us-gov-east-1 --region us-gov-west-1 + +# Export resources to CSV +aws-resource-mgmt --export --resource-type all --csv-output-dir ./exports + +# Filter by specific account +aws-resource-mgmt --stop --resource-type eks --account 123456789012 + +# Stop old ECR images +aws-resource-mgmt --stop --resource-type ecr --csv-export --csv-output-dir ./ +``` + +### Key Command Arguments + +- `--stop` / `--start` / `--export`: Action to perform +- `--resource-type`: Type of resources to manage (ec2, rds, eks, emr, ecr, all) +- `--region`: AWS regions to target (can be specified multiple times) +- `--exclude-region`: AWS regions to exclude (can be specified multiple times) +- `--account`: Specific account ID to target +- `--exclude-account`: Account IDs to exclude (can be specified multiple times) +- `--dry-run`: Simulate actions without making changes +- `--csv-export`: Export discovered resources to CSV files +- `--csv-output-dir`: Directory to save CSV exports +- `--partition`: AWS partition to operate in (aws, aws-us-gov, aws-cn) + +## Authentication + +The tool supports multiple authentication methods: + +1. **AWS Organizations**: By default, uses the current session to access AWS Organizations API and manage resources across accounts +2. **AWS Profiles**: Use the `--profile` option to specify an AWS profile from ~/.aws/config +3. **Role Assumption**: Automatically assumes appropriate roles in target accounts (OrganizationAccountAccessRole or AWSControlTowerExecution) + +## Resource Management + +### EC2 Instances + +EC2 instances can be started or stopped. The tool intelligently handles instances in Auto Scaling groups. + +### RDS Instances + +RDS database instances can be started or stopped. + +### EKS Clusters + +EKS clusters can be discovered and managed. Worker nodes are handled separately as EC2 instances. + +### EMR Clusters + +EMR clusters can be discovered and controlled. + +### ECR Images + +ECR repositories and images can be discovered. Old images (by default, older than 365 days) can be deleted. + +## CSV Export + +Resources can be exported to CSV files with detailed information: + +```bash +aws-resource-mgmt --export --resource-type all --csv-output-dir ./exports +``` + +## Configuration + +The tool uses a combination of command-line arguments and configuration files to control its behavior. + +## Troubleshooting + +Common issues: + +1. **Authentication Failures**: Check your AWS credentials and ensure your role has necessary permissions. +2. **Missing Resources**: Verify the resource type and regions specified. +3. **Performance Issues**: For large organizations, try limiting the scope with regions and account filters. ## License -This project is licensed under the MIT License. See the LICENSE file for details. + +This project is licensed under the MIT License - see the LICENSE file for details. diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py index 6636f3f8..4133ad32 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py @@ -4,18 +4,9 @@ # Re-export key modules from consolidated utility modules from aws_resource_management.utils.logging_utils import ( - LoggingContext, - configure_logging, - log_operation, - log_with_context, setup_logging, ) -from aws_resource_management.utils.file_utils import ( - initialize_action_log_csv as initialize_csv_log, - log_action_to_csv, -) - # Set up a default logger logger = setup_logging() diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli.py index 291fdb78..00ad1420 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/cli.py @@ -4,17 +4,15 @@ """ import argparse -import logging import os import sys import traceback -from typing import Any, Dict, List, Optional -from aws_resource_management.config_manager import get_config from aws_resource_management.core import ResourceManager -from aws_resource_management.logging_setup import setup_logging from aws_resource_management.reporting import export_resources_to_csv from aws_resource_management.resource_registry import get_registry +from aws_resource_management.utils.config_utils import get_config +from aws_resource_management.utils.logging_utils import setup_logging logger = setup_logging() config = get_config() @@ -28,23 +26,27 @@ def parse_args(): action_group = parser.add_mutually_exclusive_group(required=True) action_group.add_argument("--stop", action="store_true", help="Stop resources") action_group.add_argument("--start", action="store_true", help="Start resources") - action_group.add_argument("--export", action="store_true", help="Export resources to CSV without taking any action") + action_group.add_argument( + "--export", + action="store_true", + help="Export resources to CSV without taking any action", + ) # Resource type argument using registry registry = get_registry() resource_choices = registry.get_resource_type_choices() - resource_type_choices = [choice['value'] for choice in resource_choices] - + resource_type_choices = [choice["value"] for choice in resource_choices] + # Ensure 'all' is in the list of choices - if 'all' not in resource_type_choices: - resource_type_choices.append('all') - + if "all" not in resource_type_choices: + resource_type_choices.append("all") + # Format the help text with choice descriptions resource_type_help = "\n".join( f" {choice['value']}: {choice['help']}" for choice in resource_choices ) resource_type_help += "\n all: All supported resource types" - + parser.add_argument( "--resource-type", choices=resource_type_choices, @@ -100,9 +102,9 @@ def parse_args(): action="store_true", help="Show what would be done without making changes", ) - + # CSV Export options - csv_group = parser.add_argument_group('CSV Export Options') + csv_group = parser.add_argument_group("CSV Export Options") csv_group.add_argument( "--csv-export", action="store_true", @@ -121,97 +123,117 @@ def parse_args(): return parser.parse_args() -def main(): - """Main CLI entry point.""" - args = parse_args() +def process_command(args: argparse.Namespace) -> None: + """ + Process the command-line arguments and execute the appropriate action. - try: - # Determine action - action = "stop" if args.stop else "start" if args.start else "export" - - # Get registry of available resource types - registry = get_registry() - all_resource_types = registry.get_resource_types() - - # Determine resource types to process - exclude_resources = [] - if args.resource_type != "all": - # If specific resource type is selected, exclude all others - exclude_resources = [ - rt for rt in all_resource_types if rt != args.resource_type - ] - - # Validate that the requested resource type exists in the registry - if args.resource_type != "all" and args.resource_type not in all_resource_types: - logger.error(f"Invalid resource type: {args.resource_type}. Available types: {', '.join(all_resource_types)}") - sys.exit(1) - - logger.info( - f"Processing {args.resource_type} resources in {args.regions or 'all enabled'} regions for {action} action" + Args: + args: Parsed command-line arguments + """ + # Determine action + action = "stop" if args.stop else "start" if args.start else "export" + + # Get registry of available resource types + registry = get_registry() + all_resource_types = registry.get_resource_types() + + # Convert resource_type to exclude_resources format + exclude_resources = [] + if args.resource_type and args.resource_type != "all": + # Only include the specified resource type + for res_type in ["ec2", "rds", "eks", "emr", "ecr"]: + if res_type != args.resource_type: + exclude_resources.append(res_type) + + logger.debug(f"Excluding resource types: {exclude_resources}") + + # Validate that the requested resource type exists in the registry + if args.resource_type != "all" and args.resource_type not in all_resource_types: + logger.error( + f"Invalid resource type: {args.resource_type}. Available types: \ + {', '.join(all_resource_types)}" + ) + sys.exit(1) + + logger.info( + f"Processing {args.resource_type} resources in {args.regions or 'all enabled'} \ + regions for {action} action" + ) + + # Initialize resource manager + resource_manager = ResourceManager( + profile_name=args.profile, + use_profiles=args.use_profiles, + discover_regions=args.discover_regions, + partition=args.partition, + ) + + # Create account inclusion/exclusion list + exclude_accounts = args.exclude_accounts or [] + if args.account: + # If specific account is given, exclude all others by default + # But don't add the specified account to the exclusion list + logger.info(f"Processing only account {args.account}") + # We'll handle this by post-filtering the account list in resource_manager + + # For export-only action or when CSV export is requested + if action == "export" or args.csv_export: + # Discover resources across all accounts and regions + discovered_resources = resource_manager.discover_resources( + regions=args.regions or [], + exclude_accounts=exclude_accounts, + exclude_resources=exclude_resources, + exclude_regions=args.exclude_regions or [], + specific_account=args.account, ) - # Initialize resource manager - resource_manager = ResourceManager( - profile_name=args.profile, - use_profiles=args.use_profiles, - discover_regions=args.discover_regions, - partition=args.partition, + # Export to CSV + logger.info(f"Exporting resources to CSV in directory: {args.csv_output_dir}") + csv_files = export_resources_to_csv( + resources=discovered_resources, + output_dir=args.csv_output_dir, + prefix=args.csv_prefix, ) - # Create account inclusion/exclusion list - exclude_accounts = args.exclude_accounts or [] - if args.account: - # If specific account is given, exclude all others by default - # But don't add the specified account to the exclusion list - logger.info(f"Processing only account {args.account}") - # We'll handle this by post-filtering the account list in resource_manager - - # For export-only action or when CSV export is requested - if action == "export" or args.csv_export: - # Discover resources across all accounts and regions - discovered_resources = resource_manager.discover_resources( - regions=args.regions or [], - exclude_accounts=exclude_accounts, - exclude_resources=exclude_resources, - exclude_regions=args.exclude_regions or [], - specific_account=args.account + # Log success message with file locations + if csv_files: + logger.info( + f"Successfully exported {len(csv_files)} resource type(s) to CSV" ) - - # Export to CSV - logger.info(f"Exporting resources to CSV in directory: {args.csv_output_dir}") - csv_files = export_resources_to_csv( - resources=discovered_resources, - output_dir=args.csv_output_dir, - prefix=args.csv_prefix + for resource_type, filepath in csv_files.items(): + logger.info(f" - {resource_type}: {filepath}") + else: + logger.warning( + "No resources were exported to CSV. Check if resources were found." ) - - # Log success message with file locations - if csv_files: - logger.info(f"Successfully exported {len(csv_files)} resource type(s) to CSV") - for resource_type, filepath in csv_files.items(): - logger.info(f" - {resource_type}: {filepath}") - else: - logger.warning("No resources were exported to CSV. Check if resources were found.") - - # If action was export-only, we're done - if action == "export": - sys.exit(0) - - # Process accounts for start/stop actions - if action in ["start", "stop"]: - stats = resource_manager.process_accounts( - action=action, - regions=args.regions or [], - exclude_accounts=exclude_accounts, - exclude_resources=exclude_resources, - exclude_regions=args.exclude_regions or [], - dry_run=args.dry_run, - ) - - logger.info(f"Completed {action} action for {args.resource_type} resources") - # Exit with success code - sys.exit(0) + # If action was export-only, we're done + if action == "export": + sys.exit(0) + + # Process accounts for start/stop actions + if action in ["start", "stop"]: + resource_manager.process_accounts( + action=action, + regions=args.regions or [], + exclude_accounts=exclude_accounts, + exclude_resources=exclude_resources, # Use our calculated exclude_resources + exclude_regions=args.exclude_regions or [], + dry_run=args.dry_run, + ) + + logger.info(f"Completed {action} action for {args.resource_type} resources") + + # Exit with success code + sys.exit(0) + + +def main(): + """Main CLI entry point.""" + args = parse_args() + + try: + process_command(args) except KeyboardInterrupt: logger.warning("Operation interrupted by user. Exiting gracefully...") diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py index 9e3eb1b2..c238b260 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py @@ -3,36 +3,32 @@ """ import concurrent.futures -import logging -import sys -from collections import defaultdict -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional -from aws_resource_management.aws_utils import ( - detect_partition_from_credentials, - get_account_list, - get_available_regions, - get_credentials, - get_organization_accounts, -) -from aws_resource_management.config_manager import get_config from aws_resource_management.discovery import get_account_resources -from aws_resource_management.logging_setup import setup_logging -from aws_resource_management.partition_utils import ( +from aws_resource_management.reporting import initialize_stats, print_resource_summary +from aws_resource_management.resource_registry import get_registry +from aws_resource_management.utils import ( DEFAULT_PARTITION, + detect_partition_from_credentials, + ensure_valid_account_name, filter_regions_for_partition, + get_account_list, + get_config, + get_credentials, get_default_regions_for_partition, + get_enabled_regions, + get_organization_accounts, + logger, ) -from aws_resource_management.reporting import initialize_stats, print_resource_summary -from aws_resource_management.resource_registry import get_registry from botocore.exceptions import ClientError -logger = setup_logging() config = get_config() class ResourceManager: - """Main resource manager that orchestrates operations across accounts and resources.""" + """Main resource manager that orchestrates + operations across accounts and resources.""" def __init__( self, @@ -50,7 +46,7 @@ def __init__( self.partition = partition self.region_cache = {} self.MAX_CONCURRENT_ACCOUNTS = 3 # Limit concurrent account processing - # Get resource types from registry + # Get resource registry self.resource_registry = get_registry() @property @@ -76,6 +72,10 @@ def process_accounts( exclude_regions = exclude_regions or [] stats = stats or initialize_stats() + # Log which resource types are being excluded for debugging + if exclude_resources: + logger.info(f"Excluding resource types: {', '.join(exclude_resources)}") + # Get account list from the organization you're currently authenticated with accounts = self._get_accounts() @@ -86,7 +86,8 @@ def process_accounts( logger.info(f"Found {len(valid_accounts)} accessible accounts to process") if len(accounts) > len(valid_accounts): logger.info( - f"Skipped {len(accounts) - len(valid_accounts)} inaccessible accounts" + f"Skipped {len(accounts) - len(valid_accounts)} \ + inaccessible accounts" ) if exclude_accounts: logger.info(f"Excluding {len(exclude_accounts)} accounts") @@ -193,7 +194,17 @@ def _process_account_safely( ) -> Optional[Dict[str, Any]]: """Process a single account with error handling.""" account_id = account.get("account_id") - account_name = account.get("account_name", "Unknown") + # Ensure account_name is valid and update the original account dictionary + account_name = ensure_valid_account_name( + account_id, account.get("account_name") + ) + account["account_name"] = account_name # Update in original dict + + # Add debug logging to trace the account name through the process + logger.debug(f"Processing account safely with name: {account_name}") + + # Use the validated account_name when logging + logger.info(f"Processing account {account_name} ({account_id})") try: # Get credentials @@ -214,10 +225,13 @@ def _process_account_safely( account_stats = initialize_stats() account_stats["regions_processed"] = len(account_regions) account_stats["regions_by_account"][account_id] = account_regions + # Store account name in stats for better reporting + account_stats["account_name"] = account_name # Process the account with its determined regions logger.info( - f"Processing account: {account_name} ({account_id}) in {len(account_regions)} regions" + f"Processing account: {account_name} ({account_id}) in \ + {len(account_regions)} regions" ) # Auto-detect partition if not specified @@ -227,45 +241,59 @@ def _process_account_safely( else self.partition ) - # Get resources with special handling for EMR + # Log partition info + logger.info( + f"Using partition {partition} for account {account_name} ({account_id})" + ) + + # Get EMR states from registry instead of hardcoding emr_states = self._get_emr_states(exclude_resources) try: # Get all resources - this might raise AuthFailure resources = get_account_resources( - credentials, - account_regions, - exclude_resources, - account_id, - account_name, + credentials=credentials, + regions=account_regions, + exclude_resources=exclude_resources, + account_id=account_id, + account_name=account_name, # Explicitly pass as kwarg emr_cluster_states=emr_states, ) if not resources: - logger.error(f"Failed to get resources for account {account_id}") + logger.error( + f"Failed to get resources for account \ + {account_id} ({account_name})" + ) return None - # Normalize states and collect statistics - self._process_resources(resources, account_stats) + # Update statistics using the registry instead of hardcoded logic + self._process_resources_with_registry(resources, account_stats) # Log resource counts - self._log_resource_counts(account_id, resources, account_stats) - - # Create resource managers only once per account - managers = self._create_resource_managers( - credentials, account_id, account_name + self._log_resource_counts( + account_id, account_name, resources, account_stats ) - # Process actions on resources based on the selected mode - self._execute_actions( + # Execute actions using registry pattern + # instead of manual manager creation + self._execute_actions_with_registry( action, - managers, + credentials, + account_id, + account_name, # Make sure we're passing the account_name here resources, exclude_resources, dry_run, account_stats, ) + # Log discovery initialization with account name + logger.info( + f"Resource discovery initialized for account \ + {account_name} ({account_id}) in partition {partition}" + ) + return account_stats except ClientError as e: # Special handling for authentication failures @@ -277,19 +305,23 @@ def _process_account_safely( "AccessDenied", ]: logger.error( - f"Authentication failure for account {account_id}: {e}" + f"Authentication failure for account \ + {account_id} ({account_name}): {e}" ) account_stats["errors"].append( - f"Authentication failure for account {account_id}: {str(e)}" + f"Authentication failure for account \ + {account_id} ({account_name}): {str(e)}" ) return account_stats raise # Re-raise other client errors except Exception as e: - logger.error(f"Error processing account {account_id}: {str(e)}") + logger.error( + f"Error processing account {account_id} ({account_name}): {str(e)}" + ) if account_stats := locals().get("account_stats"): account_stats["errors"].append( - f"Error processing account {account_id}: {str(e)}" + f"Error processing account {account_id} ({account_name}): {str(e)}" ) return account_stats return None @@ -305,18 +337,21 @@ def _get_account_regions( if self.discover_regions: try: logger.info(f"Discovering enabled regions for account {account_id}") - discovered_regions = get_available_regions( - credentials, self.partition - ) + discovered_regions = get_enabled_regions(credentials, self.partition) # Filter discovered regions for the partition if self.partition: - discovered_regions = filter_regions_for_partition(discovered_regions, self.partition) - + discovered_regions = filter_regions_for_partition( + discovered_regions, self.partition + ) + # Apply exclusions - account_regions = [r for r in discovered_regions if r not in exclude_regions] - + account_regions = [ + r for r in discovered_regions if r not in exclude_regions + ] + logger.info( - f"Found {len(account_regions)} enabled regions in account {account_id}" + f"Found {len(account_regions)} enabled regions in \ + account {account_id}" ) self.region_cache[account_id] = account_regions return account_regions @@ -326,7 +361,8 @@ def _get_account_regions( ) if provided_regions: logger.info( - f"Falling back to provided regions: {', '.join(provided_regions)}" + f"Falling back to provided regions: \ + {', '.join(provided_regions)}" ) return provided_regions return [] @@ -334,15 +370,22 @@ def _get_account_regions( elif provided_regions: if self.partition: # Use centralized filter - valid_regions = filter_regions_for_partition(provided_regions, self.partition) - filtered_regions = [r for r in valid_regions if r not in exclude_regions] + valid_regions = filter_regions_for_partition( + provided_regions, self.partition + ) + filtered_regions = [ + r for r in valid_regions if r not in exclude_regions + ] else: - filtered_regions = [r for r in provided_regions if r not in exclude_regions] - + filtered_regions = [ + r for r in provided_regions if r not in exclude_regions + ] + if not filtered_regions: filtered_regions = self._get_default_regions(credentials) logger.info( - f"All provided regions were excluded or invalid, using defaults: {', '.join(filtered_regions)}" + f"All provided regions were excluded or invalid, using \ + defaults: {', '.join(filtered_regions)}" ) return filtered_regions else: @@ -373,260 +416,191 @@ def _get_default_regions( except Exception as e: logger.warning(f"Failed to auto-detect partition from credentials: {e}") - # If still no partition, use GovCloud AWS as fallback (default for this environment) + # If still no partition, use GovCloud AWS (default for this environment) partition = partition or DEFAULT_PARTITION - + # Use centralized utility return get_default_regions_for_partition(partition) def _get_emr_states(self, exclude_resources: List[str]) -> Optional[List[str]]: """Get valid EMR states for discovery.""" if "emr" not in exclude_resources: - emr_states = [ - "STARTING", - "BOOTSTRAPPING", - "RUNNING", - "WAITING", - "TERMINATING", - ] - logger.debug( - f"Using valid EMR states for discovery: {', '.join(emr_states)}" - ) - return emr_states + # Get EMR states from registry or config instead of hardcoding + emr_manager = self.resource_registry.get_manager_class("emr") + if hasattr(emr_manager, "VALID_CLUSTER_STATES"): + return emr_manager.VALID_CLUSTER_STATES + else: + # Fallback to hardcoded states if not defined in manager + return [ + "STARTING", + "BOOTSTRAPPING", + "RUNNING", + "WAITING", + "TERMINATING", + ] return None - def _process_resources( + def _process_resources_with_registry( self, resources: Dict[str, List[Dict[str, Any]]], stats: Dict[str, Any] ) -> None: - """Normalize resource states and collect statistics.""" - # Normalize state and status fields for consistency - for resource_key, state_mappings in [ - ("ec2_instances", "state"), - ("rds_instances", "state"), - ("eks_clusters", "state"), - ("emr_clusters", "state"), - ]: - for resource in resources.get(resource_key, []): - self._normalize_resource_state(resource) - - # Count service-managed EC2 instances - eks_tag = self.config.get("eks_tag", "kubernetes.io/cluster/") - exclusion_tag = self.config.get("exclusion_tag", "DoNotStop") - emr_tag = self.config.get("emr_tag", "aws:elasticmapreduce:job-flow-id") - ec2_instances = resources.get("ec2_instances", []) - - # Fixed: Safely check for EKS tag prefixes in instance tags - eks_managed = sum( - 1 - for i in ec2_instances - if any( - isinstance(tag_key, str) and tag_key.startswith(eks_tag) - for tag_key in i.get("tags", {}) - ) - ) + """Process resources using resource registry instead of hardcoded logic.""" + # Use registry to process resources and update stats + for resource_type in self.RESOURCE_TYPES: + manager_class = self.resource_registry.get_manager_class(resource_type) + if not manager_class: + continue + # Get the appropriate resource key based on resource type + resource_key = self.resource_registry.get_resource_key(resource_type) + resource_list = resources.get(resource_key, []) - # Fixed: Safely check for EMR tags in instance tags - emr_managed = sum(1 for i in ec2_instances if emr_tag in i.get("tags", {})) + # Update stats - use the proper key format (resource_type_found) + stats_key = f"{resource_type}_found" + if stats_key not in stats: + stats[stats_key] = 0 + stats[stats_key] += len(resource_list) - excluded_ec2 = sum( - 1 for i in ec2_instances if exclusion_tag in i.get("tags", {}) - ) + # Log the resource count for debugging + logger.debug( + f"Added {len(resource_list)} {resource_type} resources to stats" + ) - # Update resource counts - stats["ec2_found"] += len(ec2_instances) - stats["ec2_skipped"] += excluded_ec2 - stats["rds_found"] += len(resources.get("rds_instances", [])) - stats["emr_found"] += len(resources.get("emr_clusters", [])) - stats["eks_found"] += len(resources.get("eks_clusters", [])) - stats["ecr_images_found"] += len(resources.get("ecr_images", [])) - stats["ecr_old_images_found"] += len(resources.get("ecr_old_images", [])) - - # Track RDS engines - for instance in resources.get("rds_instances", []): - if instance and "engine" in instance: - engine = instance["engine"] - stats["rds_engines"][engine] = stats["rds_engines"].get(engine, 0) + 1 - - def _normalize_resource_state(self, resource: Dict[str, Any]) -> None: - """Ensure resources have both state and status fields.""" - if "state" not in resource and "status" in resource: - resource["state"] = resource["status"] - elif "status" not in resource and "state" in resource: - resource["status"] = resource["state"] - elif "state" not in resource and "status" not in resource: - resource["status"] = "unknown" - resource["state"] = "unknown" + # Let the manager normalize resources if needed + if hasattr(manager_class, "normalize_resources"): + manager_class.normalize_resources(resource_list) + # Let the manager update specific stats + if hasattr(manager_class, "update_stats"): + manager_class.update_stats(resource_list, stats) def _log_resource_counts( self, account_id: str, + account_name: str, # Added account_name parameter resources: Dict[str, List[Dict[str, Any]]], stats: Dict[str, Any], ) -> None: """Log discovered resource counts.""" - total_ec2 = len(resources.get("ec2_instances", [])) - excluded_ec2 = sum( - 1 - for i in resources.get("ec2_instances", []) - if self.config.get("exclusion_tag") in i.get("tags", {}) - ) - - eks_tag = self.config.get("eks_tag", "kubernetes.io/cluster/") - emr_tag = self.config.get("emr_tag", "aws:elasticmapreduce:job-flow-id") - - # Fixed: Safely check for EKS tag prefixes in instance tags - eks_managed = sum( - 1 - for i in resources.get("ec2_instances", []) - if any( - isinstance(tag_key, str) and tag_key.startswith(eks_tag) - for tag_key in i.get("tags", {}) - ) - ) - - emr_managed = sum( - 1 - for i in resources.get("ec2_instances", []) - if emr_tag in i.get("tags", {}) - ) - - logger.info( - f"Account {account_id} - Found {total_ec2} EC2 instances ({excluded_ec2} explicitly excluded, {eks_managed} EKS-managed, {emr_managed} EMR-managed)" - ) - logger.info( - f"Account {account_id} - Found {len(resources.get('rds_instances', []))} RDS instances" - ) - logger.info( - f"Account {account_id} - Found {len(resources.get('eks_clusters', []))} EKS clusters" - ) - logger.info( - f"Account {account_id} - Found {len(resources.get('emr_clusters', []))} EMR clusters" - ) - logger.info( - f"Account {account_id} - Found {len(resources.get('ecr_images', []))} ECR images " - f"({len(resources.get('ecr_old_images', []))} older than 1 year)" - ) - - def _create_resource_managers( - self, credentials: Dict[str, str], account_id: str, account_name: str - ) -> Dict[str, Any]: - """ - Create resource managers for all supported resource types. - - Args: - credentials: AWS credentials - account_id: AWS account ID - account_name: AWS account name - - Returns: - Dictionary of resource managers by type - """ - managers = {} - + # Use registry to get resource keys and log counts for resource_type in self.RESOURCE_TYPES: - manager_class = self.resource_registry.get_manager_class(resource_type) - if manager_class: - try: - # Create an instance for each region - managers[resource_type] = {} - for region in self.region_cache.get(account_id, []): - managers[resource_type][region] = manager_class(account_id, region) - except Exception as e: - logger.error(f"Error creating {resource_type} manager: {str(e)}") - - return managers - - def _execute_actions( + resource_key = self.resource_registry.get_resource_key(resource_type) + # Special handling for ECR which may have both regular and old images + if resource_type == "ecr": + ecr_images = len(resources.get(resource_key, [])) + if ecr_images > 0: + logger.info(f"Total ecr resources discovered: {ecr_images}") + # Update the account-specific stats + stats["ecr_found"] = stats.get("ecr_found", 0) + ecr_images + # Also track old images if available + old_images = resources.get("ecr_old_images", []) + if old_images: + stats["ecr_old_images"] = stats.get("ecr_old_images", 0) + len( + old_images + ) + else: + resource_count = len(resources.get(resource_key, [])) + if resource_count > 0: + logger.info( + f"Total {resource_type} resources discovered: {resource_count}" + ) + # Update the account-specific stats with the explicit key name + stats[f"{resource_type}_found"] = ( + stats.get(f"{resource_type}_found", 0) + resource_count + ) + # Debug: log the updated stat + logger.debug( + f"Updated stats[{resource_type}_found] to \ + {stats.get(f'{resource_type}_found')}" + ) + + def _execute_actions_with_registry( self, action: str, - managers: Dict[str, Any], + credentials: Dict[str, str], + account_id: str, + account_name: str, # Ensure we're using account_name resources: Dict[str, List[Dict[str, Any]]], exclude_resources: List[str], dry_run: bool, stats: Dict[str, Any], ) -> None: - """Execute start/stop actions on resources.""" - - # Helper function to safely process manager results - def safe_get_result(result: Optional[Dict[str, int]], key: str) -> int: - if result is None: - return 0 - return result.get(key, 0) - - # Execute actions on each resource type + """Execute resource actions using the resource registry.""" for resource_type in self.RESOURCE_TYPES: - # Special handling for ECR - it has different resource keys and different actions - if resource_type == "ecr": - if resource_type in exclude_resources: - stats[f"{resource_type}_skipped"] += len(resources.get("ecr_images", [])) - continue - - # Get ECR manager - ecr_manager = managers.get(resource_type) - if not ecr_manager: - logger.warning(f"No manager found for {resource_type}, skipping") - continue - - # ECR has cleanup action instead of start/stop - if action == "stop" and resources.get("ecr_old_images"): - # Clean up old images when "stop" action is used - old_images = resources.get("ecr_old_images", []) - if old_images: - logger.info(f"Cleaning up {len(old_images)} old ECR images") - # Get a region-specific manager for each region with images - regions_with_images = set(img.get("region") for img in old_images if img.get("region")) - for region in regions_with_images: - region_images = [img for img in old_images if img.get("region") == region] - if region_images and region in ecr_manager: - result = ecr_manager[region].cleanup_old_images(region_images, dry_run) - stats["ecr_images_deleted"] = stats.get("ecr_images_deleted", 0) + safe_get_result(result, "success") - stats["ecr_errors"] = stats.get("ecr_errors", 0) + safe_get_result(result, "errors") - stats["ecr_skipped"] = stats.get("ecr_skipped", 0) + safe_get_result(result, "skipped") - else: - # For "start" action or if no old images, just note that ECR has no relevant action - stats[f"{resource_type}_skipped"] += len(resources.get("ecr_images", [])) - - # Skip the rest of the loop for ECR - continue - if resource_type in exclude_resources: # Skip this resource type - resource_list = resources.get( - ( - f"{resource_type}_instances" - if resource_type != "eks" - else "eks_clusters" - ), - [], - ) - stats[f"{resource_type}_skipped"] += len(resource_list) + resource_key = self.resource_registry.get_resource_key(resource_type) + resource_list = resources.get(resource_key, []) + stats[f"{resource_type}_skipped"] = stats.get( + f"{resource_type}_skipped", 0 + ) + len(resource_list) continue - # Get the appropriate manager and resource list - manager = managers.get(resource_type) - if not manager: - logger.warning(f"No manager found for {resource_type}, skipping") + # Get the resource manager from registry + manager_class = self.resource_registry.get_manager_class(resource_type) + if not manager_class: continue - # Get the right resource list name - resource_key = ( - f"{resource_type}_instances" - if resource_type != "eks" - else "eks_clusters" - ) + # Get appropriate resource list + resource_key = self.resource_registry.get_resource_key(resource_type) resource_list = resources.get(resource_key, []) - # Execute the action - if action == "stop": - result = manager.stop(resource_list, dry_run) - stats[f"{resource_type}_stopped"] += safe_get_result(result, "success") - stats[f"{resource_type}_errors"] += safe_get_result(result, "errors") - stats[f"{resource_type}_skipped"] += safe_get_result(result, "skipped") - else: # action == "start" - result = manager.start(resource_list, dry_run) - stats[f"{resource_type}_started"] += safe_get_result(result, "success") - stats[f"{resource_type}_errors"] += safe_get_result(result, "errors") - stats[f"{resource_type}_skipped"] += safe_get_result(result, "skipped") + # Special handling for ECR images - + # ensure they're counted in stats even if no action is performed + if resource_type == "ecr" and resource_list: + # Count total ECR images + stats["ecr_found"] = stats.get("ecr_found", 0) + len(resource_list) + + # Count old images if available + old_images = resources.get("ecr_old_images", []) + stats["ecr_old_images"] = stats.get("ecr_old_images", 0) + len( + old_images + ) + + if not resource_list: + continue + + for region in self.region_cache.get(account_id, []): + # Filter resources for this region + region_resources = [ + r for r in resource_list if r.get("region") == region + ] + if not region_resources: + continue + + # Create manager instance for this region + manager = manager_class(account_id, region, credentials) + + # Set account name in manager if supported + if hasattr(manager, "account_name"): + manager.account_name = account_name + + # Execute appropriate action + if action == "stop": + result = manager.stop(region_resources, dry_run) + stats[f"{resource_type}_stopped"] = stats.get( + f"{resource_type}_stopped", 0 + ) + self._safe_get_result(result, "success") + stats[f"{resource_type}_errors"] = stats.get( + f"{resource_type}_errors", 0 + ) + self._safe_get_result(result, "errors") + stats[f"{resource_type}_skipped"] = stats.get( + f"{resource_type}_skipped", 0 + ) + self._safe_get_result(result, "skipped") + else: # action == "start" + result = manager.start(region_resources, dry_run) + stats[f"{resource_type}_started"] = stats.get( + f"{resource_type}_started", 0 + ) + self._safe_get_result(result, "success") + stats[f"{resource_type}_errors"] = stats.get( + f"{resource_type}_errors", 0 + ) + self._safe_get_result(result, "errors") + stats[f"{resource_type}_skipped"] = stats.get( + f"{resource_type}_skipped", 0 + ) + self._safe_get_result(result, "skipped") + + def _safe_get_result(self, result: Optional[Dict[str, int]], key: str) -> int: + """Safely get a result value from a dictionary.""" + if result is None: + return 0 + return result.get(key, 0) def _merge_stats( self, main_stats: Dict[str, Any], account_stats: Dict[str, Any] @@ -636,6 +610,16 @@ def _merge_stats( for key, value in account_stats.items(): if isinstance(value, (int, float)) and key in main_stats: main_stats[key] += value + # Special handling for ECR stats that might + # not be initialized in main_stats yet + elif key in [ + "ecr_found", + "ecr_old_images", + "ecr_deleted", + "ecr_skipped", + "ecr_errors", + ] and isinstance(value, (int, float)): + main_stats[key] = main_stats.get(key, 0) + value # Merge errors list main_stats["errors"].extend(account_stats.get("errors", [])) @@ -676,14 +660,14 @@ def discover_resources( ) -> Dict[str, List[Dict[str, Any]]]: """ Discover resources across accounts and regions. - + Args: regions: List of regions to discover resources in exclude_accounts: List of accounts to exclude exclude_resources: List of resource types to exclude exclude_regions: List of regions to exclude specific_account: Optional specific account ID to focus on - + Returns: Dictionary of resources by type """ @@ -691,27 +675,40 @@ def discover_resources( exclude_resources = exclude_resources or [] exclude_regions = exclude_regions or [] regions = regions or [] - + + # Log which resource types are being excluded for debugging + if exclude_resources: + resource_types_to_discover = [ + rt for rt in self.RESOURCE_TYPES if rt not in exclude_resources + ] + logger.info(f"Discovering only: {', '.join(resource_types_to_discover)}") + # Get account list accounts = self._get_accounts() - + # Filter accounts if specific_account is provided if specific_account: - accounts = [acc for acc in accounts if acc.get("account_id") == specific_account] + accounts = [ + acc for acc in accounts if acc.get("account_id") == specific_account + ] if not accounts: - logger.error(f"Specified account {specific_account} not found or not accessible") + logger.error( + f"Specified account {specific_account} not found or not accessible" + ) return {} - + # Pre-filter accounts that we can authenticate with accounts = self._pre_validate_accounts(accounts) - + # Apply account exclusions - accounts = [acc for acc in accounts if acc.get("account_id") not in exclude_accounts] - + accounts = [ + acc for acc in accounts if acc.get("account_id") not in exclude_accounts + ] + if not accounts: logger.error("No accounts available for resource discovery") return {} - + # Initialize consolidated resources dictionary all_resources = { "ec2_instances": [], @@ -721,60 +718,63 @@ def discover_resources( "ecr_images": [], "ecr_old_images": [], } - + # Process each account for account in accounts: account_id = account.get("account_id") - account_name = account.get("account_name", "Unknown") - - logger.info(f"Discovering resources in account {account_name} ({account_id})") - + # Ensure account_name is valid and consistent + account_name = ensure_valid_account_name( + account_id, account.get("account_name") + ) + account["account_name"] = account_name # Update in the source + + logger.info( + f"Discovering resources in account {account_name} ({account_id})" + ) + try: # Get credentials for the account credentials = get_credentials(account_id, self.profile_name) if not credentials: - logger.warning(f"Could not obtain credentials for account {account_id}") + logger.warning( + f"Could not obtain credentials for account {account_id}" + ) continue - + # Determine regions for this account account_regions = self._get_account_regions( account_id, credentials, regions, exclude_regions ) - + if not account_regions: logger.warning(f"No valid regions for account {account_id}") continue - - # Auto-detect partition if not specified - partition = ( - detect_partition_from_credentials(credentials) - if not self.partition - else self.partition - ) - + # Get resources with special handling for EMR emr_states = self._get_emr_states(exclude_resources) - + # Get all resources for the account account_resources = get_account_resources( - credentials, - account_regions, - exclude_resources, - account_id, - account_name, + credentials=credentials, + regions=account_regions, + exclude_resources=exclude_resources, + account_id=account_id, + account_name=account_name, emr_cluster_states=emr_states, ) - + # Merge resources into the consolidated dictionary for resource_type, resources in account_resources.items(): all_resources.setdefault(resource_type, []).extend(resources) - + except Exception as e: - logger.error(f"Error discovering resources for account {account_id}: {str(e)}") - + logger.error( + f"Error discovering resources for account {account_id}: {str(e)}" + ) + # Log summary of discovered resources for resource_type, resources in all_resources.items(): if resources: logger.info(f"Total {resource_type}: {len(resources)}") - + return all_resources diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py index 313a0564..ee42c594 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py @@ -3,45 +3,27 @@ """ import concurrent.futures -import logging -from typing import Any, Dict, List, Optional -from concurrent.futures import ThreadPoolExecutor, as_completed -from aws_resource_management.aws_utils import ( - detect_partition_from_credentials, - get_available_regions, - get_session_for_account, - is_valid_region, - list_enabled_regions, -) -from aws_resource_management.logging_setup import setup_logging -from aws_resource_management.partition_utils import ( +from typing import Any, Dict, List, Optional, Tuple + +import boto3 +from aws_resource_management.utils import ( DEFAULT_REGIONS, - get_regions_for_partition, - is_region_in_partition, + create_boto3_client, + detect_partition_from_credentials, + ensure_valid_account_name, filter_regions_for_partition, -) -from aws_resource_management.discovery_utils import ( - paginate_aws_response, # Import centralized pagination utility format_tags, - add_account_info, - normalize_resource_state -) -import boto3 -from aws_resource_management.config_manager import get_config -from aws_resource_management.logging_setup import log_with_context, setup_logging -from aws_resource_management.managers import ( - EC2Manager, - RDSManager, - ResourceManager, + get_config, + get_session_for_account, + logger, + paginate_aws_response, ) -from botocore.exceptions import ClientError, EndpointConnectionError # Try to import optional managers try: from aws_resource_management.managers import EKSManager except ImportError: EKSManager = None - try: from aws_resource_management.managers import EMRManager except ImportError: @@ -51,28 +33,43 @@ except ImportError: ECRManager = None -logger = logging.getLogger("aws_resource_management") config = get_config() class ResourceDiscovery: - """Discovers AWS resources across accounts and regions.""" + """Resource discovery class.""" + def __init__( - self, credentials: Dict[str, str], account_id: str, account_name: Optional[str] = None - ) -> None: - """Initialize resource discovery for an account. - + self, + credentials, + account_id: str = "unknown", + account_name: Optional[str] = None, + ): + """ + Initialize resource discovery. + Args: - credentials: AWS credentials dictionary + credentials: AWS credentials account_id: AWS account ID account_name: AWS account name (optional) """ self.credentials = credentials self.account_id = account_id - self.account_name = account_name or account_id # Default to account_id if name not provided + # Ensure account_name is never "unknown" or None + self.account_name = ( + account_name if account_name and account_name != "Unknown" else account_id + ) + self.session = boto3.Session( + aws_access_key_id=credentials.get("aws_access_key_id"), + aws_secret_access_key=credentials.get("aws_secret_access_key"), + aws_session_token=credentials.get("aws_session_token"), + ) self.partition = detect_partition_from_credentials(credentials) + + # Log with account name and ID logger.info( - f"Resource discovery initialized for account {account_id} in partition {self.partition}" + f"Resource discovery initialized for account \ + {self.account_name} ({self.account_id}) in partition {self.partition}" ) def discover_resources( @@ -85,7 +82,7 @@ def discover_resources( """Discover resources of a specific type across multiple regions. Args: - resource_type: Type of resource to discover (ec2, rds, eks, emr, ecr) + resource_type: Type of resource to discover (ec2, rds, eks, emr, ecr) regions: List of regions to discover resources in resource_ids: Optional list of specific resource IDs to discover max_workers: Maximum number of concurrent worker threads @@ -94,29 +91,34 @@ def discover_resources( List of discovered resources """ logger.debug(f"Discovering {resource_type} resources in {len(regions)} regions") - + # Validate regions for this partition valid_regions = filter_regions_for_partition(regions, self.partition) - + # If we filtered out all regions, use default regions for the partition if not valid_regions: valid_regions = DEFAULT_REGIONS.get(self.partition, DEFAULT_REGIONS["aws"]) logger.warning( - f"No valid regions found for partition {self.partition}, using defaults: {', '.join(valid_regions)}" + f"No valid regions found for partition {self.partition}, \ + using defaults: {', '.join(valid_regions)}" ) - + # Use validated regions regions = valid_regions # Validate resource type discovery_method = getattr(self, f"_discover_{resource_type}", None) if not discovery_method: - logger.error(f"No discovery method available for resource type: {resource_type}") + logger.error( + f"No discovery method available for resource type: {resource_type}" + ) return [] # Use thread pool to speed up discovery across regions resources = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=min(max_workers, len(regions))) as executor: + with concurrent.futures.ThreadPoolExecutor( + max_workers=min(max_workers, len(regions)) + ) as executor: # Create a future for each region future_to_region = { executor.submit(discovery_method, region, resource_ids): region @@ -130,7 +132,8 @@ def discover_resources( region_resources = future.result() resources.extend(region_resources) logger.debug( - f"Discovered {len(region_resources)} {resource_type} resources in {region}" + f"Discovered {len(region_resources)} {resource_type} \ + resources in {region}" ) except Exception as e: logger.error(f"Error discovering {resource_type} in {region}: {e}") @@ -150,14 +153,8 @@ def _discover_ec2( Returns: List of EC2 instance dictionaries """ - # Create EC2 client - ec2_client = boto3.client( - "ec2", - region_name=region, - aws_access_key_id=self.credentials.get("aws_access_key_id"), - aws_secret_access_key=self.credentials.get("aws_secret_access_key"), - aws_session_token=self.credentials.get("aws_session_token"), - ) + # Create EC2 client using central utility + ec2_client = create_boto3_client("ec2", self.credentials, region) # Build filters filters = [] @@ -173,31 +170,38 @@ def process_ec2_page(page: Dict[str, Any]) -> List[Dict[str, Any]]: # Convert tags to dictionary tags = format_tags(instance.get("Tags", [])) # Create a simplified instance dictionary - instances.append({ - "id": instance["InstanceId"], - "name": tags.get("Name", "Unnamed"), - "type": instance["InstanceType"], - "status": instance["State"]["Name"], - "region": region, - "tags": tags, - "launch_time": ( - instance.get("LaunchTime", "").isoformat() - if instance.get("LaunchTime") - else None - ), - "public_ip": instance.get("PublicIpAddress"), - "private_ip": instance.get("PrivateIpAddress"), - }) + instances.append( + { + "id": instance["InstanceId"], + "name": tags.get("Name", "Unnamed"), + "type": instance["InstanceType"], + "status": instance["State"]["Name"], + "region": region, + "tags": tags, + "launch_time": ( + instance.get("LaunchTime", "").isoformat() + if instance.get("LaunchTime") + else None + ), + "public_ip": instance.get("PublicIpAddress"), + "private_ip": instance.get("PrivateIpAddress"), + } + ) return instances # Use the centralized pagination utility with our custom processor instances = paginate_aws_response( - ec2_client, - "describe_instances", + ec2_client, + "describe_instances", page_processor=process_ec2_page, - Filters=filters + Filters=filters, ) + # Before returning instances, add account info + for instance in instances: + instance["accountId"] = self.account_id + instance["accountName"] = self.account_name + return instances except Exception as e: @@ -216,14 +220,8 @@ def _discover_rds( Returns: List of RDS instance dictionaries """ - # Create RDS client - rds_client = boto3.client( - "rds", - region_name=region, - aws_access_key_id=self.credentials.get("aws_access_key_id"), - aws_secret_access_key=self.credentials.get("aws_secret_access_key"), - aws_session_token=self.credentials.get("aws_session_token"), - ) + # Create RDS client using central utility + rds_client = create_boto3_client("rds", self.credentials, region) try: # Get instances @@ -278,6 +276,11 @@ def _discover_rds( } instances.append(instance_dict) + # Before returning instances, add account info + for instance in instances: + instance["accountId"] = self.account_id + instance["accountName"] = self.account_name + return instances except Exception as e: @@ -296,18 +299,13 @@ def _discover_emr( Returns: List of EMR cluster dictionaries """ - # Create EMR client - emr_client = boto3.client( - "emr", - region_name=region, - aws_access_key_id=self.credentials.get("aws_access_key_id"), - aws_secret_access_key=self.credentials.get("aws_secret_access_key"), - aws_session_token=self.credentials.get("aws_session_token"), - ) + # Create EMR client using central utility + emr_client = create_boto3_client("emr", self.credentials, region) try: # Build filter for cluster states - # Include all states: STARTING, BOOTSTRAPPING, RUNNING, WAITING, TERMINATING, TERMINATED, TERMINATED_WITH_ERRORS + # Include all states: STARTING, BOOTSTRAPPING, RUNNING, WAITING, + # TERMINATING, TERMINATED, TERMINATED_WITH_ERRORS cluster_states = [ "STARTING", "BOOTSTRAPPING", @@ -389,6 +387,11 @@ def _discover_emr( f"Error getting details for EMR cluster {cluster['Id']}: {e}" ) + # Before returning clusters, add account info + for cluster in detailed_clusters: + cluster["accountId"] = self.account_id + cluster["accountName"] = self.account_name + return detailed_clusters except Exception as e: @@ -407,14 +410,8 @@ def _discover_eks( Returns: List of EKS cluster dictionaries """ - # Create EKS client - eks_client = boto3.client( - "eks", - region_name=region, - aws_access_key_id=self.credentials.get("aws_access_key_id"), - aws_secret_access_key=self.credentials.get("aws_secret_access_key"), - aws_session_token=self.credentials.get("aws_session_token"), - ) + # Create EKS client using central utility + eks_client = create_boto3_client("eks", self.credentials, region) try: # Get clusters @@ -465,6 +462,11 @@ def _discover_eks( except Exception as e: logger.warning(f"Error getting details for EKS cluster {name}: {e}") + # Before returning clusters, add account info + for cluster in clusters: + cluster["accountId"] = self.account_id + cluster["accountName"] = self.account_name + return clusters except Exception as e: @@ -483,14 +485,8 @@ def _discover_ecr( Returns: List of ECR image dictionaries """ - # Create ECR client - ecr_client = boto3.client( - "ecr", - region_name=region, - aws_access_key_id=self.credentials.get("aws_access_key_id"), - aws_secret_access_key=self.credentials.get("aws_secret_access_key"), - aws_session_token=self.credentials.get("aws_session_token"), - ) + # Create ECR client using central utility + ecr_client = create_boto3_client("ecr", self.credentials, region) try: # Get all repositories @@ -566,12 +562,16 @@ def _discover_ecr( else None ), "digest": image.get("imageDigest"), + # Add account info directly + "accountId": self.account_id, + "accountName": self.account_name, } all_images.append(image_dict) except Exception as e: logger.warning( - f"Error describing images for repository {repo_name} chunk {i//chunk_size + 1}: {e}" + f"Error describing images for repository \ + {repo_name} chunk {i//chunk_size + 1}: {e}" ) except Exception as e: @@ -579,12 +579,69 @@ def _discover_ecr( f"Error listing images for repository {repo_name}: {e}" ) + # No need for the loop to add account info as we did it inline return all_images except Exception as e: logger.error(f"Error discovering ECR repositories in {region}: {e}") return [] + +def discover_ecr_images( + credentials: Dict[str, str], + regions: List[str], + account_id: str, + account_name: Optional[str] = None, +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """ + Discover ECR repositories and their images. + + Args: + credentials: AWS credentials dictionary + regions: List of AWS regions + account_id: AWS account ID + account_name: AWS account name + + Returns: + Tuple containing (list of all ECR images, list of old ECR images) + """ + if not ECRManager: + logger.warning("ECRManager not available, skipping ECR discovery") + return [], [] + + all_ecr_images = [] + + # Ensure we have a valid account name + if not account_name or account_name == "Unknown": + account_name = account_id + + try: + for region in regions: + try: + # Create ECR manager with provided credentials + ecr_manager = ECRManager( + account_id=account_id, region=region, credentials=credentials + ) + + # Set account name + ecr_manager.account_name = account_name + + # Discover resources in this region + region_images = ecr_manager.discover_resources([region]) + all_ecr_images.extend(region_images) + + except Exception as e: + logger.error(f"Error discovering ECR images in {region}: {e}") + + except Exception as e: + logger.error(f"Error during ECR discovery: {e}") + + # Separate old images + old_images = [img for img in all_ecr_images if img.get("isOld", False)] + + return all_ecr_images, old_images + + # Backward compatibility wrapper for get_account_resources # This function accepts both positional and keyword arguments def get_account_resources(*args, **kwargs) -> Dict[str, List[Dict[str, Any]]]: @@ -632,6 +689,7 @@ def get_account_resources(*args, **kwargs) -> Dict[str, List[Dict[str, Any]]]: emr_cluster_states=emr_cluster_states, ) + def _get_account_resources_impl( credentials: Dict[str, str], regions: List[str], @@ -655,9 +713,19 @@ def _get_account_resources_impl( Returns: Dictionary of resource lists by type """ + # Add a debugging log to confirm what account name is being used + logger.debug( + f"_get_account_resources_impl called with account_name: {account_name}" + ) + if exclude_resources is None: exclude_resources = [] + # Immediately ensure account_name is valid right at the start + # and log to confirm what value we're using + account_name = ensure_valid_account_name(account_id, account_name) + logger.debug(f"Using account name: {account_name} for account {account_id}") + resources = { "ec2_instances": [], "rds_instances": [], @@ -698,67 +766,115 @@ def _get_account_resources_impl( # Create resource discovery instance discovery = ResourceDiscovery(discovery_credentials, account_id, account_name) - # Discover EC2 instances + # Only discover resources that aren't excluded + + # Discover EC2 instances - only if not excluded if "ec2" not in exclude_resources: logger.info( - f"Discovering EC2 instances for account {account_id} in {len(regions)} regions" + f"Discovering EC2 instances for account {account_name} ({account_id}) \ + in {len(regions)} regions" ) resources["ec2_instances"] = discovery.discover_resources("ec2", regions) + else: + logger.debug("Skipping EC2 discovery as per resource type filter") - # Discover RDS instances + # Discover RDS instances - only if not excluded if "rds" not in exclude_resources: logger.info( - f"Discovering RDS instances for account {account_id} in {len(regions)} regions" + f"Discovering RDS instances for account {account_name} ({account_id}) \ + in {len(regions)} regions" ) resources["rds_instances"] = discovery.discover_resources("rds", regions) + else: + logger.debug("Skipping RDS discovery as per resource type filter") - # Discover EKS clusters + # Discover EKS clusters - only if not excluded if "eks" not in exclude_resources: logger.info( - f"Discovering EKS clusters for account {account_id} in {len(regions)} regions" + f"Discovering EKS clusters for account {account_name} ({account_id}) \ + in {len(regions)} regions" ) try: resources["eks_clusters"] = discovery.discover_resources("eks", regions) except Exception as e: logger.error(f"Error discovering EKS clusters: {e}") resources["eks_clusters"] = [] + else: + logger.debug("Skipping EKS discovery as per resource type filter") - # Discover EMR clusters + # Discover EMR clusters - only if not excluded if "emr" not in exclude_resources: logger.info( - f"Discovering EMR clusters for account {account_id} in {len(regions)} regions" + f"Discovering EMR clusters for account {account_name} ({account_id}) \ + in {len(regions)} regions" ) try: resources["emr_clusters"] = discovery.discover_resources("emr", regions) except Exception as e: logger.error(f"Error discovering EMR clusters: {e}") resources["emr_clusters"] = [] + else: + logger.debug("Skipping EMR discovery as per resource type filter") - # Discover ECR images + # Discover ECR images - only if not excluded if "ecr" not in exclude_resources: logger.info( - f"Discovering ECR images for account {account_id} in {len(regions)} regions" + f"Discovering ECR images for account {account_name} ({account_id}) \ + in {len(regions)} regions" ) try: - ecr_resources = discovery.discover_resources("ecr", regions) - resources["ecr_images"] = ecr_resources - # Filter old images based on age - only if image has pushed_at attribute - resources["ecr_old_images"] = [ - img for img in ecr_resources - if img.get("is_old", False) or - (img.get("age_days", 0) > 365) - ] + # Use our helper function that properly enriches ECR resources + all_images, old_images = discover_ecr_images( + discovery_credentials, regions, account_id, account_name + ) + resources["ecr_images"] = all_images + resources["ecr_old_images"] = old_images except Exception as e: logger.error(f"Error discovering ECR images: {e}") resources["ecr_images"] = [] resources["ecr_old_images"] = [] + else: + logger.debug("Skipping ECR discovery as per resource type filter") # Add account information to all resources for resource_type, resource_list in resources.items(): for resource in resource_list: resource["accountId"] = account_id - if account_name: - resource["accountName"] = account_name + resource["accountName"] = account_name + + # After all discoveries are complete, log the counts for each resource type + # to ensure they're properly tracked in the logs + for resource_type, resource_list in resources.items(): + if resource_list: + # Include the actual count in a consistent way for each resource type + # This helps with debugging resource counts + if resource_type in [ + "ec2_instances", + "rds_instances", + "eks_clusters", + "emr_clusters", + ]: + logger.info( + f"Total {resource_type.split('_')[0]} resources discovered: \ + {len(resource_list)}" + ) + elif resource_type == "ecr_images": + # ECR images are already logged in discover_ecr_images + logger.info( + f"Total {resource_type.split('_')[0]} resources discovered: \ + {len(resource_list)}" + ) + + # Also add a log entry for the total resources + # discovered, which helps with debugging + total_resources = sum( + len(resources[resource_type]) + for resource_type in resources + if resource_type != "ecr_old_images" + ) # Don't double-count old images + logger.debug( + f"Total resources discovered for account {account_id}: {total_resources}" + ) except Exception as e: logger.error(f"Error discovering resources for account {account_id}: {e}") diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py index 349a02c4..e4ad45ca 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py @@ -4,19 +4,14 @@ import logging from datetime import datetime -from typing import Any, Dict, List, Optional, Type, Callable +from typing import Any, Callable, Dict, List, Optional -import boto3 -from aws_resource_management.aws_utils import get_session_for_account -from aws_resource_management.config_manager import get_config -from aws_resource_management.discovery_utils import ( - add_account_info, +from aws_resource_management.utils import ( + get_config, + get_session_for_account, normalize_resource_state, - paginate_aws_response, # Use the enhanced utility - safe_api_call, - should_exclude_resource + paginate_aws_response, ) -from aws_resource_management.partition_utils import get_partition_for_region from botocore.config import Config from botocore.exceptions import ClientError, NoRegionError @@ -29,12 +24,31 @@ class ResourceManager: # Class attributes for resource type identification and registration # These should be overridden by subclasses _RESOURCE_TYPE = None # e.g., 'ec2' - used for registry lookup - _DISPLAY_NAME = None # e.g., 'EC2 Instances' - user-friendly name - _DESCRIPTION = None # e.g., 'Amazon Elastic Compute Cloud instances' + _DISPLAY_NAME = None # e.g., 'EC2 Instances' - user-friendly name + _DESCRIPTION = None # e.g., 'Amazon Elastic Compute Cloud instances' + _RESOURCE_KEY = None # e.g., 'ec2_instances' - used for resource lists - def __init__(self, account_id: str, region: str): + def __init__( + self, + account_id: str, + region: str, + credentials: Optional[Dict[str, str]] = None, + account_name: Optional[str] = None, + ): + """ + Initialize resource manager. + + Args: + account_id: AWS account ID + region: AWS region + credentials: Optional AWS credentials + account_name: Optional AWS account name + """ self.account_id = account_id self.region = region + self.credentials = credentials + # Make sure account_name is never None to avoid 'Unknown' in logs + self.account_name = account_name if account_name else account_id self.session = None self._clients = {} # Cache for boto3 clients self.config = get_config() @@ -88,18 +102,21 @@ def get_client(self, service_name: str) -> Any: except NoRegionError: logger.error( - f"No region specified for {service_name} client and no default region found" + f"No region specified for {service_name} client \ + and no default region found" ) raise except ClientError as e: error_code = e.response.get("Error", {}).get("Code", "Unknown") logger.error( - f"Error creating {service_name} client in {self.region}: {error_code} - {str(e)}" + f"Error creating {service_name} client in \ + {self.region}: {error_code} - {str(e)}" ) raise except Exception as e: logger.error( - f"Error creating {service_name} client in {self.region}: {str(e)}" + f"Error creating {service_name} client \ + in {self.region}: {str(e)}" ) raise @@ -117,7 +134,8 @@ def _handle_api_error(self, operation: str, error: Exception) -> None: if error_code == "AuthFailure": logger.error( - f"Authentication failure during {operation} in account {self.account_id} " + f"Authentication failure during {operation} in \ + account {self.account_id} " f"region {self.region}. Check IAM permissions." ) elif error_code == "AccessDenied": @@ -136,7 +154,9 @@ def _handle_api_error(self, operation: str, error: Exception) -> None: f"region {self.region}: {str(error)}" ) - def start(self, resources: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + def start( + self, resources: List[Dict[str, Any]], dry_run: bool = False + ) -> Dict[str, int]: """ Start resources. Must be implemented by derived classes. @@ -149,7 +169,9 @@ def start(self, resources: List[Dict[str, Any]], dry_run: bool = False) -> Dict[ """ raise NotImplementedError("Subclasses must implement start method") - def stop(self, resources: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + def stop( + self, resources: List[Dict[str, Any]], dry_run: bool = False + ) -> Dict[str, int]: """ Stop resources. Must be implemented by derived classes. @@ -173,19 +195,19 @@ def discover_resources(self, regions: List[str]) -> List[Dict[str, Any]]: List of dictionaries containing resource information """ raise NotImplementedError("Subclasses must implement discover_resources method") - + def paginate_boto3( self, client: Any, operation: str, result_key: str, **kwargs ) -> List[Dict[str, Any]]: """ Paginate through AWS API responses. - + Args: client: Boto3 client operation: Operation name (e.g., 'describe_instances') result_key: Key in the response that contains the results **kwargs: Additional arguments for the operation - + Returns: Combined list of results from all pages """ @@ -223,9 +245,6 @@ def get_boto3_client(self, service_name: str, region: str) -> Any: logger.error(f"Error creating {service_name} client in {region}: {str(e)}") return None - # Alias for backward compatibility - many manager implementations use create_client() - create_client = get_boto3_client - def log_action( self, resource_id: str, @@ -254,14 +273,15 @@ def log_action( schedule_str = f" [Schedule: {existing_schedule}]" if existing_schedule else "" logger.info( - f"{dry_run_prefix}{action.capitalize()} {self._RESOURCE_TYPE} {resource_id} {name_str} " + f"{dry_run_prefix}{action.capitalize()} {self._RESOURCE_TYPE} \ + {resource_id} {name_str} " f"in {region}{details_str}{schedule_str}" ) - + # Also log to CSV if tracking is enabled try: - from aws_resource_management.logging_setup import log_action_to_csv - + from aws_resource_management.utils.logging_utils import log_action_to_csv + log_action_to_csv( account_id=self.account_id, account_name=getattr(self, "account_name", self.account_id), @@ -296,21 +316,21 @@ def should_exclude(self, resource: Dict[str, Any]) -> bool: return True return False - + def run_with_retries( - self, - func: Callable, - max_retries: int = 3, - error_message: str = "Operation failed" + self, + func: Callable, + max_retries: int = 3, + error_message: str = "Operation failed", ) -> Any: """ Run a function with retries. - + Args: func: Function to run max_retries: Maximum number of retries error_message: Error message to log on failure - + Returns: Result of the function or None on failure """ @@ -329,31 +349,35 @@ def run_with_retries( return None def process_resources_in_batches( - self, + self, resources: List[Dict[str, Any]], batch_action: Callable[[List[Dict[str, Any]]], bool], batch_size: int = 20, - description: str = "Processing resources" + description: str = "Processing resources", ) -> Dict[str, int]: """ Process resources in batches to avoid API limits. - + Args: resources: List of resources to process batch_action: Function to call for each batch batch_size: Size of each batch description: Description for logging - + Returns: Dictionary with success and error counts """ stats = {"success": 0, "errors": 0, "skipped": 0} - + # Process in batches for i in range(0, len(resources), batch_size): - batch = resources[i:i+batch_size] - logger.info(f"{description} batch {i//batch_size + 1}/{(len(resources)-1)//batch_size + 1} ({len(batch)} resources)") - + batch = resources[i : i + batch_size] + logger.info( + f"{description} batch \ + {i//batch_size + 1}/{(len(resources)-1)//batch_size + 1} \ + ({len(batch)} resources)" + ) + try: if batch_action(batch): stats["success"] += len(batch) @@ -362,8 +386,27 @@ def process_resources_in_batches( except Exception as e: logger.error(f"Error processing batch: {str(e)}") stats["errors"] += len(batch) - + return stats - # Use centralized partition utility - get_partition_for_region = get_partition_for_region + @staticmethod + def normalize_resources(resources: List[Dict[str, Any]]) -> None: + """ + Normalize resource state fields for consistency. + + Args: + resources: List of resources to normalize + """ + for resource in resources: + normalize_resource_state(resource) + + # Ensure account information is present + if "accountId" not in resource and hasattr(resource, "accountId"): + resource["accountId"] = resource.accountId + + if "accountName" not in resource and hasattr(resource, "accountName"): + resource["accountName"] = resource.accountName + + # If still no account name, use account ID as fallback + if "accountName" not in resource and "accountId" in resource: + resource["accountName"] = resource["accountId"] diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ec2.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ec2.py index b44fd177..e10b7b94 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ec2.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ec2.py @@ -3,18 +3,16 @@ """ from collections import defaultdict -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List -from aws_resource_management.config_manager import get_config -from aws_resource_management.discovery_utils import ( - add_account_info, +from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.utils import ( format_tags, - normalize_resource_state, - paginate_aws_response, # Use the centralized utility - safe_api_call + get_config, + paginate_aws_response, + safe_api_call, + setup_logging, ) -from aws_resource_management.logging_setup import setup_logging -from aws_resource_management.managers.base import ResourceManager logger = setup_logging() config = get_config() @@ -47,7 +45,8 @@ def should_exclude(self, instance: Dict[str, Any]) -> bool: if exclusion_tag in tags: logger.debug( - f"Skipping EC2 instance {instance['id']} with exclusion tag {exclusion_tag}" + f"Skipping EC2 instance {instance['id']} with \ + exclusion tag {exclusion_tag}" ) return True @@ -79,7 +78,8 @@ def stop( running_instances_by_region[instance["region"]].append(instance) else: logger.debug( - f"Skipping EC2 instance {instance['id']} in state {instance['state']} (not running)" + f"Skipping EC2 instance {instance['id']} in \ + state {instance['state']} (not running)" ) stats["skipped"] += 1 @@ -96,26 +96,26 @@ def stop( # Process instances in batches using the utility method def process_batch(batch): instance_ids = [instance["id"] for instance in batch] - + # Tag and stop instances if not dry_run: # Using the safe_api_call utility for error handling tag_success = safe_api_call( lambda: ec2_client.create_tags( Resources=instance_ids, - Tags=[{"Key": config.get("stop_tag"), "Value": timestamp}] + Tags=[{"Key": config.get("stop_tag"), "Value": timestamp}], ), - f"Failed to tag EC2 instances in {region}" + f"Failed to tag EC2 instances in {region}", ) - + stop_success = safe_api_call( lambda: ec2_client.stop_instances(InstanceIds=instance_ids), - f"Failed to stop EC2 instances in {region}" + f"Failed to stop EC2 instances in {region}", ) - + if not tag_success or not stop_success: return False - + # Log individual instances for instance in batch: self.log_action( @@ -123,25 +123,26 @@ def process_batch(batch): region, "stop", resource_name=instance.get("name", "Unnamed"), - dry_run=dry_run + dry_run=dry_run, ) - + return True - + # Using the process_resources_in_batches utility batch_stats = self.process_resources_in_batches( region_instances, process_batch, self.MAX_INSTANCES_PER_API_CALL, - f"Stopping EC2 instances in {region}" + f"Stopping EC2 instances in {region}", ) - + # Merge stats for key in ["success", "errors", "skipped"]: stats[key] += batch_stats[key] logger.info( - f"EC2 stop summary: {stats['success']} stopped, {stats['skipped']} skipped, {stats['errors']} errors" + f"EC2 stop summary: {stats['success']} stopped, {stats['skipped']} \ + skipped, {stats['errors']} errors" ) return stats @@ -163,7 +164,8 @@ def start( stopped_instances_by_region[instance["region"]].append(instance) else: logger.debug( - f"Skipping EC2 instance {instance['id']} in state {instance['state']} (not stopped)" + f"Skipping EC2 instance {instance['id']} in state \ + {instance['state']} (not stopped)" ) stats["skipped"] += 1 @@ -189,13 +191,15 @@ def start( ec2_client.start_instances(InstanceIds=instance_ids) else: logger.info( - f"[DRY RUN] Would start {len(instance_ids)} EC2 instances in {region}" + f"[DRY RUN] Would start {len(instance_ids)} EC2 \ + instances in {region}" ) # Remove tags in batch if not dry_run: logger.info( - f"Removing stop tag from {len(instance_ids)} EC2 instances in {region}" + f"Removing stop tag from {len(instance_ids)} EC2 \ + instances in {region}" ) ec2_client.delete_tags( Resources=instance_ids, @@ -218,7 +222,8 @@ def start( stats["errors"] += len(batch) logger.info( - f"EC2 start summary: {stats['success']} started, {stats['skipped']} skipped, {stats['errors']} errors" + f"EC2 start summary: {stats['success']} started, {stats['skipped']} \ + skipped, {stats['errors']} errors" ) return stats @@ -239,35 +244,35 @@ def process_ec2_page(page: Dict[str, Any]) -> List[Dict[str, Any]]: for instance in reservation.get("Instances", []): # Convert tags to dictionary using shared utility tags = format_tags(instance.get("Tags", [])) - + # Create a standardized instance dictionary - instances.append({ - "id": instance["InstanceId"], - "name": tags.get("Name", "Unnamed"), - "type": instance["InstanceType"], - "status": instance["State"]["Name"], - "state": instance["State"]["Name"], - "region": region, - "tags": tags, - "launch_time": ( - instance["LaunchTime"].isoformat() - if "LaunchTime" in instance - else None - ), - "public_ip": instance.get("PublicIpAddress"), - "private_ip": instance.get("PrivateIpAddress"), - "accountId": self.account_id, - "accountName": self.account_name, - }) + instances.append( + { + "id": instance["InstanceId"], + "name": tags.get("Name", "Unnamed"), + "type": instance["InstanceType"], + "status": instance["State"]["Name"], + "state": instance["State"]["Name"], + "region": region, + "tags": tags, + "launch_time": ( + instance["LaunchTime"].isoformat() + if "LaunchTime" in instance + else None + ), + "public_ip": instance.get("PublicIpAddress"), + "private_ip": instance.get("PrivateIpAddress"), + "accountId": self.account_id, + "accountName": self.account_name, + } + ) return instances - + # Use centralized pagination utility with custom processor instances = paginate_aws_response( - ec2_client, - "describe_instances", - page_processor=process_ec2_page + ec2_client, "describe_instances", page_processor=process_ec2_page ) - + logger.info(f"Found {len(instances)} EC2 instances in {region}") all_instances.extend(instances) diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ecr.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ecr.py index 500487d6..cc20d297 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ecr.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/ecr.py @@ -2,46 +2,75 @@ ECR resource manager for AWS Resource Management. """ -import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional -from aws_resource_management.config_manager import get_config -from aws_resource_management.discovery_utils import ( - format_tags, paginate_aws_response, safe_api_call -) -from aws_resource_management.logging_setup import setup_logging +import boto3 from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.utils import ( + create_boto3_client, + get_config, + parse_iso_datetime, + setup_logging, +) logger = setup_logging() config = get_config() +# Default age threshold in days for "old" images +DEFAULT_AGE_THRESHOLD_DAYS = 365 # 1 year + class ECRManager(ResourceManager): - """Manager for ECR repositories and images.""" + """Manager for Amazon ECR images.""" # Resource type information for registry _RESOURCE_TYPE = "ecr" - _DISPLAY_NAME = "ECR Repositories" - _DESCRIPTION = "Amazon Elastic Container Registry repositories and images" + _DISPLAY_NAME = "ECR Images" + _DESCRIPTION = "Amazon Elastic Container Registry images" + _RESOURCE_KEY = "ecr_images" def __init__( self, account_id: str, region: str, + credentials: Optional[Dict[str, Any]] = None, ): - """Initialize ECR manager.""" + """ + Initialize ECR manager. + + Args: + account_id: AWS account ID + region: AWS region + credentials: Optional AWS credentials dictionary + """ super().__init__(account_id, region) - self.account_name = None - self.old_image_threshold_days = config.get("ecr_old_image_threshold_days", 365) + self.account_name = None # Will be set by the caller if available + self.credentials = credentials + self.age_threshold = config.get( + "ecr_age_threshold_days", DEFAULT_AGE_THRESHOLD_DAYS + ) + + def get_ecr_client(self, region: str = None) -> boto3.client: + """ + Get an ECR client for the specified region using central utility. + + Args: + region: AWS region (defaults to the manager's region) + + Returns: + Boto3 ECR client + """ + region = region or self.region + return create_boto3_client("ecr", self.credentials, region) def discover_resources(self, regions: List[str]) -> List[Dict[str, Any]]: """ - Discover ECR repositories and their images. - + Discover ECR repositories and their images across regions. + Args: - regions: List of AWS regions to scan - + regions: List of AWS regions to search + Returns: List of ECR image dictionaries """ @@ -49,232 +78,242 @@ def discover_resources(self, regions: List[str]) -> List[Dict[str, Any]]: for region in regions: try: - ecr_client = self.get_boto3_client("ecr", region) - if not ecr_client: - logger.warning(f"Could not create ECR client in {region}, skipping") - continue - - # Discover repositories - repositories = paginate_aws_response( - ecr_client, - "describe_repositories", - "repositories" - ) + ecr_client = self.get_ecr_client(region) + + # Get all repositories + repositories = [] + paginator = ecr_client.get_paginator("describe_repositories") + for page in paginator.paginate(): + repositories.extend(page.get("repositories", [])) logger.info(f"Found {len(repositories)} ECR repositories in {region}") - - # Process each repository to get its images + + # Get images from each repository for repo in repositories: repo_name = repo.get("repositoryName", "unknown") - - try: - # Get image IDs (more efficient than getting all details at once) - image_ids = paginate_aws_response( - ecr_client, - "list_images", - "imageIds", - repositoryName=repo_name - ) - - logger.debug(f"Found {len(image_ids)} images in repository {repo_name}") - - # Process images in batches (ECR API has limits on batch size) - for i in range(0, len(image_ids), 100): - batch = image_ids[i:i + 100] - if not batch: - continue - - try: - # Get detailed information about images - image_details = safe_api_call( - lambda: ecr_client.describe_images( - repositoryName=repo_name, - imageIds=batch - ), - f"Error describing images in {repo_name}", - {"imageDetails": []} - ) - - # Process each image - for image in image_details.get("imageDetails", []): - try: - # Get primary tag (first in list) or use digest - image_tags = image.get("imageTags", []) - primary_tag = image_tags[0] if image_tags else "untagged" - - # Calculate image age - pushed_at = image.get("imagePushedAt") - is_old = False - age_days = 0 - - if pushed_at: - age = datetime.now().astimezone() - pushed_at.astimezone() - age_days = age.days - is_old = age_days > self.old_image_threshold_days - - # Create simplified image object - image_obj = { - "id": image.get("imageDigest", ""), - "name": f"{repo_name}:{primary_tag}", - "repositoryName": repo_name, - "region": region, - "tags": image_tags, - "size_mb": round(image.get("imageSizeInBytes", 0) / (1024 * 1024), 2), - "pushed_at": pushed_at.isoformat() if pushed_at else None, - "age_days": age_days, - "is_old": is_old, - "digest": image.get("imageDigest"), - "accountId": self.account_id, - "accountName": self.account_name - } - - all_images.append(image_obj) - except Exception as e: - logger.warning(f"Error processing image in {repo_name}: {e}") - - except Exception as e: - logger.warning(f"Error describing batch of images in {repo_name}: {e}") - - except Exception as e: - logger.warning(f"Error listing images in repository {repo_name}: {e}") + images = self._get_repository_images(ecr_client, repo_name, region) + all_images.extend(images) except Exception as e: logger.error(f"Error discovering ECR repositories in {region}: {e}") - - logger.info(f"Discovered {len(all_images)} ECR images across {len(regions)} regions") + + # Enrich all images with age and old status + all_images = self.enrich_resources(all_images) + + logger.info( + f"Discovered {len(all_images)} ECR images across {len(regions)} regions" + ) return all_images - - def start(self, resources: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + + def _get_repository_images( + self, ecr_client, repo_name: str, region: str + ) -> List[Dict[str, Any]]: """ - ECR images don't have a start operation, so this is a no-op. - + Get all images from a specific ECR repository. + Args: - resources: List of ECR image dictionaries - dry_run: Whether to simulate the action - + ecr_client: Boto3 ECR client + repo_name: Name of the ECR repository + region: AWS region + Returns: - Dictionary with success and error counts + List of ECR image dictionaries """ - logger.info("ECR images don't support start operations") - return {"success": 0, "errors": 0, "skipped": len(resources)} - - def stop(self, resources: List[Dict[str, Any]], dry_run: bool = False) -> Dict[str, int]: + images = [] + + try: + paginator = ecr_client.get_paginator("describe_images") + for page in paginator.paginate(repositoryName=repo_name): + for image in page.get("imageDetails", []): + # Get a valid image tag for the name + image_tag = "untagged" + if image.get("imageTags") and len(image["imageTags"]) > 0: + image_tag = image["imageTags"][0] + + image_dict = { + "id": image.get("imageDigest", "unknown"), + "name": f"{repo_name}:{image_tag}", + "region": region, + "repositoryName": repo_name, + "tags": image.get("imageTags", []), + "imagePushedAt": image.get("imagePushedAt"), + "imageSizeInBytes": image.get("imageSizeInBytes", 0), + "status": "AVAILABLE", + # Add account info directly + "accountId": self.account_id, + "accountName": self.account_name or self.account_id, + } + + images.append(image_dict) + except Exception as e: + logger.error(f"Error getting images for repository {repo_name}: {e}") + + return images + + def enrich_resources(self, resources: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ - ECR images don't have a stop operation, so this is a no-op. - + Enrich ECR images with age information and identify old images. + Args: resources: List of ECR image dictionaries - dry_run: Whether to simulate the action - + Returns: - Dictionary with success and error counts + Enriched list of ECR image dictionaries + """ + now = datetime.now(timezone.utc) + threshold = timedelta(days=self.age_threshold) + + old_images = [] + for image in resources: + # Calculate age if pushed date exists + if pushed_at := image.get("imagePushedAt"): + if isinstance(pushed_at, str): + pushed_at = parse_iso_datetime(pushed_at) + + if pushed_at: + age = now - pushed_at + image["ageInDays"] = age.days + + # Identify images older than threshold + if age > threshold: + image["isOld"] = True + old_images.append(image) + else: + image["isOld"] = False + + logger.info( + f"Found {len(old_images)} images older than {self.age_threshold} days" + ) + return resources + + @staticmethod + def normalize_resources(resources: List[Dict[str, Any]]) -> None: + """ + Normalize ECR resources to ensure consistent format. + + Args: + resources: List of ECR image dictionaries + """ + for resource in resources: + # Ensure status field is present + if "status" not in resource: + resource["status"] = "AVAILABLE" + + # Ensure region is a string + if "region" not in resource: + resource["region"] = "unknown" + + @staticmethod + def update_stats(resources: List[Dict[str, Any]], stats: Dict[str, Any]) -> None: + """ + Update statistics with ECR-specific information. + + Args: + resources: List of ECR image resources + stats: Statistics dictionary to update """ - logger.info("ECR images don't support stop operations") - return {"success": 0, "errors": 0, "skipped": len(resources)} - - def cleanup_old_images( - self, - images: List[Dict[str, Any]], - dry_run: bool = False, - keep_tagged: bool = True + # Update the found count in stats + stats["ecr_found"] = len(resources) + + # Count old images + old_images = [r for r in resources if r.get("isOld", False)] + stats["ecr_old_images"] = len(old_images) + + # Make sure we increment the global resource counters properly + if "resources_processed" in stats: + stats["resources_processed"] += len(resources) + + def stop( + self, resources: List[Dict[str, Any]], dry_run: bool = False ) -> Dict[str, int]: """ - Delete old ECR images to save storage costs. - + Delete old ECR images. For ECR, 'stop' means deleting old images. + Args: - images: List of ECR image dictionaries - dry_run: Whether to simulate the action - keep_tagged: Whether to preserve images with tags - + resources: List of ECR image dictionaries + dry_run: If True, don't actually delete images + Returns: - Dictionary with success and error counts + Dictionary with success, skipped and error counts """ - stats = {"success": 0, "errors": 0, "skipped": 0} - - # Group images by repository and region for more efficient processing - images_by_repo_region = {} - for image in images: - repo_name = image.get("repositoryName") - region = image.get("region") - - if not repo_name or not region: - stats["skipped"] += 1 - continue - - key = f"{region}:{repo_name}" - if key not in images_by_repo_region: - images_by_repo_region[key] = [] - - images_by_repo_region[key].append(image) - - # Process each repository - for key, repo_images in images_by_repo_region.items(): - region, repo_name = key.split(":", 1) - - # Only process images that meet deletion criteria - to_delete = [] - for image in repo_images: - # Skip images that aren't old - if not image.get("is_old", False): - stats["skipped"] += 1 - continue - - # Skip tagged images if keep_tagged is True - if keep_tagged and image.get("tags"): - stats["skipped"] += 1 + result = {"success": 0, "skipped": 0, "errors": 0} + old_images = [r for r in resources if r.get("isOld", True)] + + if not old_images: + logger.info("No old ECR images found to delete") + return result + + # Create ECR client for this region + try: + ecr_client = self.get_ecr_client() + + for image in old_images: + if image.get("region") != self.region: + result["skipped"] += 1 continue - - to_delete.append(image) - - if not to_delete: - continue - - # Get ECR client for this region - ecr_client = self.get_boto3_client("ecr", region) - if not ecr_client: - logger.error(f"Failed to create ECR client for region {region}") - stats["errors"] += len(to_delete) - continue - - # Delete images in batches (ECR API limit is 100 per call) - for i in range(0, len(to_delete), 100): - batch = to_delete[i:i + 100] - - # Create image identifiers for the batch - image_ids = [] - for image in batch: - digest = image.get("digest") - if digest: - image_ids.append({"imageDigest": digest}) - - if not image_ids: + + repo_name = image.get("repositoryName") + image_id = image.get("id") + + if not repo_name or not image_id: + logger.warning(f"Incomplete image data: {image}") + result["skipped"] += 1 continue - - # Delete the batch - if not dry_run: - try: - logger.info(f"Deleting {len(image_ids)} old images from {repo_name} in {region}") + + try: + if dry_run: + self.log_action( + resource_id=image_id, + region=self.region, + action="delete", + resource_name=f"{repo_name}:{image_id[:12]}", + details=f"Age: {image.get('ageInDays', 'unknown')} days", + dry_run=True, + ) + result["success"] += 1 + else: + # Delete the image ecr_client.batch_delete_image( repositoryName=repo_name, - imageIds=image_ids + imageIds=[{"imageDigest": image_id}], ) - stats["success"] += len(image_ids) - except Exception as e: - logger.error(f"Error deleting images from {repo_name} in {region}: {e}") - stats["errors"] += len(image_ids) - else: - logger.info(f"[DRY RUN] Would delete {len(image_ids)} old images from {repo_name} in {region}") - stats["success"] += len(image_ids) - - # Log individual images for tracking - for image in batch: - self.log_action( - image.get("digest", "unknown")[:12], - region, - "delete", - resource_name=image.get("name", "Unknown"), - details=f"Age: {image.get('age_days')} days", - dry_run=dry_run - ) - - return stats + + self.log_action( + resource_id=image_id, + region=self.region, + action="delete", + resource_name=f"{repo_name}:{image_id[:12]}", + details=f"Age: {image.get('ageInDays', 'unknown')} days", + dry_run=False, + ) + result["success"] += 1 + + except Exception as e: + logger.error(f"Error deleting image {image_id}: {e}") + result["errors"] += 1 + + except Exception as e: + logger.error(f"Error creating ECR client for {self.region}: {e}") + result["errors"] += len(old_images) + + # Ensure we update the stats for deleted items properly + # This ensures both "ecr_deleted" and "ecr_stopped" keys are set + # for backward compatibility + result["deleted"] = result["success"] + return result + + def start( + self, resources: List[Dict[str, Any]], dry_run: bool = False + ) -> Dict[str, int]: + """ + Start operation is not applicable for ECR images. + + Args: + resources: List of ECR image dictionaries + dry_run: If True, simulate the operation + + Returns: + Dictionary with success, skipped and error counts + """ + logger.info("Start operation not applicable for ECR images") + return {"success": 0, "skipped": len(resources), "errors": 0} diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py index ba62637a..30b5afef 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py @@ -2,11 +2,11 @@ EKS resource manager class. """ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional -from aws_resource_management.config_manager import get_config -from aws_resource_management.logging_setup import setup_logging from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.utils.config_utils import get_config +from aws_resource_management.utils.logging_utils import setup_logging logger = setup_logging() config = get_config() @@ -39,7 +39,7 @@ def __init__( """ super().__init__(account_id, region) self.resource_type = "eks_cluster" - self.account_name = None # This can be updated if needed + self.account_name = None def stop( self, clusters: List[Dict[str, Any]], dry_run: bool = False @@ -99,7 +99,8 @@ def scale_down(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> N # Skip clusters with the exclusion tag if self.should_exclude(cluster): logger.info( - f"Skipping EKS cluster {cluster['name']} with exclusion tag {config.get('exclusion_tag')}" + f"Skipping EKS cluster {cluster['name']} with exclusion tag \ + {config.get('exclusion_tag')}" ) skipped_count += 1 continue @@ -120,7 +121,8 @@ def scale_down(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> N # Only proceed if we have some scaling values to work with if min_nodes or max_nodes or desired_nodes or has_combined_size_tag: logger.info( - f"{'[DRY RUN] Would scale down' if dry_run else 'Scaling down'} EKS cluster {cluster['name']} in region {region}" + f"{'[DRY RUN] Would scale down' if dry_run else 'Scaling down'}\ + EKS cluster {cluster['name']} in region {region}" ) if not dry_run: @@ -151,12 +153,15 @@ def scale_down(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> N # Apply the tag updates if tags_to_update: logger.info( - f"Updating scaling tags for EKS cluster {cluster['name']}: {tags_to_update}" + f"Updating scaling tags for EKS cluster \ + {cluster['name']}: {tags_to_update}" ) eks_client.tag_resource( resourceArn=cluster.get( "arn", - f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}", + f"arn:{self.get_partition_for_region(region)}:eks:\ + {region}:{self.account_id}:\ + cluster/{cluster['name']}", ), tags=tags_to_update, ) @@ -170,15 +175,19 @@ def scale_down(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> N if has_combined_size_tag: original_size = cluster["tags"][CLUSTER_SIZE_TAG] logger.info( - f"[DRY RUN] Would backup combined size tag: {original_size} and set to min:0-max:0-desired:0" + f"[DRY RUN] Would backup combined size tag: \ + {original_size} and set to min:0-max:0-desired:0" ) else: logger.info( - f"[DRY RUN] Would backup scaling values: Min={min_nodes}, Max={max_nodes}, Desired={desired_nodes}" + f"[DRY RUN] Would backup scaling values: \ + Min={min_nodes}, Max={max_nodes}, \ + Desired={desired_nodes}" ) logger.info( - f"[DRY RUN] Would set scaling values to 0 for EKS cluster {cluster['name']}" + f"[DRY RUN] Would set scaling values to \ + 0 for EKS cluster {cluster['name']}" ) self._scale_nodegroups( eks_client, cluster["name"], 0, region, dry_run=True @@ -190,24 +199,28 @@ def scale_down(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> N cluster["name"], region, "scale_down", - details=f"Min={min_nodes}->{0}, Max={max_nodes}->{0}, Desired={desired_nodes}->{0}{schedule_info}", + details=f"Min={min_nodes}->{0}, Max={max_nodes}->{0}, \ + Desired={desired_nodes}->{0}{schedule_info}", dry_run=dry_run, existing_schedule=schedule, ) scaled_count += 1 else: logger.info( - f"Skipping EKS cluster {cluster['name']}, no scaling tags found" + f"Skipping EKS cluster {cluster['name']}, \ + no scaling tags found" ) skipped_count += 1 except Exception as e: logger.error( - f"Failed to {'simulate scaling down' if dry_run else 'scale down'} EKS cluster {cluster['name']}: {e}" + f"Failed to {'simulate scaling down' if dry_run else 'scale down'} \ + EKS cluster {cluster['name']}: {e}" ) logger.info( - f"EKS scale down summary: {scaled_count} clusters processed, {skipped_count} clusters skipped" + f"EKS scale down summary: {scaled_count} clusters processed, \ + {skipped_count} clusters skipped" ) def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> None: @@ -226,7 +239,8 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non # Skip clusters with the exclusion tag if self.should_exclude(cluster): logger.info( - f"Skipping EKS cluster {cluster['name']} with exclusion tag {config.get('exclusion_tag')}" + f"Skipping EKS cluster {cluster['name']} with exclusion tag \ + {config.get('exclusion_tag')}" ) skipped_count += 1 continue @@ -253,7 +267,8 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non or has_original_combined_tag ): logger.info( - f"{'[DRY RUN] Would scale up' if dry_run else 'Scaling up'} EKS cluster {cluster['name']} in region {region}" + f"{'[DRY RUN] Would scale up' if dry_run else 'Scaling up'} \ + EKS cluster {cluster['name']} in region {region}" ) if not dry_run: @@ -279,10 +294,12 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non if desired_str.isdigit() else None ) - except: + except Exception as e: logger.warning( - f"Could not parse desired value from combined size tag: {original_size_tag}" + f"Could not parse desired value from combined \ + size tag: {original_size_tag} - {str(e)}" ) + pass else: # Restore original values for individual tags if original_min: @@ -305,12 +322,15 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non # Apply the tag updates if tags_to_update: logger.info( - f"Restoring original scaling tags for EKS cluster {cluster['name']}: {tags_to_update}" + f"Restoring original scaling tags for EKS cluster \ + {cluster['name']}: {tags_to_update}" ) eks_client.tag_resource( resourceArn=cluster.get( "arn", - f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}", + f"arn:{self.get_partition_for_region(region)}:eks\ + :{region}:{self.account_id}:\ + cluster/{cluster['name']}", ), tags=tags_to_update, ) @@ -332,7 +352,9 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non eks_client.untag_resource( resourceArn=cluster.get( "arn", - f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}", + f"arn:{self.get_partition_for_region(region)}:\ + eks:{region}:{self.account_id}:\ + cluster/{cluster['name']}", ), tagKeys=tags_to_remove, ) @@ -341,7 +363,8 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non if has_original_combined_tag: original_size = tags[EKS_ORIGINAL_CLUSTER_SIZE_TAG] logger.info( - f"[DRY RUN] Would restore combined size tag: {original_size}" + f"[DRY RUN] Would restore combined size\ + tag: {original_size}" ) # Try to parse desired value for nodegroup scaling @@ -356,11 +379,17 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non if desired_str.isdigit() else None ) - except: + except Exception as e: + logger.warning( + f"Could not parse desired value from combined \ + size tag: {original_size} - {str(e)}" + ) pass else: logger.info( - f"[DRY RUN] Would restore scaling values: Min={original_min}, Max={original_max}, Desired={original_desired}" + f"[DRY RUN] Would restore scaling values: \ + Min={original_min}, Max={original_max}, \ + Desired={original_desired}" ) desired = ( int(original_desired) @@ -380,7 +409,8 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non ) scale_details = f"Size tag: 0->'{original_size}'" else: - scale_details = f"Min=0->{original_min}, Max=0->{original_max}, Desired=0->{original_desired}" + scale_details = f"Min=0->{original_min}, Max=0->{original_max},\ + Desired=0->{original_desired}" schedule_info = f", Schedule: {schedule}" if schedule else "" self.log_action( @@ -394,17 +424,20 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non scaled_count += 1 else: logger.info( - f"Skipping EKS cluster {cluster['name']}, no original scaling values found in tags" + f"Skipping EKS cluster {cluster['name']}, no original \ + scaling values found in tags" ) skipped_count += 1 except Exception as e: logger.error( - f"Failed to {'simulate scaling up' if dry_run else 'scale up'} EKS cluster {cluster['name']}: {e}" + f"Failed to {'simulate scaling up' if dry_run else 'scale up'} EKS \ + cluster {cluster['name']}: {e}" ) logger.info( - f"EKS scale up summary: {scaled_count} clusters processed, {skipped_count} clusters skipped" + f"EKS scale up summary: {scaled_count} clusters processed, {skipped_count}\ + clusters skipped" ) def _scale_nodegroups( @@ -455,7 +488,8 @@ def _scale_nodegroups( "desiredSize" ) - # Use current min and max when scaling up if desired capacity is provided + # Use current min and max when scaling + # up if desired capacity is provided if desired_capacity is not None: # When scaling up, we need a valid min and max new_min = ( @@ -472,7 +506,8 @@ def _scale_nodegroups( if dry_run: logger.info( - f"[DRY RUN] Would update nodegroup {nodegroup_name} scaling: " + f"[DRY RUN] Would update nodegroup \ + {nodegroup_name} scaling: " + f"Min: {current_min}->{new_min}, " + f"Max: {current_max}->{new_max}, " + f"Desired: {current_desired}->{desired_capacity}" @@ -500,7 +535,8 @@ def _scale_nodegroups( except Exception as e: logger.error( - f"Error listing nodegroups for cluster {cluster_name} in region {region}: {e}" + f"Error listing nodegroups for cluster {cluster_name} in \ + region {region}: {e}" ) def discover_clusters(self, regions: List[str]) -> List[Dict[str, Any]]: @@ -547,7 +583,8 @@ def discover_clusters(self, regions: List[str]) -> List[Dict[str, Any]]: "name": name, "arn": cluster.get( "arn", - f"arn:aws:eks:{region}:{self.account_id}:cluster/{name}", + f"arn:aws-us-gov:eks:\ + {region}:{self.account_id}:cluster/{name}", ), "status": cluster.get("status", "UNKNOWN"), "state": cluster.get( diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/emr.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/emr.py index 5ff40c28..3524cf33 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/emr.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/emr.py @@ -4,15 +4,13 @@ from typing import Any, Dict, List, Optional -from aws_resource_management.aws_utils import ( +from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.utils import ( detect_partition_from_credentials, + get_config, get_session_for_account, + setup_logging, ) -from aws_resource_management.config_manager import get_config -from aws_resource_management.logging_setup import setup_logging -from aws_resource_management.managers.base import ResourceManager -from botocore.exceptions import ClientError -from aws_resource_management.partition_utils import get_partition_for_region logger = setup_logging() config = get_config() @@ -35,7 +33,7 @@ def __init__( """ super().__init__(account_id, region) self.resource_type = "emr_cluster" - self.account_name = None # This can be updated if needed + self.account_name = None def discover_clusters( self, regions: List[str], cluster_states: Optional[List[str]] = None @@ -120,7 +118,8 @@ def discover_clusters( processed_clusters.append(cluster_dict) except Exception as e: logger.warning( - f"Error getting details for EMR cluster {cluster['Id']}: {e}" + f"Error getting details for EMR cluster \ + {cluster['Id']}: {e}" ) logger.info(f"Found {len(processed_clusters)} EMR clusters in {region}") @@ -165,7 +164,8 @@ def discover_all_clusters(self, regions: List[str]) -> List[Dict[str, Any]]: for page in page_iterator: if "Clusters" in page: for cluster_summary in page["Clusters"]: - # Check if this was a STOPPED cluster (vs actually terminated) + # Check if this was a STOPPED cluster + # (vs actually terminated) cluster_id = cluster_summary.get("Id") if cluster_id: try: @@ -179,7 +179,8 @@ def discover_all_clusters(self, regions: List[str]) -> List[Dict[str, Any]]: status_reason = response["Cluster"][ "Status" ].get("StateChangeReason", {}) - # EMR uses the TERMINATED state for stopped clusters with a specific code + # EMR uses the TERMINATED state for + # stopped clusters with a specific code if ( status_reason.get("Code") == "USER_REQUEST" and "stop clusters" @@ -190,7 +191,7 @@ def discover_all_clusters(self, regions: List[str]) -> List[Dict[str, Any]]: "name": cluster_summary.get( "Name", "Unknown" ), - "status": "STOPPED", # Override with our interpretation + "status": "STOPPED", "region": region, "account_id": self.account_id, "account_name": self.account_name, @@ -211,7 +212,8 @@ def discover_all_clusters(self, regions: List[str]) -> List[Dict[str, Any]]: except Exception as e: logger.warning( - f"Error checking if cluster {cluster_id} is stopped: {str(e)}" + f"Error checking if cluster \ + {cluster_id} is stopped: {str(e)}" ) except Exception as e: @@ -264,7 +266,7 @@ def validate_credentials(self, region: str = None) -> bool: emr_client = self.get_boto3_client("emr", region) # Just list a small number of clusters to verify permissions - response = emr_client.list_clusters(MaxResults=1) + emr_client.list_clusters(MaxResults=1) logger.info(f"EMR credentials valid in region {region}") return True except Exception as e: @@ -303,7 +305,8 @@ def stop( cluster_status = cluster.get("status", cluster.get("state")) if not cluster_status: logger.warning( - f"Cannot determine status for EMR cluster {cluster_id}, assuming UNKNOWN" + f"Cannot determine status for EMR cluster {cluster_id}\ + , assuming UNKNOWN" ) cluster_status = "UNKNOWN" @@ -317,7 +320,8 @@ def stop( # Skip clusters with the exclusion tag if self.should_exclude(cluster): logger.info( - f"Skipping EMR cluster {cluster_id} with exclusion tag {config.get('exclusion_tag')}" + f"Skipping EMR cluster {cluster_id} with \ + exclusion tag {config.get('exclusion_tag')}" ) continue @@ -330,7 +334,9 @@ def stop( # Tag the cluster before stopping logger.info( - f"{'[DRY RUN] Would tag' if dry_run else 'Tagging'} EMR cluster {cluster_name} ({cluster_id}) with {config.get('stop_tag')}={timestamp}" + f"{'[DRY RUN] Would tag' if dry_run else 'Tagging'} \ + EMR cluster {cluster_name} ({cluster_id}) with \ + {config.get('stop_tag')}={timestamp}" ) if not dry_run: emr_client.add_tags( @@ -339,7 +345,8 @@ def stop( ) logger.info( - f"{'[DRY RUN] Would stop' if dry_run else 'Stopping'} EMR cluster {cluster_name} ({cluster_id}) in region {region}" + f"{'[DRY RUN] Would stop' if dry_run else 'Stopping'} EMR \ + cluster {cluster_name} ({cluster_id}) in region {region}" ) if not dry_run: emr_client.stop_cluster(ClusterId=cluster_id) @@ -356,11 +363,13 @@ def stop( except Exception as e: logger.error( - f"Failed to {'simulate stopping' if dry_run else 'stop'} EMR cluster {cluster_id}: {e}" + f"Failed to {'simulate stopping' if dry_run else 'stop'} \ + EMR cluster {cluster_id}: {e}" ) error_count += 1 - # For STARTING and BOOTSTRAPPING clusters, we need to terminate them as they can't be stopped + # For STARTING and BOOTSTRAPPING clusters, + # we need to terminate them as they can't be stopped elif cluster_status in ["STARTING", "BOOTSTRAPPING"]: try: emr_client = self.create_client("emr", region) @@ -369,7 +378,9 @@ def stop( # Tag the cluster before terminating logger.info( - f"{'[DRY RUN] Would tag' if dry_run else 'Tagging'} EMR cluster {cluster_name} ({cluster_id}) with {config.get('stop_tag')}={timestamp}" + f"{'[DRY RUN] Would tag' if dry_run else 'Tagging'} \ + EMR cluster {cluster_name} ({cluster_id}) with \ + {config.get('stop_tag')}={timestamp}" ) if not dry_run: emr_client.add_tags( @@ -378,7 +389,10 @@ def stop( ) logger.info( - f"{'[DRY RUN] Would terminate' if dry_run else 'Terminating'} EMR cluster {cluster_name} ({cluster_id}) in region {region} (cannot stop a cluster in {cluster_status} state)" + f"{'[DRY RUN] Would terminate' if dry_run else 'Terminating'} \ + EMR cluster {cluster_name} ({cluster_id}) in region \ + {region} (cannot stop a cluster \ + in {cluster_status} state)" ) if not dry_run: emr_client.terminate_job_flows(JobFlowIds=[cluster_id]) @@ -395,12 +409,15 @@ def stop( except Exception as e: logger.error( - f"Failed to {'simulate terminating' if dry_run else 'terminate'} EMR cluster {cluster_id}: {e}" + f"Failed to \ + {'simulate terminating' if dry_run else 'terminate'} \ + EMR cluster {cluster_id}: {e}" ) error_count += 1 else: logger.info( - f"Skipping EMR cluster {cluster_id} in state {cluster_status} (not eligible for stop)" + f"Skipping EMR cluster {cluster_id} in \ + state {cluster_status} (not eligible for stop)" ) return {"success": success_count, "errors": error_count} @@ -436,7 +453,8 @@ def start( cluster_status = cluster.get("status", cluster.get("state")) if not cluster_status: logger.warning( - f"Cannot determine status for EMR cluster {cluster_id}, assuming UNKNOWN" + f"Cannot determine status for EMR cluster {cluster_id}, \ + assuming UNKNOWN" ) cluster_status = "UNKNOWN" @@ -450,7 +468,8 @@ def start( # Skip clusters with the exclusion tag if self.should_exclude(cluster): logger.info( - f"Skipping EMR cluster {cluster_id} with exclusion tag {config.get('exclusion_tag')}" + f"Skipping EMR cluster {cluster_id} with exclusion tag \ + {config.get('exclusion_tag')}" ) continue @@ -459,14 +478,16 @@ def start( emr_client = self.create_client("emr", region) logger.info( - f"{'[DRY RUN] Would start' if dry_run else 'Starting'} EMR cluster {cluster_name} ({cluster_id}) in region {region}" + f"{'[DRY RUN] Would start' if dry_run else 'Starting'} EMR \ + cluster {cluster_name} ({cluster_id}) in region {region}" ) if not dry_run: emr_client.start_cluster(ClusterId=cluster_id) # Remove the stop tag after successful start logger.info( - f"Removing {config.get('stop_tag')} tag from EMR cluster {cluster_id}" + f"Removing {config.get('stop_tag')} tag from EMR\ + cluster {cluster_id}" ) emr_client.remove_tags( ResourceId=cluster_id, TagKeys=[config.get("stop_tag")] @@ -484,15 +505,14 @@ def start( except Exception as e: logger.error( - f"Failed to {'simulate starting' if dry_run else 'start'} EMR cluster {cluster_id}: {e}" + f"Failed to {'simulate starting' if dry_run else 'start'} EMR \ + cluster {cluster_id}: {e}" ) error_count += 1 else: logger.info( - f"Skipping EMR cluster {cluster_id} in state {cluster_status} (not in STOPPED state)" + f"Skipping EMR cluster {cluster_id} in state {cluster_status} \ + (not in STOPPED state)" ) return {"success": success_count, "errors": error_count} - - def _some_method_using_partition(self): - partition = get_partition_for_region(self.region) diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/rds.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/rds.py index c38bf37b..9e0e1f2d 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/rds.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/rds.py @@ -3,11 +3,11 @@ """ from collections import defaultdict -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List -from aws_resource_management.config_manager import get_config -from aws_resource_management.logging_setup import setup_logging from aws_resource_management.managers.base import ResourceManager +from aws_resource_management.utils.config_utils import get_config +from aws_resource_management.utils.logging_utils import setup_logging logger = setup_logging() config = get_config() @@ -45,7 +45,8 @@ def stop( available_instances_by_region[instance["region"]].append(instance) else: logger.debug( - f"Skipping RDS instance {instance['id']} in state {instance['status']} (not available)" + f"Skipping RDS instance {instance['id']} in state\ + {instance['status']} (not available)" ) stats["skipped"] += 1 @@ -68,7 +69,8 @@ def stop( for instance in batch: try: db_id = instance["id"] - arn = f"arn:{self.get_partition_for_region(region)}:rds:{region}:{self.account_id}:db:{db_id}" + arn = f"arn:{self.get_partition_for_region(region)}:rds:\ + {region}:{self.account_id}:db:{db_id}" # Tag the instance before stopping if not dry_run: @@ -102,7 +104,8 @@ def stop( stats["errors"] += 1 logger.info( - f"RDS stop summary: {stats['success']} stopped, {stats['skipped']} skipped, {stats['errors']} errors" + f"RDS stop summary: {stats['success']} stopped,\ + {stats['skipped']} skipped, {stats['errors']} errors" ) return stats @@ -124,7 +127,8 @@ def start( stopped_instances_by_region[instance["region"]].append(instance) else: logger.debug( - f"Skipping RDS instance {instance['id']} in state {instance['status']} (not stopped)" + f"Skipping RDS instance {instance['id']} in state\ + {instance['status']} (not stopped)" ) stats["skipped"] += 1 @@ -144,7 +148,8 @@ def start( for instance in batch: try: db_id = instance["id"] - arn = f"arn:{self.get_partition_for_region(region)}:rds:{region}:{self.account_id}:db:{db_id}" + arn = f"arn:{self.get_partition_for_region(region)}:rds:\ + {region}:{self.account_id}:db:{db_id}" # Start the instance if not dry_run: @@ -159,7 +164,8 @@ def start( ) else: logger.info( - f"[DRY RUN] Would start RDS instance {db_id} in {region}" + f"[DRY RUN] Would start RDS instance \ + {db_id} in {region}" ) self.log_action( @@ -177,7 +183,8 @@ def start( stats["errors"] += 1 logger.info( - f"RDS start summary: {stats['success']} started, {stats['skipped']} skipped, {stats['errors']} errors" + f"RDS start summary: {stats['success']} started,\ + {stats['skipped']} skipped, {stats['errors']} errors" ) return stats @@ -210,7 +217,8 @@ def discover_instances(self, regions: List[str]) -> List[Dict[str, Any]]: } except Exception as e: logger.warning( - f"Error getting tags for RDS instance {db['DBInstanceIdentifier']}: {e}" + f"Error getting tags for RDS instance \ + {db['DBInstanceIdentifier']}: {e}" ) tags = {} diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/reporting.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/reporting.py index 1ef4f3ed..207e7725 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/reporting.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/reporting.py @@ -1,12 +1,13 @@ """ Reporting functionality for AWS resource management. """ + import logging import os from typing import Any, Dict, List, Optional -from aws_resource_management.csv_utils import ( - ensure_csv_dir, +from aws_resource_management.utils.file_utils import ( + ensure_directory, format_tags_for_csv, generate_timestamp_filename, initialize_csv_file, @@ -21,23 +22,23 @@ def print_resource_summary( ) -> None: """ Print a detailed summary of resources processed, broken down by resource type. - + Args: stats: Statistics dictionary action: Action that was performed (stop or start) dry_run: Whether this was a dry run """ - logger.info(f"\n{'=' * 30} SUMMARY {'=' * 30}") + logger.info("\n" + "=" * 30 + " SUMMARY " + "=" * 30) logger.info(f"ACTION: {action.upper()} {'(DRY RUN)' if dry_run else ''}") # General stats - logger.info(f"\nGENERAL STATISTICS:") + logger.info("\nGENERAL STATISTICS:") logger.info(f"Accounts processed: {stats.get('accounts_processed', 0)}") logger.info(f"Accounts skipped: {stats.get('accounts_skipped', 0)}") logger.info(f"Regions processed: {stats.get('regions_processed', 0)}") # EC2 resources - logger.info(f"\nEC2 INSTANCES:") + logger.info("\nEC2 INSTANCES:") logger.info(f"Found: {stats.get('ec2_found', 0)}") if action == "stop": logger.info(f"Stopped: {stats.get('ec2_stopped', 0)}") @@ -47,7 +48,7 @@ def print_resource_summary( logger.info(f"Errors: {stats.get('ec2_errors', 0)}") # RDS resources - logger.info(f"\nRDS INSTANCES:") + logger.info("\nRDS INSTANCES:") logger.info(f"Found: {stats.get('rds_found', 0)}") if action == "stop": logger.info(f"Stopped: {stats.get('rds_stopped', 0)}") @@ -59,12 +60,11 @@ def print_resource_summary( # RDS engine breakdown if available rds_engines = stats.get("rds_engines", {}) if rds_engines: - logger.info( - f"RDS engines: {', '.join(f'{engine}({count})' for engine, count in rds_engines.items())}" - ) + eng = ", ".join(f"{engine}({count})" for engine, count in rds_engines.items()) + logger.info(f"RDS engines: {eng}") # EKS resources - logger.info(f"\nEKS CLUSTERS:") + logger.info("\nEKS CLUSTERS:") logger.info(f"Found: {stats.get('eks_found', 0)}") if action == "stop": logger.info(f"Stopped: {stats.get('eks_stopped', 0)}") @@ -74,7 +74,7 @@ def print_resource_summary( logger.info(f"Errors: {stats.get('eks_errors', 0)}") # EMR resources - logger.info(f"\nEMR CLUSTERS:") + logger.info("\nEMR CLUSTERS:") logger.info(f"Found: {stats.get('emr_found', 0)}") if action == "stop": logger.info(f"Stopped: {stats.get('emr_stopped', 0)}") @@ -83,29 +83,42 @@ def print_resource_summary( logger.info(f"Skipped: {stats.get('emr_skipped', 0)}") logger.info(f"Errors: {stats.get('emr_errors', 0)}") - # ECR images - logger.info(f"\nECR IMAGES:") - logger.info(f"Total images found: {stats.get('ecr_images_found', 0)}") - logger.info(f"Images older than 1 year: {stats.get('ecr_old_images_found', 0)}") + # ECR images - correctly handle both key names for compatibility + logger.info("\nECR IMAGES:") + logger.info(f"Total images found: {stats.get('ecr_found', 0)}") + logger.info(f"Images older than 1 year: {stats.get('ecr_old_images', 0)}") if action == "stop": - logger.info(f"Old images deleted: {stats.get('ecr_images_deleted', 0)}") + # Use either ecr_deleted or ecr_stopped (for backward compatibility) + deleted = stats.get("ecr_deleted", 0) or stats.get("ecr_stopped", 0) + logger.info(f"Old images deleted: {deleted}") logger.info(f"Skipped: {stats.get('ecr_skipped', 0)}") logger.info(f"Errors: {stats.get('ecr_errors', 0)}") # Error summary if there were any errors print_error_summary(stats.get("errors", [])) - logger.info(f"{'=' * 68}") + logger.info("=" * 65) + + # Debug: log the stats values for each resource type before printing summary + ec2_found = stats.get("ec2_found", 0) + rds_found = stats.get("rds_found", 0) + eks_found = stats.get("eks_found", 0) + emr_found = stats.get("emr_found", 0) + ecr_found = stats.get("ecr_found", 0) + logger.debug( + f"Summary stats: ec2_found={ec2_found}, rds_found={rds_found}, " + f"eks_found={eks_found}, emr_found={emr_found}, ecr_found={ecr_found}" + ) def print_error_summary(errors: List[str]) -> None: """ Print a summary of errors that occurred during processing. - + Args: errors: List of error messages """ if errors: - logger.info(f"\nERROR SUMMARY:") + logger.info("\nERROR SUMMARY:") logger.info(f"Total errors: {len(errors)}") for i, error in enumerate(errors[:5], 1): # Show first 5 errors logger.info(f" {i}. {error}") @@ -118,7 +131,7 @@ def print_error_summary(errors: List[str]) -> None: def initialize_stats() -> Dict[str, Any]: """ Initialize a statistics dictionary with default values. - + Returns: A dictionary with initialized statistics counters """ @@ -131,13 +144,17 @@ def initialize_stats() -> Dict[str, Any]: "regions_by_account": {}, "errors": [], "rds_engines": {}, - "ecr_images_found": 0, - "ecr_old_images_found": 0, - "ecr_images_deleted": 0, + # Make sure all ECR metrics are properly initialized + "ecr_found": 0, + "ecr_old_images": 0, + "ecr_deleted": 0, + "ecr_stopped": 0, # For compatibility with both key names + "ecr_skipped": 0, + "ecr_errors": 0, } # Initialize resource-specific counters - for resource_type in ["ec2", "rds", "eks", "emr", "ecr"]: + for resource_type in ["ec2", "rds", "eks", "emr"]: for stat_type in [ "found", "skipped", @@ -147,33 +164,41 @@ def initialize_stats() -> Dict[str, Any]: ]: stats[f"{resource_type}_{stat_type}"] = 0 + # Log the initialized stats for debugging + stats_to_log = [ + f"{k}={v}" + for k, v in stats.items() + if k in ["ec2_found", "rds_found", "eks_found", "emr_found", "ecr_found"] + ] + logger.debug(f"Initialized stats: {', '.join(stats_to_log)}") + return stats def export_resources_to_csv( resources: Dict[str, List[Dict[str, Any]]], output_dir: Optional[str] = None, - prefix: Optional[str] = None + prefix: Optional[str] = None, ) -> Dict[str, str]: """ Export discovered resources to CSV files. - + Args: resources: Dictionary of resource lists by type output_dir: Directory to save CSV files (defaults to current directory) prefix: Optional prefix for CSV filenames - + Returns: Dictionary mapping resource types to their CSV file paths """ if not output_dir: output_dir = os.getcwd() - + # Ensure output directory exists - output_dir = ensure_csv_dir(output_dir) - + output_dir = ensure_directory(output_dir) + csv_files = {} - + # Process each resource type for resource_type, resource_list in resources.items(): if not resource_list: @@ -183,39 +208,47 @@ def export_resources_to_csv( # Create CSV filename using utility function filename = generate_timestamp_filename(resource_type, prefix) filepath = os.path.join(output_dir, filename) - + try: # Extract all unique keys to use as CSV headers all_keys = set() for resource in resource_list: all_keys.update(resource.keys()) - + # Define common fields to appear first in the CSV common_fields = [ - "id", "name", "accountId", "accountName", "region", - "status", "type", "tags" + "id", + "name", + "accountId", + "accountName", + "region", + "status", + "type", + "tags", ] # Sort headers with common fields first, then alphabetically headers = [h for h in common_fields if h in all_keys] headers.extend(sorted(k for k in all_keys if k not in common_fields)) - + # Initialize the CSV file with headers initialize_csv_file(filepath, headers, overwrite=True) - + # Process and write each resource for resource in resource_list: # Handle tags special case (convert dict to string) if "tags" in resource and isinstance(resource["tags"], dict): resource["tags"] = format_tags_for_csv(resource["tags"]) - + # Use the write_csv_row utility - write_csv_row(filepath, - {k: resource.get(k, "") for k in headers}, - headers) - - logger.info(f"Exported {len(resource_list)} {resource_type} resources to {filepath}") + write_csv_row( + filepath, {k: resource.get(k, "") for k in headers}, headers + ) + + logger.info( + f"Exported {len(resource_list)} {resource_type} resources to {filepath}" + ) csv_files[resource_type] = filepath - + except Exception as e: logger.error(f"Error exporting {resource_type} to CSV: {e}") diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/resource_registry.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/resource_registry.py index 7c524e50..d7fae21a 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/resource_registry.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/resource_registry.py @@ -1,15 +1,14 @@ """ -Resource manager registry for dynamically discovering and registering AWS resource types. +Resource manager registry for dynamically +discovering and registering AWS resource types. """ import importlib import inspect -import logging import os -import sys from typing import Any, Dict, List, Optional, Type -from aws_resource_management.logging_setup import setup_logging +from aws_resource_management.utils.logging_utils import setup_logging logger = setup_logging() @@ -34,11 +33,12 @@ def __init__(self): # Only initialize once (singleton pattern) if ResourceRegistry._initialized: return - + self._managers = {} # Dict of resource_type -> manager_class self._display_names = {} # Dict of resource_type -> display_name self._descriptions = {} # Dict of resource_type -> description - + self._resource_keys = {} # Dict of resource_type -> resource_key + # Discover managers on initialization self._discover_managers() ResourceRegistry._initialized = True @@ -48,50 +48,91 @@ def _discover_managers(self) -> None: try: # Import the managers package to scan it import aws_resource_management.managers as managers_pkg - + # Get the package directory package_dir = os.path.dirname(managers_pkg.__file__) - + # Find Python files in the package (excluding __init__.py) for filename in os.listdir(package_dir): - if filename.endswith('.py') and filename != '__init__.py' and not filename.startswith('_'): + if ( + filename.endswith(".py") + and filename != "__init__.py" + and not filename.startswith("_") + ): module_name = filename[:-3] # Remove '.py' - + try: # Import the module - module = importlib.import_module(f"aws_resource_management.managers.{module_name}") - + module = importlib.import_module( + f"aws_resource_management.managers.{module_name}" + ) + # Scan for manager classes in the module for name, obj in inspect.getmembers(module): # Look for classes that have a name ending with "Manager" # and have a "_RESOURCE_TYPE" attribute - if (inspect.isclass(obj) and name.endswith('Manager') and - hasattr(obj, '_RESOURCE_TYPE') and obj._RESOURCE_TYPE): - + if ( + inspect.isclass(obj) + and name.endswith("Manager") + and hasattr(obj, "_RESOURCE_TYPE") + and obj._RESOURCE_TYPE + ): + resource_type = obj._RESOURCE_TYPE - display_name = getattr(obj, '_DISPLAY_NAME', resource_type.upper()) - description = getattr(obj, '_DESCRIPTION', f"AWS {resource_type.upper()} resources") - - self.register_manager(resource_type, obj, display_name, description) - logger.debug(f"Auto-discovered resource manager: {name} for type {resource_type}") - + display_name = getattr( + obj, "_DISPLAY_NAME", resource_type.upper() + ) + description = getattr( + obj, + "_DESCRIPTION", + f"AWS {resource_type.upper()} resources", + ) + + # Add resource key mapping based on resource type + # For example: 'ec2' -> 'ec2_instances' + resource_key = getattr( + obj, "_RESOURCE_KEY", f"{resource_type}_instances" + ) + # For special cases like 'eks' -> 'eks_clusters' + if resource_type == "eks": + resource_key = "eks_clusters" + elif resource_type == "emr": + resource_key = "emr_clusters" + elif resource_type == "ecr": + resource_key = "ecr_images" + + self._resource_keys[resource_type] = resource_key + + self.register_manager( + resource_type, obj, display_name, description + ) + logger.debug( + f"Auto-discovered resource manager: {name}\ + for type {resource_type}\ + with key {resource_key}" + ) + except (ImportError, AttributeError) as e: - logger.warning(f"Error importing manager from {module_name}: {str(e)}") - + logger.warning( + f"Error importing manager from {module_name}: {str(e)}" + ) + if not self._managers: logger.warning("No resource managers were discovered automatically.") - + except Exception as e: logger.error(f"Error discovering resource managers: {str(e)}") - - def register_manager(self, - resource_type: str, - manager_class: Type, - display_name: Optional[str] = None, - description: Optional[str] = None) -> None: + + def register_manager( + self, + resource_type: str, + manager_class: Type, + display_name: Optional[str] = None, + description: Optional[str] = None, + ) -> None: """ Register a resource manager for a specific resource type. - + Args: resource_type: Resource type identifier (e.g., 'ec2', 'rds') manager_class: Manager class for the resource type @@ -100,88 +141,149 @@ def register_manager(self, """ resource_type = resource_type.lower() self._managers[resource_type] = manager_class - + if display_name: self._display_names[resource_type] = display_name else: self._display_names[resource_type] = resource_type.upper() - + if description: self._descriptions[resource_type] = description else: self._descriptions[resource_type] = f"AWS {resource_type.upper()} resources" - + logger.debug(f"Registered resource manager for {resource_type}") - + def get_manager_class(self, resource_type: str) -> Optional[Type]: """ Get the manager class for a specific resource type. - + Args: resource_type: Resource type identifier - + Returns: Manager class or None if not found """ return self._managers.get(resource_type.lower()) - + def get_resource_types(self) -> List[str]: """ Get a list of all registered resource types. - + Returns: List of resource type identifiers """ return list(self._managers.keys()) - + def get_display_name(self, resource_type: str) -> str: """ Get the display name for a resource type. - + Args: resource_type: Resource type identifier - + Returns: Display name """ return self._display_names.get(resource_type.lower(), resource_type.upper()) - + def get_description(self, resource_type: str) -> str: """ Get the description for a resource type. - + Args: resource_type: Resource type identifier - + Returns: Description """ - return self._descriptions.get(resource_type.lower(), f"AWS {resource_type.upper()} resources") - + return self._descriptions.get( + resource_type.lower(), f"AWS {resource_type.upper()} resources" + ) + def get_resource_type_choices(self) -> List[Dict[str, str]]: """ Get a list of resource type choices suitable for CLI argument choices. - + Returns: List of dictionaries with 'name', 'value', and 'help' keys """ choices = [] for resource_type in sorted(self.get_resource_types()): - choices.append({ - 'name': self.get_display_name(resource_type), - 'value': resource_type, - 'help': self.get_description(resource_type) - }) - + choices.append( + { + "name": self.get_display_name(resource_type), + "value": resource_type, + "help": self.get_description(resource_type), + } + ) + # Add the "all" option - choices.append({ - 'name': 'ALL', - 'value': 'all', - 'help': 'All supported resource types' - }) - + choices.append( + {"name": "ALL", "value": "all", "help": "All supported resource types"} + ) + return choices + def create_manager( + self, + resource_type: str, + account_id: str, + region: str, + credentials: Dict[str, Any], + account_name: str = None, + ) -> Any: + """ + Create a manager instance for a specific resource type. + + Args: + resource_type: Type of resource + account_id: AWS account ID + region: AWS region + credentials: AWS credentials + account_name: AWS account name (optional) + + Returns: + Resource manager instance + """ + manager_class = self.get_manager_class(resource_type) + if not manager_class: + return None + + manager = manager_class(account_id, region, credentials) + + # Set account_name if provided and manager supports it + if account_name and hasattr(manager, "account_name"): + manager.account_name = account_name + + return manager + + def get_resource_key(self, resource_type: str) -> str: + """ + Get the resource key used for resource lists. + + Args: + resource_type: Resource type identifier + + Returns: + Resource key string (e.g., 'ec2' -> 'ec2_instances') + """ + resource_type = resource_type.lower() + + # Use stored mapping or generate default + if resource_type in self._resource_keys: + return self._resource_keys[resource_type] + + # Fallback logic for resource keys + if resource_type == "eks": + return "eks_clusters" + elif resource_type == "emr": + return "emr_clusters" + elif resource_type == "ecr": + return "ecr_images" + else: + return f"{resource_type}_instances" + # Create singleton instance for import registry = ResourceRegistry() @@ -190,7 +292,7 @@ def get_resource_type_choices(self) -> List[Dict[str, str]]: def get_registry() -> ResourceRegistry: """ Get the resource registry singleton. - + Returns: ResourceRegistry instance """ diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/__init__.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/__init__.py index 613e8ee8..09b5e9e7 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/__init__.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/__init__.py @@ -7,47 +7,51 @@ # Import frequently used functions for convenience from aws_resource_management.utils.api_utils import ( - paginate_aws_response, - safe_api_call, + add_account_info, format_tags, normalize_resource_state, - add_account_info + paginate_aws_response, + safe_api_call, ) - from aws_resource_management.utils.aws_core_utils import ( + create_boto3_client, + ensure_valid_account_name, + get_account_list, get_credentials, + get_organization_accounts, get_session_for_account, - get_account_list, - create_boto3_client ) - from aws_resource_management.utils.config_utils import ( get_config, get_config_value, - update_config + update_config, ) - +from aws_resource_management.utils.datetime_utils import parse_iso_datetime from aws_resource_management.utils.file_utils import ( log_action_to_csv, + read_json_file, write_csv_row, write_json_file, - read_json_file ) +# Make logger available for direct import from aws_resource_management.utils.logging_utils import ( - setup_logging, + LoggingContext, configure_logging, log_operation, - LoggingContext + logger, + setup_logging, ) - from aws_resource_management.utils.region_utils import ( + DEFAULT_PARTITION, + DEFAULT_REGIONS, + detect_partition_from_credentials, + filter_regions_for_partition, get_all_regions, + get_default_regions_for_partition, get_enabled_regions, - is_valid_region, get_partition_for_region, - DEFAULT_PARTITION + get_regions_for_partition, + is_valid_region, + list_enabled_regions, ) - -# Make logger available for direct import -from aws_resource_management.utils.logging_utils import logger diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/api_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/api_utils.py index 0dc7c3fa..fd266278 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/api_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/api_utils.py @@ -4,27 +4,28 @@ This module provides utilities for working with AWS APIs, including pagination, error handling, and resource normalization. """ + import logging -from typing import Any, Dict, List, Optional, Callable, Iterator, Tuple, Union +from typing import Any, Callable, Dict, List, Optional import boto3 from botocore.exceptions import ClientError -from botocore.paginate import PageIterator # Configure logger logger = logging.getLogger(__name__) + def paginate_aws_response( - client: Any, - operation: str, - result_key: Optional[str] = None, + client: Any, + operation: str, + result_key: Optional[str] = None, max_items: Optional[int] = None, page_processor: Optional[Callable[[Dict[str, Any]], List[Dict[str, Any]]]] = None, - **kwargs + **kwargs, ) -> List[Dict[str, Any]]: """ Paginate through AWS API responses with flexible options. - + Args: client: Boto3 client operation: Operation name (e.g., 'describe_instances') @@ -32,31 +33,35 @@ def paginate_aws_response( max_items: Maximum number of items to return (optional) page_processor: Optional function to process each page before extracting results **kwargs: Additional arguments for the operation - + Returns: Combined list of results from all pages """ try: # Check if the operation supports pagination - if not hasattr(client, 'get_paginator') or not hasattr(client.get_paginator, '__call__'): + if not hasattr(client, "get_paginator") or not hasattr( + client.get_paginator, "__call__" + ): # Fall back to direct call if pagination isn't supported - logger.debug(f"Pagination not supported for {operation}, making direct call") + logger.debug( + f"Pagination not supported for {operation}, making direct call" + ) response = getattr(client, operation)(**kwargs) - + if result_key and result_key in response: return response[result_key] return [response] # Return response as a list item - + # Initialize pagination paginator = client.get_paginator(operation) - + # Configure pagination pagination_config = {} if max_items: - pagination_config['MaxItems'] = max_items - + pagination_config["MaxItems"] = max_items + page_iterator = paginator.paginate(**kwargs, PaginationConfig=pagination_config) - + # Process pages and collect results results = [] for page in page_iterator: @@ -65,77 +70,91 @@ def paginate_aws_response( processed_items = page_processor(page) results.extend(processed_items) continue - + # Extract items using result_key if provided if result_key: if result_key in page: results.extend(page[result_key]) else: - logger.warning(f"Result key '{result_key}' not found in {operation} response") + logger.warning( + f"Result key '{result_key}' not found in {operation} response" + ) else: # If no result_key provided, add the entire page as a result item results.append(page) - + return results - + except Exception as e: logger.error(f"Error paginating {operation}: {str(e)}") return [] -def safe_api_call(func: Callable, error_message: str, default_return: Any = None) -> Any: + +def safe_api_call( + func: Callable[..., Any], description: str, log_success: bool = False, **kwargs: Any +) -> Dict[str, Any]: """ - Safely execute an AWS API call with error handling. - + Execute an AWS API call safely with error handling. + Args: - func: Function to execute (usually a lambda) - error_message: Error message to log on failure - default_return: Value to return on failure - + func: Function to call (e.g., ec2_client.describe_instances) + description: Description of the call for logging + log_success: Whether to log successful calls + **kwargs: Arguments to pass to the function + Returns: - API call result or default value on failure + Response dictionary from API call, or empty dict on error """ try: - return func() + response = func(**kwargs) + if log_success: + logger.info(f"Successfully executed AWS API call: {description}") + return response except ClientError as e: error_code = e.response.get("Error", {}).get("Code", "Unknown") - logger.error(f"{error_message}: {error_code} - {str(e)}") + error_msg = e.response.get("Error", {}).get("Message", str(e)) + + logger.error(f"AWS API error during {description}: {error_code} - {error_msg}") + return {} except Exception as e: - logger.error(f"{error_message}: {str(e)}") - - return default_return + logger.error(f"Unexpected error during {description}: {str(e)}") + return {} + def format_tags(tags_list: List[Dict[str, str]]) -> Dict[str, str]: """ Convert AWS tags list format to dictionary. - + Args: tags_list: List of tag dictionaries with Key and Value - + Returns: Dictionary of tag key-value pairs """ if not tags_list: return {} - + return {tag.get("Key", ""): tag.get("Value", "") for tag in tags_list} + def should_exclude_resource(tags: Dict[str, str], exclusion_tag: str) -> bool: """ Check if a resource should be excluded based on tags. - + Args: tags: Resource tags exclusion_tag: Tag key to check for exclusion - + Returns: True if resource should be excluded, False otherwise """ return exclusion_tag in tags + def normalize_resource_state(resource: Dict[str, Any]) -> None: """ Normalize resource state/status fields for consistency. - + Args: resource: Resource dictionary to normalize """ @@ -147,14 +166,13 @@ def normalize_resource_state(resource: Dict[str, Any]) -> None: resource["status"] = "unknown" resource["state"] = "unknown" + def add_account_info( - resources: List[Dict[str, Any]], - account_id: str, - account_name: Optional[str] = None + resources: List[Dict[str, Any]], account_id: str, account_name: Optional[str] = None ) -> None: """ Add account information to resources. - + Args: resources: List of resource dictionaries account_id: AWS account ID @@ -165,19 +183,18 @@ def add_account_info( if account_name: resource["accountName"] = account_name + def create_boto3_client( - service: str, - credentials: Dict[str, str], - region: Optional[str] = None + service: str, credentials: Dict[str, str], region: Optional[str] = None ) -> boto3.client: """ Create a boto3 client with provided credentials. - + Args: service: AWS service name credentials: Dict containing AWS credentials region: AWS region name - + Returns: Boto3 client """ @@ -189,6 +206,7 @@ def create_boto3_client( aws_session_token=credentials.get("aws_session_token"), ) + def retry_with_backoff( func: Callable, max_retries: int = 3, @@ -198,25 +216,25 @@ def retry_with_backoff( ) -> Any: """ Retry a function with exponential backoff. - + Args: func: Function to call max_retries: Maximum number of retries initial_backoff: Initial backoff time in seconds backoff_multiplier: Multiplier for backoff time after each retry retryable_exceptions: List of exception types to retry on - + Returns: Result of the function call """ import time - + if retryable_exceptions is None: retryable_exceptions = [ClientError] - + retries = 0 backoff = initial_backoff - + while True: try: return func() @@ -225,20 +243,33 @@ def retry_with_backoff( if retries >= max_retries: logger.warning(f"Max retries ({max_retries}) reached: {str(e)}") raise - + if isinstance(e, ClientError): error_code = e.response.get("Error", {}).get("Code", "") - if error_code in ["ThrottlingException", "RequestLimitExceeded", "Throttling"]: - logger.info(f"Request throttled, retrying ({retries+1}/{max_retries}) after {backoff}s") + if error_code in [ + "ThrottlingException", + "RequestLimitExceeded", + "Throttling", + ]: + logger.info( + f"Request throttled, retrying ({retries+1}/{max_retries})\ + after {backoff}s" + ) else: # Don't retry on permission errors if error_code in ["AccessDenied", "UnauthorizedOperation"]: logger.warning(f"Permission error, not retrying: {error_code}") raise - logger.info(f"Retrying on error {error_code} ({retries+1}/{max_retries}) after {backoff}s") + logger.info( + f"Retrying on error {error_code} ({retries+1}/{max_retries})\ + after {backoff}s" + ) else: - logger.info(f"Retrying on error: {str(e)} ({retries+1}/{max_retries}) after {backoff}s") - + logger.info( + f"Retrying on error: {str(e)} ({retries+1}/{max_retries})\ + after {backoff}s" + ) + # Sleep and retry time.sleep(backoff) retries += 1 diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/aws_core_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/aws_core_utils.py index 57a95f0d..da9e3c8c 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/aws_core_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/aws_core_utils.py @@ -1,29 +1,21 @@ """ Core AWS utilities for credential management and session handling. -This module provides centralized functionality for AWS authentication, credential management, -and session handling with optimized caching. +This module provides centralized functionality for AWS authentication, +credential management, and session handling with optimized caching. """ + import configparser -import logging import os import threading import time from functools import lru_cache -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import boto3 import botocore -from botocore.exceptions import ClientError, NoCredentialsError, ProfileNotFound - -# Import utilities from other modules -from aws_resource_management.utils.region_utils import ( - DEFAULT_PARTITION, - detect_partition_from_credentials, - get_all_regions, - get_default_regions_for_partition, -) from aws_resource_management.utils.logging_utils import setup_logging +from botocore.exceptions import ClientError, NoCredentialsError, ProfileNotFound # Configure logger logger = setup_logging(__name__) @@ -38,6 +30,7 @@ # Cache expiration time in seconds (1 hour) CACHE_EXPIRY = 3600 + # --------------------------------------------------------------------------- # String utility functions for safer string handling # --------------------------------------------------------------------------- @@ -45,6 +38,7 @@ def is_string(value: Any) -> bool: """Check if a value is a string.""" return isinstance(value, str) + def safe_startswith(value: Any, prefix: Union[str, Tuple[str, ...]]) -> bool: """Safely check if a value starts with a prefix, handling None values.""" if not is_string(value): @@ -55,6 +49,7 @@ def safe_startswith(value: Any, prefix: Union[str, Tuple[str, ...]]) -> bool: return value.startswith(prefix) return False + def safe_in(substring: Any, container: Any) -> bool: """Safely check if a substring is in a container, handling None values.""" if substring is None or container is None: @@ -64,6 +59,7 @@ def safe_in(substring: Any, container: Any) -> bool: except (TypeError, ValueError): return False + # --------------------------------------------------------------------------- # Simplified credential management # --------------------------------------------------------------------------- @@ -72,42 +68,43 @@ def get_credentials( ) -> Optional[Dict[str, Any]]: """ Get AWS credentials with simplified logic and better caching. - + Args: account_id: AWS account ID profile_name: Optional AWS profile name region: Optional AWS region - + Returns: Dictionary of credentials or None if not found """ cache_key = f"creds:{account_id}:{profile_name or 'default'}" - + with _session_cache_lock: if ( cache_key in _session_cache and _session_cache[cache_key]["timestamp"] > time.time() - CACHE_EXPIRY ): return _session_cache[cache_key]["credentials"] - + # Try profile approach first credentials = _get_credentials_from_profile(account_id, profile_name) if credentials: _cache_credentials(cache_key, credentials) return credentials - + # Fall back to role assumption credentials = _assume_role_for_account(account_id) if credentials: _cache_credentials(cache_key, credentials) return credentials - + return None + def _cache_credentials(cache_key: str, credentials: Dict[str, Any]) -> None: """ Cache credentials with timestamp. - + Args: cache_key: Cache key credentials: Credentials dictionary to cache @@ -118,16 +115,17 @@ def _cache_credentials(cache_key: str, credentials: Dict[str, Any]) -> None: "timestamp": time.time(), } + def _get_credentials_from_profile( account_id: str, profile_name: Optional[str] = None ) -> Optional[Dict[str, Any]]: """ Get credentials from an AWS profile. - + Args: account_id: AWS account ID profile_name: Optional profile name to use - + Returns: Dictionary of credentials or None if not found """ @@ -136,28 +134,31 @@ def _get_credentials_from_profile( profile_to_use = profile_name or _find_profile_for_account(account_id) if not profile_to_use: return None - + # Get credentials from the profile session = boto3.Session(profile_name=profile_to_use) if session.get_credentials() is None: return None - + return { "aws_access_key_id": session.get_credentials().access_key, "aws_secret_access_key": session.get_credentials().secret_key, "aws_session_token": session.get_credentials().token, } except Exception as e: - logger.debug(f"Error getting credentials from profile for {account_id}: {str(e)}") + logger.debug( + f"Error getting credentials from profile for {account_id}: {str(e)}" + ) return None + def _find_profile_for_account(account_id: str) -> Optional[str]: """ Find an AWS profile for a specific account. - + Args: account_id: AWS account ID - + Returns: Profile name or None if not found """ @@ -165,10 +166,10 @@ def _find_profile_for_account(account_id: str) -> Optional[str]: config_path = os.path.expanduser("~/.aws/config") if not os.path.exists(config_path): return None - + config = configparser.ConfigParser() config.read(config_path) - + # Prioritized profile patterns profile_patterns = [ f"{account_id}.AdministratorAccess", @@ -176,13 +177,15 @@ def _find_profile_for_account(account_id: str) -> Optional[str]: f"{account_id}-gov.administratoraccess", f"{account_id}-gov.inf-admin-t2", ] - + # Check for exact matches of preferred profiles first for section in config.sections(): - section_name = section[8:] if safe_startswith(section, "profile ") else section + section_name = ( + section[8:] if safe_startswith(section, "profile ") else section + ) if section_name.lower() in [p.lower() for p in profile_patterns]: return section_name - + # Then check for any profile with matching account ID for section in config.sections(): if ( @@ -190,18 +193,19 @@ def _find_profile_for_account(account_id: str) -> Optional[str]: and config.get(section, "sso_account_id") == account_id ): return section[8:] if safe_startswith(section, "profile ") else section - + return None except Exception: return None + def _assume_role_for_account(account_id: str) -> Optional[Dict[str, Any]]: """ Assume a role to get credentials for an account. - + Args: account_id: AWS account ID - + Returns: Dictionary of credentials or None if failed """ @@ -225,6 +229,44 @@ def _assume_role_for_account(account_id: str) -> Optional[Dict[str, Any]]: except Exception: return None + +def _try_assume_role( + account_id: str, role_name: str, session: boto3.Session +) -> Optional[Dict[str, str]]: + """ + Try to assume a specific role in an account. + + Args: + account_id: AWS account ID + role_name: Role name to assume + session: boto3 session to use + + Returns: + Dictionary with credential information or None on failure + """ + try: + sts_client = session.client("sts") + role_arn = f"arn:aws:iam::{account_id}:role/{role_name}" + + # Log attempt at debug level + logger.debug(f"Attempting to assume role {role_arn} for account {account_id}") + + response = sts_client.assume_role( + RoleArn=role_arn, RoleSessionName="ResourceManagementSession" + ) + credentials = response["Credentials"] + return { + "aws_access_key_id": credentials["AccessKeyId"], + "aws_secret_access_key": credentials["SecretAccessKey"], + "aws_session_token": credentials["SessionToken"], + } + except botocore.exceptions.ClientError as e: + logger.error( + f"Failed to assume role {role_name} for account {account_id}: {str(e)}" + ) + return None + + # --------------------------------------------------------------------------- # Simplified session management # --------------------------------------------------------------------------- @@ -234,17 +276,19 @@ def get_session_for_account( ) -> Optional[boto3.Session]: """ Get a boto3 session for the specified account with proper role assumption. - + Args: account_id: AWS account ID region: AWS region name (optional) profile_name: AWS profile name to use (optional) - + Returns: Boto3 session with appropriate credentials """ - cache_key = f"session:{account_id}:{region or 'default'}:{profile_name or 'default'}" - + cache_key = ( + f"session:{account_id}:{region or 'default'}:{profile_name or 'default'}" + ) + # Return cached session if available with _session_cache_lock: if ( @@ -252,17 +296,22 @@ def get_session_for_account( and _session_cache[cache_key]["timestamp"] > time.time() - CACHE_EXPIRY ): return _session_cache[cache_key]["session"] - + # Try getting a session using standard methods try: - # Method 1: Try using a profile named after the account ID or the specified profile + # Method 1: Try using a profile named after + # the account ID or the specified profile try: - session = boto3.Session(profile_name=profile_name or account_id, region_name=region) + session = boto3.Session( + profile_name=profile_name or account_id, region_name=region + ) # Validate session by checking identity sts = session.client("sts") identity = sts.get_caller_identity() if identity.get("Account") == account_id: - logger.debug(f"Using profile {profile_name or account_id} for authentication") + logger.debug( + f"Using profile {profile_name or account_id} for authentication" + ) # Cache the session with _session_cache_lock: _session_cache[cache_key] = { @@ -273,7 +322,7 @@ def get_session_for_account( except (ProfileNotFound, NoCredentialsError, ClientError): # Profile not found or invalid, continue to next method pass - + # Method 2: Use credentials to create a session credentials = get_credentials(account_id, profile_name) if credentials: @@ -291,11 +340,11 @@ def get_session_for_account( } logger.debug(f"Created session for account {account_id} using credentials") return session - + # Method 3: Try to assume role in target account using default session default_session = boto3.Session(region_name=region) sts = default_session.client("sts") - + # Try common cross-account roles for role_name in ["OrganizationAccountAccessRole", "AWSControlTowerExecution"]: try: @@ -303,7 +352,7 @@ def get_session_for_account( response = sts.assume_role( RoleArn=role_arn, RoleSessionName="ResourceManagementSession" ) - + # Create session with temporary credentials credentials = response["Credentials"] session = boto3.Session( @@ -312,7 +361,7 @@ def get_session_for_account( aws_session_token=credentials["SessionToken"], region_name=region, ) - + # Cache the session with _session_cache_lock: _session_cache[cache_key] = { @@ -324,25 +373,26 @@ def get_session_for_account( except ClientError: # Role assumption failed, try the next role continue - + # If all methods fail, raise exception raise Exception(f"Could not authenticate to account {account_id}") except Exception as e: logger.error(f"Failed to get session for account {account_id}: {str(e)}") raise + # --------------------------------------------------------------------------- # Account discovery with caching # --------------------------------------------------------------------------- def get_account_list() -> List[Dict[str, str]]: """ Get AWS accounts with caching to minimize API calls. - + Returns: List of dictionaries with account information """ cache_key = "accounts" - + # Check cache first with _session_cache_lock: if ( @@ -350,12 +400,14 @@ def get_account_list() -> List[Dict[str, str]]: and _session_cache[cache_key]["timestamp"] > time.time() - CACHE_EXPIRY ): return _session_cache[cache_key]["accounts"] - + # Try Organizations API first try: session = boto3.Session() accounts = [] - for account in session.client("organizations").get_paginator("list_accounts").paginate(): + for account in ( + session.client("organizations").get_paginator("list_accounts").paginate() + ): accounts.extend( [ {"account_id": acc["Id"], "account_name": acc["Name"]} @@ -363,26 +415,27 @@ def get_account_list() -> List[Dict[str, str]]: if acc["Status"] == "ACTIVE" ] ) - + # Cache the accounts with _session_cache_lock: _session_cache[cache_key] = {"accounts": accounts, "timestamp": time.time()} return accounts except Exception as e: logger.warning(f"Failed to get accounts from Organizations API: {str(e)}") - + # Fall back to profiles accounts = _get_accounts_from_profiles() - + # Cache these accounts too with _session_cache_lock: _session_cache[cache_key] = {"accounts": accounts, "timestamp": time.time()} return accounts + def get_organization_accounts() -> List[Dict[str, str]]: """ Get list of accounts from AWS Organization. - + Returns: List of dictionaries with account information """ @@ -391,7 +444,7 @@ def get_organization_accounts() -> List[Dict[str, str]]: session = boto3.Session() org_client = session.client("organizations") accounts = [] - + # Use pagination to get all accounts paginator = org_client.get_paginator("list_accounts") for page in paginator.paginate(): @@ -400,7 +453,7 @@ def get_organization_accounts() -> List[Dict[str, str]]: accounts.append( {"account_id": account["Id"], "account_name": account["Name"]} ) - + logger.info(f"Found {len(accounts)} accounts in Organizations API") return accounts except Exception as e: @@ -408,10 +461,11 @@ def get_organization_accounts() -> List[Dict[str, str]]: logger.info("Falling back to accounts from profiles") return _get_accounts_from_profiles() + def _get_accounts_from_profiles() -> List[Dict[str, str]]: """ Get accounts from local AWS config. - + Returns: List of dictionaries with account information """ @@ -420,30 +474,30 @@ def _get_accounts_from_profiles() -> List[Dict[str, str]]: config_path = os.path.expanduser("~/.aws/config") if not os.path.exists(config_path): return [] - + config = configparser.ConfigParser() config.read(config_path) - + for section in config.sections(): if not config.has_option(section, "sso_account_id"): continue - + account_id = config.get(section, "sso_account_id") account_name = ( config.get(section, "sso_account_name") if config.has_option(section, "sso_account_name") else account_id ) - + # Get profile name profile = section[8:] if safe_startswith(section, "profile ") else section - + # Keep track if we should replace an existing entry replace_existing = account_id in accounts_by_id if replace_existing and config.has_option(section, "sso_role_name"): role_name = config.get(section, "sso_role_name") replace_existing = role_name in ["AdministratorAccess", "inf-admin-t2"] - + # Store or replace account info if replace_existing or account_id not in accounts_by_id: accounts_by_id[account_id] = { @@ -451,17 +505,80 @@ def _get_accounts_from_profiles() -> List[Dict[str, str]]: "account_name": account_name, "profile": profile, } - + return list(accounts_by_id.values()) except Exception as e: logger.error(f"Error reading AWS profiles: {str(e)}") return [] -# Function alias for backward compatibility -create_boto3_client = lambda service, credentials, region=None: boto3.client( - service, - region_name=region, - aws_access_key_id=credentials.get("aws_access_key_id"), - aws_secret_access_key=credentials.get("aws_secret_access_key"), - aws_session_token=credentials.get("aws_session_token"), -) + +def ensure_valid_account_name( + account_id: str, account_name: Optional[str] = None +) -> str: + """ + Ensure account name is valid and not Unknown. + + Args: + account_id: AWS account ID + account_name: AWS account name (optional) + + Returns: + Valid account name (using account_id as fallback) + """ + if not account_name or account_name == "Unknown": + return account_id + return account_name + + +def is_pending_subscription(credentials: Dict[str, str], region: str) -> bool: + """Check if there's a pending marketplace subscription.""" + + # Replace lambda with named function as per E731 + def check_subscription_status(): + """Check marketplace subscription status.""" + ec2_client = boto3.client( + "ec2", + aws_access_key_id=credentials["aws_access_key_id"], + aws_secret_access_key=credentials["aws_secret_access_key"], + aws_session_token=credentials.get("aws_session_token"), + region_name=region, + ) + try: + ec2_client.describe_instances(MaxResults=5) + return False + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "") + error_msg = e.response.get("Error", {}).get("Message", "") + return ( + "pending subscription" in error_msg.lower() + or "subscription" in error_code.lower() + ) + + return check_subscription_status() + + +def create_boto3_client_from_creds( + service: str, credentials: Dict[str, str], region: Optional[str] = None +) -> boto3.client: + """ + Create a boto3 client from credentials. + + Args: + service: AWS service name + credentials: Dictionary containing AWS credentials + region: AWS region name (optional) + + Returns: + Boto3 client for the specified service + """ + return boto3.client( + service, + region_name=region, + aws_access_key_id=credentials.get("aws_access_key_id"), + aws_secret_access_key=credentials.get("aws_secret_access_key"), + aws_session_token=credentials.get("aws_session_token"), + ) + + +# Replace the lambda with a proper function (fixing E731) +create_boto3_client = create_boto3_client_from_creds diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/config_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/config_utils.py index cb1e17c1..00d1a22c 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/config_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/config_utils.py @@ -4,9 +4,10 @@ This module provides functions for loading and managing configuration from multiple sources. """ + import os -from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional + import yaml # Default configuration @@ -24,28 +25,33 @@ # Global config cache _config = None + def get_config() -> Dict[str, Any]: """ Get configuration with defaults merged with user settings. - + Returns: Configuration dictionary """ global _config - + # Return cached config if available if _config is not None: return _config - + # Start with default config config = DEFAULT_CONFIG.copy() - + # Look for config files in multiple locations config_paths = [ - os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "..", "config.yaml"), + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "..", + "config.yaml", + ), os.path.expanduser("~/.aws-resource-management/config.yaml"), ] - + # Try to load and merge configs from files for path in config_paths: try: @@ -58,19 +64,20 @@ def get_config() -> Dict[str, Any]: except Exception: # Ignore errors reading config files pass - + # Cache the config _config = config return config + def _deep_merge(base: Dict, update: Dict) -> Dict: """ Recursively merge dictionaries. - + Args: base: Base dictionary to update update: Dictionary with values to merge - + Returns: Updated base dictionary """ @@ -81,19 +88,20 @@ def _deep_merge(base: Dict, update: Dict) -> Dict: base[key] = value return base + def get_config_value(key: str, default: Any = None) -> Any: """ Get a specific configuration value. - + Args: key: Configuration key to retrieve default: Default value if key not found - + Returns: Configuration value or default """ config = get_config() - + # Handle nested keys with dot notation (e.g., "aws.region") if "." in key: parts = key.split(".") @@ -103,53 +111,55 @@ def get_config_value(key: str, default: Any = None) -> Any: return default current = current[part] return current - + return config.get(key, default) + def save_config(config: Dict[str, Any], config_path: Optional[str] = None) -> bool: """ Save configuration to a file. - + Args: config: Configuration dictionary to save config_path: Path to save config to (default: user config path) - + Returns: True if saved successfully, False otherwise """ if config_path is None: config_path = os.path.expanduser("~/.aws-resource-management/config.yaml") - + try: # Ensure directory exists os.makedirs(os.path.dirname(config_path), exist_ok=True) - + # Write config with open(config_path, "w") as f: yaml.safe_dump(config, f, default_flow_style=False) - + # Update cache global _config _config = config - + return True except Exception: return False + def update_config(key: str, value: Any, config_path: Optional[str] = None) -> bool: """ Update a specific configuration value and save. - + Args: key: Configuration key to update value: New value config_path: Path to save config to (default: user config path) - + Returns: True if updated successfully, False otherwise """ config = get_config() - + # Handle nested keys with dot notation if "." in key: parts = key.split(".") @@ -161,5 +171,5 @@ def update_config(key: str, value: Any, config_path: Optional[str] = None) -> bo current[parts[-1]] = value else: config[key] = value - + return save_config(config, config_path) diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/datetime_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/datetime_utils.py new file mode 100644 index 00000000..6ac6c1c8 --- /dev/null +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/datetime_utils.py @@ -0,0 +1,62 @@ +""" +Date and time utilities for AWS Resource Management. +""" + +import logging +from datetime import datetime, timezone +from typing import Optional + +logger = logging.getLogger(__name__) + + +def parse_iso_datetime(timestamp_str: str) -> Optional[datetime]: + """ + Parse an ISO format timestamp string to a datetime object. + + Args: + timestamp_str: ISO format timestamp string + + Returns: + Datetime object or None if parsing fails + """ + if not timestamp_str: + return None + + try: + # Handle different ISO formats + if "Z" in timestamp_str: + # UTC timezone marker + return datetime.fromisoformat(timestamp_str.replace("Z", "+00:00")) + elif "+" in timestamp_str or "-" in timestamp_str and "T" in timestamp_str: + # ISO format with timezone + return datetime.fromisoformat(timestamp_str) + else: + # Try direct parsing and assume UTC if no timezone + dt = datetime.fromisoformat(timestamp_str) + if not dt.tzinfo: + dt = dt.replace(tzinfo=timezone.utc) + return dt + except (ValueError, AttributeError) as e: + logger.warning(f"Failed to parse timestamp {timestamp_str}: {e}") + return None + + +def get_age_days(dt: Optional[datetime]) -> int: + """ + Get the age in days between a datetime and now. + + Args: + dt: Datetime to check age of + + Returns: + Age in days, or 0 if datetime is None + """ + if not dt: + return 0 + + now = datetime.now(timezone.utc) + if not dt.tzinfo: + dt = dt.replace(tzinfo=timezone.utc) + + delta = now - dt + return delta.days diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/file_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/file_utils.py index 2a91d8b0..15cd04a3 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/file_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/file_utils.py @@ -4,24 +4,27 @@ This module provides functions for file operations, including CSV file handling and reporting file management. """ + import csv import json + +# Logger for this module +import logging import os from datetime import datetime from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Union +from typing import Any, Dict, List, Optional, Union -# Logger for this module -import logging logger = logging.getLogger(__name__) + def ensure_directory(directory_path: Union[str, Path]) -> str: """ Ensure a directory exists and create it if it doesn't. - + Args: directory_path: Directory path - + Returns: Absolute path to the directory """ @@ -29,19 +32,18 @@ def ensure_directory(directory_path: Union[str, Path]) -> str: path.mkdir(parents=True, exist_ok=True) return str(path.absolute()) + def generate_timestamp_filename( - base_name: str, - prefix: Optional[str] = None, - extension: str = "csv" + base_name: str, prefix: Optional[str] = None, extension: str = "csv" ) -> str: """ Generate a filename with timestamp. - + Args: base_name: Base name for the file prefix: Optional prefix extension: File extension without dot - + Returns: Timestamped filename """ @@ -49,61 +51,61 @@ def generate_timestamp_filename( prefix_str = f"{prefix}_" if prefix else "" return f"{prefix_str}{base_name}_{timestamp}.{extension}" + def get_logs_directory(base_dir: Optional[str] = None) -> str: """ Get the logs directory path, creating it if it doesn't exist. - + Args: base_dir: Base directory to place logs in, defaults to module path - + Returns: Absolute path to logs directory """ if base_dir is None: # Default to a logs directory in the parent of the current module base_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "logs" + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "logs" ) - + return ensure_directory(base_dir) + # CSV Handling Functions def initialize_csv_file( - filepath: str, - headers: List[str], - overwrite: bool = False + filepath: str, headers: List[str], overwrite: bool = False ) -> bool: """ Initialize a CSV file with headers if it doesn't exist or overwrite is True. - + Args: filepath: Path to CSV file headers: List of column headers overwrite: Whether to overwrite existing file - + Returns: True if file was created/initialized, False if file already existed """ # Create parent directories if they don't exist os.makedirs(os.path.dirname(filepath), exist_ok=True) - + # Check if file exists file_exists = os.path.isfile(filepath) - + # Create new file or overwrite existing file if not file_exists or overwrite: - with open(filepath, 'w', newline='') as csvfile: + with open(filepath, "w", newline="") as csvfile: writer = csv.writer(csvfile) writer.writerow(headers) return True - + return False + def write_csv_row(filepath: str, row_data: Dict[str, Any], headers: List[str]) -> None: """ Write a row to a CSV file. - + Args: filepath: Path to CSV file row_data: Dictionary containing row data @@ -112,20 +114,19 @@ def write_csv_row(filepath: str, row_data: Dict[str, Any], headers: List[str]) - # Create file with headers if it doesn't exist if not os.path.isfile(filepath): initialize_csv_file(filepath, headers) - + # Write the row - with open(filepath, 'a', newline='') as csvfile: + with open(filepath, "a", newline="") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headers) writer.writerow(row_data) + def write_multiple_csv_rows( - filepath: str, - rows: List[Dict[str, Any]], - headers: List[str] + filepath: str, rows: List[Dict[str, Any]], headers: List[str] ) -> None: """ Write multiple rows to a CSV file. - + Args: filepath: Path to CSV file rows: List of dictionaries containing row data @@ -134,41 +135,43 @@ def write_multiple_csv_rows( # Create file with headers if it doesn't exist if not os.path.isfile(filepath): initialize_csv_file(filepath, headers) - + # Write the rows - with open(filepath, 'a', newline='') as csvfile: + with open(filepath, "a", newline="") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headers) writer.writerows(rows) + def read_csv_file(filepath: str) -> List[Dict[str, str]]: """ Read a CSV file and return rows as dictionaries. - + Args: filepath: Path to CSV file - + Returns: List of dictionaries containing row data """ if not os.path.exists(filepath): logger.warning(f"CSV file not found: {filepath}") return [] - + try: - with open(filepath, 'r', newline='') as csvfile: + with open(filepath, "r", newline="") as csvfile: reader = csv.DictReader(csvfile) return list(reader) except Exception as e: logger.error(f"Error reading CSV file {filepath}: {str(e)}") return [] + def format_tags_for_csv(tags: Dict[str, str]) -> str: """ Format AWS resource tags for CSV output. - + Args: tags: Dictionary of tag key-value pairs - + Returns: Formatted string representation of tags """ @@ -176,29 +179,29 @@ def format_tags_for_csv(tags: Dict[str, str]) -> str: return "" return "; ".join([f"{k}={v}" for k, v in tags.items()]) + def initialize_action_log_csv( - csv_file: Optional[str] = None, - log_dir: Optional[str] = None + csv_file: Optional[str] = None, log_dir: Optional[str] = None ) -> str: """ Initialize CSV log file for resource actions with headers. - + Args: csv_file: CSV file name (default: resource_actions.csv) log_dir: Directory to place log file (default: project logs dir) - + Returns: Path to the CSV log file """ if csv_file is None: csv_file = "resource_actions.csv" - + # Configure CSV logging directory if log_dir is None: log_dir = get_logs_directory() - + csv_path = os.path.join(log_dir, csv_file) - + # Initialize with headers headers = [ "timestamp", @@ -217,6 +220,7 @@ def initialize_action_log_csv( initialize_csv_file(csv_path, headers, overwrite=False) return csv_path + def log_action_to_csv( account_id: str, account_name: str, @@ -234,7 +238,7 @@ def log_action_to_csv( ) -> None: """ Log resource action to CSV file for tracking. - + Args: account_id: AWS account ID account_name: AWS account name @@ -251,10 +255,10 @@ def log_action_to_csv( log_dir: Custom log directory """ timestamp = datetime.now().isoformat() - + # Initialize CSV log file csv_path = initialize_action_log_csv(csv_file, log_dir) - + # Prepare row data row_data = { "timestamp": timestamp, @@ -270,58 +274,57 @@ def log_action_to_csv( "dry_run": "Yes" if dry_run else "No", "schedule": existing_schedule or "", } - + # Headers must match the keys in row_data headers = list(row_data.keys()) - + # Write to CSV write_csv_row(csv_path, row_data, headers) + def write_json_file( - filepath: str, - data: Any, - indent: int = 2, - ensure_dir: bool = True + filepath: str, data: Any, indent: int = 2, ensure_dir: bool = True ) -> bool: """ Write data to a JSON file. - + Args: filepath: Path to JSON file data: Data to write indent: Indentation level for JSON ensure_dir: Whether to ensure the directory exists - + Returns: True if successful, False if failed """ try: if ensure_dir: os.makedirs(os.path.dirname(filepath), exist_ok=True) - - with open(filepath, 'w') as f: + + with open(filepath, "w") as f: json.dump(data, f, indent=indent) return True except Exception as e: logger.error(f"Error writing JSON file {filepath}: {str(e)}") return False + def read_json_file(filepath: str, default: Any = None) -> Any: """ Read data from a JSON file. - + Args: filepath: Path to JSON file default: Default value to return if file doesn't exist or can't be parsed - + Returns: Parsed JSON data or default value """ if not os.path.exists(filepath): return default - + try: - with open(filepath, 'r') as f: + with open(filepath, "r") as f: return json.load(f) except Exception as e: logger.error(f"Error reading JSON file {filepath}: {str(e)}") diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/logging_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/logging_utils.py index a1894a07..c374018b 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/logging_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/logging_utils.py @@ -4,47 +4,46 @@ This module provides a centralized logging configuration and context-aware logging functionality. """ -import csv -import datetime + import json import logging import os import sys import time from contextlib import contextmanager -from pathlib import Path -from typing import Any, Callable, Dict, Optional +from typing import Any, Dict, Optional # Import our file utilities from aws_resource_management.utils.file_utils import ( - ensure_directory, - initialize_action_log_csv, - log_action_to_csv + ensure_directory, ) + # Global context data that can be included in log messages class LoggingContext: """Context manager for logging additional data with log messages.""" + _context = {} - + @classmethod def get(cls) -> Dict[str, Any]: """Get the current context dictionary.""" return cls._context.copy() - + @classmethod def set(cls, **kwargs) -> None: """Set context values.""" cls._context.update(kwargs) - + @classmethod def clear(cls) -> None: """Clear the context.""" cls._context.clear() + class JSONFormatter(logging.Formatter): """Format log records as JSON.""" - + def format(self, record: logging.LogRecord) -> str: """Format the log record as JSON.""" log_data = { @@ -53,37 +52,40 @@ def format(self, record: logging.LogRecord) -> str: "logger": record.name, "message": record.getMessage(), } - + # Add exception info if present if record.exc_info: log_data["exception"] = self.formatException(record.exc_info) - + # Add context data if present if hasattr(record, "context") and record.context: log_data.update(record.context) - + # Add LoggingContext data log_data.update(LoggingContext.get()) - + return json.dumps(log_data) -def setup_logging(name: str = "aws_resource_management", level: int = logging.INFO) -> logging.Logger: + +def setup_logging( + name: str = "aws_resource_management", level: int = logging.INFO +) -> logging.Logger: """ Set up basic logging with simple configuration. - + Args: name: Logger name level: Logging level - + Returns: Configured logger """ logger = logging.getLogger(name) - + # Only configure if handlers aren't already set up if not logger.handlers: logger.setLevel(level) - + # Create console handler with formatter handler = logging.StreamHandler(sys.stdout) handler.setFormatter( @@ -91,9 +93,10 @@ def setup_logging(name: str = "aws_resource_management", level: int = logging.IN ) logger.addHandler(handler) logger.propagate = False - + return logger + def configure_logging( app_name: str, log_level: str = "INFO", @@ -104,7 +107,7 @@ def configure_logging( ) -> logging.Logger: """ Configure logging with advanced features. - + Args: app_name: Application name log_level: Log level name @@ -112,22 +115,22 @@ def configure_logging( log_file: Path to log file console_output: Whether to log to console aws_context: AWS context information - + Returns: Configured logger instance """ # Convert log level string to logging level level = getattr(logging, log_level.upper(), logging.INFO) - + # Create logger logger = logging.getLogger(app_name) logger.setLevel(level) logger.handlers = [] # Remove any existing handlers - + # Set global AWS context if provided if aws_context: LoggingContext.set(**aws_context) - + # Create formatter if json_format: formatter = JSONFormatter() @@ -135,33 +138,34 @@ def configure_logging( formatter = logging.Formatter( "%(asctime)s [%(levelname)s] %(name)s - %(message)s" ) - + # Add console handler if console_output: console_handler = logging.StreamHandler(sys.stdout) console_handler.setFormatter(formatter) logger.addHandler(console_handler) - + # Add file handler if log_file: # Ensure directory exists log_dir = os.path.dirname(log_file) if log_dir: ensure_directory(log_dir) - + file_handler = logging.FileHandler(log_file) file_handler.setFormatter(formatter) logger.addHandler(file_handler) - + # Prevent propagation to avoid duplicate logs logger.propagate = False - + return logger + def log_with_context(logger: logging.Logger, level: int, msg: str, **context) -> None: """ Log with additional context data. - + Args: logger: Logger to use level: Log level @@ -171,13 +175,14 @@ def log_with_context(logger: logging.Logger, level: int, msg: str, **context) -> # Add logging context combined_context = LoggingContext.get() combined_context.update(context) - + # Create a record with extra context extra = {"context": combined_context} - + # Log with context logger.log(level, msg, extra=extra) + @contextmanager def log_operation( logger: logging.Logger, @@ -189,7 +194,7 @@ def log_operation( ): """ Context manager to log the start, end, and any exceptions for an operation. - + Args: logger: Logger to use operation: Operation name @@ -200,21 +205,21 @@ def log_operation( """ success_level = success_level or level start_time = time.time() - + # Set operation context operation_context = { "operation": operation, "operation_status": "started", } operation_context.update(context) - + # Log start with context log_with_context(logger, level, f"Started {operation}", **operation_context) - + try: # Execute the operation yield - + # Log success with duration duration = time.time() - start_time operation_context.update( @@ -245,5 +250,6 @@ def log_operation( ) raise + # Create a default logger instance for direct import logger = setup_logging() diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/region_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/region_utils.py index 50356719..d4c02d73 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/region_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/region_utils.py @@ -1,12 +1,12 @@ """ AWS region and partition utilities. -This module provides centralized functionality for working with AWS regions and partitions, -consolidating functionality previously spread across multiple modules. +This module provides centralized functionality for working with AWS regions """ + import logging import time -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional import boto3 @@ -15,11 +15,26 @@ "aws-us-gov": ["us-gov-east-1", "us-gov-west-1"], "aws-cn": ["cn-north-1", "cn-northwest-1"], "aws": [ - "us-east-1", "us-east-2", "us-west-1", "us-west-2", - "eu-west-1", "eu-central-1", "eu-west-2", "eu-west-3", - "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", - "ca-central-1", "sa-east-1", "eu-north-1", "eu-south-1", - "af-south-1", "ap-east-1", "ap-south-1", "me-south-1" + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "eu-west-1", + "eu-central-1", + "eu-west-2", + "eu-west-3", + "ap-northeast-1", + "ap-northeast-2", + "ap-southeast-1", + "ap-southeast-2", + "ca-central-1", + "sa-east-1", + "eu-north-1", + "eu-south-1", + "af-south-1", + "ap-east-1", + "ap-south-1", + "me-south-1", ], } @@ -37,7 +52,7 @@ REGION_PREFIX_TO_PARTITION = { "us-gov-": "aws-us-gov", "gov-": "aws-us-gov", - "cn-": "aws-cn" + "cn-": "aws-cn", } # All known AWS regions (flattened from PARTITION_REGIONS) @@ -59,13 +74,14 @@ # Logger for this module logger = logging.getLogger(__name__) + def get_partition_for_region(region_name: str) -> str: """ Determine AWS partition based on region name. - + Args: region_name: AWS region name - + Returns: AWS partition name (e.g., 'aws', 'aws-us-gov', 'aws-cn') """ @@ -73,125 +89,140 @@ def get_partition_for_region(region_name: str) -> str: for prefix, partition in REGION_PREFIX_TO_PARTITION.items(): if region_name.startswith(prefix): return partition - + # Default to standard AWS partition return "aws" + def get_regions_for_partition(partition: str) -> List[str]: """ Get the list of regions for a specific partition. - + Args: partition: AWS partition name - + Returns: List of region names in the partition """ return PARTITION_REGIONS.get(partition, PARTITION_REGIONS["aws"]) + def get_default_regions_for_partition(partition: str) -> List[str]: """ Get the default regions commonly used for a partition. - + Args: partition: AWS partition name - + Returns: List of default region names """ return DEFAULT_REGIONS.get(partition, DEFAULT_REGIONS["aws"]) + def is_region_in_partition(region: str, partition: str) -> bool: """ Check if a region belongs to the specified partition. - + Args: region: AWS region name partition: AWS partition name - + Returns: True if the region is in the partition, False otherwise """ return get_partition_for_region(region) == partition + def is_valid_region(region: str) -> bool: """ Check if a region is known to be valid without making an API call. - + Args: region: Region name to validate - + Returns: True if the region is valid, False otherwise """ if not isinstance(region, str): return False - + # First check if it's in our known regions list if region in ALL_KNOWN_REGIONS: return True - + # Check if it follows expected naming patterns for newer regions # Commercial regions follow patterns like us-east-1, eu-west-2 - if region.startswith(("us-", "eu-", "ap-", "sa-", "ca-", "me-", "af-")) and len(region.split("-")) >= 3: + if ( + region.startswith(("us-", "eu-", "ap-", "sa-", "ca-", "me-", "af-")) + and len(region.split("-")) >= 3 + ): return True - + # GovCloud regions if region.startswith("us-gov-") and len(region.split("-")) >= 3: return True - + # China regions if region.startswith("cn-") and len(region.split("-")) >= 3: return True - + return False + def filter_regions_for_partition(regions: List[str], partition: str) -> List[str]: """ Filter regions to include only those belonging to the specified partition. - + Args: regions: List of region names partition: AWS partition name - + Returns: List of valid region names within the partition """ # First validate the regions and filter out invalid ones valid_regions = [r for r in regions if is_valid_region(r)] - + # Then filter for the specific partition - partition_regions = [r for r in valid_regions if is_region_in_partition(r, partition)] - + partition_regions = [ + r for r in valid_regions if is_region_in_partition(r, partition) + ] + return partition_regions + def detect_partition_from_credentials(credentials: Dict[str, str]) -> str: """ Detect partition from credentials with minimal API calls. - + Args: credentials: Dictionary containing AWS credentials - + Returns: AWS partition name """ if not credentials: return DEFAULT_PARTITION # Default for environment - + # Check credential format first (fast, no API calls) access_key = str(credentials.get("aws_access_key_id", "")).lower() session_token = str(credentials.get("aws_session_token", "")).lower() - + # Check for GovCloud indicators - if any(indicator in access_key or indicator in session_token - for indicator in ["gov", "usgovcloud"]): + if any( + indicator in access_key or indicator in session_token + for indicator in ["gov", "usgovcloud"] + ): return "aws-us-gov" - + # Check for China indicators - if any(indicator in access_key or indicator in session_token - for indicator in ["cn-", "china"]): + if any( + indicator in access_key or indicator in session_token + for indicator in ["cn-", "china"] + ): return "aws-cn" - + # Try one API call to most likely partition based on environment try: boto3.client( @@ -217,20 +248,21 @@ def detect_partition_from_credentials(credentials: Dict[str, str]) -> str: # Default to the environment's default partition return DEFAULT_PARTITION + def get_all_regions(partition: Optional[str] = None) -> List[str]: """ Get all AWS regions, using cache to minimize API calls. - + Args: partition: Optional partition to filter regions by - + Returns: List of region names """ # Use predefined regions for special partitions if partition in ("aws-us-gov", "aws-cn"): return get_regions_for_partition(partition) - + # Check cache first current_time = time.time() cache_key = partition or "all" @@ -239,67 +271,69 @@ def get_all_regions(partition: Optional[str] = None) -> List[str]: and current_time - _region_cache["timestamp"] < CACHE_EXPIRY ): return _region_cache["all_regions"][cache_key] - + # Cache miss: fetch regions try: ec2 = boto3.client("ec2", region_name="us-east-1") all_regions = [ region["RegionName"] for region in ec2.describe_regions()["Regions"] ] - + # Cache the full list _region_cache["all_regions"]["all"] = all_regions _region_cache["timestamp"] = current_time - + # If partition specified, filter and cache that too if partition: filtered_regions = [ - r for r in all_regions + r + for r in all_regions if isinstance(r, str) and get_partition_for_region(r) == partition ] _region_cache["all_regions"][partition] = filtered_regions return filtered_regions - + return all_regions except Exception as e: logger.warning(f"Error getting regions: {str(e)}") return get_regions_for_partition(partition or DEFAULT_PARTITION) + def get_enabled_regions( - credentials: Dict[str, str], - partition: Optional[str] = None + credentials: Dict[str, str], partition: Optional[str] = None ) -> List[str]: """ Get enabled regions with minimized API calls using cache. - + Args: credentials: Dictionary containing AWS credentials partition: Optional partition to filter regions by - + Returns: List of enabled region names """ # Determine partition if not specified if not partition: partition = detect_partition_from_credentials(credentials) - + # For GovCloud/China, return predefined regions to avoid API calls if partition in ("aws-us-gov", "aws-cn"): return get_regions_for_partition(partition) - + # Generate cache key from credentials creds_hash = hash( - f"{credentials.get('aws_access_key_id', '')}:{credentials.get('aws_secret_access_key', '')}" + f"{credentials.get('aws_access_key_id', '')}:\ + {credentials.get('aws_secret_access_key', '')}" ) cache_key = f"enabled_regions:{creds_hash}:{partition}" - + # Check cache if ( cache_key in _region_cache["enabled_regions"] and time.time() - _region_cache["timestamp"] < CACHE_EXPIRY ): return _region_cache["enabled_regions"][cache_key] - + # Make a single API call to describe_regions rather than checking each region try: session = boto3.Session( @@ -313,36 +347,40 @@ def get_enabled_regions( "ec2", region_name="us-east-1" ).describe_regions()["Regions"] ] - + # Cache the result _region_cache["enabled_regions"][cache_key] = regions _region_cache["timestamp"] = time.time() - + return regions except Exception: # Fall back to default regions return get_default_regions_for_partition(partition) + def list_enabled_regions( - session: boto3.Session, - exclude_regions: Optional[List[str]] = None + session: boto3.Session, exclude_regions: Optional[List[str]] = None ) -> List[str]: """ List all enabled regions for the account, respecting partition. - + Args: session: Boto3 session exclude_regions: List of regions to exclude - + Returns: List of enabled region names """ if exclude_regions is None: exclude_regions = [] - + try: - ec2 = session.client("ec2", region_name="us-east-1") # Use US East 1 for global operations - regions_response = ec2.describe_regions(AllRegions=False) # Only get enabled regions + ec2 = session.client( + "ec2", region_name="us-east-1" + ) # Use US East 1 for global operations + regions_response = ec2.describe_regions( + AllRegions=False + ) # Only get enabled regions regions = [ r["RegionName"] for r in regions_response["Regions"] @@ -351,13 +389,37 @@ def list_enabled_regions( return regions except Exception as e: logger.warning(f"Could not retrieve regions, using defaults: {str(e)}") - + # For GovCloud, provide sensible defaults if session.region_name and session.region_name.startswith("us-gov-"): return ["us-gov-east-1", "us-gov-west-1"] - + # Default to common commercial regions return ["us-east-1", "us-east-2", "us-west-1", "us-west-2"] -# Alias for backward compatibility -get_available_regions = get_enabled_regions + +def discover_account_regions(client: Any, include_disabled: bool = False) -> List[str]: + """ + Discover regions enabled for an account using EC2 client. + + Args: + client: EC2 client to use for region discovery + include_disabled: Whether to include disabled regions + + Returns: + List of region names + """ + try: + # Call describe_regions API + if include_disabled: + response = client.describe_regions(AllRegions=True) + else: + response = client.describe_regions() + + # Extract region names + regions = [region["RegionName"] for region in response.get("Regions", [])] + return regions + + except Exception as e: + logger.error(f"Error discovering account regions: {str(e)}") + return []