diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py index 4133ad32..fd935b42 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/__init__.py @@ -1,13 +1,90 @@ """ -AWS Resource Management package. +AWS Resource Management Tool + +A Python-based tool for discovering, starting, stopping, and managing AWS resources +across multiple accounts, primarily designed for GovCloud environments to help +with cost management. """ -# Re-export key modules from consolidated utility modules -from aws_resource_management.utils.logging_utils import ( - setup_logging, +__version__ = "1.0.0" +__author__ = "AWS Resource Management Team" + +# Import core components and expose them at the package level +from aws_resource_management.core import ResourceManager +from aws_resource_management.resource_registry import get_registry +from aws_resource_management.discovery import ResourceDiscovery, get_account_resources + +# Import utility functions from the new consolidated utils package +from aws_resource_management.utils import ( + # AWS Core utilities + get_credentials, + get_session_for_account, + get_account_list, + get_organization_accounts, + + # Logging utilities + logger, + log_operation, + configure_logging, + + # File utilities + log_action_to_csv, + write_csv_row, + + # API utilities + paginate_aws_response, + format_tags, + + # Region utilities + # Note: These would be imported from region_utils, but we don't have that file's content +) + +# Import the managers package for resource-specific implementations +from aws_resource_management.managers import ( + ResourceManager as BaseResourceManager, + EC2Manager, + RDSManager, ) -# Set up a default logger -logger = setup_logging() +# Try importing optional managers +try: + from aws_resource_management.managers import EKSManager +except ImportError: + EKSManager = None + +try: + from aws_resource_management.managers import EMRManager +except ImportError: + EMRManager = None + +try: + from aws_resource_management.managers import ECRManager +except ImportError: + ECRManager = None + +# Get the resource registry instance +registry = get_registry() + +# Export all important components +__all__ = [ + "ResourceManager", + "ResourceDiscovery", + "get_account_resources", + "get_registry", + "registry", + "BaseResourceManager", + "EC2Manager", + "RDSManager", + "get_credentials", + "get_session_for_account", + "logger", + "configure_logging", +] -__version__ = "0.1.0" +# Conditionally add optional managers to __all__ +if EKSManager: + __all__.append("EKSManager") +if EMRManager: + __all__.append("EMRManager") +if ECRManager: + __all__.append("ECRManager") diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py index f933f469..eb0cf000 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py @@ -115,7 +115,7 @@ def _pre_validate_accounts( for account in accounts: account_id = account.get("account_id") try: - credentials = get_credentials(account_id, self) + credentials = get_credentials(account_id, self.profile_name) if credentials: valid_accounts.append(account) else: diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py index f8b6e6b3..b31ebafe 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/discovery.py @@ -68,8 +68,7 @@ def __init__( # Log with account name and ID logger.info( - f"Resource discovery initialized for account \ - {self.account_name} ({self.account_id}) in partition {self.partition}" + f"Resource discovery initialized for account {self.account_name} ({self.account_id}) in partition {self.partition}" ) def discover_resources( @@ -99,8 +98,7 @@ def discover_resources( if not valid_regions: valid_regions = DEFAULT_REGIONS.get(self.partition, DEFAULT_REGIONS["aws"]) logger.warning( - f"No valid regions found for partition {self.partition}, \ - using defaults: {', '.join(valid_regions)}" + f"No valid regions found for partition {self.partition}, using defaults: {', '.join(valid_regions)}" ) # Use validated regions @@ -132,8 +130,7 @@ def discover_resources( region_resources = future.result() resources.extend(region_resources) logger.debug( - f"Discovered {len(region_resources)} {resource_type} \ - resources in {region}" + f"Discovered {len(region_resources)} {resource_type} resources in {region}" ) except Exception as e: logger.error(f"Error discovering {resource_type} in {region}: {e}") @@ -360,7 +357,7 @@ def _discover_emr( "id": cluster["Id"], "name": cluster.get("Name", "Unnamed"), "status": state, - "state": state, # Add both fields to ensure compatibility + "state": state, "region": region, "tags": tags, "creation_time": ( @@ -732,7 +729,7 @@ def _get_account_resources_impl( "eks_clusters": [], "emr_clusters": [], "ecr_images": [], - "ecr_old_images": [], # Add a new key for old ECR images + "ecr_old_images": [], } try: @@ -771,8 +768,7 @@ def _get_account_resources_impl( # Discover EC2 instances - only if not excluded if "ec2" not in exclude_resources: logger.info( - f"Discovering EC2 instances for account {account_name} ({account_id}) \ - in {len(regions)} regions" + f"Discovering EC2 instances for account {account_name} ({account_id}) in {len(regions)} regions" ) resources["ec2_instances"] = discovery.discover_resources("ec2", regions) else: @@ -781,8 +777,7 @@ def _get_account_resources_impl( # Discover RDS instances - only if not excluded if "rds" not in exclude_resources: logger.info( - f"Discovering RDS instances for account {account_name} ({account_id}) \ - in {len(regions)} regions" + f"Discovering RDS instances for account {account_name} ({account_id}) in {len(regions)} regions" ) resources["rds_instances"] = discovery.discover_resources("rds", regions) else: @@ -791,8 +786,7 @@ def _get_account_resources_impl( # Discover EKS clusters - only if not excluded if "eks" not in exclude_resources: logger.info( - f"Discovering EKS clusters for account {account_name} ({account_id}) \ - in {len(regions)} regions" + f"Discovering EKS clusters for account {account_name} ({account_id}) in {len(regions)} regions" ) try: resources["eks_clusters"] = discovery.discover_resources("eks", regions) @@ -805,8 +799,7 @@ def _get_account_resources_impl( # Discover EMR clusters - only if not excluded if "emr" not in exclude_resources: logger.info( - f"Discovering EMR clusters for account {account_name} ({account_id}) \ - in {len(regions)} regions" + f"Discovering EMR clusters for account {account_name} ({account_id}) in {len(regions)} regions" ) try: resources["emr_clusters"] = discovery.discover_resources("emr", regions) @@ -819,8 +812,7 @@ def _get_account_resources_impl( # Discover ECR images - only if not excluded if "ecr" not in exclude_resources: logger.info( - f"Discovering ECR images for account {account_name} ({account_id}) \ - in {len(regions)} regions" + f"Discovering ECR images for account {account_name} ({account_id}) in {len(regions)} regions" ) try: # Use our helper function that properly enriches ECR resources @@ -855,14 +847,12 @@ def _get_account_resources_impl( "emr_clusters", ]: logger.info( - f"Total {resource_type.split('_')[0]} resources discovered: \ - {len(resource_list)}" + f"Total {resource_type.split('_')[0]} resources discovered: {len(resource_list)}" ) elif resource_type == "ecr_images": # ECR images are already logged in discover_ecr_images logger.info( - f"Total {resource_type.split('_')[0]} resources discovered: \ - {len(resource_list)}" + f"Total {resource_type.split('_')[0]} resources discovered: {len(resource_list)}" ) # Also add a log entry for the total resources diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py index ab5ac906..bedf36ed 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/base.py @@ -7,6 +7,7 @@ from typing import Any, Callable, Dict, List, Optional from aws_resource_management.utils import ( + get_account_alias, # Add the new import get_config, get_session_for_account, normalize_resource_state, @@ -346,3 +347,13 @@ def normalize_resources(resources: List[Dict[str, Any]]) -> None: # If still no account name, use account ID as fallback if "accountName" not in resource and "accountId" in resource: resource["accountName"] = resource["accountId"] + + def get_account_alias(self) -> str: + """ + Get the AWS account alias for cleaner reporting. + + Returns: + Account alias or account ID if alias not found + """ + # Call the centralized utility function + return get_account_alias(self.account_id, self.region, self.credentials) diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/__init__.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/__init__.py index e1b23a27..8b95a2d3 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/__init__.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/__init__.py @@ -16,12 +16,14 @@ from aws_resource_management.utils.aws_core_utils import ( create_client, ensure_valid_account_name, + get_account_alias, get_account_list, get_credentials, get_organization_accounts, get_session_for_account, ) from aws_resource_management.utils.config_utils import ( + ConfigManager, get_config, get_config_value, update_config, @@ -43,6 +45,7 @@ setup_logging, ) from aws_resource_management.utils.region_utils import ( + RegionManager, DEFAULT_PARTITION, DEFAULT_REGIONS, detect_partition_from_credentials, diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/aws_core_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/aws_core_utils.py index 56f287c4..bbf4d3c1 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/aws_core_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/aws_core_utils.py @@ -562,3 +562,47 @@ def create_client( except Exception as e: logger.error(f"Error creating {service_name} client: {e}") raise + + +def get_account_alias(account_id: str, region: Optional[str] = None, credentials: Optional[Dict[str, Any]] = None) -> str: + """ + Get the AWS account alias for an account. + + Args: + account_id: AWS account ID + region: AWS region to use for the API call (optional) + credentials: AWS credentials to use (optional) + + Returns: + Account alias or account ID if no alias is found + """ + try: + # Try to get a session for the account + if credentials: + session = boto3.Session( + aws_access_key_id=credentials.get("aws_access_key_id"), + aws_secret_access_key=credentials.get("aws_secret_access_key"), + aws_session_token=credentials.get("aws_session_token"), + region_name=region or "us-east-1" # Default to us-east-1 if no region provided + ) + else: + session = get_session_for_account(account_id, region=region) + + if not session: + logger.debug(f"Could not create session for account {account_id}") + return account_id + + # Try to get account aliases from IAM + iam_client = session.client("iam") + response = iam_client.list_account_aliases() + aliases = response.get("AccountAliases", []) + + # Return the first alias if any exist + if aliases: + return aliases[0] + + except Exception as e: + logger.debug(f"Failed to get account alias for account {account_id}: {str(e)}") + + # Default to account ID + return account_id diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/config_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/config_utils.py index bb54efbb..ec58890e 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/config_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/config_utils.py @@ -29,14 +29,14 @@ class ConfigManager: """Manages configuration for resource operations.""" def __init__(self, profile_name=None, use_profiles=False, discover_regions=False, partition=None): - self.config = get_config() + self.config = self.get_config() self.profile_name = profile_name self.use_profiles = use_profiles self.discover_regions = discover_regions self.partition = partition - def get_config() -> Dict[str, Any]: + def get_config(self) -> Dict[str, Any]: """ Get configuration with defaults merged with user settings. @@ -70,7 +70,7 @@ def get_config() -> Dict[str, Any]: user_config = yaml.safe_load(f) if user_config and isinstance(user_config, dict): # Merge with default config - _deep_merge(config, user_config) + self._deep_merge(config, user_config) except Exception: # Ignore errors reading config files pass @@ -80,7 +80,7 @@ def get_config() -> Dict[str, Any]: return config - def _deep_merge(base: Dict, update: Dict) -> Dict: + def _deep_merge(self, base: Dict, update: Dict) -> Dict: """ Recursively merge dictionaries. @@ -93,13 +93,13 @@ def _deep_merge(base: Dict, update: Dict) -> Dict: """ for key, value in update.items(): if isinstance(value, dict) and key in base and isinstance(base[key], dict): - _deep_merge(base[key], value) + self._deep_merge(base[key], value) else: base[key] = value return base - def get_config_value(key: str, default: Any = None) -> Any: + def get_config_value(self, key: str, default: Any = None) -> Any: """ Get a specific configuration value. @@ -110,10 +110,10 @@ def get_config_value(key: str, default: Any = None) -> Any: Returns: Configuration value or default """ - config = get_config() + config = self.get_config() # Handle nested keys with dot notation (e.g., "aws.region") - if "." in key: + if ("." in key): parts = key.split(".") current = config for part in parts: @@ -125,7 +125,7 @@ def get_config_value(key: str, default: Any = None) -> Any: return config.get(key, default) - def save_config(config: Dict[str, Any], config_path: Optional[str] = None) -> bool: + def save_config(self, config: Dict[str, Any], config_path: Optional[str] = None) -> bool: """ Save configuration to a file. @@ -156,7 +156,7 @@ def save_config(config: Dict[str, Any], config_path: Optional[str] = None) -> bo return False - def update_config(key: str, value: Any, config_path: Optional[str] = None) -> bool: + def update_config(self, key: str, value: Any, config_path: Optional[str] = None) -> bool: """ Update a specific configuration value and save. @@ -168,7 +168,7 @@ def update_config(key: str, value: Any, config_path: Optional[str] = None) -> bo Returns: True if updated successfully, False otherwise """ - config = get_config() + config = self.get_config() # Handle nested keys with dot notation if "." in key: @@ -182,4 +182,47 @@ def update_config(key: str, value: Any, config_path: Optional[str] = None) -> bo else: config[key] = value - return save_config(config, config_path) + return self.save_config(config, config_path) + + +# Create module-level functions that instantiate the class +def get_config() -> Dict[str, Any]: + """ + Module-level function to get configuration. + + Returns: + Configuration dictionary with default values merged with user settings + """ + config_manager = ConfigManager() + return config_manager.get_config() + + +def get_config_value(key: str, default: Any = None) -> Any: + """ + Module-level function to get a specific configuration value. + + Args: + key: Configuration key to retrieve + default: Default value if key not found + + Returns: + Configuration value or default + """ + config_manager = ConfigManager() + return config_manager.get_config_value(key, default) + + +def update_config(key: str, value: Any, config_path: Optional[str] = None) -> bool: + """ + Module-level function to update a configuration value and save it. + + Args: + key: Configuration key to update + value: New value + config_path: Path to save config to (default: user config path) + + Returns: + True if updated successfully, False otherwise + """ + config_manager = ConfigManager() + return config_manager.update_config(key, value, config_path) diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/region_utils.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/region_utils.py index 73bcdca9..5a8ab9db 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/region_utils.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/utils/region_utils.py @@ -428,3 +428,142 @@ def discover_account_regions(client: Any, include_disabled: bool = False) -> Lis except Exception as e: logger.error(f"Error discovering account regions: {str(e)}") return [] + +# Module-level functions to provide easier access to the RegionManager functionality + +def get_partition_for_region(region_name: str) -> str: + """ + Determine AWS partition based on region name. + + Args: + region_name: AWS region name + + Returns: + AWS partition name (e.g., 'aws', 'aws-us-gov', 'aws-cn') + """ + return RegionManager.get_partition_for_region(region_name) + +def get_regions_for_partition(partition: str) -> List[str]: + """ + Get the list of regions for a specific partition. + + Args: + partition: AWS partition name + + Returns: + List of region names in the partition + """ + return RegionManager.get_regions_for_partition(partition) + +def get_default_regions_for_partition(partition: str) -> List[str]: + """ + Get the default regions commonly used for a partition. + + Args: + partition: AWS partition name + + Returns: + List of default region names + """ + return RegionManager.get_default_regions_for_partition(partition) + +def is_region_in_partition(region: str, partition: str) -> bool: + """ + Check if a region belongs to the specified partition. + + Args: + region: AWS region name + partition: AWS partition name + + Returns: + True if the region is in the partition, False otherwise + """ + return RegionManager.is_region_in_partition(region, partition) + +def is_valid_region(region: str) -> bool: + """ + Check if a region is known to be valid without making an API call. + + Args: + region: Region name to validate + + Returns: + True if the region is valid, False otherwise + """ + return RegionManager.is_valid_region(region) + +def filter_regions_for_partition(regions: List[str], partition: str) -> List[str]: + """ + Filter regions to include only those belonging to the specified partition. + + Args: + regions: List of region names + partition: AWS partition name + + Returns: + List of valid region names within the partition + """ + return RegionManager.filter_regions_for_partition(regions, partition) + +def detect_partition_from_credentials(credentials: Dict[str, str]) -> str: + """ + Detect partition from credentials with minimal API calls. + + Args: + credentials: Dictionary containing AWS credentials + + Returns: + AWS partition name + """ + return RegionManager.detect_partition_from_credentials(credentials) + +def get_all_regions(partition: Optional[str] = None) -> List[str]: + """ + Get all AWS regions, using cache to minimize API calls. + + Args: + partition: Optional partition to filter regions by + + Returns: + List of region names + """ + return RegionManager.get_all_regions(partition) + +def get_enabled_regions(credentials: Dict[str, str], partition: Optional[str] = None) -> List[str]: + """ + Get enabled regions with minimized API calls using cache. + + Args: + credentials: Dictionary containing AWS credentials + partition: Optional partition to filter regions by + + Returns: + List of enabled region names + """ + return RegionManager.get_enabled_regions(credentials, partition) + +def list_enabled_regions(session: boto3.Session, exclude_regions: Optional[List[str]] = None) -> List[str]: + """ + List all enabled regions for the account, respecting partition. + + Args: + session: Boto3 session + exclude_regions: List of regions to exclude + + Returns: + List of enabled region names + """ + return RegionManager.list_enabled_regions(session, exclude_regions) + +def discover_account_regions(client: Any, include_disabled: bool = False) -> List[str]: + """ + Discover regions enabled for an account using EC2 client. + + Args: + client: EC2 client to use for region discovery + include_disabled: Whether to include disabled regions + + Returns: + List of region names + """ + return RegionManager.discover_account_regions(client, include_disabled)