diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management.py deleted file mode 100644 index c7913ec9..00000000 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -""" -Runner script for AWS Resource Management. -Acts as a wrapper for the main.py script. -""" - -import concurrent.futures -import logging -import os -import subprocess -import sys -import tempfile - -# Configure logging -logging.basicConfig( - level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s - %(message)s" -) -logger = logging.getLogger("aws_resource_management") - - -def process_account(account_id, script_path, script_dir, args): - """Process a single account with the main script.""" - try: - cmd = [sys.executable, script_path] + args - env = os.environ.copy() - env["PYTHONPATH"] = script_dir + os.pathsep + env.get("PYTHONPATH", "") - env["AWS_ACCOUNT"] = account_id - - result = subprocess.run(cmd, env=env, capture_output=True, text=True) - if result.returncode != 0: - logger.error(f"Error processing account {account_id}: {result.stderr}") - return result.returncode - except Exception as e: - logger.error(f"Error processing account {account_id}: {str(e)}") - return 1 - - -if __name__ == "__main__": - # Get the current script directory - script_dir = os.path.dirname(os.path.abspath(__file__)) - - # Path to main.py - main_path = os.path.join(script_dir, "main.py") - - # Check if the main script exists - if not os.path.exists(main_path): - print(f"Error: Main script not found at {main_path}") - sys.exit(1) - - # Create a temporary version of main.py with absolute imports - with open(main_path, "r") as f: - content = f.read() - - # Replace relative imports with absolute imports - content = content.replace("from . import config", "import config") - content = content.replace("from .logging_utils", "from logging_utils") - content = content.replace("from .aws_utils", "from aws_utils") - content = content.replace("from .discovery", "from discovery") - content = content.replace("from .managers", "from managers") - - # Write to temporary file - temp_file = tempfile.NamedTemporaryFile(suffix=".py", delete=False) - temp_file.write(content.encode("utf-8")) - temp_file.close() - - try: - # Get list of accounts to process - accounts = [os.environ.get("AWS_ACCOUNT_ID", "")] - - # Create a thread pool for parallel execution - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: - # Submit account processing tasks - futures = { - executor.submit( - process_account, account, temp_file.name, script_dir, sys.argv[1:] - ): account - for account in accounts - if account - } - - # Wait for all tasks to complete - results = [] - for future in concurrent.futures.as_completed(futures): - account = futures[future] - try: - results.append(future.result()) - except Exception as e: - logger.error( - f"Thread execution error for account {account}: {str(e)}" - ) - results.append(1) - - # Exit with error if any account processing failed - sys.exit(1 if any(result != 0 for result in results) else 0) - finally: - # Clean up the temporary file - os.unlink(temp_file.name) diff --git a/local-app/python-tools/gfl-resource-actions/aws_utils.py b/local-app/python-tools/gfl-resource-actions/aws_utils.py deleted file mode 100644 index 31bf0db5..00000000 --- a/local-app/python-tools/gfl-resource-actions/aws_utils.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -AWS utility functions for resource management. -""" - -import boto3 -import config -from logging_utils import setup_logging - -logger = setup_logging() - - -def create_boto3_client(service, credentials, region): - """ - Create a boto3 client for a specific service using the provided credentials. - - Args: - service (str): AWS service name (e.g., 'ec2', 'rds') - credentials (dict): AWS credentials dict with access key, secret key, and token - region (str): AWS region name - - Returns: - boto3.client: Configured boto3 client - """ - return boto3.client( - service, - aws_access_key_id=credentials["AccessKeyId"], - aws_secret_access_key=credentials["SecretAccessKey"], - aws_session_token=credentials["SessionToken"], - region_name=region, - ) - - -def get_available_regions(): - """ - Get a list of all available AWS regions. - - Returns: - list: List of region names - """ - try: - ec2_client = boto3.client("ec2") - response = ec2_client.describe_regions() - return [region["RegionName"] for region in response["Regions"]] - except Exception as e: - logger.error(f"Error getting available regions: {e}") - # Return a default list of common regions as fallback - return ["us-east-1", "us-east-2", "us-west-1", "us-west-2"] - - -def assume_role(account_id): - """ - Assume the OrganizationAccountAccessRole in the specified account. - - Args: - account_id (str): AWS account ID to assume role in - - Returns: - dict: Credentials dictionary or None if failed - """ - try: - role_arn = f"arn:aws:iam::{account_id}:role/{config.ASSUME_ROLE_NAME}" - sts_client = boto3.client("sts") - - response = sts_client.assume_role( - RoleArn=role_arn, RoleSessionName="ResourceManagementSession" - ) - - return response["Credentials"] - except Exception as e: - logger.error(f"Error assuming role in account {account_id}: {e}") - return None - - -def get_organization_accounts(exclude_accounts=None): - """ - Get a list of accounts in the AWS organization. - - Args: - exclude_accounts (list): List of account IDs to exclude - - Returns: - list: List of account dicts with id and name - """ - if exclude_accounts is None: - exclude_accounts = [] - - try: - org_client = boto3.client("organizations") - accounts = [] - - # Get all accounts in the organization - paginator = org_client.get_paginator("list_accounts") - for page in paginator.paginate(): - for account in page["Accounts"]: - # Skip suspended accounts and accounts in exclude list - if ( - account["Status"] == "ACTIVE" - and account["Id"] not in exclude_accounts - ): - accounts.append({"id": account["Id"], "name": account["Name"]}) - - return accounts - except Exception as e: - logger.error(f"Error getting organization accounts: {e}") - return [] diff --git a/local-app/python-tools/gfl-resource-actions/logging_setup.py b/local-app/python-tools/gfl-resource-actions/logging_setup.py deleted file mode 100644 index f51e7f5f..00000000 --- a/local-app/python-tools/gfl-resource-actions/logging_setup.py +++ /dev/null @@ -1,337 +0,0 @@ -import json -import logging -import os -import sys -import threading -import time -import uuid -from contextlib import contextmanager -from datetime import datetime -from logging.handlers import RotatingFileHandler -from typing import Any, Dict, List, Optional, Set, Union - -# Store sensitive field patterns to mask in logs -SENSITIVE_FIELDS = { - "password", - "secret", - "token", - "key", - "passphrase", - "credential", - "auth", - "jwt", - "private_key", - "access_key", - "secret_key", -} - - -class LoggingContext: - """Thread-local storage for logging context information.""" - - _context = threading.local() - - @classmethod - def set(cls, **kwargs) -> None: - """Set context values.""" - if not hasattr(cls._context, "data"): - cls._context.data = {} - cls._context.data.update(kwargs) - - @classmethod - def get(cls) -> Dict[str, Any]: - """Get all context values.""" - return getattr(cls._context, "data", {}).copy() - - @classmethod - def get_value(cls, key: str, default: Any = None) -> Any: - """Get a specific context value.""" - return getattr(cls._context, "data", {}).get(key, default) - - @classmethod - def clear(cls) -> None: - """Clear all context values.""" - if hasattr(cls._context, "data"): - cls._context.data = {} - - -class JSONFormatter(logging.Formatter): - """Format log records as JSON with additional context.""" - - def __init__(self, sanitize_fields: Optional[Set[str]] = None): - super().__init__() - self.sanitize_fields = sanitize_fields or SENSITIVE_FIELDS - - def format(self, record: logging.LogRecord) -> str: - """Format the record as a JSON string with context.""" - # Start with basic log information - log_data = { - "timestamp": datetime.utcfromtimestamp(record.created).isoformat() + "Z", - "level": record.levelname, - "logger": record.name, - "message": record.getMessage(), - "path": record.pathname, - "function": record.funcName, - "line": record.lineno, - "process": record.process, - "thread": record.thread, - "hostname": os.uname().nodename, - } - - # Add correlation ID if not already present - if not hasattr(record, "correlation_id"): - log_data["correlation_id"] = str(uuid.uuid4()) - - # Add exception info if available - if record.exc_info: - log_data["exception"] = { - "type": record.exc_info[0].__name__, - "message": str(record.exc_info[1]), - "traceback": self.formatException(record.exc_info), - } - - # Add extra fields from the record - if hasattr(record, "extra") and record.extra: - for key, value in record.extra.items(): - log_data[key] = value - - # Add context from LoggingContext - context_data = LoggingContext.get() - if context_data: - for key, value in context_data.items(): - if key not in log_data: # Don't overwrite existing fields - log_data[key] = value - - # Sanitize sensitive fields - self._sanitize_data(log_data) - - # Convert to JSON - return json.dumps(log_data) - - def _sanitize_data(self, data: Dict[str, Any], path: str = "") -> None: - """Recursively sanitize sensitive data.""" - if isinstance(data, dict): - for key, value in list(data.items()): - current_path = f"{path}.{key}" if path else key - - # Check if this is a sensitive field - is_sensitive = any( - sensitive in current_path.lower() - for sensitive in self.sanitize_fields - ) - - if is_sensitive and isinstance(value, (str, int, float, bool)): - data[key] = "********" - elif isinstance(value, (dict, list)): - self._sanitize_data(value, current_path) - elif isinstance(data, list): - for i, item in enumerate(data): - current_path = f"{path}[{i}]" - if isinstance(item, (dict, list)): - self._sanitize_data(item, current_path) - - -class ContextFilter(logging.Filter): - """Filter for automatically adding AWS context to log records.""" - - def __init__( - self, - account_id: Optional[str] = None, - region: Optional[str] = None, - service_name: Optional[str] = None, - ): - super().__init__() - self.account_id = account_id - self.region = region - self.service_name = service_name - - def filter(self, record: logging.LogRecord) -> bool: - """Add AWS context to the log record.""" - # Add basic context if not already present - if not hasattr(record, "aws"): - record.aws = {} - - # Add AWS account ID - if self.account_id and not record.aws.get("account_id"): - record.aws["account_id"] = self.account_id - - # Add AWS region - if self.region and not record.aws.get("region"): - record.aws["region"] = self.region - - # Add service name - if self.service_name and not record.aws.get("service_name"): - record.aws["service_name"] = self.service_name - - # Add context from LoggingContext if relevant - context = LoggingContext.get() - for field in ["resource_type", "resource_id", "operation_name"]: - if field in context and not record.aws.get(field): - record.aws[field] = context[field] - - # Always return True as we're just enriching, not filtering out - return True - - -@contextmanager -def log_operation( - logger: logging.Logger, name: str, level: int = logging.INFO, **context -) -> None: - """ - Context manager for logging operations with timing and status. - - Args: - logger: The logger to use - name: Name of the operation - level: Log level to use - **context: Additional context to include in logs - """ - # Generate correlation ID for this operation if not provided - correlation_id = context.get("correlation_id", str(uuid.uuid4())) - - # Set initial context - LoggingContext.set( - operation_name=name, - operation_start_time=time.time(), - correlation_id=correlation_id, - **context, - ) - - logger.log( - level, f"Starting operation: {name}", extra={"correlation_id": correlation_id} - ) - - try: - yield - # Calculate duration and update context on successful completion - duration = ( - time.time() - LoggingContext.get_value("operation_start_time", 0) - ) * 1000 - LoggingContext.set(operation_status="success", duration_ms=duration) - logger.log( - level, - f"Operation {name} completed successfully in {duration:.2f}ms", - extra={"correlation_id": correlation_id, "duration_ms": duration}, - ) - except Exception as e: - # Calculate duration and update context on failure - duration = ( - time.time() - LoggingContext.get_value("operation_start_time", 0) - ) * 1000 - LoggingContext.set( - operation_status="failed", duration_ms=duration, error=str(e) - ) - logger.error( - f"Operation {name} failed after {duration:.2f}ms: {str(e)}", - exc_info=True, - extra={"correlation_id": correlation_id, "duration_ms": duration}, - ) - raise - finally: - # Clear context to avoid leaking between operations - LoggingContext.clear() - - -def log_with_context(logger: logging.Logger, level: int, msg: str, **context) -> None: - """ - Log a message with the current context plus any additional context provided. - - Args: - logger: Logger to use - level: Log level - msg: Message to log - **context: Additional context to include - """ - # Merge with existing context - merged_context = LoggingContext.get() - merged_context.update(context) - - # Log with the merged context - logger.log(level, msg, extra={"extra": merged_context}) - - -def configure_logging( - app_name: str, - log_level: Union[int, str] = logging.INFO, - json_format: bool = True, - log_file: Optional[str] = None, - max_log_size: int = 10 * 1024 * 1024, # 10 MB - backup_count: int = 5, - console_output: bool = True, - aws_context: Optional[Dict[str, str]] = None, -) -> logging.Logger: - """ - Configure application logging with advanced features. - - Args: - app_name: Application name used for the logger - log_level: Logging level (name or constant) - json_format: Whether to use JSON formatting - log_file: Path to log file (if None, file logging is disabled) - max_log_size: Maximum size of log files before rotation - backup_count: Number of backup log files to keep - console_output: Whether to output logs to console - aws_context: Dict with AWS context (account_id, region, service_name) - - Returns: - Configured logger instance - """ - # Convert string log level to int if needed - if isinstance(log_level, str): - log_level = getattr(logging, log_level.upper(), logging.INFO) - - # Create logger - logger = logging.getLogger(app_name) - logger.setLevel(log_level) - logger.handlers = [] # Remove existing handlers - - # Create formatters - if json_format: - json_formatter = JSONFormatter() - console_formatter = json_formatter - else: - # Human-readable format for console - console_formatter = logging.Formatter( - "%(asctime)s [%(levelname)s] [%(name)s] [%(correlation_id)s] %(message)s" - ) - # JSON for file even if console is human-readable - json_formatter = JSONFormatter() - - # Add AWS context filter if provided - if aws_context: - context_filter = ContextFilter( - account_id=aws_context.get("account_id"), - region=aws_context.get("region"), - service_name=aws_context.get("service_name"), - ) - logger.addFilter(context_filter) - - # Add console handler if requested - if console_output: - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setFormatter(console_formatter) - logger.addHandler(console_handler) - - # Add file handler if requested - if log_file: - os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True) - file_handler = RotatingFileHandler( - log_file, maxBytes=max_log_size, backupCount=backup_count - ) - file_handler.setFormatter(json_formatter) - logger.addHandler(file_handler) - - return logger - - -def get_logger(name: str) -> logging.Logger: - """ - Get a logger with the given name, inheriting configuration from the root logger. - - Args: - name: Logger name - - Returns: - Logger instance - """ - return logging.getLogger(name)