diff --git a/local-app/python-tools/gfl-resource-actions/logging_utils.py b/local-app/python-tools/gfl-resource-actions/logging_utils.py index 04f8c508..6f84b6a9 100644 --- a/local-app/python-tools/gfl-resource-actions/logging_utils.py +++ b/local-app/python-tools/gfl-resource-actions/logging_utils.py @@ -17,40 +17,27 @@ ACTION_CSV_FILE = 'resource_actions.csv' CSV_FIELDS = ['timestamp', 'account_id', 'account_name', 'resource_type', 'resource_id', 'resource_name', 'action', 'region', 'status', 'details'] -def setup_logging(log_level=None, log_format=None): - """ - Set up logging configuration. +def setup_logging(name='resource_management', level=logging.INFO): + """Set up and configure logging.""" + logger = logging.getLogger(name) - Args: - log_level (int, optional): Logging level (default: INFO) - log_format (str, optional): Log message format + # Only configure the logger if it hasn't been configured already + if not logger.handlers: + logger.setLevel(level) - Returns: - logging.Logger: Configured logger - """ - if log_level is None: - log_level = DEFAULT_LOG_LEVEL + # Create console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(level) - if log_format is None: - log_format = DEFAULT_LOG_FORMAT - - # Create logger - logger = logging.getLogger('resource-management') - logger.setLevel(log_level) - - # Create console handler - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setLevel(log_level) - - # Create formatter and add it to the handler - formatter = logging.Formatter(log_format) - console_handler.setFormatter(formatter) - - # Add handler to logger - logger.addHandler(console_handler) - - # Ensure CSV log directory exists - os.makedirs(CSV_LOG_DIR, exist_ok=True) + # Create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + console_handler.setFormatter(formatter) + + # Add handler to logger + logger.addHandler(console_handler) + + # Prevent propagation to the root logger to avoid duplicate messages + logger.propagate = False return logger diff --git a/local-app/python-tools/gfl-resource-actions/main.py b/local-app/python-tools/gfl-resource-actions/main.py index 3f20674b..fc65bc2c 100644 --- a/local-app/python-tools/gfl-resource-actions/main.py +++ b/local-app/python-tools/gfl-resource-actions/main.py @@ -6,6 +6,7 @@ import argparse import config import sys +from collections import defaultdict from logging_utils import setup_logging from aws_utils import get_available_regions, assume_role, get_organization_accounts from discovery import get_account_resources @@ -88,6 +89,12 @@ def display_summary(stats, action, was_interrupted=False): print(f" {action.capitalize()}ed: {stats['rds_' + action + 'ed']}") print(f" Errors: {stats['rds_errors']}") + # RDS by engine type + if stats['rds_engines'] and stats['rds_found'] > 0: + print("\n RDS by Engine Type:") + for engine, count in stats['rds_engines'].items(): + print(f" {engine}: {count}") + # EKS summary print("\nEKS Clusters:") print(f" Found: {stats['eks_found']}") @@ -110,26 +117,35 @@ def process_account(account, credentials, regions, action, dry_run, exclude_reso # Get resources resources = get_account_resources(credentials, regions, exclude_resources) + if not resources: + logger.error(f"Failed to get resources for account {account['id']}") + return # Count service-managed EC2 instances - eks_managed = sum(1 for i in resources['ec2_instances'] if config.EKS_TAG in i.get('tags', {})) - emr_managed = sum(1 for i in resources['ec2_instances'] if config.EMR_TAG in i.get('tags', {})) + eks_managed = sum(1 for i in resources.get('ec2_instances', []) if config.EKS_TAG in i.get('tags', {})) + emr_managed = sum(1 for i in resources.get('ec2_instances', []) if config.EMR_TAG in i.get('tags', {})) # Update resource counts - total_ec2 = len(resources['ec2_instances']) - excluded_ec2 = sum(1 for i in resources['ec2_instances'] if config.EXCLUSION_TAG in i.get('tags', {})) + total_ec2 = len(resources.get('ec2_instances', [])) + excluded_ec2 = sum(1 for i in resources.get('ec2_instances', []) if config.EXCLUSION_TAG in i.get('tags', {})) stats['ec2_found'] += total_ec2 stats['ec2_skipped'] += excluded_ec2 - stats['rds_found'] += len(resources['rds_instances']) - stats['eks_found'] += len(resources['eks_clusters']) - stats['emr_found'] += len(resources['emr_clusters']) + stats['rds_found'] += len(resources.get('rds_instances', [])) + stats['eks_found'] += len(resources.get('eks_clusters', [])) + stats['emr_found'] += len(resources.get('emr_clusters', [])) + + # Track RDS engines + for instance in resources.get('rds_instances', []): + if instance and 'engine' in instance: + engine = instance['engine'] + stats['rds_engines'][engine] += 1 # Log discovered resources with service-managed counts logger.info(f"Account {account['id']} - Found {total_ec2} EC2 instances ({excluded_ec2} explicitly excluded, {eks_managed} EKS-managed, {emr_managed} EMR-managed)") - logger.info(f"Account {account['id']} - Found {len(resources['rds_instances'])} RDS instances") - logger.info(f"Account {account['id']} - Found {len(resources['eks_clusters'])} EKS clusters") - logger.info(f"Account {account['id']} - Found {len(resources['emr_clusters'])} EMR clusters") + logger.info(f"Account {account['id']} - Found {len(resources.get('rds_instances', []))} RDS instances") + logger.info(f"Account {account['id']} - Found {len(resources.get('eks_clusters', []))} EKS clusters") + logger.info(f"Account {account['id']} - Found {len(resources.get('emr_clusters', []))} EMR clusters") # Initialize resource managers ec2_manager = EC2Manager(credentials, account['id']) @@ -137,67 +153,73 @@ def process_account(account, credentials, regions, action, dry_run, exclude_reso eks_manager = EKSManager(credentials, account['id']) emr_manager = EMRManager(credentials, account['id']) + # Helper function to safely process manager results + def safe_get_result(result, key): + if result is None: + return 0 + return result.get(key, 0) + # Perform actions based on selected mode if action == "stop": if "ec2" not in exclude_resources: - result = ec2_manager.stop(resources['ec2_instances'], dry_run) - stats['ec2_stopped'] += result.get('success', 0) - stats['ec2_errors'] += result.get('errors', 0) + result = ec2_manager.stop(resources.get('ec2_instances', []), dry_run) + stats['ec2_stopped'] += safe_get_result(result, 'success') + stats['ec2_errors'] += safe_get_result(result, 'errors') else: stats['ec2_skipped'] += total_ec2 - excluded_ec2 if "rds" not in exclude_resources: - result = rds_manager.stop(resources['rds_instances'], dry_run) - stats['rds_stopped'] += result.get('success', 0) - stats['rds_errors'] += result.get('errors', 0) + result = rds_manager.stop(resources.get('rds_instances', []), dry_run) + stats['rds_stopped'] += safe_get_result(result, 'success') + stats['rds_errors'] += safe_get_result(result, 'errors') else: - stats['rds_skipped'] += len(resources['rds_instances']) + stats['rds_skipped'] += len(resources.get('rds_instances', [])) if "eks" not in exclude_resources: - result = eks_manager.stop(resources['eks_clusters'], dry_run) - stats['eks_stopped'] += result.get('success', 0) - stats['eks_errors'] += result.get('errors', 0) + result = eks_manager.stop(resources.get('eks_clusters', []), dry_run) + stats['eks_stopped'] += safe_get_result(result, 'success') + stats['eks_errors'] += safe_get_result(result, 'errors') else: - stats['eks_skipped'] += len(resources['eks_clusters']) + stats['eks_skipped'] += len(resources.get('eks_clusters', [])) if "emr" not in exclude_resources: - result = emr_manager.stop(resources['emr_clusters'], dry_run) - stats['emr_stopped'] += result.get('success', 0) - stats['emr_errors'] += result.get('errors', 0) + result = emr_manager.stop(resources.get('emr_clusters', []), dry_run) + stats['emr_stopped'] += safe_get_result(result, 'success') + stats['emr_errors'] += safe_get_result(result, 'errors') else: - stats['emr_skipped'] += len(resources['emr_clusters']) + stats['emr_skipped'] += len(resources.get('emr_clusters', [])) else: # action == "start" if "ec2" not in exclude_resources: - result = ec2_manager.start(resources['ec2_instances'], dry_run) - stats['ec2_started'] += result.get('success', 0) - stats['ec2_errors'] += result.get('errors', 0) + result = ec2_manager.start(resources.get('ec2_instances', []), dry_run) + stats['ec2_started'] += safe_get_result(result, 'success') + stats['ec2_errors'] += safe_get_result(result, 'errors') else: stats['ec2_skipped'] += total_ec2 - excluded_ec2 if "rds" not in exclude_resources: - result = rds_manager.start(resources['rds_instances'], dry_run) - stats['rds_started'] += result.get('success', 0) - stats['rds_errors'] += result.get('errors', 0) + result = rds_manager.start(resources.get('rds_instances', []), dry_run) + stats['rds_started'] += safe_get_result(result, 'success') + stats['rds_errors'] += safe_get_result(result, 'errors') else: - stats['rds_skipped'] += len(resources['rds_instances']) + stats['rds_skipped'] += len(resources.get('rds_instances', [])) if "eks" not in exclude_resources: - result = eks_manager.start(resources['eks_clusters'], dry_run) - stats['eks_started'] += result.get('success', 0) - stats['eks_errors'] += result.get('errors', 0) + result = eks_manager.start(resources.get('eks_clusters', []), dry_run) + stats['eks_started'] += safe_get_result(result, 'success') + stats['eks_errors'] += safe_get_result(result, 'errors') else: - stats['eks_skipped'] += len(resources['eks_clusters']) + stats['eks_skipped'] += len(resources.get('eks_clusters', [])) if "emr" not in exclude_resources: - result = emr_manager.start(resources['emr_clusters'], dry_run) - stats['emr_started'] += result.get('success', 0) - stats['emr_errors'] += result.get('errors', 0) + result = emr_manager.start(resources.get('emr_clusters', []), dry_run) + stats['emr_started'] += safe_get_result(result, 'success') + stats['emr_errors'] += safe_get_result(result, 'errors') else: - stats['emr_skipped'] += len(resources['emr_clusters']) + stats['emr_skipped'] += len(resources.get('emr_clusters', [])) def initialize_stats(): """Initialize the statistics tracking dictionary""" - return { + stats = { 'accounts_processed': 0, 'accounts_skipped': 0, 'ec2_found': 0, @@ -221,6 +243,9 @@ def initialize_stats(): 'emr_started': 0, 'emr_errors': 0, } + # Add RDS engine tracking using defaultdict + stats['rds_engines'] = defaultdict(int) + return stats def main(): """Main execution function."""