Skip to content

Commit

Permalink
update output
Browse files Browse the repository at this point in the history
  • Loading branch information
@morga471
morga471 committed Mar 14, 2025
1 parent 0ccd2eb commit 8088f08
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 72 deletions.
49 changes: 18 additions & 31 deletions local-app/python-tools/gfl-resource-actions/logging_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,40 +17,27 @@
ACTION_CSV_FILE = 'resource_actions.csv'
CSV_FIELDS = ['timestamp', 'account_id', 'account_name', 'resource_type', 'resource_id', 'resource_name', 'action', 'region', 'status', 'details']

def setup_logging(log_level=None, log_format=None):
"""
Set up logging configuration.
def setup_logging(name='resource_management', level=logging.INFO):
"""Set up and configure logging."""
logger = logging.getLogger(name)

Args:
log_level (int, optional): Logging level (default: INFO)
log_format (str, optional): Log message format
# Only configure the logger if it hasn't been configured already
if not logger.handlers:
logger.setLevel(level)

Returns:
logging.Logger: Configured logger
"""
if log_level is None:
log_level = DEFAULT_LOG_LEVEL
# Create console handler
console_handler = logging.StreamHandler()
console_handler.setLevel(level)

if log_format is None:
log_format = DEFAULT_LOG_FORMAT

# Create logger
logger = logging.getLogger('resource-management')
logger.setLevel(log_level)

# Create console handler
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(log_level)

# Create formatter and add it to the handler
formatter = logging.Formatter(log_format)
console_handler.setFormatter(formatter)

# Add handler to logger
logger.addHandler(console_handler)

# Ensure CSV log directory exists
os.makedirs(CSV_LOG_DIR, exist_ok=True)
# Create formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
console_handler.setFormatter(formatter)

# Add handler to logger
logger.addHandler(console_handler)

# Prevent propagation to the root logger to avoid duplicate messages
logger.propagate = False

return logger

Expand Down
107 changes: 66 additions & 41 deletions local-app/python-tools/gfl-resource-actions/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import argparse
import config
import sys
from collections import defaultdict
from logging_utils import setup_logging
from aws_utils import get_available_regions, assume_role, get_organization_accounts
from discovery import get_account_resources
Expand Down Expand Up @@ -88,6 +89,12 @@ def display_summary(stats, action, was_interrupted=False):
print(f" {action.capitalize()}ed: {stats['rds_' + action + 'ed']}")
print(f" Errors: {stats['rds_errors']}")

# RDS by engine type
if stats['rds_engines'] and stats['rds_found'] > 0:
print("\n RDS by Engine Type:")
for engine, count in stats['rds_engines'].items():
print(f" {engine}: {count}")

# EKS summary
print("\nEKS Clusters:")
print(f" Found: {stats['eks_found']}")
Expand All @@ -110,94 +117,109 @@ def process_account(account, credentials, regions, action, dry_run, exclude_reso

# Get resources
resources = get_account_resources(credentials, regions, exclude_resources)
if not resources:
logger.error(f"Failed to get resources for account {account['id']}")
return

# Count service-managed EC2 instances
eks_managed = sum(1 for i in resources['ec2_instances'] if config.EKS_TAG in i.get('tags', {}))
emr_managed = sum(1 for i in resources['ec2_instances'] if config.EMR_TAG in i.get('tags', {}))
eks_managed = sum(1 for i in resources.get('ec2_instances', []) if config.EKS_TAG in i.get('tags', {}))
emr_managed = sum(1 for i in resources.get('ec2_instances', []) if config.EMR_TAG in i.get('tags', {}))

# Update resource counts
total_ec2 = len(resources['ec2_instances'])
excluded_ec2 = sum(1 for i in resources['ec2_instances'] if config.EXCLUSION_TAG in i.get('tags', {}))
total_ec2 = len(resources.get('ec2_instances', []))
excluded_ec2 = sum(1 for i in resources.get('ec2_instances', []) if config.EXCLUSION_TAG in i.get('tags', {}))
stats['ec2_found'] += total_ec2
stats['ec2_skipped'] += excluded_ec2

stats['rds_found'] += len(resources['rds_instances'])
stats['eks_found'] += len(resources['eks_clusters'])
stats['emr_found'] += len(resources['emr_clusters'])
stats['rds_found'] += len(resources.get('rds_instances', []))
stats['eks_found'] += len(resources.get('eks_clusters', []))
stats['emr_found'] += len(resources.get('emr_clusters', []))

# Track RDS engines
for instance in resources.get('rds_instances', []):
if instance and 'engine' in instance:
engine = instance['engine']
stats['rds_engines'][engine] += 1

# Log discovered resources with service-managed counts
logger.info(f"Account {account['id']} - Found {total_ec2} EC2 instances ({excluded_ec2} explicitly excluded, {eks_managed} EKS-managed, {emr_managed} EMR-managed)")
logger.info(f"Account {account['id']} - Found {len(resources['rds_instances'])} RDS instances")
logger.info(f"Account {account['id']} - Found {len(resources['eks_clusters'])} EKS clusters")
logger.info(f"Account {account['id']} - Found {len(resources['emr_clusters'])} EMR clusters")
logger.info(f"Account {account['id']} - Found {len(resources.get('rds_instances', []))} RDS instances")
logger.info(f"Account {account['id']} - Found {len(resources.get('eks_clusters', []))} EKS clusters")
logger.info(f"Account {account['id']} - Found {len(resources.get('emr_clusters', []))} EMR clusters")

# Initialize resource managers
ec2_manager = EC2Manager(credentials, account['id'])
rds_manager = RDSManager(credentials, account['id'])
eks_manager = EKSManager(credentials, account['id'])
emr_manager = EMRManager(credentials, account['id'])

# Helper function to safely process manager results
def safe_get_result(result, key):
if result is None:
return 0
return result.get(key, 0)

# Perform actions based on selected mode
if action == "stop":
if "ec2" not in exclude_resources:
result = ec2_manager.stop(resources['ec2_instances'], dry_run)
stats['ec2_stopped'] += result.get('success', 0)
stats['ec2_errors'] += result.get('errors', 0)
result = ec2_manager.stop(resources.get('ec2_instances', []), dry_run)
stats['ec2_stopped'] += safe_get_result(result, 'success')
stats['ec2_errors'] += safe_get_result(result, 'errors')
else:
stats['ec2_skipped'] += total_ec2 - excluded_ec2

if "rds" not in exclude_resources:
result = rds_manager.stop(resources['rds_instances'], dry_run)
stats['rds_stopped'] += result.get('success', 0)
stats['rds_errors'] += result.get('errors', 0)
result = rds_manager.stop(resources.get('rds_instances', []), dry_run)
stats['rds_stopped'] += safe_get_result(result, 'success')
stats['rds_errors'] += safe_get_result(result, 'errors')
else:
stats['rds_skipped'] += len(resources['rds_instances'])
stats['rds_skipped'] += len(resources.get('rds_instances', []))

if "eks" not in exclude_resources:
result = eks_manager.stop(resources['eks_clusters'], dry_run)
stats['eks_stopped'] += result.get('success', 0)
stats['eks_errors'] += result.get('errors', 0)
result = eks_manager.stop(resources.get('eks_clusters', []), dry_run)
stats['eks_stopped'] += safe_get_result(result, 'success')
stats['eks_errors'] += safe_get_result(result, 'errors')
else:
stats['eks_skipped'] += len(resources['eks_clusters'])
stats['eks_skipped'] += len(resources.get('eks_clusters', []))

if "emr" not in exclude_resources:
result = emr_manager.stop(resources['emr_clusters'], dry_run)
stats['emr_stopped'] += result.get('success', 0)
stats['emr_errors'] += result.get('errors', 0)
result = emr_manager.stop(resources.get('emr_clusters', []), dry_run)
stats['emr_stopped'] += safe_get_result(result, 'success')
stats['emr_errors'] += safe_get_result(result, 'errors')
else:
stats['emr_skipped'] += len(resources['emr_clusters'])
stats['emr_skipped'] += len(resources.get('emr_clusters', []))
else: # action == "start"
if "ec2" not in exclude_resources:
result = ec2_manager.start(resources['ec2_instances'], dry_run)
stats['ec2_started'] += result.get('success', 0)
stats['ec2_errors'] += result.get('errors', 0)
result = ec2_manager.start(resources.get('ec2_instances', []), dry_run)
stats['ec2_started'] += safe_get_result(result, 'success')
stats['ec2_errors'] += safe_get_result(result, 'errors')
else:
stats['ec2_skipped'] += total_ec2 - excluded_ec2

if "rds" not in exclude_resources:
result = rds_manager.start(resources['rds_instances'], dry_run)
stats['rds_started'] += result.get('success', 0)
stats['rds_errors'] += result.get('errors', 0)
result = rds_manager.start(resources.get('rds_instances', []), dry_run)
stats['rds_started'] += safe_get_result(result, 'success')
stats['rds_errors'] += safe_get_result(result, 'errors')
else:
stats['rds_skipped'] += len(resources['rds_instances'])
stats['rds_skipped'] += len(resources.get('rds_instances', []))

if "eks" not in exclude_resources:
result = eks_manager.start(resources['eks_clusters'], dry_run)
stats['eks_started'] += result.get('success', 0)
stats['eks_errors'] += result.get('errors', 0)
result = eks_manager.start(resources.get('eks_clusters', []), dry_run)
stats['eks_started'] += safe_get_result(result, 'success')
stats['eks_errors'] += safe_get_result(result, 'errors')
else:
stats['eks_skipped'] += len(resources['eks_clusters'])
stats['eks_skipped'] += len(resources.get('eks_clusters', []))

if "emr" not in exclude_resources:
result = emr_manager.start(resources['emr_clusters'], dry_run)
stats['emr_started'] += result.get('success', 0)
stats['emr_errors'] += result.get('errors', 0)
result = emr_manager.start(resources.get('emr_clusters', []), dry_run)
stats['emr_started'] += safe_get_result(result, 'success')
stats['emr_errors'] += safe_get_result(result, 'errors')
else:
stats['emr_skipped'] += len(resources['emr_clusters'])
stats['emr_skipped'] += len(resources.get('emr_clusters', []))

def initialize_stats():
"""Initialize the statistics tracking dictionary"""
return {
stats = {
'accounts_processed': 0,
'accounts_skipped': 0,
'ec2_found': 0,
Expand All @@ -221,6 +243,9 @@ def initialize_stats():
'emr_started': 0,
'emr_errors': 0,
}
# Add RDS engine tracking using defaultdict
stats['rds_engines'] = defaultdict(int)
return stats

def main():
"""Main execution function."""
Expand Down

0 comments on commit 8088f08

Please sign in to comment.