Skip to content

Commit

Permalink
refactor for clarity
Browse files Browse the repository at this point in the history
  • Loading branch information
@morga471
morga471 committed Apr 3, 2025
1 parent 60c40e7 commit 2c8af08
Show file tree
Hide file tree
Showing 10 changed files with 952 additions and 932 deletions.
151 changes: 21 additions & 130 deletions local-app/python-tools/gfl-resource-actions/aws_resource_management/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,13 @@
EMRManager,
RDSManager,
)
from aws_resource_management.reporting import print_resource_summary, initialize_stats

logger = setup_logging()
config = get_config()


class ResourceManager:
"""Main resource manager that orchestrates operations across accounts and resources."""

def __init__(
self,
profile_name: Optional[str] = None,
Expand All @@ -56,7 +55,6 @@ def __init__(
self.partition = partition
# Cache for discovered regions by account
self.region_cache = {}

# If no partition was specified, we'll auto-detect based on credentials
# when we process the first account

Expand Down Expand Up @@ -88,34 +86,13 @@ def process_accounts(
try:
if exclude_accounts is None:
exclude_accounts = []

if exclude_resources is None:
exclude_resources = []

if exclude_regions is None:
exclude_regions = []

# Initialize stats if not provided
if stats is None:
stats = {
"accounts_processed": 0,
"accounts_skipped": 0,
"resources_processed": 0,
"resources_skipped": 0,
"regions_processed": 0,
"regions_by_account": {},
}

# Initialize resource-specific counters
for resource_type in ["ec2", "rds", "eks", "emr"]:
for stat_type in [
"found",
"skipped",
"stopped",
"started",
"errors",
]:
stats[f"{resource_type}_{stat_type}"] = 0
stats = initialize_stats()

# Get account list - either from profiles or Organizations API
if self.use_profiles:
Expand All @@ -132,9 +109,8 @@ def process_accounts(

# Process each account
for account in accounts:
account_id = account.get("account_id")
account_name = account.get("account_name", "Unknown")

account_id = account.get("account_id")
# Skip excluded accounts
if account_id in exclude_accounts:
logger.info(
Expand All @@ -155,8 +131,6 @@ def process_accounts(

# Determine regions to scan for this account
account_regions = regions

# If auto-discovery of regions is enabled, get all enabled regions for this account
if self.discover_regions:
try:
logger.info(
Expand All @@ -174,7 +148,6 @@ def process_accounts(
logger.info(
f"Found {len(account_regions)} enabled regions in account {account_id}"
)

# Cache discovered regions
self.region_cache[account_id] = account_regions
except Exception as e:
Expand All @@ -198,8 +171,8 @@ def process_accounts(
]

# Track regions processed for this account
stats["regions_by_account"][account_id] = account_regions
stats["regions_processed"] += len(account_regions)
stats["regions_by_account"][account_id] = account_regions

# Process this account with its regions
logger.info(
Expand All @@ -211,11 +184,10 @@ def process_accounts(
credentials=credentials,
regions=account_regions,
action=action,
dry_run=dry_run,
exclude_resources=exclude_resources,
dry_run=dry_run,
stats=stats,
)

stats["accounts_processed"] += 1

except Exception as e:
Expand All @@ -239,97 +211,16 @@ def process_accounts(
logger.debug(
f"Account {account_id} regions: {', '.join(account_regions)}"
)

# Print detailed resource summary
self._print_resource_summary(stats, action)

return stats

except KeyboardInterrupt:
# Log the interruption but re-raise to ensure application exit
logger.warning("Account processing interrupted by user")
raise

def _print_resource_summary(self, stats: Dict[str, Any], action: str) -> None:
"""
Print a detailed summary of resources processed, broken down by resource type.
Args:
stats: Statistics dictionary
action: Action that was performed (stop or start)
"""
logger.info(f"\n{'=' * 30} SUMMARY {'=' * 30}")
logger.info(
f"ACTION: {action.upper()} {'(DRY RUN)' if stats.get('dry_run', False) else ''}"
)

# General stats
logger.info(f"\nGENERAL STATISTICS:")
logger.info(f"Accounts processed: {stats.get('accounts_processed', 0)}")
logger.info(f"Accounts skipped: {stats.get('accounts_skipped', 0)}")
logger.info(f"Regions processed: {stats.get('regions_processed', 0)}")

# EC2 resources
logger.info(f"\nEC2 INSTANCES:")
logger.info(f"Found: {stats.get('ec2_found', 0)}")
if action == "stop":
logger.info(f"Stopped: {stats.get('ec2_stopped', 0)}")
else:
logger.info(f"Started: {stats.get('ec2_started', 0)}")
logger.info(f"Skipped: {stats.get('ec2_skipped', 0)}")
logger.info(f"Errors: {stats.get('ec2_errors', 0)}")

# RDS resources
logger.info(f"\nRDS INSTANCES:")
logger.info(f"Found: {stats.get('rds_found', 0)}")
if action == "stop":
logger.info(f"Stopped: {stats.get('rds_stopped', 0)}")
else:
logger.info(f"Started: {stats.get('rds_started', 0)}")
logger.info(f"Skipped: {stats.get('rds_skipped', 0)}")
logger.info(f"Errors: {stats.get('rds_errors', 0)}")

# RDS engine breakdown if available
rds_engines = stats.get("rds_engines", {})
if rds_engines:
logger.info(
f"RDS engines: {', '.join(f'{engine}({count})' for engine, count in rds_engines.items())}"
)

# EKS resources
logger.info(f"\nEKS CLUSTERS:")
logger.info(f"Found: {stats.get('eks_found', 0)}")
if action == "stop":
logger.info(f"Stopped: {stats.get('eks_stopped', 0)}")
else:
logger.info(f"Started: {stats.get('eks_started', 0)}")
logger.info(f"Skipped: {stats.get('eks_skipped', 0)}")
logger.info(f"Errors: {stats.get('eks_errors', 0)}")

# EMR resources
logger.info(f"\nEMR CLUSTERS:")
logger.info(f"Found: {stats.get('emr_found', 0)}")
if action == "stop":
logger.info(f"Stopped: {stats.get('emr_stopped', 0)}")
else:
logger.info(f"Started: {stats.get('emr_started', 0)}")
logger.info(f"Skipped: {stats.get('emr_skipped', 0)}")
logger.info(f"Errors: {stats.get('emr_errors', 0)}")

# Error summary if there were any errors
if stats.get("errors", []):
logger.info(f"\nERROR SUMMARY:")
logger.info(f"Total errors: {len(stats.get('errors', []))}")
for i, error in enumerate(
stats.get("errors", [])[:5], 1
): # Show first 5 errors
logger.info(f" {i}. {error}")
if len(stats.get("errors", [])) > 5:
logger.info(
f" ... and {len(stats.get('errors', [])) - 5} more errors (see log for details)"
)

logger.info(f"{'=' * 68}")
"""Print a summary of the resources processed."""
print_resource_summary(stats, action, stats.get("dry_run", False))

def _process_single_account(
self,
Expand Down Expand Up @@ -408,22 +299,19 @@ def _process_single_account(
f"No regions specified, using defaults for partition {self.partition}: {', '.join(regions)}"
)

# Only log this once - removing duplicate message
logger.info(
f"Processing account: {account_name} ({account_id}) in {len(regions)} regions"
)

# Define special handling for EMR clusters - remove 'STOPPED' from valid states as it's not accepted by the API
emr_states = None
if "emr" not in exclude_resources:
# Only include valid EMR states for the ListClusters API call
# Note: 'STOPPED' is not a valid state for ListClusters API
emr_states = [
"STARTING",
"BOOTSTRAPPING",
"RUNNING",
"WAITING",
"TERMINATING",
"WAITING",
]
logger.debug(
f"Using valid EMR states for discovery: {', '.join(emr_states)}"
Expand Down Expand Up @@ -464,6 +352,8 @@ def _process_single_account(
"emr_stopped",
"emr_started",
"emr_errors",
"ecr_images_found",
"ecr_old_images_found",
]:
if stat_type not in stats:
stats[stat_type] = 0
Expand All @@ -477,40 +367,36 @@ def _process_single_account(
stats["errors"] = []

# Normalize state and status fields for all resource types
# For EC2 instances
ec2_instances = resources.get("ec2_instances", [])
for instance in ec2_instances:
if "state" not in instance and "status" in instance:
instance["state"] = instance["status"]
elif "status" not in instance and "state" in instance:
instance["status"] = instance["state"]
elif "state" not in instance and "status" not in instance:
instance["state"] = "unknown"
instance["status"] = "unknown"
instance["state"] = "unknown"

# For RDS instances
rds_instances = resources.get("rds_instances", [])
for instance in rds_instances:
if "state" not in instance and "status" in instance:
instance["state"] = instance["status"]
elif "status" not in instance and "state" in instance:
instance["status"] = instance["state"]
elif "state" not in instance and "status" not in instance:
instance["state"] = "unknown"
instance["status"] = "unknown"
instance["state"] = "unknown"

# For EKS clusters
eks_clusters = resources.get("eks_clusters", [])
for cluster in eks_clusters:
if "state" not in cluster and "status" in cluster:
cluster["state"] = cluster["status"]
elif "status" not in cluster and "state" in cluster:
cluster["status"] = cluster["state"]
elif "state" not in cluster and "status" not in cluster:
cluster["state"] = "unknown"
cluster["status"] = "unknown"
cluster["state"] = "unknown"

# For EMR clusters
emr_clusters = resources.get("emr_clusters", [])
for cluster in emr_clusters:
if "state" not in cluster:
Expand All @@ -520,10 +406,9 @@ def _process_single_account(

# Count service-managed EC2 instances
eks_tag = self.config.get("eks_tag", "kubernetes.io/cluster/")
emr_tag = self.config.get("emr_tag", "aws:elasticmapreduce:job-flow-id")
exclusion_tag = self.config.get("exclusion_tag", "DoNotStop")
emr_tag = self.config.get("emr_tag", "aws:elasticmapreduce:job-flow-id")

# Continue with existing code
eks_managed = sum(
1
for i in resources.get("ec2_instances", [])
Expand All @@ -546,8 +431,10 @@ def _process_single_account(
stats["ec2_skipped"] += excluded_ec2

stats["rds_found"] += len(resources.get("rds_instances", []))
stats["eks_found"] += len(resources.get("eks_clusters", []))
stats["emr_found"] += len(resources.get("emr_clusters", []))
stats["eks_found"] += len(resources.get("eks_clusters", []))
stats["ecr_images_found"] += len(resources.get("ecr_images", []))
stats["ecr_old_images_found"] += len(resources.get("ecr_old_images", []))

# Track RDS engines
for instance in resources.get("rds_instances", []):
Expand All @@ -571,6 +458,10 @@ def _process_single_account(
logger.info(
f"Account {account_id} - Found {len(resources.get('emr_clusters', []))} EMR clusters"
)
logger.info(
f"Account {account_id} - Found {len(resources.get('ecr_images', []))} ECR images "
f"({len(resources.get('ecr_old_images', []))} older than 1 year)"
)

# Initialize resource managers with account name
ec2_manager = EC2Manager(credentials, account_id, account_name)
Expand Down

0 comments on commit 2c8af08

Please sign in to comment.