Skip to content

Commit

Permalink
outputs csv
Browse files Browse the repository at this point in the history
  • Loading branch information
@morga471
morga471 committed Apr 3, 2025
1 parent c11dc9b commit 0591971
Show file tree
Hide file tree
Showing 17 changed files with 1,715 additions and 860 deletions.
2 changes: 1 addition & 1 deletion local-app/python-tools/gfl-resource-actions/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ dist:

# Run in dry-run mode
run-dry-run:
$(PYTHON) -m $(PACKAGE_NAME).cli --stop --dry-run --resource-type all
$(PYTHON) -m $(PACKAGE_NAME).cli --stop --dry-run --resource-type ecr --csv-output-dir ./ --csv-export

# Run to stop resources
run-stop:
Expand Down
10 changes: 0 additions & 10 deletions local-app/python-tools/gfl-resource-actions/aws_resource_management/__main__.py
View file

This file was deleted.

100 changes: 34 additions & 66 deletions local-app/python-tools/gfl-resource-actions/aws_resource_management/aws_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,16 @@
import boto3
import botocore
from aws_resource_management.logging_setup import setup_logging
from aws_resource_management.partition_utils import (
DEFAULT_PARTITION,
DEFAULT_REGIONS,
PARTITION_REGIONS,
get_default_regions_for_partition,
get_partition_for_region,
get_regions_for_partition,
is_region_in_partition,
is_valid_region,
)
from botocore.exceptions import ClientError, NoCredentialsError, ProfileNotFound

logger = setup_logging()
Expand All @@ -30,24 +40,8 @@
"timestamp": 0,
"all_regions": {},
"enabled_regions": {},
"partition_regions": {
"aws-us-gov": ["us-gov-east-1", "us-gov-west-1"],
"aws-cn": ["cn-north-1", "cn-northwest-1"],
"aws": [
"us-east-1",
"us-east-2",
"us-west-1",
"us-west-2",
"eu-west-1",
"eu-central-1",
"eu-west-2",
"eu-west-3",
"ap-northeast-1",
"ap-northeast-2",
"ap-southeast-1",
"ap-southeast-2",
],
},
# Use PARTITION_REGIONS from partition_utils for partition region data
"partition_regions": PARTITION_REGIONS,
}

# Cache expiration time in seconds (1 hour)
Expand Down Expand Up @@ -91,33 +85,9 @@ def safe_in(substring: Any, container: Any) -> bool:
# Simplified region and partition handling
# ---------------------------------------------------------------------------


@lru_cache(maxsize=128)
def get_partition_for_region(region_name: str) -> str:
"""
Determine AWS partition based on region name.
Args:
region_name: AWS region name
Returns:
AWS partition name (e.g., 'aws', 'aws-us-gov', 'aws-cn')
"""
if region_name.startswith("us-gov-"):
return "aws-us-gov"
elif region_name.startswith("cn-"):
return "aws-cn"
return "aws"


def get_regions_for_partition(partition: str) -> List[str]:
"""Get regions for a partition using cached data instead of API calls."""
if partition in _region_cache["partition_regions"]:
return _region_cache["partition_regions"][partition]

# Default to GovCloud
return _region_cache["partition_regions"]["aws-us-gov"]

# Export partition functions directly from partition_utils for backwards compatibility
# These explicit re-exports make it clearer that they're from partition_utils
# rather than just happening to have the same name

def get_all_regions(partition: Optional[str] = None) -> List[str]:
"""Get all regions, using cache to minimize API calls."""
Expand Down Expand Up @@ -159,7 +129,7 @@ def get_all_regions(partition: Optional[str] = None) -> List[str]:
return all_regions
except Exception as e:
logger.warning(f"Error getting regions: {str(e)}")
return get_regions_for_partition(partition or "aws-us-gov")
return get_regions_for_partition(partition or DEFAULT_PARTITION)


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -299,7 +269,7 @@ def _assume_role_for_account(account_id: str) -> Optional[Dict[str, Any]]:
def detect_partition_from_credentials(credentials: Dict[str, str]) -> str:
"""Detect partition from credentials with minimal API calls."""
if not credentials:
return "aws-us-gov" # Default for environment
return DEFAULT_PARTITION # Default for environment from partition_utils

# Check credential format first (fast, no API calls)
access_key = str(credentials.get("aws_access_key_id", "")).lower()
Expand Down Expand Up @@ -342,7 +312,7 @@ def detect_partition_from_credentials(credentials: Dict[str, str]) -> str:
return "aws"
except Exception:
# Default to GovCloud for environment
return "aws-us-gov"
return DEFAULT_PARTITION


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -644,8 +614,8 @@ def get_enabled_regions(

return regions
except Exception:
# Fall back to default regions
return get_regions_for_partition(partition)
# Fall back to default regions from partition_utils
return get_default_regions_for_partition(partition)


def list_enabled_regions(
Expand Down Expand Up @@ -689,29 +659,27 @@ def list_enabled_regions(
return ["us-east-1", "us-east-2", "us-west-1", "us-west-2"]


# Replace is_valid_region with import from partition_utils
def is_valid_region(region: str) -> bool:
"""Check if a region is valid without making an API call."""
if not is_string(region):
return False

# Check against our cached region lists
for partition, regions in _region_cache["partition_regions"].items():
if region in regions:
return True

# If not in predefined lists, make an API call as last resort
try:
boto3.session.Session().client("ec2", region_name=region)
return True
except:
return False
"""
Check if a region is valid without making an API call.
Legacy function - delegates to partition_utils.is_valid_region.
Args:
region: Region name to check
Returns:
True if the region is valid, False otherwise
"""
from aws_resource_management.partition_utils import is_valid_region as _is_valid_region
return _is_valid_region(region)


# ---------------------------------------------------------------------------
# Function aliases for backward compatibility
# ---------------------------------------------------------------------------

# Alias get_enabled_regions as get_available_regions for backward compatibility
# Explicitly document this is an alias for get_enabled_regions
get_available_regions = get_enabled_regions


Expand Down
109 changes: 92 additions & 17 deletions local-app/python-tools/gfl-resource-actions/aws_resource_management/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@

import argparse
import logging
import os
import sys
import traceback
from typing import Any, Dict, List, Optional

from aws_resource_management.config_manager import get_config
from aws_resource_management.core import ResourceManager
from aws_resource_management.logging_setup import setup_logging
from aws_resource_management.reporting import export_resources_to_csv
from aws_resource_management.resource_registry import get_registry

logger = setup_logging()
config = get_config()
Expand All @@ -25,13 +28,28 @@ def parse_args():
action_group = parser.add_mutually_exclusive_group(required=True)
action_group.add_argument("--stop", action="store_true", help="Stop resources")
action_group.add_argument("--start", action="store_true", help="Start resources")

# Resource selection
action_group.add_argument("--export", action="store_true", help="Export resources to CSV without taking any action")

# Resource type argument using registry
registry = get_registry()
resource_choices = registry.get_resource_type_choices()
resource_type_choices = [choice['value'] for choice in resource_choices]

# Ensure 'all' is in the list of choices
if 'all' not in resource_type_choices:
resource_type_choices.append('all')

# Format the help text with choice descriptions
resource_type_help = "\n".join(
f" {choice['value']}: {choice['help']}" for choice in resource_choices
)
resource_type_help += "\n all: All supported resource types"

parser.add_argument(
"--resource-type",
choices=["ecr", "ec2", "rds", "eks", "emr", "all"],
choices=resource_type_choices,
default="all",
help="Resource type to manage (default: all)",
help=f"Resource type to manage. Available types:\n{resource_type_help}",
)

# Account filtering
Expand Down Expand Up @@ -82,6 +100,23 @@ def parse_args():
action="store_true",
help="Show what would be done without making changes",
)

# CSV Export options
csv_group = parser.add_argument_group('CSV Export Options')
csv_group.add_argument(
"--csv-export",
action="store_true",
help="Export discovered resources to CSV",
)
csv_group.add_argument(
"--csv-output-dir",
help="Directory to save CSV files (default: current directory)",
default=os.getcwd(),
)
csv_group.add_argument(
"--csv-prefix",
help="Prefix for CSV filenames",
)

return parser.parse_args()

Expand All @@ -92,17 +127,25 @@ def main():

try:
# Determine action
action = "stop" if args.stop else "start"
action = "stop" if args.stop else "start" if args.start else "export"

# Get registry of available resource types
registry = get_registry()
all_resource_types = registry.get_resource_types()

# Determine resource types to process
exclude_resources = []
if args.resource_type != "all":
# If specific resource type is selected, exclude all others
all_resource_types = ["ecr", "ec2", "rds", "eks", "emr"]
exclude_resources = [
rt for rt in all_resource_types if rt != args.resource_type
]

# Validate that the requested resource type exists in the registry
if args.resource_type != "all" and args.resource_type not in all_resource_types:
logger.error(f"Invalid resource type: {args.resource_type}. Available types: {', '.join(all_resource_types)}")
sys.exit(1)

logger.info(
f"Processing {args.resource_type} resources in {args.regions or 'all enabled'} regions for {action} action"
)
Expand All @@ -123,17 +166,49 @@ def main():
logger.info(f"Processing only account {args.account}")
# We'll handle this by post-filtering the account list in resource_manager

# Process accounts - single scan for all resource types
stats = resource_manager.process_accounts(
action=action,
regions=args.regions or [],
exclude_accounts=exclude_accounts,
exclude_resources=exclude_resources,
exclude_regions=args.exclude_regions or [],
dry_run=args.dry_run,
)

logger.info(f"Completed {action} action for {args.resource_type} resources")
# For export-only action or when CSV export is requested
if action == "export" or args.csv_export:
# Discover resources across all accounts and regions
discovered_resources = resource_manager.discover_resources(
regions=args.regions or [],
exclude_accounts=exclude_accounts,
exclude_resources=exclude_resources,
exclude_regions=args.exclude_regions or [],
specific_account=args.account
)

# Export to CSV
logger.info(f"Exporting resources to CSV in directory: {args.csv_output_dir}")
csv_files = export_resources_to_csv(
resources=discovered_resources,
output_dir=args.csv_output_dir,
prefix=args.csv_prefix
)

# Log success message with file locations
if csv_files:
logger.info(f"Successfully exported {len(csv_files)} resource type(s) to CSV")
for resource_type, filepath in csv_files.items():
logger.info(f" - {resource_type}: {filepath}")
else:
logger.warning("No resources were exported to CSV. Check if resources were found.")

# If action was export-only, we're done
if action == "export":
sys.exit(0)

# Process accounts for start/stop actions
if action in ["start", "stop"]:
stats = resource_manager.process_accounts(
action=action,
regions=args.regions or [],
exclude_accounts=exclude_accounts,
exclude_resources=exclude_resources,
exclude_regions=args.exclude_regions or [],
dry_run=args.dry_run,
)

logger.info(f"Completed {action} action for {args.resource_type} resources")

# Exit with success code
sys.exit(0)
Expand Down

0 comments on commit 0591971

Please sign in to comment.