diff --git a/local-app/python-tools/gfl-resource-actions/discovery.py b/local-app/python-tools/gfl-resource-actions/discovery.py index 6925a2a4..ec191280 100644 --- a/local-app/python-tools/gfl-resource-actions/discovery.py +++ b/local-app/python-tools/gfl-resource-actions/discovery.py @@ -3,11 +3,11 @@ """ import config from aws_utils import create_boto3_client -from logging_utils import setup_logging +from logging_utils import setup_logging, log_action_to_csv logger = setup_logging() -def get_ec2_instances(credentials, region): +def get_ec2_instances(credentials, region, account_id=None, account_name=None): """Get EC2 instances in a region.""" try: ec2_client = create_boto3_client('ec2', credentials, region) @@ -19,12 +19,34 @@ def get_ec2_instances(credentials, region): # Capture tags for exclusion check tags = {tag['Key']: tag['Value'] for tag in instance.get('Tags', [])} - instances.append({ + # Extract name from tags + name = tags.get('Name', '') + + instance_data = { "id": instance['InstanceId'], + "name": name, "state": instance['State']['Name'], "region": region, + "private_ip": instance.get('PrivateIpAddress', ''), + "public_ip": instance.get('PublicIpAddress', ''), "tags": tags - }) + } + + instances.append(instance_data) + + # Log the discovery action to CSV + if account_id: + log_action_to_csv( + account_id=account_id, + account_name=account_name or "Unknown", + resource_type="ec2", + resource_id=instance['InstanceId'], + resource_name=name, + action="discover", + region=region, + status="success", + details=f"State: {instance['State']['Name']}, Private IP: {instance.get('PrivateIpAddress', 'N/A')}" + ) # Handle pagination while 'NextToken' in response: @@ -34,19 +56,57 @@ def get_ec2_instances(credentials, region): # Capture tags for exclusion check tags = {tag['Key']: tag['Value'] for tag in instance.get('Tags', [])} - instances.append({ + # Extract name from tags + name = tags.get('Name', '') + + instance_data = { "id": instance['InstanceId'], + "name": name, "state": instance['State']['Name'], "region": region, + "private_ip": instance.get('PrivateIpAddress', ''), + "public_ip": instance.get('PublicIpAddress', ''), "tags": tags - }) + } + + instances.append(instance_data) + + # Log the discovery action to CSV + if account_id: + log_action_to_csv( + account_id=account_id, + account_name=account_name or "Unknown", + resource_type="ec2", + resource_id=instance['InstanceId'], + resource_name=name, + action="discover", + region=region, + status="success", + details=f"State: {instance['State']['Name']}, Private IP: {instance.get('PrivateIpAddress', 'N/A')}" + ) return instances except Exception as e: - logger.error(f"Error getting EC2 instances in region {region}: {e}") + error_msg = f"Error getting EC2 instances in region {region}: {e}" + logger.error(error_msg) + + # Log the error to CSV + if account_id: + log_action_to_csv( + account_id=account_id, + account_name=account_name or "Unknown", + resource_type="ec2", + resource_id="N/A", + resource_name="N/A", + action="discover", + region=region, + status="failed", + details=error_msg + ) + return [] -def get_rds_instances(credentials, region): +def get_rds_instances(credentials, region, account_id=None, account_name=None): """Get RDS instances in a region.""" try: rds_client = create_boto3_client('rds', credentials, region) @@ -64,12 +124,35 @@ def get_rds_instances(credentials, region): except Exception as tag_e: logger.warning(f"Error getting tags for RDS instance {instance['DBInstanceIdentifier']}: {tag_e}") - instances.append({ + # Get endpoint address + endpoint_address = "" + if 'Endpoint' in instance and 'Address' in instance['Endpoint']: + endpoint_address = instance['Endpoint']['Address'] + + instance_data = { "id": instance['DBInstanceIdentifier'], + "name": instance['DBInstanceIdentifier'], # Using ID as name "status": instance['DBInstanceStatus'], "region": region, + "endpoint": endpoint_address, "tags": tags - }) + } + + instances.append(instance_data) + + # Log the discovery action to CSV + if account_id: + log_action_to_csv( + account_id=account_id, + account_name=account_name or "Unknown", + resource_type="rds", + resource_id=instance['DBInstanceIdentifier'], + resource_name=instance['DBInstanceIdentifier'], + action="discover", + region=region, + status="success", + details=f"Status: {instance['DBInstanceStatus']}, Endpoint: {endpoint_address}" + ) # Handle pagination while 'Marker' in response: @@ -85,21 +168,60 @@ def get_rds_instances(credentials, region): except Exception as tag_e: logger.warning(f"Error getting tags for RDS instance {instance['DBInstanceIdentifier']}: {tag_e}") - instances.append({ + # Get endpoint address + endpoint_address = "" + if 'Endpoint' in instance and 'Address' in instance['Endpoint']: + endpoint_address = instance['Endpoint']['Address'] + + instance_data = { "id": instance['DBInstanceIdentifier'], + "name": instance['DBInstanceIdentifier'], # Using ID as name "status": instance['DBInstanceStatus'], "region": region, + "endpoint": endpoint_address, "tags": tags - }) + } + + instances.append(instance_data) + + # Log the discovery action to CSV + if account_id: + log_action_to_csv( + account_id=account_id, + account_name=account_name or "Unknown", + resource_type="rds", + resource_id=instance['DBInstanceIdentifier'], + resource_name=instance['DBInstanceIdentifier'], + action="discover", + region=region, + status="success", + details=f"Status: {instance['DBInstanceStatus']}, Endpoint: {endpoint_address}" + ) return instances except Exception as e: - logger.error(f"Error getting RDS instances in region {region}: {e}") + error_msg = f"Error getting RDS instances in region {region}: {e}" + logger.error(error_msg) + + # Log the error to CSV + if account_id: + log_action_to_csv( + account_id=account_id, + account_name=account_name or "Unknown", + resource_type="rds", + resource_id="N/A", + resource_name="N/A", + action="discover", + region=region, + status="failed", + details=error_msg + ) + return [] # Similarly implement get_eks_clusters() and get_emr_clusters()... -def get_account_resources(credentials, regions, exclude_resources=None): +def get_account_resources(credentials, regions, account_id=None, account_name=None, exclude_resources=None): """Get all resources from an account across specified regions.""" if exclude_resources is None: exclude_resources = [] @@ -114,11 +236,11 @@ def get_account_resources(credentials, regions, exclude_resources=None): for region in regions: # Get EC2 instances if "ec2" not in exclude_resources: - resources["ec2_instances"].extend(get_ec2_instances(credentials, region)) + resources["ec2_instances"].extend(get_ec2_instances(credentials, region, account_id, account_name)) # Get RDS instances if "rds" not in exclude_resources: - resources["rds_instances"].extend(get_rds_instances(credentials, region)) + resources["rds_instances"].extend(get_rds_instances(credentials, region, account_id, account_name)) # Get EKS clusters if "eks" not in exclude_resources: diff --git a/local-app/python-tools/gfl-resource-actions/logging_utils.py b/local-app/python-tools/gfl-resource-actions/logging_utils.py index 180b7c08..04f8c508 100644 --- a/local-app/python-tools/gfl-resource-actions/logging_utils.py +++ b/local-app/python-tools/gfl-resource-actions/logging_utils.py @@ -1,18 +1,124 @@ """ -Logging utilities for AWS Resource Management. +Logging utilities for resource management. """ import logging +import os import sys -import config +import csv +import datetime +from pathlib import Path -def setup_logging(): - """Configure logging for the application.""" - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler(config.LOG_FILE), - logging.StreamHandler(sys.stdout) - ] - ) - return logging.getLogger("AWS-Resource-Management") +# Configure log formats +DEFAULT_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +DEFAULT_LOG_LEVEL = logging.INFO + +# Configure CSV logging +CSV_LOG_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logs') +ACTION_CSV_FILE = 'resource_actions.csv' +CSV_FIELDS = ['timestamp', 'account_id', 'account_name', 'resource_type', 'resource_id', 'resource_name', 'action', 'region', 'status', 'details'] + +def setup_logging(log_level=None, log_format=None): + """ + Set up logging configuration. + + Args: + log_level (int, optional): Logging level (default: INFO) + log_format (str, optional): Log message format + + Returns: + logging.Logger: Configured logger + """ + if log_level is None: + log_level = DEFAULT_LOG_LEVEL + + if log_format is None: + log_format = DEFAULT_LOG_FORMAT + + # Create logger + logger = logging.getLogger('resource-management') + logger.setLevel(log_level) + + # Create console handler + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(log_level) + + # Create formatter and add it to the handler + formatter = logging.Formatter(log_format) + console_handler.setFormatter(formatter) + + # Add handler to logger + logger.addHandler(console_handler) + + # Ensure CSV log directory exists + os.makedirs(CSV_LOG_DIR, exist_ok=True) + + return logger + +def initialize_csv_log(csv_file=None): + """ + Initialize CSV log file with headers if it doesn't exist. + + Args: + csv_file (str, optional): CSV file name (default: resource_actions.csv) + + Returns: + str: Path to the CSV log file + """ + if csv_file is None: + csv_file = ACTION_CSV_FILE + + csv_path = os.path.join(CSV_LOG_DIR, csv_file) + + # Create directory if it doesn't exist + Path(CSV_LOG_DIR).mkdir(parents=True, exist_ok=True) + + # Check if file exists, if not create with headers + file_exists = os.path.isfile(csv_path) + + if not file_exists: + with open(csv_path, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(CSV_FIELDS) + + return csv_path + +def log_action_to_csv(account_id, account_name, resource_type, resource_id, + resource_name, action, region, status, details='', csv_file=None): + """ + Log an action to the CSV file. + + Args: + account_id (str): AWS account ID + account_name (str): AWS account name + resource_type (str): Type of resource (e.g., 'ec2', 'rds') + resource_id (str): Resource identifier + resource_name (str): Resource name + action (str): Action performed (e.g., 'discover', 'stop', 'start') + region (str): AWS region + status (str): Status of the action (e.g., 'success', 'failed') + details (str, optional): Additional details + csv_file (str, optional): CSV file name (default: resource_actions.csv) + """ + if csv_file is None: + csv_file = ACTION_CSV_FILE + + csv_path = initialize_csv_log(csv_file) + timestamp = datetime.datetime.now().isoformat() + + with open(csv_path, 'a', newline='') as f: + writer = csv.writer(f) + writer.writerow([ + timestamp, + account_id, + account_name, + resource_type, + resource_id, + resource_name, + action, + region, + status, + details + ]) + +# Create a logger instance for direct import +logger = setup_logging()