Skip to content

Commit

Permalink
expand logging, add csv output
Browse files Browse the repository at this point in the history
  • Loading branch information
morga471 committed Mar 14, 2025
1 parent 355fb16 commit 71e3325
Show file tree
Hide file tree
Showing 2 changed files with 257 additions and 29 deletions.
154 changes: 138 additions & 16 deletions local-app/python-tools/gfl-resource-actions/discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
"""
import config
from aws_utils import create_boto3_client
from logging_utils import setup_logging
from logging_utils import setup_logging, log_action_to_csv

logger = setup_logging()

def get_ec2_instances(credentials, region):
def get_ec2_instances(credentials, region, account_id=None, account_name=None):
"""Get EC2 instances in a region."""
try:
ec2_client = create_boto3_client('ec2', credentials, region)
Expand All @@ -19,12 +19,34 @@ def get_ec2_instances(credentials, region):
# Capture tags for exclusion check
tags = {tag['Key']: tag['Value'] for tag in instance.get('Tags', [])}

instances.append({
# Extract name from tags
name = tags.get('Name', '')

instance_data = {
"id": instance['InstanceId'],
"name": name,
"state": instance['State']['Name'],
"region": region,
"private_ip": instance.get('PrivateIpAddress', ''),
"public_ip": instance.get('PublicIpAddress', ''),
"tags": tags
})
}

instances.append(instance_data)

# Log the discovery action to CSV
if account_id:
log_action_to_csv(
account_id=account_id,
account_name=account_name or "Unknown",
resource_type="ec2",
resource_id=instance['InstanceId'],
resource_name=name,
action="discover",
region=region,
status="success",
details=f"State: {instance['State']['Name']}, Private IP: {instance.get('PrivateIpAddress', 'N/A')}"
)

# Handle pagination
while 'NextToken' in response:
Expand All @@ -34,19 +56,57 @@ def get_ec2_instances(credentials, region):
# Capture tags for exclusion check
tags = {tag['Key']: tag['Value'] for tag in instance.get('Tags', [])}

instances.append({
# Extract name from tags
name = tags.get('Name', '')

instance_data = {
"id": instance['InstanceId'],
"name": name,
"state": instance['State']['Name'],
"region": region,
"private_ip": instance.get('PrivateIpAddress', ''),
"public_ip": instance.get('PublicIpAddress', ''),
"tags": tags
})
}

instances.append(instance_data)

# Log the discovery action to CSV
if account_id:
log_action_to_csv(
account_id=account_id,
account_name=account_name or "Unknown",
resource_type="ec2",
resource_id=instance['InstanceId'],
resource_name=name,
action="discover",
region=region,
status="success",
details=f"State: {instance['State']['Name']}, Private IP: {instance.get('PrivateIpAddress', 'N/A')}"
)

return instances
except Exception as e:
logger.error(f"Error getting EC2 instances in region {region}: {e}")
error_msg = f"Error getting EC2 instances in region {region}: {e}"
logger.error(error_msg)

# Log the error to CSV
if account_id:
log_action_to_csv(
account_id=account_id,
account_name=account_name or "Unknown",
resource_type="ec2",
resource_id="N/A",
resource_name="N/A",
action="discover",
region=region,
status="failed",
details=error_msg
)

return []

def get_rds_instances(credentials, region):
def get_rds_instances(credentials, region, account_id=None, account_name=None):
"""Get RDS instances in a region."""
try:
rds_client = create_boto3_client('rds', credentials, region)
Expand All @@ -64,12 +124,35 @@ def get_rds_instances(credentials, region):
except Exception as tag_e:
logger.warning(f"Error getting tags for RDS instance {instance['DBInstanceIdentifier']}: {tag_e}")

instances.append({
# Get endpoint address
endpoint_address = ""
if 'Endpoint' in instance and 'Address' in instance['Endpoint']:
endpoint_address = instance['Endpoint']['Address']

instance_data = {
"id": instance['DBInstanceIdentifier'],
"name": instance['DBInstanceIdentifier'], # Using ID as name
"status": instance['DBInstanceStatus'],
"region": region,
"endpoint": endpoint_address,
"tags": tags
})
}

instances.append(instance_data)

# Log the discovery action to CSV
if account_id:
log_action_to_csv(
account_id=account_id,
account_name=account_name or "Unknown",
resource_type="rds",
resource_id=instance['DBInstanceIdentifier'],
resource_name=instance['DBInstanceIdentifier'],
action="discover",
region=region,
status="success",
details=f"Status: {instance['DBInstanceStatus']}, Endpoint: {endpoint_address}"
)

# Handle pagination
while 'Marker' in response:
Expand All @@ -85,21 +168,60 @@ def get_rds_instances(credentials, region):
except Exception as tag_e:
logger.warning(f"Error getting tags for RDS instance {instance['DBInstanceIdentifier']}: {tag_e}")

instances.append({
# Get endpoint address
endpoint_address = ""
if 'Endpoint' in instance and 'Address' in instance['Endpoint']:
endpoint_address = instance['Endpoint']['Address']

instance_data = {
"id": instance['DBInstanceIdentifier'],
"name": instance['DBInstanceIdentifier'], # Using ID as name
"status": instance['DBInstanceStatus'],
"region": region,
"endpoint": endpoint_address,
"tags": tags
})
}

instances.append(instance_data)

# Log the discovery action to CSV
if account_id:
log_action_to_csv(
account_id=account_id,
account_name=account_name or "Unknown",
resource_type="rds",
resource_id=instance['DBInstanceIdentifier'],
resource_name=instance['DBInstanceIdentifier'],
action="discover",
region=region,
status="success",
details=f"Status: {instance['DBInstanceStatus']}, Endpoint: {endpoint_address}"
)

return instances
except Exception as e:
logger.error(f"Error getting RDS instances in region {region}: {e}")
error_msg = f"Error getting RDS instances in region {region}: {e}"
logger.error(error_msg)

# Log the error to CSV
if account_id:
log_action_to_csv(
account_id=account_id,
account_name=account_name or "Unknown",
resource_type="rds",
resource_id="N/A",
resource_name="N/A",
action="discover",
region=region,
status="failed",
details=error_msg
)

return []

# Similarly implement get_eks_clusters() and get_emr_clusters()...

def get_account_resources(credentials, regions, exclude_resources=None):
def get_account_resources(credentials, regions, account_id=None, account_name=None, exclude_resources=None):
"""Get all resources from an account across specified regions."""
if exclude_resources is None:
exclude_resources = []
Expand All @@ -114,11 +236,11 @@ def get_account_resources(credentials, regions, exclude_resources=None):
for region in regions:
# Get EC2 instances
if "ec2" not in exclude_resources:
resources["ec2_instances"].extend(get_ec2_instances(credentials, region))
resources["ec2_instances"].extend(get_ec2_instances(credentials, region, account_id, account_name))

# Get RDS instances
if "rds" not in exclude_resources:
resources["rds_instances"].extend(get_rds_instances(credentials, region))
resources["rds_instances"].extend(get_rds_instances(credentials, region, account_id, account_name))

# Get EKS clusters
if "eks" not in exclude_resources:
Expand Down
132 changes: 119 additions & 13 deletions local-app/python-tools/gfl-resource-actions/logging_utils.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,124 @@
"""
Logging utilities for AWS Resource Management.
Logging utilities for resource management.
"""
import logging
import os
import sys
import config
import csv
import datetime
from pathlib import Path

def setup_logging():
"""Configure logging for the application."""
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(config.LOG_FILE),
logging.StreamHandler(sys.stdout)
]
)
return logging.getLogger("AWS-Resource-Management")
# Configure log formats
DEFAULT_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
DEFAULT_LOG_LEVEL = logging.INFO

# Configure CSV logging
CSV_LOG_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logs')
ACTION_CSV_FILE = 'resource_actions.csv'
CSV_FIELDS = ['timestamp', 'account_id', 'account_name', 'resource_type', 'resource_id', 'resource_name', 'action', 'region', 'status', 'details']

def setup_logging(log_level=None, log_format=None):
"""
Set up logging configuration.
Args:
log_level (int, optional): Logging level (default: INFO)
log_format (str, optional): Log message format
Returns:
logging.Logger: Configured logger
"""
if log_level is None:
log_level = DEFAULT_LOG_LEVEL

if log_format is None:
log_format = DEFAULT_LOG_FORMAT

# Create logger
logger = logging.getLogger('resource-management')
logger.setLevel(log_level)

# Create console handler
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(log_level)

# Create formatter and add it to the handler
formatter = logging.Formatter(log_format)
console_handler.setFormatter(formatter)

# Add handler to logger
logger.addHandler(console_handler)

# Ensure CSV log directory exists
os.makedirs(CSV_LOG_DIR, exist_ok=True)

return logger

def initialize_csv_log(csv_file=None):
"""
Initialize CSV log file with headers if it doesn't exist.
Args:
csv_file (str, optional): CSV file name (default: resource_actions.csv)
Returns:
str: Path to the CSV log file
"""
if csv_file is None:
csv_file = ACTION_CSV_FILE

csv_path = os.path.join(CSV_LOG_DIR, csv_file)

# Create directory if it doesn't exist
Path(CSV_LOG_DIR).mkdir(parents=True, exist_ok=True)

# Check if file exists, if not create with headers
file_exists = os.path.isfile(csv_path)

if not file_exists:
with open(csv_path, 'w', newline='') as f:
writer = csv.writer(f)
writer.writerow(CSV_FIELDS)

return csv_path

def log_action_to_csv(account_id, account_name, resource_type, resource_id,
resource_name, action, region, status, details='', csv_file=None):
"""
Log an action to the CSV file.
Args:
account_id (str): AWS account ID
account_name (str): AWS account name
resource_type (str): Type of resource (e.g., 'ec2', 'rds')
resource_id (str): Resource identifier
resource_name (str): Resource name
action (str): Action performed (e.g., 'discover', 'stop', 'start')
region (str): AWS region
status (str): Status of the action (e.g., 'success', 'failed')
details (str, optional): Additional details
csv_file (str, optional): CSV file name (default: resource_actions.csv)
"""
if csv_file is None:
csv_file = ACTION_CSV_FILE

csv_path = initialize_csv_log(csv_file)
timestamp = datetime.datetime.now().isoformat()

with open(csv_path, 'a', newline='') as f:
writer = csv.writer(f)
writer.writerow([
timestamp,
account_id,
account_name,
resource_type,
resource_id,
resource_name,
action,
region,
status,
details
])

# Create a logger instance for direct import
logger = setup_logging()

0 comments on commit 71e3325

Please sign in to comment.