From 2ce1265f61d2086aad2dd3686127cba076adf30c Mon Sep 17 00:00:00 2001 From: arnol377 Date: Wed, 12 Feb 2025 15:49:12 -0500 Subject: [PATCH] adding some useful scripts --- scripts/manage_ec2_instances.md | 61 ++++++++++++++ scripts/manage_ec2_instances.py | 141 ++++++++++++++++++++++++++++++++ scripts/requirements.txt | 14 ++++ scripts/terraform-venv.md | 47 +++++++++++ scripts/terraform-venv.sh | 89 ++++++++++++++++++++ 5 files changed, 352 insertions(+) create mode 100644 scripts/manage_ec2_instances.md create mode 100755 scripts/manage_ec2_instances.py create mode 100644 scripts/requirements.txt create mode 100644 scripts/terraform-venv.md create mode 100755 scripts/terraform-venv.sh diff --git a/scripts/manage_ec2_instances.md b/scripts/manage_ec2_instances.md new file mode 100644 index 0000000..e1d8b6c --- /dev/null +++ b/scripts/manage_ec2_instances.md @@ -0,0 +1,61 @@ +# EC2 Instance Management Script + +This script helps manage EC2 instances that have a specific security group attached. It can: +1. Find all instances with a given security group +2. Disable both stop and termination protection +3. Terminate the instances + +## Prerequisites + +- Python 3.6+ +- Boto3 installed (`pip install boto3`) +- AWS credentials configured with appropriate permissions +- Required IAM permissions: + - ec2:DescribeInstances + - ec2:ModifyInstanceAttribute + - ec2:TerminateInstances + +## Usage + +Basic usage: +```bash +./manage_ec2_instances.py --security-group sg-02f69cb0a8d862745 +``` + +With different region: +```bash +./manage_ec2_instances.py --security-group sg-02f69cb0a8d862745 --region us-west-2 +``` + +Dry run (show what would happen without making changes): +```bash +./manage_ec2_instances.py --security-group sg-02f69cb0a8d862745 --dry-run +``` + +## Command Line Arguments + +- `--security-group`, `-s`: (Required) Security Group ID to filter instances +- `--region`, `-r`: AWS region (default: us-east-1) +- `--dry-run`, `-d`: Only show what would be done without making changes + +## Example Output + +``` +Found 2 instances with security group sg-02f69cb0a8d862745 + +Processing instance i-1234567890abcdef0 (Current state: running) +Successfully disabled protection for instance i-1234567890abcdef0 +Successfully initiated termination for instance i-1234567890abcdef0 + +Processing instance i-0987654321fedcba0 (Current state: running) +Successfully disabled protection for instance i-0987654321fedcba0 +Successfully initiated termination for instance i-0987654321fedcba0 +``` + +## Error Handling + +The script will: +- Continue processing remaining instances if one fails +- Report specific errors for each operation (protection modification, termination) +- Skip termination if protection modification fails +- Handle AWS API errors gracefully \ No newline at end of file diff --git a/scripts/manage_ec2_instances.py b/scripts/manage_ec2_instances.py new file mode 100755 index 0000000..b9c4e82 --- /dev/null +++ b/scripts/manage_ec2_instances.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +import argparse +import json +import subprocess +from typing import List + +def run_aws_command(command: List[str]) -> dict: + """Run an AWS CLI command and return the JSON output.""" + try: + full_command = ['aws'] + command + result = subprocess.run(full_command, capture_output=True, text=True, check=True) + return json.loads(result.stdout) if result.stdout else {} + except subprocess.CalledProcessError as e: + print(f"Error running AWS command: {e}") + print(f"Error output: {e.stderr}") + return {} + except json.JSONDecodeError as e: + print(f"Error parsing AWS command output: {e}") + return {} + +def get_instances_by_security_group(security_group_id: str, region: str) -> List[dict]: + """Find EC2 instances that have the specified security group attached.""" + command = [ + 'ec2', 'describe-instances', + '--region', region, + '--filters', f'Name=instance.group-id,Values={security_group_id}', + '--output', 'json' + ] + + result = run_aws_command(command) + instances = [] + + for reservation in result.get('Reservations', []): + instances.extend(reservation.get('Instances', [])) + + return instances + +def get_instance_info(instance: dict) -> str: + """Format instance information for display.""" + instance_id = instance['InstanceId'] + state = instance['State']['Name'] + + # Get instance name from tags + name = next((tag['Value'] for tag in instance.get('Tags', []) + if tag['Key'] == 'Name'), 'No Name') + + # Get key pair name + key_name = instance.get('KeyName', 'No Key Pair') + + # Format tags + tags = ', '.join([f"{tag['Key']}={tag['Value']}" + for tag in instance.get('Tags', [])]) + tags = tags if tags else 'No Tags' + + return (f"Instance ID: {instance_id}\n" + f"Name: {name}\n" + f"State: {state}\n" + f"Key Pair: {key_name}\n" + f"Tags: {tags}") + +def modify_instance_protection(instance_id: str, region: str, disable_protection: bool = True) -> bool: + """Modify instance stop and termination protection.""" + try: + # Modify stop protection + stop_command = [ + 'ec2', 'modify-instance-attribute', + '--region', region, + '--instance-id', instance_id, + '--no-disable-api-stop' if disable_protection else '--disable-api-stop' + ] + run_aws_command(stop_command) + + # Modify termination protection + term_command = [ + 'ec2', 'modify-instance-attribute', + '--region', region, + '--instance-id', instance_id, + '--no-disable-api-termination' if disable_protection else '--disable-api-termination' + ] + run_aws_command(term_command) + return True + except Exception as e: + print(f"Error modifying protection for instance {instance_id}: {e}") + return False + +def terminate_instance(instance_id: str, region: str) -> bool: + """Terminate an EC2 instance.""" + try: + command = [ + 'ec2', 'terminate-instances', + '--region', region, + '--instance-ids', instance_id + ] + run_aws_command(command) + return True + except Exception as e: + print(f"Error terminating instance {instance_id}: {e}") + return False + +def main(): + parser = argparse.ArgumentParser(description='Manage EC2 instances with specific security group') + parser.add_argument('--security-group', '-s', required=True, + help='Security Group ID to filter instances') + parser.add_argument('--region', '-r', default='us-gov-west-1', + help='AWS region (default: us-gov-west-1)') + parser.add_argument('--dry-run', '-d', action='store_true', + help='Dry run mode - only show what would be done') + args = parser.parse_args() + + # Get instances with the specified security group + instances = get_instances_by_security_group(args.security_group, args.region) + + if not instances: + print(f"No instances found with security group {args.security_group}") + return + + print(f"Found {len(instances)} instances with security group {args.security_group}") + + for instance in instances: + instance_id = instance['InstanceId'] + print(f"\n{get_instance_info(instance)}") + + if args.dry_run: + print("DRY RUN: Would modify instance protection and terminate instance") + continue + + # Modify instance protection + if modify_instance_protection(instance_id, args.region, True): + print(f"Successfully disabled protection for instance {instance_id}") + else: + print(f"Skipping termination for instance {instance_id} due to protection modification failure") + continue + + # Terminate instance + if terminate_instance(instance_id, args.region): + print(f"Successfully initiated termination for instance {instance_id}") + else: + print(f"Failed to terminate instance {instance_id}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..992f06c --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,14 @@ +# Core AWS packages +boto3==1.24.96 +botocore==1.27.96 +awscli==1.25.97 + +# Terraform interaction +python-terraform==0.10.1 + +# Utilities +PyYAML==6.0 +requests==2.28.1 +urllib3==1.26.12 +six==1.16.0 +jmespath==1.0.1 \ No newline at end of file diff --git a/scripts/terraform-venv.md b/scripts/terraform-venv.md new file mode 100644 index 0000000..be45a49 --- /dev/null +++ b/scripts/terraform-venv.md @@ -0,0 +1,47 @@ +# Terraform Virtual Environment Setup + +This script sets up a Python virtual environment configured for Terraform development. + +## Features + +- Creates and activates a dedicated Python virtualenv for Terraform work +- Installs required Python packages (boto3, python-terraform, awscli, etc.) +- Sets up useful Terraform aliases +- Configures proxy settings automatically +- Activates automatically on login + +## Installation + +1. The script has been installed at: `~/git/aws-image-pipeline/scripts/terraform-venv.sh` + +2. Activation on login has been configured in your `~/.bash_profile` + +3. To activate manually in any terminal: +```bash +source ~/git/aws-image-pipeline/scripts/terraform-venv.sh +``` + +## Available Aliases + +- `tf` - shorthand for `terraform` +- `tfp` - terraform plan +- `tfa` - terraform apply +- `tfd` - terraform destroy +- `tfi` - terraform init +- `tfw` - terraform workspace +- `tfmt` - terraform fmt +- `tfv` - terraform validate + +## Environment Variables + +The script sets up: +- `AWS_PAGER=""` - Disables AWS CLI pagination +- `PYTHONPATH` - Includes virtualenv packages +- Proxy settings (if available in environment) + +## Troubleshooting + +If you encounter pip install errors: +1. Check your proxy settings +2. Try running `pip config set global.index-url https://pypi.org/simple` +3. Ensure you have network access to pypi.org \ No newline at end of file diff --git a/scripts/terraform-venv.sh b/scripts/terraform-venv.sh new file mode 100755 index 0000000..e8d6254 --- /dev/null +++ b/scripts/terraform-venv.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Set Census Bureau proxy settings first +export http_proxy="http://proxy.tco.census.gov:3128" +export https_proxy="http://proxy.tco.census.gov:3128" +export no_proxy="pypi.org,localhost,127.0.0.1,.census.gov,.local" + +# Function to configure pip settings +configure_pip() { + mkdir -p ~/.pip + cat > ~/.pip/pip.conf << EOL +[global] +index-url = http://repository/DataScience-Group/simple/ +trusted-host = repository +proxy = ${http_proxy} +timeout = 60 +retries = 3 +EOL +} + +# Create and configure virtualenv +setup_virtualenv() { + if [ ! -d "$HOME/.virtualenvs/terraform" ]; then + echo "Creating terraform virtualenv..." + python3 -m venv "$HOME/.virtualenvs/terraform" + fi + + source "$HOME/.virtualenvs/terraform/bin/activate" + + # Configure pip + # configure_pip + + # Install base pip first + pip install --upgrade pip --trusted-host repository + + # Try installing requirements + if [ -f "$SCRIPT_DIR/requirements.txt" ]; then + pip install -r "$SCRIPT_DIR/requirements.txt" \ + --trusted-host repository \ + --trusted-host pypi.org \ + --trusted-host files.pythonhosted.org \ + || echo "Warning: Some packages failed to install. You may need to install them manually." + fi +} + +# Check if conda is available, if not use venv +if command -v conda &> /dev/null; then + # Create conda environment if it doesn't exist + if ! conda env list | grep -q "^terraform "; then + echo "Creating terraform conda environment..." + conda create -n terraform python=3.8 -y + fi + + # Activate conda environment + conda activate terraform + + # Install packages using conda where possible + conda install -y boto3 pyyaml requests + pip install python-terraform +else + # Fall back to virtualenv if conda isn't available + # Get the directory where this script is located + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + + # Setup virtualenv + setup_virtualenv +fi + +# Set AWS environment variables +export AWS_PAGER="" +export PYTHONPATH="$PYTHONPATH:$VIRTUAL_ENV/lib/python3.8/site-packages" + +# Set terraform aliases +alias tf='terraform' +alias tfp='terraform plan' +alias tfa='terraform apply' +alias tfd='terraform destroy' +alias tfi='terraform init' +alias tfw='terraform workspace' +alias tfmt='terraform fmt' +alias tfv='terraform validate' + +# Print status +echo "Terraform environment activated using $([ -n "$CONDA_DEFAULT_ENV" ] && echo "conda: $CONDA_DEFAULT_ENV" || echo "virtualenv: $VIRTUAL_ENV")" +echo "Proxy settings:" +echo " http_proxy: $http_proxy" +echo " https_proxy: $https_proxy" +echo " no_proxy: $no_proxy" +echo "Pip config location: ~/.pip/pip.conf" \ No newline at end of file