adding some useful scripts

scripts/manage_ec2_instances.md

@@ -0,0 +1,61 @@
+# EC2 Instance Management Script
+
+This script helps manage EC2 instances that have a specific security group attached. It can:
+1. Find all instances with a given security group
+2. Disable both stop and termination protection
+3. Terminate the instances
+
+## Prerequisites
+
+- Python 3.6+
+- Boto3 installed (`pip install boto3`)
+- AWS credentials configured with appropriate permissions
+- Required IAM permissions:
+  - ec2:DescribeInstances
+  - ec2:ModifyInstanceAttribute
+  - ec2:TerminateInstances
+
+## Usage
+
+Basic usage:
+```bash
+./manage_ec2_instances.py --security-group sg-02f69cb0a8d862745
+```
+
+With different region:
+```bash
+./manage_ec2_instances.py --security-group sg-02f69cb0a8d862745 --region us-west-2
+```
+
+Dry run (show what would happen without making changes):
+```bash
+./manage_ec2_instances.py --security-group sg-02f69cb0a8d862745 --dry-run
+```
+
+## Command Line Arguments
+
+- `--security-group`, `-s`: (Required) Security Group ID to filter instances
+- `--region`, `-r`: AWS region (default: us-east-1)
+- `--dry-run`, `-d`: Only show what would be done without making changes
+
+## Example Output
+
+```
+Found 2 instances with security group sg-02f69cb0a8d862745
+
+Processing instance i-1234567890abcdef0 (Current state: running)
+Successfully disabled protection for instance i-1234567890abcdef0
+Successfully initiated termination for instance i-1234567890abcdef0
+
+Processing instance i-0987654321fedcba0 (Current state: running)
+Successfully disabled protection for instance i-0987654321fedcba0
+Successfully initiated termination for instance i-0987654321fedcba0
+```
+
+## Error Handling
+
+The script will:
+- Continue processing remaining instances if one fails
+- Report specific errors for each operation (protection modification, termination)
+- Skip termination if protection modification fails
+- Handle AWS API errors gracefully

scripts/manage_ec2_instances.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import subprocess
+from typing import List
+
+def run_aws_command(command: List[str]) -> dict:
+    """Run an AWS CLI command and return the JSON output."""
+    try:
+        full_command = ['aws'] + command
+        result = subprocess.run(full_command, capture_output=True, text=True, check=True)
+        return json.loads(result.stdout) if result.stdout else {}
+    except subprocess.CalledProcessError as e:
+        print(f"Error running AWS command: {e}")
+        print(f"Error output: {e.stderr}")
+        return {}
+    except json.JSONDecodeError as e:
+        print(f"Error parsing AWS command output: {e}")
+        return {}
+
+def get_instances_by_security_group(security_group_id: str, region: str) -> List[dict]:
+    """Find EC2 instances that have the specified security group attached."""
+    command = [
+        'ec2', 'describe-instances',
+        '--region', region,
+        '--filters', f'Name=instance.group-id,Values={security_group_id}',
+        '--output', 'json'
+    ]
+
+    result = run_aws_command(command)
+    instances = []
+
+    for reservation in result.get('Reservations', []):
+        instances.extend(reservation.get('Instances', []))
+
+    return instances
+
+def get_instance_info(instance: dict) -> str:
+    """Format instance information for display."""
+    instance_id = instance['InstanceId']
+    state = instance['State']['Name']
+
+    # Get instance name from tags
+    name = next((tag['Value'] for tag in instance.get('Tags', []) 
+                if tag['Key'] == 'Name'), 'No Name')
+
+    # Get key pair name
+    key_name = instance.get('KeyName', 'No Key Pair')
+
+    # Format tags
+    tags = ', '.join([f"{tag['Key']}={tag['Value']}" 
+                     for tag in instance.get('Tags', [])])
+    tags = tags if tags else 'No Tags'
+
+    return (f"Instance ID: {instance_id}\n"
+            f"Name: {name}\n"
+            f"State: {state}\n"
+            f"Key Pair: {key_name}\n"
+            f"Tags: {tags}")
+
+def modify_instance_protection(instance_id: str, region: str, disable_protection: bool = True) -> bool:
+    """Modify instance stop and termination protection."""
+    try:
+        # Modify stop protection
+        stop_command = [
+            'ec2', 'modify-instance-attribute',
+            '--region', region,
+            '--instance-id', instance_id,
+            '--no-disable-api-stop' if disable_protection else '--disable-api-stop'
+        ]
+        run_aws_command(stop_command)
+
+        # Modify termination protection
+        term_command = [
+            'ec2', 'modify-instance-attribute',
+            '--region', region,
+            '--instance-id', instance_id,
+            '--no-disable-api-termination' if disable_protection else '--disable-api-termination'
+        ]
+        run_aws_command(term_command)
+        return True
+    except Exception as e:
+        print(f"Error modifying protection for instance {instance_id}: {e}")
+        return False
+
+def terminate_instance(instance_id: str, region: str) -> bool:
+    """Terminate an EC2 instance."""
+    try:
+        command = [
+            'ec2', 'terminate-instances',
+            '--region', region,
+            '--instance-ids', instance_id
+        ]
+        run_aws_command(command)
+        return True
+    except Exception as e:
+        print(f"Error terminating instance {instance_id}: {e}")
+        return False
+
+def main():
+    parser = argparse.ArgumentParser(description='Manage EC2 instances with specific security group')
+    parser.add_argument('--security-group', '-s', required=True,
+                      help='Security Group ID to filter instances')
+    parser.add_argument('--region', '-r', default='us-gov-west-1',
+                      help='AWS region (default: us-gov-west-1)')
+    parser.add_argument('--dry-run', '-d', action='store_true',
+                      help='Dry run mode - only show what would be done')
+    args = parser.parse_args()
+
+    # Get instances with the specified security group
+    instances = get_instances_by_security_group(args.security_group, args.region)
+
+    if not instances:
+        print(f"No instances found with security group {args.security_group}")
+        return
+
+    print(f"Found {len(instances)} instances with security group {args.security_group}")
+
+    for instance in instances:
+        instance_id = instance['InstanceId']
+        print(f"\n{get_instance_info(instance)}")
+
+        if args.dry_run:
+            print("DRY RUN: Would modify instance protection and terminate instance")
+            continue
+
+        # Modify instance protection
+        if modify_instance_protection(instance_id, args.region, True):
+            print(f"Successfully disabled protection for instance {instance_id}")
+        else:
+            print(f"Skipping termination for instance {instance_id} due to protection modification failure")
+            continue
+
+        # Terminate instance
+        if terminate_instance(instance_id, args.region):
+            print(f"Successfully initiated termination for instance {instance_id}")
+        else:
+            print(f"Failed to terminate instance {instance_id}")
+
+if __name__ == "__main__":
+    main()

scripts/requirements.txt

@@ -0,0 +1,14 @@
+# Core AWS packages
+boto3==1.24.96
+botocore==1.27.96
+awscli==1.25.97
+
+# Terraform interaction
+python-terraform==0.10.1
+
+# Utilities
+PyYAML==6.0
+requests==2.28.1
+urllib3==1.26.12
+six==1.16.0
+jmespath==1.0.1

scripts/terraform-venv.md

@@ -0,0 +1,47 @@
+# Terraform Virtual Environment Setup
+
+This script sets up a Python virtual environment configured for Terraform development.
+
+## Features
+
+- Creates and activates a dedicated Python virtualenv for Terraform work
+- Installs required Python packages (boto3, python-terraform, awscli, etc.)
+- Sets up useful Terraform aliases
+- Configures proxy settings automatically
+- Activates automatically on login
+
+## Installation
+
+1. The script has been installed at: `~/git/aws-image-pipeline/scripts/terraform-venv.sh`
+
+2. Activation on login has been configured in your `~/.bash_profile`
+
+3. To activate manually in any terminal:
+```bash
+source ~/git/aws-image-pipeline/scripts/terraform-venv.sh
+```
+
+## Available Aliases
+
+- `tf` - shorthand for `terraform`
+- `tfp` - terraform plan
+- `tfa` - terraform apply
+- `tfd` - terraform destroy
+- `tfi` - terraform init
+- `tfw` - terraform workspace
+- `tfmt` - terraform fmt
+- `tfv` - terraform validate
+
+## Environment Variables
+
+The script sets up:
+- `AWS_PAGER=""` - Disables AWS CLI pagination
+- `PYTHONPATH` - Includes virtualenv packages
+- Proxy settings (if available in environment)
+
+## Troubleshooting
+
+If you encounter pip install errors:
+1. Check your proxy settings
+2. Try running `pip config set global.index-url https://pypi.org/simple`
+3. Ensure you have network access to pypi.org

scripts/terraform-venv.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Set Census Bureau proxy settings first
+export http_proxy="http://proxy.tco.census.gov:3128"
+export https_proxy="http://proxy.tco.census.gov:3128"
+export no_proxy="pypi.org,localhost,127.0.0.1,.census.gov,.local"
+
+# Function to configure pip settings
+configure_pip() {
+    mkdir -p ~/.pip
+    cat > ~/.pip/pip.conf << EOL
+[global]
+index-url = http://repository/DataScience-Group/simple/
+trusted-host = repository
+proxy = ${http_proxy}
+timeout = 60
+retries = 3
+EOL
+}
+
+# Create and configure virtualenv
+setup_virtualenv() {
+    if [ ! -d "$HOME/.virtualenvs/terraform" ]; then
+        echo "Creating terraform virtualenv..."
+        python3 -m venv "$HOME/.virtualenvs/terraform"
+    fi
+
+    source "$HOME/.virtualenvs/terraform/bin/activate"
+
+    # Configure pip
+    # configure_pip
+
+    # Install base pip first
+    pip install --upgrade pip --trusted-host repository
+
+    # Try installing requirements
+    if [ -f "$SCRIPT_DIR/requirements.txt" ]; then
+        pip install -r "$SCRIPT_DIR/requirements.txt" \
+            --trusted-host repository \
+            --trusted-host pypi.org \
+            --trusted-host files.pythonhosted.org \
+            || echo "Warning: Some packages failed to install. You may need to install them manually."
+    fi
+}
+
+# Check if conda is available, if not use venv
+if command -v conda &> /dev/null; then
+    # Create conda environment if it doesn't exist
+    if ! conda env list | grep -q "^terraform "; then
+        echo "Creating terraform conda environment..."
+        conda create -n terraform python=3.8 -y
+    fi
+
+    # Activate conda environment
+    conda activate terraform
+
+    # Install packages using conda where possible
+    conda install -y boto3 pyyaml requests
+    pip install python-terraform
+else
+    # Fall back to virtualenv if conda isn't available
+    # Get the directory where this script is located
+    SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+    # Setup virtualenv
+    setup_virtualenv
+fi
+
+# Set AWS environment variables
+export AWS_PAGER=""
+export PYTHONPATH="$PYTHONPATH:$VIRTUAL_ENV/lib/python3.8/site-packages"
+
+# Set terraform aliases
+alias tf='terraform'
+alias tfp='terraform plan'
+alias tfa='terraform apply'
+alias tfd='terraform destroy'
+alias tfi='terraform init'
+alias tfw='terraform workspace'
+alias tfmt='terraform fmt'
+alias tfv='terraform validate'
+
+# Print status
+echo "Terraform environment activated using $([ -n "$CONDA_DEFAULT_ENV" ] && echo "conda: $CONDA_DEFAULT_ENV" || echo "virtualenv: $VIRTUAL_ENV")"
+echo "Proxy settings:"
+echo "  http_proxy: $http_proxy"
+echo "  https_proxy: $https_proxy"
+echo "  no_proxy: $no_proxy"
+echo "Pip config location: ~/.pip/pip.conf"