diff --git a/meeting_docs b/meeting_docs new file mode 100644 index 0000000..bb0fa74 --- /dev/null +++ b/meeting_docs @@ -0,0 +1,36 @@ +AMI Build Pipeline +Copy Morpheus.tf into a new file +Input new variables +Execute the Morpheus.tf +- Builds a new Code pipeline +- Downloads the packer Template +- Downloads the Anbile playbooks +- Downloads the Goss testing sutie +- Builds 3 new build jobs – +o Build Job calls Packer +o Packer creates the EC23 instance +o Packer calls ansible- Morpheus -arem:baseline.yaml + Installs Morpheus and configs on the EC2 + Makes sure Python is in the environment + Regiters and deregisters from Satellite + + +- Packer builds a new AMI from EC2 +o Register AMI with AWS +o Registers AMI with parameter store +- Packer deletes the EC2 instance + Sets up PIP +- Test Build +o Uses TF to create a instance of the AMI, image gets registered with AWS +o Logs into AMI and uploads the testing template to the instance +o Executes the test and ssh is working. +o If test fails then it de registers the image from AWS +o If pass AMI id get uploaded to Parameter Store + +If creating a new image may need to tweak packer +Will need a new playbook to call roles + (Cloud have a standard sets for roles shared across all builds) +Customer will need to create the playbook to install and configure their app + +Customer create a EC2 instance from Morpheus\Service Catalog and selects the respective AMI + diff --git a/scripts/manage_ec2_instances.py b/scripts/manage_ec2_instances.py index b9c4e82..4dba4e6 100755 --- a/scripts/manage_ec2_instances.py +++ b/scripts/manage_ec2_instances.py @@ -18,6 +18,43 @@ def run_aws_command(command: List[str]) -> dict: print(f"Error parsing AWS command output: {e}") return {} +def get_parameter_value(parameter_name: str, region: str) -> str: + """Get a parameter value from AWS Parameter Store.""" + command = [ + 'ssm', 'get-parameter', + '--name', parameter_name, + '--region', region, + '--output', 'json' + ] + result = run_aws_command(command) + return result.get('Parameter', {}).get('Value', '') + +def get_security_group_by_project(project_name: str, region: str) -> str: + """Get security group ID from Parameter Store and filter by project name.""" + parameter_name = f"/image-pipeline/{project_name}/security_group_ids" + value = get_parameter_value(parameter_name, region) + if not value: + return None + + # Split comma-separated security group IDs + sg_ids = [sg.strip() for sg in value.split(',')] + + # Get details of all security groups + command = [ + 'ec2', 'describe-security-groups', + '--group-ids'] + sg_ids + [ + '--region', region, + '--output', 'json' + ] + result = run_aws_command(command) + + # Filter security groups by project name in their name + for sg in result.get('SecurityGroups', []): + print(f"Found security group: {sg['GroupName']} ({sg['GroupId']})") + if project_name.lower() in sg.get('GroupName', '').lower(): + return sg['GroupId'] + return None + def get_instances_by_security_group(security_group_id: str, region: str) -> List[dict]: """Find EC2 instances that have the specified security group attached.""" command = [ @@ -99,22 +136,28 @@ def terminate_instance(instance_id: str, region: str) -> bool: def main(): parser = argparse.ArgumentParser(description='Manage EC2 instances with specific security group') - parser.add_argument('--security-group', '-s', required=True, - help='Security Group ID to filter instances') + parser.add_argument('--project-name', '-p', required=True, + help='Project name to get security group from parameter store') parser.add_argument('--region', '-r', default='us-gov-west-1', help='AWS region (default: us-gov-west-1)') parser.add_argument('--dry-run', '-d', action='store_true', help='Dry run mode - only show what would be done') args = parser.parse_args() + # Get security group ID that matches project name + security_group_id = get_security_group_by_project(args.project_name, args.region) + if not security_group_id: + print(f"No security group found containing project name {args.project_name}") + return + # Get instances with the specified security group - instances = get_instances_by_security_group(args.security_group, args.region) + instances = get_instances_by_security_group(security_group_id, args.region) if not instances: - print(f"No instances found with security group {args.security_group}") + print(f"No instances found with security group {security_group_id}") return - print(f"Found {len(instances)} instances with security group {args.security_group}") + print(f"Found {len(instances)} instances with security group {security_group_id}") for instance in instances: instance_id = instance['InstanceId'] diff --git a/summary.md b/summary.md new file mode 100644 index 0000000..bcfc7fc --- /dev/null +++ b/summary.md @@ -0,0 +1,109 @@ +# Project Summary + +This document summarizes the AWS Image Pipeline project, outlining its purpose, structure, and key processes. + +## Overview + +The AWS Image Pipeline is an automated solution for building, configuring, and testing Amazon Machine Images (AMIs) using AWS services and open-source tools. It streamlines the AMI creation process, ensuring consistency and security. + +## Key Components + +* **Terraform (terraform-aws-image-pipeline):** Manages the AWS infrastructure required for the pipeline, including CodePipeline, CodeBuild, IAM roles, S3 buckets, and KMS keys. +* **Packer (linux-image-pipeline):** Defines the AMI build process, including the base image, configuration steps, and testing. +* **Ansible (image-pipeline-ansible-playbooks):** Automates the configuration of the EC2 instance during the AMI build process, including application installation and registration with services like Satellite. +* **Goss (image-pipeline-goss-testing):** Validates the AMI after it is built, ensuring it meets the required specifications. + +## Technical Implementation + +### Infrastructure Setup (Terraform) +* Creates AWS CodePipeline with multiple stages for orchestration +* Sets up CodeBuild projects for building and testing AMIs +* Manages IAM roles and policies for secure access +* Configures S3 buckets for storing artifacts and build outputs +* Sets up KMS keys for encryption of sensitive data +* Manages AWS Parameter Store entries for configuration + +### Build Process (Packer) +* Uses HCL-based templates (e.g., build.pkr.hcl, morpheus-build.pkr.hcl) +* Fetches configurations from AWS Parameter Store +* Integrates with AWS Secrets Manager for credentials +* Launches EC2 instances with specified configurations +* Coordinates with Ansible for instance configuration +* Handles AMI creation and registration +* Uses post-processors to update Parameter Store with AMI IDs + +### Configuration Management (Ansible) +* Organizes configurations using role-based architecture +* Provides roles for: + * Base system configuration + * Application installation + * Satellite registration/deregistration + * Python/PIP setup + * Volume mapping +* Integrates with AWS Parameter Store for dynamic configurations +* Handles both Linux and Windows environments +* Supports multiple deployment scenarios (Morpheus, GitHub Runner, etc.) + +### Testing Framework (Goss) +* Implements validation rules for AMI testing +* Executes tests on newly created instances +* Validates system configurations and installed software +* Integrates with CodePipeline for automated testing +* Supports both pass/fail scenarios with appropriate cleanup + +## Pipeline Workflow Details + +1. **Infrastructure Preparation** + * Terraform creates/updates required AWS resources + * Sets up secure communication channels and access controls + * Configures pipeline stages and build environments + +2. **Source Stage** + * Downloads Packer templates from S3/CodeCommit + * Retrieves Ansible playbooks and roles + * Fetches Goss testing suite + +3. **Build Stage** + * CodeBuild executes Packer with appropriate templates + * Launches EC2 instance with security groups and networking + * Retrieves secure credentials from Secrets Manager + * Manages SSH keys and access configurations + +4. **Configuration Stage** + * Ansible applies roles and configurations + * Installs required software and dependencies + * Configures system settings and services + * Handles registration with external services + +5. **AMI Creation** + * Packer creates AMI from configured instance + * Registers AMI with AWS + * Updates Parameter Store with AMI information + * Cleans up temporary resources + +6. **Testing Stage** + * Creates test instance from new AMI + * Deploys Goss test suite + * Executes validation tests + * Reports test results to pipeline + +7. **Completion** + * Success: AMI is ready for use, ID stored in Parameter Store + * Failure: AMI is deregistered, resources cleaned up + * Pipeline status updated with results + +## Usage Workflow + +1. Developers modify configuration code in any of the component repositories +2. Changes trigger the pipeline through CodePipeline +3. Pipeline executes build, configure, and test processes +4. Resulting AMI is available through Parameter Store +5. Users can create EC2 instances from Morpheus/Service Catalog using the new AMI + +## Security Features + +* KMS encryption for artifacts and secrets +* IAM role-based access control +* Secure credential management through Secrets Manager +* Automated cleanup of temporary resources +* Secure network configurations for build processes