diff --git a/aws-image-pipeline.code-workspace b/aws-image-pipeline.code-workspace deleted file mode 100644 index ba0cc0e..0000000 --- a/aws-image-pipeline.code-workspace +++ /dev/null @@ -1,29 +0,0 @@ -{ - "folders": [ - { - "path": ".", - "name": "aws-image-pipeline" - }, - { - "path": "../image-pipeline-ansible-playbooks", - "name": "ansible/ansible-playbooks" - }, - { - "path": "../linux-image-pipeline", - "name": "packer/linux-image-pipeline" - }, - { - "path": "../terraform-aws-image-pipeline", - "name": "modules/terraform-aws-image-pipeline" - }, - { - "path": "../image-pipeline-goss-testing", - "name": "test/goss-testing" - }, - { - "path": "../packer-plugin-amazon/docs" - } - ], - "settings": { - } -} diff --git a/external-dependencies.md b/external-dependencies.md new file mode 100644 index 0000000..fde19a6 --- /dev/null +++ b/external-dependencies.md @@ -0,0 +1,137 @@ +# External Dependencies Documentation + +This document lists all external dependencies that are not managed by Terraform in the aws-image-pipeline project. + +## Hardcoded Values + +### Build Configuration +- Builder Image: `aws/codebuild/standard:7.0` +- Terraform Version: `1.8.5` +- Packer Version: `1.10.3` (default) +- SSH User: `ec2-user` + +### AMI Configuration +- Base AMI ID: `ami-03fadeeea589a106b` +- Instance Type: `t2.micro` + +### Repository Names (Defaults) +- `linux-image-pipeline` +- `image-pipeline-ansible-playbooks` +- `image-pipeline-goss-testing` + +### Network Configuration +- Proxy Server: `proxy.tco.census.gov:3128` +- Allowed Domains: + - `.census.gov` + - `.eks.amazonaws.com` + - `.s3.amazonaws.com` + - `.amazonaws.com` + - `.gcr.io` + - `.pkg.dev` + - `downloads.morpheusdata.com` + - `169.254.169.254` + - Various internal network ranges (148.129.*, 10.*, 172.18-25.*) + +## External AWS Resources + +### AWS Parameter Store Values +- AMI configuration parameters +- Subnet configuration +- Security group settings +- Region settings +- Source AMI information +- Instance type configurations +- SSH user parameters +- Docker repository configurations +- AWS account ID parameters +- Shared accounts parameters +- Userdata parameters + +### S3 Bucket Artifacts +- Packer configurations (`linux-image-pipeline.zip`) +- Ansible playbooks (`image-pipeline-ansible-playbooks.zip`) +- Goss testing files (`image-pipeline-goss-testing.zip`) + +### AWS Secrets Manager +- WinRM credentials +- AWS credentials for build process +- SSH private keys + +### Security Groups +- Existing security group: `it-linux-base` + +### IAM Dependencies +- AWS managed policies (referenced in IAM policy documents) +- Cross-account roles and permissions + +### AWS Service Dependencies +- AWS Partition data (`aws_partition.current`) +- AWS Caller Identity (`aws_caller_identity.current`) +- AWS Region data (`aws_region.current`) +- KMS keys (referenced via ARNs) + +### VPC Dependencies +- Pre-existing Security Group IDs +- Pre-existing Subnet IDs +- Pre-existing VPC ID + +### State Backend Requirements +- Pre-existing S3 bucket for state storage +- Pre-existing DynamoDB table for state locking + +### Cross-Account Resources +- AMI sharing account IDs +- Cross-region S3 bucket replication configurations + +## Build Dependencies + +### Environment Variables +- HTTP_PROXY +- HTTPS_PROXY +- NO_PROXY + +### Source Control +- CodeCommit repository ARNs (when not using S3) +- Default branch names (defaulting to "main") + +### Build Resources +- Pre-existing ECR/Docker images + - `aws/codebuild/standard:7.0` + +### Configuration Files +- Ansible playbook: `hello-world.yaml` +- Goss profile: `base-test` + +## Workspace-Managed Resources +Resources that are created in this workspace but outside the terraform-aws-image-pipeline module: + +### S3 Resources +- Assets bucket (`aws_s3_bucket.assets_bucket`) - Used to store pipeline artifacts +- Associated bucket policies and access controls + +### IAM Resources +- Morpheus build user policy (`aws_iam_user_policy.morpheus_build_user`) +- AMI sharing policies and roles + +### Parameter Store Resources +- RHEL9 AMI parameters +- Base image parameters +- Ansible-related parameters + +### VPC Resources +- VPC endpoints for AWS services +- Associated security groups and routing configurations + +### Pipeline Configurations +- Multiple pipeline definitions: + - Amazon Linux pipeline + - RHEL pipeline + - Morpheus application pipeline + - Docker image pipeline + - GitHub runner pipeline + +### Volume Configurations +- Custom EBS volume mappings (e.g., for Morpheus deployments) + - Root volumes + - Application volumes + - Data volumes \ No newline at end of file diff --git a/external-dependencies.tf b/external-dependencies.tf new file mode 100644 index 0000000..934b2eb --- /dev/null +++ b/external-dependencies.tf @@ -0,0 +1,25 @@ +module "external_dependencies" { + source = "../terraform-aws-image-pipeline-external" + + project_name = "aws-image-pipeline" + assets_bucket_name = aws_s3_bucket.assets_bucket.bucket + state_bucket_name = local.state_config.bucket + + pipeline_iam_arns = [ + module.amazon_linux.iam_arn, + module.morpheus.iam_arn + ] + + vpc_config = { + vpc_id = local._vpc_config.vpc_id + region = local._vpc_config.region + security_group_ids = local._vpc_config.security_group_ids + subnets = local._vpc_config.subnets + } + + # Add common tags + tags = { + Project = "aws-image-pipeline" + Environment = local.environment + } +} \ No newline at end of file diff --git a/linux-images.code-workspace b/linux-images.code-workspace new file mode 100644 index 0000000..7424916 --- /dev/null +++ b/linux-images.code-workspace @@ -0,0 +1,13 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "../terraform-aws-image-pipeline" + }, + { + "path": "../../terraform-aws-image-pipeline-external" + } + ] +} \ No newline at end of file diff --git a/morpheus.tf b/morpheus.tf index 0b6f06a..8f5b510 100644 --- a/morpheus.tf +++ b/morpheus.tf @@ -43,7 +43,6 @@ module "morpheus" { morpheus_version = "7.0.10-1", shutdown_behavior = "stop" } - assets_bucket_name = aws_s3_bucket.assets_bucket.bucket image_volume_mapping = [ { device_name = "/dev/sda1" # Root device diff --git a/moved.tf b/moved.tf new file mode 100644 index 0000000..9f20bbe --- /dev/null +++ b/moved.tf @@ -0,0 +1,41 @@ +# S3 Bucket moves +moved { + from = aws_s3_bucket.state_bucket + to = module.external_dependencies.aws_s3_bucket.state_bucket +} + +moved { + from = aws_s3_bucket.assets_bucket + to = module.external_dependencies.aws_s3_bucket.assets_bucket +} + +moved { + from = aws_s3_bucket_server_side_encryption_configuration.state_bucket_encryption["state_bucket"] + to = module.external_dependencies.aws_s3_bucket_server_side_encryption_configuration.state_bucket_encryption +} + +moved { + from = aws_s3_bucket_server_side_encryption_configuration.state_bucket_encryption["assets_bucket"] + to = module.external_dependencies.aws_s3_bucket_server_side_encryption_configuration.assets_bucket_encryption +} + +moved { + from = aws_s3_bucket_policy.assets_bucket_policy + to = module.external_dependencies.aws_s3_bucket_policy.assets_bucket_policy +} + +# Security group moves +moved { + from = aws_security_group.allow_amznlinux_cdn + to = module.external_dependencies.aws_security_group.pipeline_security_group +} + +moved { + from = aws_vpc_security_group_egress_rule.allow_all_traffic_ipv4 + to = module.external_dependencies.aws_vpc_security_group_egress_rule.allow_all_traffic_ipv4 +} + +moved { + from = aws_vpc_security_group_ingress_rule.allow_all_between_self + to = module.external_dependencies.aws_vpc_security_group_ingress_rule.allow_self_traffic +} \ No newline at end of file diff --git a/scripts/__pycache__/syncrepo.cpython-39.pyc b/scripts/__pycache__/syncrepo.cpython-39.pyc new file mode 100644 index 0000000..c59c66b Binary files /dev/null and b/scripts/__pycache__/syncrepo.cpython-39.pyc differ diff --git a/scripts/__pycache__/syncrepos.cpython-39.pyc b/scripts/__pycache__/syncrepos.cpython-39.pyc new file mode 100644 index 0000000..9ba16b6 Binary files /dev/null and b/scripts/__pycache__/syncrepos.cpython-39.pyc differ diff --git a/scripts/__pycache__/test_sync_repos.cpython-39.pyc b/scripts/__pycache__/test_sync_repos.cpython-39.pyc new file mode 100644 index 0000000..1e5b083 Binary files /dev/null and b/scripts/__pycache__/test_sync_repos.cpython-39.pyc differ diff --git a/scripts/sync-repos.py b/scripts/sync-repos.py deleted file mode 100755 index 89f2dfd..0000000 --- a/scripts/sync-repos.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import subprocess -import argparse -from typing import Dict, Optional, List - -# Base directory containing all repositories -BASE_DIR = "/home/a/arnol377/git" - -# Default repository configurations -DEFAULT_REPOS: Dict[str, Dict[str, str]] = { - "aws-image-pipeline": { - "branch": "main" - }, - "image-pipeline-ansible-playbooks": { - "branch": "main" - }, - "linux-image-pipeline": { - "branch": "main" - } -} - -def parse_args(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser(description='Sync git repositories with multiple remotes') - parser.add_argument('--remote', '-r', - choices=['all', 'origin', 'hpw'], - default='all', - help='Remote to sync with (default: all)') - parser.add_argument('--message', '-m', - help='Commit message to use for all repositories') - return parser.parse_args() - -def get_remotes(args) -> List[str]: - """Get list of remotes based on command line argument.""" - return ['origin', 'hpw'] if args.remote == 'all' else [args.remote] - -def run_command(cmd: list[str], cwd: Optional[str] = None) -> tuple[int, str, str]: - """Run a shell command and return the exit code, stdout, and stderr.""" - try: - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=cwd, - text=True - ) - stdout, stderr = process.communicate() - return process.returncode, stdout, stderr - except Exception as e: - return 1, "", str(e) - -def sync_repo(repo_name: str, config: dict, remotes: List[str], commit_message: Optional[str] = None) -> bool: - """Sync (pull/push) a single repository.""" - repo_path = os.path.join(BASE_DIR, repo_name) - if not os.path.isdir(repo_path): - print(f"Error: Directory {repo_path} does not exist") - return False - - print(f"\nProcessing {repo_name}...") - - # Pull from specified remotes - for remote in remotes: - print(f"Pulling from {remote}...") - code, out, err = run_command( - ["git", "pull", remote, config["branch"]], - repo_path - ) - if code != 0: - print(f"Warning: Failed to pull from {remote}") - print(f"Error: {err}") - - # Check for changes - code, out, err = run_command(["git", "status", "--porcelain"], repo_path) - if code != 0: - print(f"Error checking git status: {err}") - return False - - if out.strip() and commit_message: - print(f"Changes detected in {repo_name}, committing...") - - # Add all changes - code, out, err = run_command(["git", "add", "."], repo_path) - if code != 0: - print(f"Error adding files: {err}") - return False - - # Commit changes - code, out, err = run_command( - ["git", "commit", "-m", commit_message], - repo_path - ) - if code != 0: - print(f"Error committing changes: {err}") - return False - - # Always try to push to specified remotes - for remote in remotes: - print(f"Pushing to {remote}...") - code, out, err = run_command( - ["git", "push", remote, config["branch"]], - repo_path - ) - if code != 0: - # Only warn if the error indicates nothing to push - if "Everything up-to-date" not in err: - print(f"Warning: Failed to push to {remote}") - print(f"Error: {err}") - - return True - -def main(): - """Main function to sync all repositories.""" - args = parse_args() - remotes = get_remotes(args) - success = True - - # Process each repository - for repo_name, config in DEFAULT_REPOS.items(): - try: - if not sync_repo(repo_name, config, remotes, args.message): - success = False - except Exception as e: - print(f"Error processing {repo_name}: {e}") - success = False - - if success: - print("\nAll repositories processed successfully!") - sys.exit(0) - else: - print("\nSome repositories failed to process") - sys.exit(1) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/syncrepos.py b/scripts/syncrepos.py new file mode 100755 index 0000000..b9c33e5 --- /dev/null +++ b/scripts/syncrepos.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +import os +import sys +import subprocess +import argparse +import json +from typing import Dict, Optional, List + +class GitSync: + def __init__(self, base_dir: str): + self.base_dir = base_dir + + def parse_args(self): + """Parse command line arguments.""" + parser = argparse.ArgumentParser(description='Sync git repositories with multiple remotes') + parser.add_argument('--base-dir', + default=os.environ.get("PWD"), + help='Base directory for repositories') + parser.add_argument('--remote', '-r', + choices=['all', 'origin', 'hpw'], + default='all', + help='Remote to sync with (default: all)') + parser.add_argument('--message', '-m', + help='Commit message to use for all repositories') + parser.add_argument('--workspace-file', + help='Path to VS Code workspace file') + parser.add_argument('--repo-paths', + help='Comma-separated list of repository paths') + parser.add_argument('--branch', + help='Branch to checkout before syncing') + return parser.parse_args() + + def get_remotes(self, args, repo_path: str) -> List[str]: + """Get list of remotes based on command line argument.""" + # If a specific remote is specified, return it as a single-element list + if args.remote != 'all': + return [args.remote] + + # Otherwise, get all configured remotes for the repository + code, out, err = self.run_command(["git", "remote"], repo_path) + if code != 0: + print(f"Error getting remotes: {err}") + return [] + + # Return the list of remotes + return out.strip().split('\n') + + def run_command(self, cmd: list[str], cwd: Optional[str] = None) -> tuple[int, str, str]: + """Run a shell command and return the exit code, stdout, and stderr.""" + try: + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cwd, + text=True + ) + stdout, stderr = process.communicate() + return process.returncode, stdout, stderr + except Exception as e: + return 1, "", str(e) + + def create_branch(self, branch_name: str, repo_path: str) -> bool: + """Create a new branch in a git repository.""" + code, out, err = self.run_command(["git", "checkout", "-b", branch_name], repo_path) + if code != 0: + print(f"Error creating branch: {err}") + return False + + def get_current_branch(self, repo_path: str) -> Optional[str]: + """Get the current branch of a git repository.""" + code, out, err = self.run_command(["git", "rev-parse", "--abbrev-ref", "HEAD"], repo_path) + if code == 0: + return out.strip() + else: + print(f"Error getting current branch: {err}") + return None + + def parse_workspace_file(self, workspace_file: str) -> List[str]: + """Parse the VS Code workspace file to get the list of folders.""" + with open(workspace_file, 'r') as file: + workspace_data = json.load(file) + repo_paths = [] + for folder in workspace_data['folders']: + path = folder['path'] + if path == ".": + path = os.getcwd() + repo_paths.append(path) + return repo_paths + + def is_git_repo(self, repo_path: str) -> bool: + """Check if a directory is a Git repository.""" + return os.path.isdir(os.path.join(repo_path, ".git")) + + def sync_repo(self, repo_path: str, remotes: List[str], commit_message: Optional[str] = None) -> bool: + """Sync (pull/push) a single repository.""" + if not os.path.isdir(repo_path): + print(f"Error: Directory {repo_path} does not exist") + return False + + if not self.is_git_repo(repo_path): + print(f"Error: Directory {repo_path} is not a Git repository") + return False + + print(f"\nProcessing {repo_path}...") + + current_branch = self.get_current_branch(repo_path) + if not current_branch: + return False + + # Pull from specified remotes + for remote in remotes: + print(f"Pulling from {remote} on branch {current_branch}...") + code, out, err = self.run_command( + ["git", "pull", remote, current_branch], + repo_path + ) + if code != 0: + print(f"Warning: Failed to pull from {remote}") + print(f"Error: {err}") + + # Check for changes + code, out, err = self.run_command(["git", "status", "--porcelain"], repo_path) + if code != 0: + print(f"Error checking git status: {err}") + return False + + if out.strip() and commit_message: + print(f"Changes detected in {repo_path}, committing...") + + # Add all changes, including deletions + code, out, err = self.run_command(["git", "add", "-A"], repo_path) + if code != 0: + print(f"Error adding files: {err}") + return False + + # Commit changes + code, out, err = self.run_command( + ["git", "commit", "-m", commit_message], + repo_path + ) + if code != 0: + print(f"Error committing changes: {err}") + return False + + # Always try to push to specified remotes + for remote in remotes: + print(f"Pushing to {remote} on branch {current_branch}...") + code, out, err = self.run_command( + ["git", "push", remote, current_branch], + repo_path + ) + if code != 0: + # Only warn if the error indicates nothing to push + if "Everything up-to-date" not in err: + print(f"Warning: Failed to push to {remote}") + print(f"Error: {err}") + + return True + + def main(self): + """Main function to sync all repositories.""" + args = self.parse_args() + success = True + self.base_dir = args.base_dir + + # Determine repository paths + if args.repo_paths: + repo_paths = args.repo_paths.split(',') + else: + # Find the VS Code workspace file + workspace_file = args.workspace_file + if not workspace_file: + workspace_files = [f for f in os.listdir(self.base_dir) if f.endswith('.code-workspace')] + if not workspace_files: + print("Error: No .code-workspace file found and no repo paths provided") + sys.exit(1) + workspace_file = os.path.join(self.base_dir, workspace_files[0]) + repo_paths = self.parse_workspace_file(workspace_file) + + # Process each repository + for repo_path in repo_paths: + remotes = self.get_remotes(args, repo_path) + try: + if args.branch: + current = self.get_current_branch(repo_path) + if current != args.branch: + print(f"Checking out branch {args.branch}...") + code, out, err = self.run_command(["git", "checkout", args.branch], repo_path) + if code != 0: + print(f"Error checking out branch: {err}") + success = False + continue + + if not self.sync_repo(repo_path, remotes, args.message): + success = False + except Exception as e: + print(f"Error processing {repo_path}: {e}") + success = False + + if success: + print("\nAll repositories processed successfully!") + sys.exit(0) + else: + print("\nSome repositories failed to process") + sys.exit(1) + +if __name__ == "__main__": + base_dir = os.environ.get("PWD") + git_sync = GitSync(base_dir) + git_sync.main() \ No newline at end of file diff --git a/scripts/test_sync_repos.py b/scripts/test_sync_repos.py new file mode 100644 index 0000000..6cba7bd --- /dev/null +++ b/scripts/test_sync_repos.py @@ -0,0 +1,68 @@ +import unittest +from unittest.mock import patch, mock_open, MagicMock +import os +import json +from syncrepos import GitSync + +class TestGitSync(unittest.TestCase): + def setUp(self): + self.base_dir = "/fake/base/dir" + self.git_sync = GitSync(self.base_dir) + + @patch("os.path.isdir") + @patch("os.listdir") + def test_main_no_workspace_file(self, mock_listdir, mock_isdir): + mock_listdir.return_value = [] + with self.assertRaises(SystemExit) as cm: + self.git_sync.main() + self.assertEqual(cm.exception.code, 1) + + @patch("os.path.isdir") + @patch("os.listdir") + @patch("builtins.open", new_callable=mock_open, read_data='{"folders": [{"path": "/fake/repo1"}, {"path": "/fake/repo2"}]}') + @patch("syncrepos.GitSync.run_command") + @patch("syncrepos.GitSync.get_current_branch") + def test_main_success(self, mock_get_current_branch, mock_run_command, mock_open, mock_listdir, mock_isdir): + mock_listdir.return_value = ["workspace.code-workspace"] + mock_isdir.return_value = True + mock_get_current_branch.return_value = "main" + mock_run_command.return_value = (0, "", "") + + with patch.object(self.git_sync, 'parse_args', return_value=MagicMock(remote='all', message='test commit')): + with self.assertRaises(SystemExit) as cm: + self.git_sync.main() + self.assertEqual(cm.exception.code, 0) + + @patch("os.path.isdir") + @patch("os.listdir") + @patch("builtins.open", new_callable=mock_open, read_data='{"folders": [{"path": "/fake/repo1"}, {"path": "/fake/repo2"}]}') + @patch("syncrepos.GitSync.run_command") + @patch("syncrepos.GitSync.get_current_branch") + def test_main_failure(self, mock_get_current_branch, mock_run_command, mock_open, mock_listdir, mock_isdir): + mock_listdir.return_value = ["workspace.code-workspace"] + mock_isdir.return_value = True + mock_get_current_branch.return_value = "main" + mock_run_command.side_effect = [(0, "", ""), (0, "", ""), (1, "", "error")] + + with patch.object(self.git_sync, 'parse_args', return_value=MagicMock(remote='all', message='test commit')): + with self.assertRaises(SystemExit) as cm: + self.git_sync.main() + self.assertEqual(cm.exception.code, 1) + + @patch("syncrepos.GitSync.run_command") + def test_get_current_branch(self, mock_run_command): + mock_run_command.return_value = (0, "main", "") + branch = self.git_sync.get_current_branch("/fake/repo") + self.assertEqual(branch, "main") + + mock_run_command.return_value = (1, "", "error") + branch = self.git_sync.get_current_branch("/fake/repo") + self.assertIsNone(branch) + + @patch("builtins.open", new_callable=mock_open, read_data='{"folders": [{"path": "/fake/repo1"}, {"path": "/fake/repo2"}]}') + def test_parse_workspace_file(self, mock_open): + repo_paths = self.git_sync.parse_workspace_file("/fake/workspace.code-workspace") + self.assertEqual(repo_paths, ["/fake/repo1", "/fake/repo2"]) + +if __name__ == "__main__": + unittest.main()