diff --git a/design-docs/CUSTOM_TEMPLATES.MD b/design-docs/CUSTOM_TEMPLATES.MD new file mode 100644 index 0000000..52b42d4 --- /dev/null +++ b/design-docs/CUSTOM_TEMPLATES.MD @@ -0,0 +1,138 @@ +# Custom Template Management + +This document outlines approaches for managing custom GitHub repository templates, including support for subdirectory-based templates. + +## Template Sources + +### Full Repository Templates +The standard approach where an entire repository is used as a template. + +### Subdirectory Templates +Allows using a specific subdirectory from a template repository, enabling: +- Modular template organization +- Sharing common components +- Granular template selection + +## Usage + +### Creating from Full Repository + +```json +{ + "action": "create", + "project_name": "my-service", + "template_settings": { + "type": "service", + "environment": "prod", + "variables": { + "region": "us-west-2" + } + } +} +``` + +### Creating from Subdirectory + +```json +{ + "action": "create", + "project_name": "my-service", + "template_settings": { + "type": "service", + "environment": "prod", + "source_path": "templates/microservice", + "variables": { + "region": "us-west-2" + } + } +} +``` + +## Template Organization + +Example structure for a template repository using subdirectories: + +``` +template-repository/ +├── README.md +├── templates/ +│ ├── microservice/ # Template for microservices +│ │ ├── .github/ +│ │ │ └── workflows/ +│ │ ├── src/ +│ │ └── config/ +│ ├── terraform-module/ # Template for Terraform modules +│ │ ├── .github/ +│ │ └── examples/ +│ └── python-package/ # Template for Python packages +│ ├── .github/ +│ └── src/ +└── common/ # Shared components + ├── workflows/ + ├── scripts/ + └── config/ +``` + +## Implementation Details + +### Template Copying Logic + +1. Validate source path exists in template repository +2. If source path specified: + - Get contents of specified directory + - Strip source path prefix from target paths +3. If no source path: + - Get contents of entire repository +4. Copy files maintaining directory structure +5. Generate and store destroy token + +### Error Handling + +- Source path validation +- File copy failures +- Permission issues +- Missing files/directories + +## Security Considerations + +1. **Access Control**: + - Template repository access restrictions + - Source path validation +2. **Content Validation**: + - File type restrictions + - Size limits + - Path traversal prevention + +## Best Practices + +1. **Template Organization**: + - Use clear directory structure + - Include README in each template + - Document variables and requirements + +2. **Subdirectory Usage**: + - Group related templates + - Share common components + - Use consistent naming + +3. **Maintenance**: + - Regular template updates + - Version tagging + - Change documentation + +## Future Enhancements + +1. **Template Composition**: + - Combine multiple subdirectories + - Template inheritance + - Component overrides + +2. **Validation**: + - Template schema validation + - Required files checking + - Variable validation + +3. **Advanced Features**: + - Template versioning + - Hot-reload templates + - Template discovery API \ No newline at end of file diff --git a/design-docs/REPO_VARS_AND_SECRETS.md b/design-docs/REPO_VARS_AND_SECRETS.md new file mode 100644 index 0000000..728942b --- /dev/null +++ b/design-docs/REPO_VARS_AND_SECRETS.md @@ -0,0 +1,253 @@ +# Repository Variables and Secrets Management + +This document outlines the approach for managing GitHub Actions secrets and variables for newly created repositories using AWS Parameter Store and Secrets Manager. + +## Overview + +The template automation system will configure GitHub Actions secrets and variables by: +1. Reading secrets from AWS Secrets Manager +2. Reading variables from AWS Parameter Store +3. Setting them in the newly created repository using GitHub's API + +## Implementation + +### Parameter Structure + +#### AWS Parameter Store +``` +/template-automation/ + ├── variables/ + │ ├── global/ # Variables for all repos + │ │ ├── AWS_REGION + │ │ └── TERRAFORM_VERSION + │ └── by-type/ # Variables by repository type + │ ├── eks-cluster/ + │ │ ├── CLUSTER_VERSION + │ │ └── NODE_TYPE + │ └── terraform-module/ + │ ├── GO_VERSION + │ └── TFLINT_VERSION +``` + +#### AWS Secrets Manager +``` +template-automation/ + ├── secrets/global/ # Secrets for all repos + │ ├── AWS_ACCESS_KEY_ID + │ └── AWS_SECRET_ACCESS_KEY + └── secrets/by-type/ # Secrets by repository type + ├── eks-cluster/ + │ └── KUBECONFIG + └── terraform-module/ + └── SNYK_TOKEN +``` + +### Infrastructure Changes + +#### Lambda Configuration + +Add environment variables to the Lambda function: + +```hcl +# In terraform-aws-template-automation/main.tf +resource "aws_lambda_function" "template_automation" { + # ...existing configuration... + + environment { + variables = { + PARAM_STORE_PREFIX = "/template-automation" + SECRETS_PREFIX = "template-automation" + } + } +} +``` + +#### IAM Permissions + +Add required permissions to the Lambda role: + +```hcl +# In terraform-aws-template-automation/iam.tf +data "aws_iam_policy_document" "secrets_access" { + statement { + effect = "Allow" + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:ListSecrets" + ] + resources = [ + "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${var.secrets_prefix}/*" + ] + } +} + +data "aws_iam_policy_document" "ssm_access" { + statement { + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:GetParametersByPath" + ] + resources = [ + "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter${var.param_store_prefix}/*" + ] + } +} +``` + +### Implementation Details + +#### In GitHubClient + +The `GitHubClient` class will be extended with methods to handle secrets and variables: + +```python +class GitHubClient: + def set_repository_secrets(self, repo_name: str, repo_type: str = None) -> None: + """Set GitHub Actions secrets for a repository.""" + # Get global secrets + secrets = self._get_aws_secrets("secrets/global") + + # Get type-specific secrets + if repo_type: + type_secrets = self._get_aws_secrets(f"secrets/by-type/{repo_type}") + secrets.update(type_secrets) + + # Set secrets in repository + repo = self.org.get_repo(repo_name) + for name, value in secrets.items(): + repo.create_secret(name, value) + + def set_repository_variables(self, repo_name: str, repo_type: str = None) -> None: + """Set GitHub Actions variables for a repository.""" + # Get global variables + variables = self._get_ssm_parameters("variables/global") + + # Get type-specific variables + if repo_type: + type_vars = self._get_ssm_parameters(f"variables/by-type/{repo_type}") + variables.update(type_vars) + + # Set variables in repository + repo = self.org.get_repo(repo_name) + for name, value in variables.items(): + repo.create_variable(name, value) +``` + +#### In Lambda Handler + +The handler will be updated to set secrets and variables during repository creation: + +```python +def lambda_handler(event: dict, context) -> dict: + # ...existing initialization code... + + # Create repository + repo = github.get_repository(repo_name, create=True) + + # Set secrets and variables + repo_type = template_input.template_settings.get("type") + github.set_repository_secrets(repo_name, repo_type) + github.set_repository_variables(repo_name, repo_type) + + # ...rest of handler code... +``` + +## Security Considerations + +1. **Secret Encryption**: All secrets are encrypted at rest in AWS +2. **IAM Access Control**: Fine-grained control over who can access secrets +3. **Audit Trail**: AWS CloudTrail tracks all secret access +4. **Repository Isolation**: Each repository gets its own copy of secrets +5. **Least Privilege**: Lambda has minimal required permissions + +## Usage Examples + +### Setting Up Repository Type Secrets + +1. Store secrets in AWS: +```bash +aws secretsmanager create-secret \ + --name "template-automation/secrets/by-type/eks-cluster/KUBECONFIG" \ + --secret-string "..." +``` + +2. Store variables in Parameter Store: +```bash +aws ssm put-parameter \ + --name "/template-automation/variables/by-type/eks-cluster/CLUSTER_VERSION" \ + --value "1.27" \ + --type "String" +``` + +### Creating a Repository with Secrets + +Create a new EKS cluster repository with the Lambda function: + +```json +{ + "action": "create", + "project_name": "production-eks", + "template_settings": { + "type": "eks-cluster", + "environment": "production" + } +} +``` + +The Lambda function will: +1. Create the repository +2. Generate a secure destroy token +3. Store the token in a `.destroy-token` file in the repository root +4. Set up global secrets and variables +5. Set up EKS-specific secrets and variables +6. Configure necessary GitHub Actions environment + +The response will include the repository URL: + +```json +{ + "status": "success", + "repository_url": "https://github.com/org/production-eks", + "message": "Repository created successfully. The destroy token is stored in .destroy-token file." +} +``` + +**Important**: The destroy token is stored in the `.destroy-token` file in your repository. You'll need this token to delete the repository later. The file looks like: + +```plaintext +# This file contains the token required to delete this repository. +# Store this token securely as it will be required for repository deletion. +# DO NOT delete or modify this file unless you want to prevent repository deletion. + +ESxK2ld9J4mCpA-ghi8932jk... +``` + +### Destroying a Repository + +To clean up a repository and its associated secrets/variables: + +```json +{ + "action": "destroy", + "project_name": "production-eks", + "destroy_token": "ESxK2ld9J4mCpA-ghi8932jk..." +} +``` + +The Lambda function will: +1. Validate the provided destroy token +2. Delete all repository secrets +3. Delete all repository variables +4. Delete the repository itself + +If an invalid destroy token is provided, the operation will fail with an error. + +## Future Enhancements + +1. **Secret Rotation**: Implement automatic secret rotation +2. **Environment Support**: Add environment-specific secrets (dev/staging/prod) +3. **Organization Variables**: Support for organization-level variables +4. **Validation Rules**: Add validation for secret/variable names and values +5. **Backup/Restore**: Implement backup and restore for secrets/variables diff --git a/template_automation/models.py b/template_automation/models.py index 922ebbd..7322296 100644 --- a/template_automation/models.py +++ b/template_automation/models.py @@ -4,17 +4,21 @@ from pydantic import BaseModel, Field class GitHubConfig(BaseModel): - """Encapsulates configuration settings for interacting with the GitHub API. + """Configuration settings for GitHub API interactions. + + This class defines the settings needed to interact with the GitHub API, + including the API URL, authentication token, organization name, and template + repository information. Attributes: - api_base_url (str): The base URL for GitHub API requests. - token (str): The authentication token for GitHub. - org_name (str): The name of the GitHub organization. - commit_author_name (str): The name to use for commit authorship. - commit_author_email (str): The email to use for commit authorship. - source_version (Optional[str]): The version, tag, or SHA of the template to use. - template_repo_name (Optional[str]): The name of the template repository. - config_file_name (str): The name of the configuration file to write. + api_base_url (str): The base URL for all GitHub API requests. For example, + "https://api.github.com" for public GitHub. + token (str): Personal access token for GitHub API authentication. + org_name (str): Organization name where repositories will be created. + template_repo_name (Optional[str]): Name of the template repository to use + as a base. Default is None. + source_version (Optional[str]): Git reference (branch, tag, commit) to use + from the template repository. Default is None. """ api_base_url: str token: str @@ -26,33 +30,30 @@ class GitHubConfig(BaseModel): config_file_name: str = "config.json" class WorkflowConfig(BaseModel): - """Defines the configuration for a GitHub Actions workflow. - - This class represents a single GitHub Actions workflow configuration, - including its template source and destination paths, along with any + """Configuration for GitHub Actions workflow files. + + This class defines the structure for configuring GitHub Actions workflow files, + including the workflow name, template source and destination paths, and any variables needed for template rendering. Attributes: - name (str): Descriptive name of the workflow, used for logging and - identification purposes. - template_path (str): Path to the Jinja2 template file containing the - workflow definition. This path should be relative to the template - root directory. - output_path (str): Destination path where the rendered workflow file - will be saved in the new repository. This path should be relative - to the repository root. - variables (Dict[str, Any]): Dictionary of variables to use when - rendering the workflow template. These values will be passed to - the Jinja2 template engine. + name (str): Name of the workflow, used for identification and logging. + template_path (str): Path to the workflow template file, relative to the + template root directory. + output_path (str): Destination path where the rendered workflow file should + be written in the target repository. + variables (Dict[str, Any]): Variables to use when rendering the workflow + template with Jinja2. Keys are variable names and values can be any + type that Jinja2 can handle. Defaults to an empty dict. Example: >>> workflow = WorkflowConfig( - ... name="CI/CD Pipeline", + ... name="CI/CD", ... template_path="workflows/ci.yml.j2", ... output_path=".github/workflows/ci.yml", ... variables={ - ... "python_version": "3.9", - ... "test_commands": ["pytest", "flake8"] + ... "runner": "ubuntu-latest", + ... "python_version": "3.9" ... } ... ) """ @@ -137,8 +138,7 @@ class TemplateInput(BaseModel): ... "environment": "production", ... "region": "us-west-2" ... }, - ... trigger_init_workflow=True, - ... owning_team="platform-team" + ... trigger_init_workflow=True ... ) """ project_name: str @@ -147,35 +147,30 @@ class TemplateInput(BaseModel): owning_team: Optional[str] = None class TemplateConfig(BaseModel): - """Configuration for a template repository. + """Configuration for template repository automation. - This class defines the configuration structure for template repositories, - including pull request settings and workflow configurations. + This class defines the overall configuration for how a template repository + should be processed, including pull request settings and workflow automations. Attributes: - pr (PRConfig): Pull request configuration settings including title template, - body template, branch settings, labels, reviewers and assignees. - workflows (List[WorkflowConfig]): List of workflow configurations to apply - to the repository. Each workflow config specifies name, template path, - output path and variables. + pr (PRConfig): Configuration settings for pull request creation, including + templates for title and body, branch names, and PR metadata. + workflows (List[WorkflowConfig]): List of workflow configurations that should be + applied to repositories created from this template. Example: - ```python - config = TemplateConfig( - pr=PRConfig( - title_template="Initialize {{ repo_name }}", - base_branch="main", - labels=["automated"] - ), - workflows=[ - WorkflowConfig( - name="CI", - template_path="workflows/ci.yml", - output_path=".github/workflows/ci.yml" - ) - ] - ) - ``` + >>> config = TemplateConfig( + ... pr=PRConfig( + ... title_template="Initialize {{ repo_name }}", + ... reviewers=["team-lead"] + ... ), + ... workflows=[ + ... WorkflowConfig( + ... template_path="workflows/ci.yml", + ... variables={"runner": "ubuntu-latest"} + ... ) + ... ] + ... ) """ pr: PRConfig = Field( default_factory=lambda: PRConfig(