From 5f2284f91837904c51507f60fd38ea356e0f616f Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 28 Mar 2025 13:07:44 -0400 Subject: [PATCH] docs --- docs/api-reference.md | 607 +++++++++++++++++++++++++++++++++ docs/census-examples.md | 275 +++++++++++++++ docs/census-integration.md | 179 ++++++++++ docs/code-architecture.md | 322 ++++++++++++++++++ docs/command-reference.md | 638 +++++++++++++++++++++++++++++++++++ docs/github-workflow.md | 268 +++++++++++++++ docs/implementation-guide.md | 223 ++++++++++++ docs/index.md | 202 +++++++++++ docs/installation.md | 229 +++++++++++++ docs/overview.md | 90 +++++ docs/quick-start.md | 236 +++++++++++++ docs/testing-strategy.md | 201 +++++++++++ docs/troubleshooting.md | 361 ++++++++++++++++++++ docs/upgrade-path.md | 159 +++++++++ docs/user-guide.md | 445 ++++++++++++++++++++++++ 15 files changed, 4435 insertions(+) create mode 100644 docs/api-reference.md create mode 100644 docs/census-examples.md create mode 100644 docs/census-integration.md create mode 100644 docs/code-architecture.md create mode 100644 docs/command-reference.md create mode 100644 docs/github-workflow.md create mode 100644 docs/implementation-guide.md create mode 100644 docs/index.md create mode 100644 docs/installation.md create mode 100644 docs/overview.md create mode 100644 docs/quick-start.md create mode 100644 docs/testing-strategy.md create mode 100644 docs/troubleshooting.md create mode 100644 docs/upgrade-path.md create mode 100644 docs/user-guide.md diff --git a/docs/api-reference.md b/docs/api-reference.md new file mode 100644 index 00000000..51516768 --- /dev/null +++ b/docs/api-reference.md @@ -0,0 +1,607 @@ +# API Reference + +This document provides a technical reference for developers who want to extend or integrate with the Terraform Upgrade Tool's Python API. + +## Core Modules + +### Scanner Module + +The scanner module finds and analyzes Terraform configurations. + +```python +from tf_upgrade.scanner import DirectoryScanner + +# Create a scanner +scanner = DirectoryScanner( + root_dir="/path/to/terraform/repo", + max_depth=5, + include_patterns=["*.tf"], + exclude_patterns=["*.tfvars", "*.tfbackup"], + parallel=True +) + +# Run scanning +results = scanner.scan() + +# Process results +for directory, info in results.items(): + print(f"Directory: {directory}") + print(f" Version: {info['version']}") + print(f" Resources: {info['resource_count']}") + print(f" Files: {', '.join(info['files'])}") +``` + +#### DirectoryScanner Options + +| Option | Type | Description | Default | +|--------|------|-------------|---------| +| `root_dir` | `str` | Directory to start scanning | Required | +| `max_depth` | `int` | Maximum directory depth to scan | `5` | +| `include_patterns` | `List[str]` | File patterns to include | `["*.tf"]` | +| `exclude_patterns` | `List[str]` | File patterns to exclude | `["*.tfvars"]` | +| `parallel` | `bool` | Use parallel processing | `False` | + +### Version Detector Module + +Detects the Terraform version used in configurations and determines the upgrade path. + +```python +from tf_upgrade.version_detector import VersionDetector + +# Create detector +detector = VersionDetector() + +# Detect version in directory +result = detector.detect_version("/path/to/terraform/config") + +print(f"Detected version: {result.version}") +print(f"Upgrade path: {' -> '.join(result.upgrade_path)}") +print(f"Version constraint location: {result.constraint_file}") +print(f"Is 0.12 syntax: {result.is_0_12_syntax}") +``` + +#### VersionDetector Methods + +| Method | Parameters | Returns | Description | +|--------|------------|---------|-------------| +| `detect_version` | `directory: str` | `VersionResult` | Detects version from config files | +| `determine_upgrade_path` | `current_version: str` | `List[str]` | Determines version steps needed | +| `extract_version_constraint` | `file_path: str` | `Optional[str]` | Extracts version constraint | + +### Complexity Analyzer Module + +Analyzes Terraform configurations for complexity factors. + +```python +from tf_upgrade.complexity_analyzer import ComplexityAnalyzer + +# Create analyzer +analyzer = ComplexityAnalyzer() + +# Analyze directory +result = analyzer.analyze_directory("/path/to/terraform/config") + +print(f"Complexity score: {result.score}/10") +print(f"Risk level: {result.risk_level}") +print(f"Factors:") +for factor in result.factors: + print(f" - {factor.name}: {factor.score} ({factor.description})") +``` + +#### ComplexityAnalyzer Methods + +| Method | Parameters | Returns | Description | +|--------|------------|---------|-------------| +| `analyze_directory` | `directory: str` | `ComplexityResult` | Analyzes directory complexity | +| `analyze_file` | `file_path: str` | `FileComplexity` | Analyzes single file complexity | +| `calculate_score` | `factors: Dict[str, float]` | `float` | Calculates overall complexity score | + +### Risk Assessment Module + +Evaluates upgrade risk based on complexity, dependencies, and usage patterns. + +```python +from tf_upgrade.risk_assessment import RiskAssessor + +# Create assessor +assessor = RiskAssessor() + +# Assess directory +result = assessor.assess_directory("/path/to/terraform/config") + +print(f"Risk score: {result.score}/10 ({result.risk_level})") +print(f"Estimated upgrade time: {result.estimated_time} minutes") +print("Risk factors:") +for factor in result.risk_factors: + print(f" - {factor.name}: {factor.impact} - {factor.description}") +``` + +#### RiskAssessor Methods + +| Method | Parameters | Returns | Description | +|--------|------------|---------|-------------| +| `assess_directory` | `directory: str` | `RiskAssessment` | Full risk assessment | +| `calculate_risk_score` | `complexity: float, factors: Dict[str, float]` | `float` | Calculate risk score | +| `estimate_upgrade_time` | `score: float, resource_count: int` | `int` | Estimate minutes required | + +### Upgrade Controller Module + +Manages the end-to-end upgrade process. + +```python +from tf_upgrade.upgrade_controller import UpgradeController + +# Create controller +controller = UpgradeController( + interactive=True, + create_backup=True, + target_version="1.0" +) + +# Perform upgrade +result = controller.upgrade_directory("/path/to/terraform/config") + +print(f"Upgrade success: {result.success}") +print(f"Versions upgraded: {', '.join(result.versions_upgraded)}") +if result.errors: + print("Errors encountered:") + for error in result.errors: + print(f" - {error}") +``` + +#### UpgradeController Options + +| Option | Type | Description | Default | +|--------|------|-------------|---------| +| `interactive` | `bool` | Confirm each step | `False` | +| `create_backup` | `bool` | Create backups before changes | `True` | +| `target_version` | `str` | Target Terraform version | `"1.0"` | +| `dry_run` | `bool` | Show changes without applying | `False` | +| `git_branch` | `Optional[str]` | Git branch for changes | `None` | + +#### UpgradeController Methods + +| Method | Parameters | Returns | Description | +|--------|------------|---------|-------------| +| `upgrade_directory` | `directory: str` | `UpgradeResult` | Upgrades entire directory | +| `dry_run` | `directory: str` | `DryRunResult` | Simulates upgrade changes | +| `calculate_upgrade_path` | `directory: str` | `List[str]` | Determines required versions | +| `create_backup` | `directory: str` | `str` | Creates backup directory | +| `restore_backup` | `directory: str, backup_path: str` | `bool` | Restores from backup | + +## Utility Modules + +### File Manager + +Handles file operations, backups, and transformations. + +```python +from tf_upgrade.utils.file_manager import FileManager + +# Create file manager +fm = FileManager() + +# Create backup +backup_path = fm.create_backup("/path/to/terraform/config") +print(f"Backup created at: {backup_path}") + +# Transform files +fm.transform_files( + directory="/path/to/terraform/config", + pattern="*.tf", + transformer=lambda content: content.replace("old", "new") +) + +# Restore backup if needed +fm.restore_backup("/path/to/terraform/config", backup_path) +``` + +### HCL Transformer + +Handles HCL syntax transformations using regex patterns and callable functions. + +```python +from tf_upgrade.utils.hcl_transformer import HCLTransformer + +# Create transformer +transformer = HCLTransformer() + +# Add transformation rules +transformer.add_rule( + name="provider_version_to_required_providers", + pattern=r'provider\s+"([^"]+)"\s+{[^}]*version\s+=\s+"([^"]+)"[^}]*}', + replacement=lambda m: f'provider "{m.group(1)}" {{}}\n\nterraform {{\n required_providers {{\n {m.group(1)} = {{\n version = "{m.group(2)}"\n }}\n }}\n}}' +) + +# Apply transformations +result = transformer.transform_file("/path/to/terraform/config/main.tf") +print(f"Changes applied: {result.changed}") +print(f"Rules matched: {', '.join(result.matched_rules)}") +``` + +### Provider Migration + +Handles provider declarations during the 0.13 upgrade process. + +```python +from tf_upgrade.utils.provider_migration import ProviderMigration + +# Create migration helper +pm = ProviderMigration() + +# Generate required_providers block from provider blocks +required_providers = pm.generate_required_providers( + directory="/path/to/terraform/config" +) + +print("Generated required_providers:") +print(required_providers) + +# Update provider references in module blocks +pm.update_module_provider_references( + directory="/path/to/terraform/config" +) +``` + +### Terraform Runner + +Executes Terraform commands with appropriate version handling. + +```python +from tf_upgrade.utils.terraform_runner import TerraformRunner + +# Create runner with specific version +runner = TerraformRunner(version="0.13") + +# Run commands +init_result = runner.init("/path/to/terraform/config") +validate_result = runner.validate("/path/to/terraform/config") +plan_result = runner.plan("/path/to/terraform/config") + +print(f"Init exitcode: {init_result.exitcode}") +print(f"Validation successful: {validate_result.success}") +print(f"Plan would change {plan_result.resource_changes} resources") +``` + +## Reporter Modules + +### Base Reporter + +Base class for all reporters. + +```python +from tf_upgrade.reporters.base import BaseReporter + +class CustomReporter(BaseReporter): + def __init__(self): + super().__init__() + + def add_section(self, title, content): + # Custom implementation + + def generate_report(self): + # Custom implementation + return "Generated report" +``` + +### Console Reporter + +Outputs information to the console with formatting. + +```python +from tf_upgrade.reporters.console import ConsoleReporter + +# Create reporter +reporter = ConsoleReporter( + use_colors=True, + use_emoji=True, + verbose=True +) + +# Report information +reporter.header("Scanning Results") +reporter.success("Found 10 Terraform configurations") +reporter.warning("3 configurations have high complexity") +reporter.error("1 configuration could not be parsed") +reporter.info("Use --verbose for more details") + +# Generate detailed report +reporter.report_scan_results({ + "/path/to/config1": {"version": "0.12.29", "resources": 15}, + "/path/to/config2": {"version": "0.13.5", "resources": 8} +}) +``` + +### Markdown Reporter + +Generates Markdown reports for documentation or GitHub. + +```python +from tf_upgrade.reporters.markdown import MarkdownReporter + +# Create reporter +reporter = MarkdownReporter() + +# Add content +reporter.add_header("Upgrade Report", level=1) +reporter.add_paragraph("This report summarizes the upgrade process.") +reporter.add_code_block("terraform validate", "Success! The configuration is valid.") + +# Add table +reporter.add_table( + headers=["Directory", "Original Version", "New Version", "Status"], + rows=[ + ["/path/to/config1", "0.12.29", "1.0.0", "✓"], + ["/path/to/config2", "0.13.5", "1.0.0", "✓"] + ] +) + +# Generate and save report +report = reporter.generate_report() +with open("upgrade-report.md", "w") as f: + f.write(report) +``` + +## Version Upgraders + +### Base Upgrader + +Base class for version-specific upgraders. + +```python +from tf_upgrade.version_upgraders.base import BaseUpgrader + +class CustomUpgrader(BaseUpgrader): + def __init__(self): + super().__init__( + source_version="0.12", + target_version="0.13" + ) + + def pre_upgrade_check(self, directory): + # Check if directory is ready for upgrade + return True + + def upgrade(self, directory): + # Implement upgrade logic + return self.upgrade_result(success=True) +``` + +### 0.13 Upgrader + +Handles upgrade from 0.12.x to 0.13.x. + +```python +from tf_upgrade.version_upgraders.v0_13 import V013Upgrader + +# Create upgrader +upgrader = V013Upgrader() + +# Check if directory is ready for upgrade +is_ready = upgrader.pre_upgrade_check("/path/to/terraform/config") +if is_ready: + # Perform upgrade + result = upgrader.upgrade("/path/to/terraform/config") + print(f"Upgrade success: {result.success}") + if result.success: + print(f"Changes made: {result.changes}") + else: + print(f"Error: {result.error}") +``` + +## Extension Points + +### Custom Transformations + +You can create custom transformations by extending the transformation system: + +```python +from tf_upgrade.utils.hcl_transformer import HCLTransformer, TransformRule + +# Create custom rule set +custom_rules = [ + TransformRule( + name="census_tag_format", + pattern=r'tags\s+=\s+{([^}]*)}', + replacement=lambda m: standardize_census_tags(m.group(1)) + ), + TransformRule( + name="custom_module_source", + pattern=r'source\s+=\s+"git::([^"]+)\?ref=([^"]+)"', + replacement=lambda m: f'source = "git::{m.group(1)}?ref=v1.0.0"' + ) +] + +# Create transformer with custom rules +transformer = HCLTransformer() +for rule in custom_rules: + transformer.add_rule(rule.name, rule.pattern, rule.replacement) + +# Apply transformations +transformer.transform_directory( + directory="/path/to/terraform/config", + pattern="*.tf" +) +``` + +### Custom Version Upgrader + +You can create custom upgraders for specific upgrade scenarios: + +```python +from tf_upgrade.version_upgraders.base import BaseUpgrader +from tf_upgrade.utils.hcl_transformer import HCLTransformer + +class CustomV013Upgrader(BaseUpgrader): + def __init__(self): + super().__init__( + source_version="0.12", + target_version="0.13" + ) + self.transformer = HCLTransformer() + + # Add custom rules for Census patterns + self.transformer.add_rule( + name="census_module_source", + pattern=r'source\s+=\s+"git::https://github\.com/census-bureau-internal/([^"]+)\.git\?ref=([^"]+)"', + replacement=lambda m: f'source = "git::https://github.com/census-bureau-internal/{m.group(1)}.git?ref=tf-upgrade"' + ) + + def upgrade(self, directory): + # Create backup + self.create_backup(directory) + + try: + # Apply custom transformations + self.transformer.transform_directory(directory, "*.tf") + + # Run standard 0.13upgrade command + self.terraform_runner.run_command( + directory=directory, + command=["0.13upgrade", "-yes"] + ) + + # Validate the upgraded configuration + validation = self.terraform_runner.validate(directory) + if not validation.success: + return self.upgrade_result( + success=False, + error=f"Validation failed: {validation.stderr}" + ) + + return self.upgrade_result(success=True) + + except Exception as e: + # Restore from backup in case of error + self.restore_backup(directory) + return self.upgrade_result(success=False, error=str(e)) +``` + +## Error Handling + +The tool uses custom exceptions for better error handling: + +```python +from tf_upgrade.exceptions import ( + UpgradeError, + ValidationError, + TerraformExecutionError, + BackupError +) + +try: + # Run potentially failing operation + result = controller.upgrade_directory("/path/to/terraform/config") +except ValidationError as e: + print(f"Validation failed: {e}") + print(f"At file: {e.file_path}, line {e.line_number}") +except TerraformExecutionError as e: + print(f"Terraform execution failed: {e}") + print(f"Command: {e.command}") + print(f"Output: {e.stderr}") +except BackupError as e: + print(f"Backup operation failed: {e}") + print(f"Backup path: {e.backup_path}") +except UpgradeError as e: + print(f"General upgrade error: {e}") +``` + +## Configuration Management + +Access and modify tool configuration: + +```python +from tf_upgrade.config import ConfigManager + +# Get configuration manager +config = ConfigManager() + +# Get configuration values +backup_enabled = config.get("backup.enabled", default=True) +github_token = config.get("github.token") +log_level = config.get("logging.level", default="info") + +# Set configuration values +config.set("backup.keep_days", 30) +config.set("github.organization", "census-bureau-internal") + +# Save configuration +config.save() + +# Reset to defaults +config.reset() +``` + +## Integration with GitHub + +Integrate with GitHub for project management: + +```python +from tf_upgrade.integrations.github import GitHubClient + +# Create client +github = GitHubClient( + token=os.environ.get("GITHUB_TOKEN"), + organization="census-bureau-internal", + project_name="terraform-upgrade" +) + +# List upgrade tickets +tickets = github.list_tickets(state="In Progress") +for ticket in tickets: + print(f"Ticket: {ticket.title} ({ticket.state})") + print(f" Directory: {ticket.metadata.get('directory')}") + print(f" Risk: {ticket.metadata.get('risk_level')}") + +# Create new ticket +ticket = github.create_ticket( + title="Upgrade module1 to Terraform 1.0", + description="Terraform upgrade for VPC module", + metadata={ + "directory": "/path/to/terraform/module1", + "risk_level": "Medium", + "complexity_score": 6.5 + } +) +print(f"Created ticket #{ticket.number}: {ticket.url}") + +# Update ticket status +github.update_ticket_status(ticket.number, "In Review") + +# Create pull request +pr = github.create_pull_request( + title="Terraform Upgrade: module1", + branch="tf-upgrade-module1", + base="main", + body="This PR upgrades module1 to Terraform 1.0", + ticket_number=ticket.number +) +print(f"Created PR #{pr.number}: {pr.url}") +``` + +## Command-Line Integration + +Create custom command-line commands: + +```python +import click +from tf_upgrade.cli import cli_group + +@cli_group.command("custom-scan") +@click.argument("directory", type=click.Path(exists=True)) +@click.option("--custom-option", help="Custom scan option") +def custom_scan(directory, custom_option): + """Perform a custom scan of Terraform configurations.""" + click.echo(f"Scanning {directory} with {custom_option}") + # Custom scan implementation + +@cli_group.command("custom-upgrade") +@click.argument("directory", type=click.Path(exists=True)) +@click.option("--special-handling", is_flag=True, help="Use special handling") +def custom_upgrade(directory, special_handling): + """Perform a custom upgrade process.""" + click.echo(f"Upgrading {directory}") + if special_handling: + click.echo("Using special handling") + # Custom upgrade implementation +``` diff --git a/docs/census-examples.md b/docs/census-examples.md new file mode 100644 index 00000000..3050e616 --- /dev/null +++ b/docs/census-examples.md @@ -0,0 +1,275 @@ +# Census Bureau Usage Examples + +This guide provides real-world examples of using the Terraform Upgrade Tool specifically for Census Bureau Terraform configurations. These examples cover common patterns and scenarios encountered in Census environments. + +## Table of Contents + +- [Module Migration Best Practices](#module-migration-best-practices) +- [Census Module Upgrade Matrix](#census-module-upgrade-matrix) +- [EDL-Specific Upgrade Patterns](#edl-specific-upgrade-patterns) +- [Census Security Implementation Patterns](#census-security-implementation-patterns) +- [Census AWS Account Structure Handling](#census-aws-account-structure-handling) +- [Advanced Census Configuration Patterns](#advanced-census-configuration-patterns) +- [Best Practices for Census-Specific Terraform Upgrades](#best-practices-for-census-specific-terraform-upgrades) +- [Resources](#resources) + +## Module Migration Best Practices + +This section provides guidance on upgrading commonly used Census Bureau modules. + +### Census Module Upgrade Matrix + +| Module | Original Version | Recommended Version | Notes | +|--------|------------------|---------------------|-------| +| terraform-aws-vpc-setup | v1.x | ref=tf-upgrade | Regional subnet changes | +| terraform-aws-s3 | v0.9.x | v1.0.0+ | New encryption settings | +| terraform-aws-iam-role | v0.5.x | v1.0.0+ | Permission boundaries | +| terraform-aws-edl-launch-instance | v0.8.x | ref=tf-upgrade | Updated AMI handling | +| terraform-aws-common-security-groups | v0.4.x | v1.0.0+ | New rule structures | +| terraform-aws-tls-certificate | v0.3.x | v1.0.0+ | Updated validation method | + +## EDL-Specific Upgrade Patterns + +### EDL Data Workflow Upgrade + +EDL data workflow configurations often have special patterns. Here's an example of upgrading an EDL data pipeline: + +```bash +# First, analyze the EDL workflow directory +make analyze DIR=/path/to/edl-workflow + +# Perform a targeted dry-run for EDL patterns +make dry-run DIR=/path/to/edl-workflow EDL_AWARE=true + +# Upgrade with EDL-specific handling +make upgrade DIR=/path/to/edl-workflow EDL_AWARE=true +``` + +The `EDL_AWARE` flag triggers special handling for EDL module sources and tag patterns. + +### EDL Module Version Reference Pattern + +For EDL modules, follow this version reference pattern after upgrading to Terraform 1.x: + +```terraform +module "edl_processing" { + source = "git::https://github.com/census-bureau-internal/terraform-aws-edl-workflow.git?ref=tf-upgrade" + + // Rest of module configuration +} +``` + +The `tf-upgrade` reference points to the 1.x-compatible branch maintained by the Census EDL team. + +## Census Security Implementation Patterns + +### VPC Security Group Pattern + +Census VPC security group configurations use a specific pattern. Here's an example upgrade: + +```terraform +# Before upgrade (0.12.x) +resource "aws_security_group" "app_sg" { + name = "${var.environment}-${var.app_name}-sg" + description = "Security group for ${var.app_name}" + vpc_id = var.vpc_id + + tags = { + "boc:application" = var.app_name + "boc:environment" = var.environment + "Name" = "${var.environment}-${var.app_name}-sg" + } +} + +# After upgrade (1.x) +resource "aws_security_group" "app_sg" { + name = "${var.environment}-${var.app_name}-sg" + description = "Security group for ${var.app_name}" + vpc_id = var.vpc_id + + tags = { + "boc:application" = var.app_name + "boc:environment" = var.environment + "Name" = "${var.environment}-${var.app_name}-sg" + } +} +``` + +Note that the security group resource itself doesn't change significantly, but associated rules and provider declarations would be updated. + +### IAM Role Pattern + +Census Bureau IAM roles follow a standard pattern. Here's how they're upgraded: + +```terraform +# Before upgrade (0.12.x) +module "app_role" { + source = "git::https://github.com/census-bureau-internal/terraform-aws-iam-role.git?ref=v0.5.0" + + name = "${var.environment}-${var.app_name}-role" + assume_role_policy = data.aws_iam_policy_document.assume_role.json + + policy_arns = [ + "arn:aws-us-gov:iam::aws:policy/AmazonS3ReadOnlyAccess", + aws_iam_policy.custom_policy.arn + ] + + tags = var.tags +} + +# After upgrade (1.x) +module "app_role" { + source = "git::https://github.com/census-bureau-internal/terraform-aws-iam-role.git?ref=v1.0.0" + + name = "${var.environment}-${var.app_name}-role" + assume_role_policy = data.aws_iam_policy_document.assume_role.json + + policy_arns = [ + "arn:aws-us-gov:iam::aws:policy/AmazonS3ReadOnlyAccess", + aws_iam_policy.custom_policy.arn + ] + + tags = var.tags +} +``` + +The module reference is updated to the 1.x-compatible version. + +## Census AWS Account Structure Handling + +### Cross-Account Resource Access Pattern + +Census Bureau uses a multi-account structure. Here's how to handle cross-account resource references during upgrades: + +```terraform +# Before upgrade (0.12.x) +data "terraform_remote_state" "network" { + backend = "s3" + config = { + bucket = "census-tfstate-${var.network_account_id}" + key = "network/terraform.tfstate" + region = "us-gov-west-1" + role_arn = "arn:aws-us-gov:iam::${var.network_account_id}:role/TerraformAccessRole" + } +} + +# After upgrade (1.x) +data "terraform_remote_state" "network" { + backend = "s3" + config = { + bucket = "census-tfstate-${var.network_account_id}" + key = "network/terraform.tfstate" + region = "us-gov-west-1" + role_arn = "arn:aws-us-gov:iam::${var.network_account_id}:role/TerraformAccessRole" + } +} +``` + +The remote state data source itself doesn't change, but the provider configuration would be updated to use the required_providers block. + +### Census AWS Profile Conventions + +When using the AWS profile for different Census accounts, follow this pattern: + +```bash +# List the available profiles +aws configure list-profiles + +# Export the profile before running the upgrade tool +export AWS_PROFILE=123456789012.AdministratorAccess + +# Run the upgrade with the profile +make upgrade DIR=/path/to/terraform/config +``` + +The tool will detect Census AWS profile naming conventions (account_id.role_name) and use them appropriately. + +## Advanced Census Configuration Patterns + +### Dynamic Backend Configuration + +Census often uses dynamic backend configurations. These are handled carefully during upgrade: + +```terraform +# Before upgrade (0.12.x) +terraform { + backend "s3" { + # These values are populated from environment variables + } +} + +# After upgrade (1.x) +terraform { + backend "s3" { + # These values are populated from environment variables + } + + required_version = ">= 1.0.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + } +} +``` + +The upgrade tool preserves the backend configuration while adding the required version constraints. + +### Census Tag Standardization + +Census has standard tag conventions that are preserved during upgrade: + +```terraform +# Common pattern for Census tag variables +variable "tags" { + type = map(string) + description = "Standard Census tags" + default = {} +} + +locals { + standard_tags = merge(var.tags, { + "boc:project" = var.project_name + "boc:cost-center" = var.cost_center + "boc:technical-poc" = var.tech_poc + "Name" = var.name + }) +} + +# This pattern is preserved during upgrade +``` + +## Best Practices for Census-Specific Terraform Upgrades + +1. **Upgrade Order**: + - Start with foundation modules (vpc, networking) + - Move to security modules (iam, security groups) + - Then upgrade application infrastructure + - Finally, upgrade pipeline and deployment configurations + +2. **AWS Profile Management**: + - Use a consistent AWS profile naming convention + - Maintain role-based profiles for different environments + - Test upgraded configurations with appropriate role permissions + +3. **Module Version References**: + - For Census modules, prefer specific version tags (e.g., `?ref=v1.0.0`) + - For modules still in transition, use the `tf-upgrade` branch + - After upgrade completion, migrate all references to specific versions + +4. **Validation Strategy**: + - Use `terraform validate` after each version upgrade step + - Compare resource counts with `terraform plan` before applying + - Test both successful creation and updates for key resources + +5. **Census-Specific Code Patterns**: + - Keep `boc:` tag prefixes consistent across resources + - Update comments to reflect current Census naming conventions + - Maintain standard file organization (variables.tf, outputs.tf, main.tf) + +## Resources + +- Census Bureau Module Repository: `https://github.com/census-bureau-internal/terraform-modules` +- Census Terraform Standards: `https://wiki.census.gov/terraform-standards` +- Terraform Provider Documentation for AWS GovCloud: `https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/custom-service-endpoints#aws-govcloud` diff --git a/docs/census-integration.md b/docs/census-integration.md new file mode 100644 index 00000000..9fd0fddb --- /dev/null +++ b/docs/census-integration.md @@ -0,0 +1,179 @@ +# Census Bureau Integration Guide + +This guide explains how the Terraform Upgrade Tool integrates with existing Census Bureau tools and practices. + +## Integration with Census Bureau Tools + +To ensure this upgrade tool works seamlessly with established Census Bureau practices and tooling, we integrate with two key existing systems: + +### tf-control Integration + +The Census Bureau uses a `tf-control.sh` script system to manage Terraform execution environments. Our tool integrates with this approach: + +#### Version Selection +- Detect and use the appropriate Terraform version by checking `.tf-control` files in: + - Current directory + - Git repository root + - User's home directory +- Respect the `TFCOMMAND` variable defined in these files for running Terraform commands + +#### Logging Compatibility +- Follow the established pattern of logging to the `logs/` directory +- Use timestamped log filenames matching the tf-control format +- Include similar header information (git repository, branch, etc.) + +#### Command Execution +- Run commands through the appropriate `tf-x` wrapper when applicable +- Ensure our tool's output can be consumed by tf-control's summary features + +### tf-run Integration + +The `tf-run.sh` script provides workflow automation for Terraform operations. Our tool integrates with this approach: + +#### AWS Profile Handling +- Use the same profile detection mechanism as in tf-run.sh +- Extract profile information from .tfvars files when environment variables aren't set + +#### Module Upgrade Readiness +- Prioritize upgrades based on the known upgrade-ready module list: + ``` + aws-common-security-groups + aws-edl-launch-instance + aws-iam-role + aws-iam-user + aws-inf-setup + aws-s3 + aws-setup-s3-object-logging + aws-tls-certificate + aws-vpc-setup + dns-lookup + aws-ecr-copy-images + ``` +- Check for modules that should use `ref=tf-upgrade` but don't yet + +#### Repository Structure +- Respect the existing repository organization patterns +- Detect use of linked files/modules through LINK commands + +## Census Bureau-Specific Patterns + +### AWS Account Structure + +The upgrade tool handles Census Bureau's multi-account AWS architecture: + +1. **Account Discovery** + - Detect and validate AWS accounts used in configurations + - Map configuration references to correct AWS accounts + - Handle cross-account resource references + +2. **AWS Profile Management** + - Match AWS profiles to correct accounts + - Handle various profile naming patterns + - Prioritize profiles with admin permissions + +3. **Cross-Partition Support** + - Support both commercial and GovCloud accounts + - Handle partition-specific formatting for ARNs + - Apply appropriate region formatting + +### Module References + +The tool handles Census Bureau-specific module reference patterns: + +1. **EDL Workflow Patterns** + - Identify `edl_*` module references + - Suggest appropriate module reference versions + - Handle EDL-specific configuration patterns + +2. **Module Source Formats** + - Handle different git reference formats (git::https://, git@github) + - Process ref= parameters in module sources + - Manage branch and tag reference recommendations + +3. **Infrastructure Setup Files** + - Process tf-run.data.bkp files and bootstrapping configurations + - Handle LINKTOP commands and references + - Process COMMAND, COMMENT pattern recognition + - Maintain integration with tf-directory-setup.py references + +### Tag Structure Handling + +The tool recognizes and preserves Census-specific tag patterns: + +1. **Standard Tag Formats** + - Preserve boc: prefixed tags + - Maintain CostAllocation tag formats + - Handle special tags like boc:dns:name, boc:dns:zone + +2. **Tag Value Formats** + - Upgrade string interpolation in tag values + - Preserve tag structure during upgrade + - Handle complex tag reference patterns + +## AWS Toolkit Integration + +### Profile Configuration +- Detect and use existing AWS profiles +- Validate profile credentials +- Cache credentials for better performance + +### Cross-Account Resource Management +- Identify and maintain cross-account dependencies +- Handle role assumption vs. direct profile access +- Preserve cross-account references during upgrades + +### S3 Backend Configuration +- Handle dynamic S3 backends +- Preserve interpolation in backend configurations +- Maintain complex key paths in backend blocks + +## Census-Specific Module Compatibility + +### Module Readiness +- Validate module compatibility with target Terraform versions +- Suggest appropriate module version references +- Identify modules that need special handling + +### Dependency Resolution +- Map complex module dependency chains +- Determine optimal upgrade order +- Detect and report circular dependencies + +### VPC Code Patterns +- Compatibility with VPC upgrade scripts +- Handle curl-based file downloads in scripts +- Process git::https to git@ format transitions +- Analyze ref=tf-upgrade references + +## Census Bureau Infrastructure Testing + +### Infrastructure Pattern Testing +- Test EC2 keypair generation patterns +- Verify DNS provider configuration upgrades +- Process DynamoDB table attribute patterns +- Validate container service definitions + +### Directory Structure Support +- Support account/vpc/region/vpc# organization +- Handle file relationships within nested structures +- Process output variable references between parent/child directories +- Respect organizational boundaries when planning upgrades + +## Implementation Components + +The following components provide Census Bureau-specific integration: + +1. **tf_upgrade/env_validator.py** + - Checks for .tf-control files + - Validates terraform binaries + - Extracts AWS profile settings + +2. **tf_upgrade/utils/terraform.py** + - Generates logs in tf-control format + - Determines correct terraform binary + - Extracts configuration information + +3. **tf_upgrade/utils/git.py** + - Validates Git repository access + - Sets up Git user configuration + - Creates branches for upgrades diff --git a/docs/code-architecture.md b/docs/code-architecture.md new file mode 100644 index 00000000..9b464110 --- /dev/null +++ b/docs/code-architecture.md @@ -0,0 +1,322 @@ +# Code Architecture + +This document details the architecture and design principles of the Terraform Upgrade Tool codebase. + +## Design Philosophy + +The Terraform Upgrade Tool is designed with several key principles in mind: + +1. **Modularity**: Clear separation of concerns between components +2. **Reliability**: Robust error handling and validation +3. **Safety**: Built-in safeguards to prevent data loss +4. **Clarity**: Clear, well-documented code and interfaces +5. **Pragmatism**: Focus on practical solutions over perfect abstractions + +## Architecture Overview + +The tool follows a modular architecture with these main components: + +1. **Command Line Interface**: Entry point for user interaction +2. **Assessment Tools**: Analyze and evaluate Terraform configurations +3. **Upgrade Controller**: Orchestrate the upgrade process +4. **Version-Specific Upgraders**: Handle version-specific transformations +5. **Common Utilities**: Provide shared functionality +6. **Reporters**: Format and output results + +## Component Details + +### Command Line Interface (CLI) + +The CLI module (`cli.py`) provides the user-facing interface using Click: + +``` +cli.py +├── scan - Identify Terraform directories +├── detect-version - Determine Terraform version +├── analyze-complexity - Assess configuration complexity +├── generate-graph - Create dependency visualizations +├── assess-risk - Calculate upgrade risk scores +├── dry-run - Preview changes without modifications +├── upgrade - Perform the actual upgrade +├── github-* - GitHub integration commands +└── utils - Helper commands and utilities +``` + +### Assessment Tools + +The assessment modules analyze Terraform configurations: + +``` +scanner.py - Recursively find Terraform configurations +complexity_analyzer.py - Calculate complexity metrics +version_detector.py - Determine Terraform version +dependency_graph.py - Analyze module dependencies +risk_assessment.py - Calculate risk scores +``` + +### Upgrade Controller + +The upgrade controller (`upgrade_controller.py`) coordinates the upgrade process: + +1. Determines the current version +2. Plans the upgrade path +3. Executes the appropriate upgraders in sequence +4. Tracks progress and handles errors +5. Generates reports on completion + +### Version-Specific Upgraders + +Each version upgrader handles transformation for a specific version: + +``` +version_upgraders/ +├── __init__.py - Common interfaces and factories +├── v0_13.py - 0.12 to 0.13 upgrade logic +├── v0_14.py - 0.13 to 0.14 upgrade logic +├── v0_15.py - 0.14 to 0.15 upgrade logic +└── v1_0.py - 0.15 to 1.0 upgrade logic +``` + +Each upgrader implements: +- Version-specific syntax transformations +- Provider compatibility handling +- Required command execution +- Pre and post validation + +### Common Utilities + +The utility modules provide shared functionality: + +``` +utils/ +├── file_manager.py - File operations and backups +├── git.py - Git repository operations +├── hcl_transformer.py - HCL syntax transformation +├── parallel.py - Parallel processing +├── provider_migration.py - Provider handling +├── terraform.py - Terraform operations +├── terraform_runner.py - Terraform command execution +└── validator.py - Configuration validation +``` + +### Reporters + +The reporter modules handle output formatting: + +``` +reporters/ +├── __init__.py - Common interfaces +├── console.py - Terminal output +├── file.py - File-based reporting +└── markdown.py - Markdown report generation +``` + +## Design Patterns + +The codebase employs several design patterns: + +### Command Pattern + +The CLI uses the Command pattern to encapsulate operations: + +```python +class Command(ABC): + @abstractmethod + def execute(self, context): + pass + +class ScanCommand(Command): + def execute(self, context): + # Scanning logic +``` + +### Strategy Pattern + +Version upgraders use the Strategy pattern: + +```python +class BaseUpgrader(ABC): + @abstractmethod + def upgrade(self, directory): + pass + +class V013Upgrader(BaseUpgrader): + def upgrade(self, directory): + # 0.13 specific upgrade logic +``` + +### Template Method Pattern + +The base upgrader uses the Template Method pattern: + +```python +class BaseUpgrader(ABC): + def upgrade(self, directory): + self.backup(directory) + self.pre_validation(directory) + self.transform_configuration(directory) + self.post_validation(directory) + self.report_results(directory) + + @abstractmethod + def transform_configuration(self, directory): + pass +``` + +### Observer Pattern + +Progress reporting uses the Observer pattern: + +```python +class ProgressSubject(ABC): + def __init__(self): + self._observers = [] + + def attach(self, observer): + self._observers.append(observer) + + def detach(self, observer): + self._observers.remove(observer) + + def notify(self, message, progress_pct): + for observer in self._observers: + observer.update(message, progress_pct) +``` + +### Factory Pattern + +Creating upgraders uses the Factory pattern: + +```python +class UpgraderFactory: + @staticmethod + def create_upgrader(version): + if version == "0.13": + return V013Upgrader() + elif version == "0.14": + return V014Upgrader() + # ...and so on +``` + +## Code Consolidation + +The code base includes several areas of consolidation to reduce duplication: + +### 1. Core Utility Consolidation + +File operations are consolidated in a central FileUtils class: + +```python +class FileUtils: + @staticmethod + def read_file(path): + # Common file reading logic + + @staticmethod + def write_file(path, content): + # Common file writing logic + + @staticmethod + def backup_file(path, backup_dir): + # Common backup logic +``` + +### 2. Version-Specific Upgraders Consolidation + +Common transformation patterns are extracted into a shared TransformationLibrary: + +```python +class TransformationLibrary: + @staticmethod + def replace_provider_syntax(content): + # Common provider transformation logic + + @staticmethod + def update_interpolation_syntax(content): + # Common interpolation transformation logic +``` + +### 3. Reporter Modules Consolidation + +A base Reporter class with common functionality: + +```python +class BaseReporter(ABC): + def __init__(self): + self.results = {} + + def add_result(self, key, value): + self.results[key] = value + + def get_result(self, key, default=None): + return self.results.get(key, default) + + @abstractmethod + def generate_report(self): + pass +``` + +## Dependency Flow + +The general flow of dependencies in the codebase is: + +``` +CLI → UpgradeController → VersionSpecificUpgraders → CommonUtilities → Reporters +``` + +This ensures clean separation of concerns and minimizes circular dependencies. + +## Configuration Management + +Configuration is managed centrally through a ConfigurationManager: + +```python +class ConfigurationManager: + def __init__(self, config_file=None): + self.config = self._load_config(config_file) + self.defaults = self._load_defaults() + + def get(self, key, default=None): + # Get configuration with fallback to defaults + + def set(self, key, value): + # Set and persist configuration +``` + +## Error Handling + +Errors are handled through a unified error handling framework: + +1. **Specific Exception Types**: + - `TerraformUpgradeError` as base class + - Specific subclasses for different error scenarios + +2. **Graceful Degradation**: + - Critical errors abort operations + - Non-critical errors log warnings and continue + +3. **Comprehensive Logging**: + - All errors logged with context + - Detailed error reports for diagnosis + +## Testing Architecture + +The testing architecture follows the same modular approach: + +``` +tests/ +├── unit/ - Unit tests for individual components +├── integration/ - Tests for component interaction +├── validation/ - End-to-end tests with real configurations +└── fixtures/ - Test configuration samples +``` + +## Future Improvements + +Areas identified for future architectural improvements: + +1. **Plugin Architecture**: Allow custom upgraders and reporters +2. **Dependency Injection**: Further reduce coupling between components +3. **Event System**: Replace direct observer pattern with a more flexible event system +4. **Metadata Storage**: Add structured metadata about upgrade operations +5. **Performance Optimizations**: Improve handling of large codebases diff --git a/docs/command-reference.md b/docs/command-reference.md new file mode 100644 index 00000000..f20d2a88 --- /dev/null +++ b/docs/command-reference.md @@ -0,0 +1,638 @@ +# Command Reference + +This document provides detailed information about all commands available in the Terraform Upgrade Tool, including usage examples, options, and expected outputs. + +## Common Flags for All Commands + +These flags can be used with any command: + +| Flag | Description | Default | +|------|-------------|---------| +| `-v, --verbose` | Enable verbose output | `False` | +| `--debug` | Enable debug logging | `False` | +| `-q, --quiet` | Suppress all output except errors | `False` | +| `--log-file PATH` | Write logs to specified file | `None` | +| `--config-file PATH` | Use specific config file | `~/.tf-upgrade/config.yaml` | + +## Core Commands + +### scan + +Scan directories for Terraform configurations that need upgrading. + +```bash +tf-upgrade scan [OPTIONS] [DIRECTORY] +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `-d, --depth INT` | Maximum directory depth for scanning | `5` | +| `--include TEXT` | File pattern to include | `*.tf` | +| `--exclude TEXT` | File pattern to exclude | `*.tfvars` | +| `--parallel` | Use parallel processing | `False` | +| `--output FORMAT` | Output format (text, json, csv, md) | `text` | +| `--report-file PATH` | Path to save detailed report | `scan-report-{timestamp}.md` | + +**Examples:** + +```bash +# Scan current directory +tf-upgrade scan + +# Scan specific directory with depth limit +tf-upgrade scan /path/to/terraform/repo --depth 3 + +# Scan and output results as JSON +tf-upgrade scan --output json + +# Scan with parallel processing +tf-upgrade scan --parallel +``` + +**Output:** + +``` +Scanning for Terraform configurations... +Found 15 Terraform configuration directories: + /path/to/terraform/repo/module1 (0.12.29) + /path/to/terraform/repo/module2 (0.13.5) + /path/to/terraform/repo/environments/dev (0.12.26) + ... + +Analysis complete. See detailed report at: scan-report-20230615-123045.md +``` + +### analyze + +Analyze a specific directory to assess upgrade complexity and risks. + +```bash +tf-upgrade analyze [OPTIONS] DIRECTORY +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--risk-threshold INT` | Risk threshold (1-10) | `5` | +| `--dependency-graph` | Generate dependency graph | `False` | +| `--graph-output PATH` | Path to save dependency graph | `dependencies.png` | +| `--output FORMAT` | Output format (text, json, md) | `text` | +| `--report-file PATH` | Path to save detailed report | `analysis-report-{timestamp}.md` | + +**Examples:** + +```bash +# Analyze directory +tf-upgrade analyze /path/to/terraform/repo/module1 + +# Analyze with dependency graph +tf-upgrade analyze /path/to/terraform/repo/module1 --dependency-graph + +# Save analysis report to custom file +tf-upgrade analyze /path/to/terraform/repo/module1 --report-file module1-analysis.md +``` + +**Output:** + +``` +Analyzing /path/to/terraform/repo/module1... + +Summary: +- Current Terraform version: 0.12.29 +- Resources: 24 +- Data sources: 8 +- Modules: 3 +- Providers: 2 (aws, null) +- Risk score: 6.5/10 (MEDIUM-HIGH) +- Estimated upgrade time: 25-35 minutes + +Risk factors: +- Complex provider configurations (HIGH) +- Module dependencies (MEDIUM) +- Dynamic blocks (MEDIUM) +- Custom module sources (LOW) + +See detailed report at: analysis-report-20230615-123045.md +``` + +### dry-run + +Perform a dry-run upgrade to see what changes would be made without modifying files. + +```bash +tf-upgrade dry-run [OPTIONS] DIRECTORY +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--target-version VERSION` | Target Terraform version | `1.0` | +| `--include-plan` | Include terraform plan output | `False` | +| `--output FORMAT` | Output format (text, json, md) | `text` | +| `--report-file PATH` | Path to save detailed report | `dry-run-report-{timestamp}.md` | + +**Examples:** + +```bash +# Perform dry-run +tf-upgrade dry-run /path/to/terraform/repo/module1 + +# Dry-run to specific version +tf-upgrade dry-run /path/to/terraform/repo/module1 --target-version 0.14 + +# Dry-run with terraform plan output +tf-upgrade dry-run /path/to/terraform/repo/module1 --include-plan +``` + +**Output:** + +``` +Performing dry-run upgrade of /path/to/terraform/repo/module1... + +Files that would be modified: +- main.tf (35 changes) +- providers.tf (12 changes) +- variables.tf (2 changes) + +Example changes: +1. Converting provider blocks to required_providers +2. Updating module source references +3. Updating deprecated syntax +4. Adding dependency lock file + +See detailed report at: dry-run-report-20230615-123045.md +``` + +### upgrade + +Perform the actual upgrade of a Terraform configuration. + +```bash +tf-upgrade upgrade [OPTIONS] DIRECTORY +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--target-version VERSION` | Target Terraform version | `1.0` | +| `-i, --interactive` | Interactive mode (confirm each step) | `False` | +| `--no-backup` | Skip backup creation (not recommended) | `False` | +| `--backup-dir PATH` | Custom backup directory | `.terraform-upgrade-backup-{timestamp}` | +| `--git-branch TEXT` | Create Git branch for changes | `None` | +| `--apply` | Run terraform apply after upgrade | `False` | +| `--report-file PATH` | Path to save detailed report | `upgrade-report-{timestamp}.md` | + +**Examples:** + +```bash +# Upgrade directory +tf-upgrade upgrade /path/to/terraform/repo/module1 + +# Interactive upgrade +tf-upgrade upgrade /path/to/terraform/repo/module1 --interactive + +# Upgrade to specific version +tf-upgrade upgrade /path/to/terraform/repo/module1 --target-version 0.14 + +# Upgrade with Git branch +tf-upgrade upgrade /path/to/terraform/repo/module1 --git-branch tf-upgrade-module1 +``` + +**Output:** + +``` +Upgrading /path/to/terraform/repo/module1... + +✓ Creating backup at .terraform-upgrade-backup-20230615-123045 +✓ Upgrading to Terraform 0.13 + - Updating provider declarations + - Running 0.13upgrade command + - Validating 0.13 compatibility +✓ Upgrading to Terraform 0.14 + - Creating dependency lock file + - Updating sensitive outputs + - Validating 0.14 compatibility +✓ Upgrading to Terraform 0.15 + - Updating deprecated function calls + - Validating 0.15 compatibility +✓ Upgrading to Terraform 1.0 + - Final compatibility updates + - Validating 1.0 compatibility + +Upgrade complete! See detailed report at: upgrade-report-20230615-123045.md +``` + +### restore + +Restore files from a backup created during an upgrade. + +```bash +tf-upgrade restore [OPTIONS] DIRECTORY +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--backup TEXT` | Specific backup to restore (timestamp or 'latest') | `latest` | +| `--list` | List available backups | `False` | +| `--force` | Force restore without confirmation | `False` | + +**Examples:** + +```bash +# Restore from latest backup +tf-upgrade restore /path/to/terraform/repo/module1 + +# List available backups +tf-upgrade restore /path/to/terraform/repo/module1 --list + +# Restore from specific backup +tf-upgrade restore /path/to/terraform/repo/module1 --backup 20230615-123045 +``` + +**Output:** + +``` +Available backups for /path/to/terraform/repo/module1: +- 20230615-123045 (35 minutes ago) +- 20230614-165530 (1 day ago) + +Restoring from backup 20230615-123045... +✓ Restored 5 files from backup +``` + +## GitHub Integration Commands + +### github-list + +List upgrade tickets from the GitHub project board. + +```bash +tf-upgrade github-list [OPTIONS] +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--state TEXT` | Filter by ticket state | `None` | +| `--assigned-to TEXT` | Filter by assignee | `None` | +| `--output FORMAT` | Output format (text, json, md) | `text` | + +**Examples:** + +```bash +# List all tickets +tf-upgrade github-list + +# List tickets in "In Progress" state +tf-upgrade github-list --state "In Progress" + +# List tickets assigned to specific user +tf-upgrade github-list --assigned-to "username" +``` + +**Output:** + +``` +GitHub upgrade tickets: + +Todo (5): +- module1: High risk, not started +- environments/dev: Medium risk, not started +- environments/stage: Medium risk, not started +- module2: Low risk, not started +- module3: Low risk, not started + +In Progress (2): +- environments/prod: Medium risk, @username +- shared-modules: High risk, @otheruser + +In Review (1): +- util-module: Low risk, PR #123 open + +Done (3): +- base-infra: Medium risk, completed 2 days ago +- security-groups: Low risk, completed 5 days ago +- iam-roles: Low risk, completed 1 week ago +``` + +### github-claim + +Claim a directory for upgrading (moves ticket to "In Progress"). + +```bash +tf-upgrade github-claim [OPTIONS] DIRECTORY +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--create` | Create ticket if it doesn't exist | `False` | +| `--assignee TEXT` | GitHub username to assign | `Current user` | + +**Examples:** + +```bash +# Claim directory +tf-upgrade github-claim /path/to/terraform/repo/module1 + +# Claim and create ticket if needed +tf-upgrade github-claim /path/to/terraform/repo/module1 --create + +# Claim for someone else +tf-upgrade github-claim /path/to/terraform/repo/module1 --assignee otheruser +``` + +**Output:** + +``` +Claiming /path/to/terraform/repo/module1 for upgrade... +✓ Ticket moved to "In Progress" +✓ Assigned to @username +``` + +### github-update + +Update ticket status for a directory. + +```bash +tf-upgrade github-update [OPTIONS] DIRECTORY +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--status TEXT` | New status for ticket | `Required` | +| `--comment TEXT` | Add comment to ticket | `None` | + +**Examples:** + +```bash +# Update status to "In Review" +tf-upgrade github-update /path/to/terraform/repo/module1 --status "In Review" + +# Update status with comment +tf-upgrade github-update /path/to/terraform/repo/module1 --status "Done" --comment "Upgrade completed successfully" +``` + +**Output:** + +``` +Updating status for /path/to/terraform/repo/module1... +✓ Ticket moved to "In Review" +✓ Comment added +``` + +### github-pr + +Create or update pull request for an upgraded directory. + +```bash +tf-upgrade github-pr [OPTIONS] DIRECTORY +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--branch TEXT` | Git branch to use | `tf-upgrade-{dirname}-{timestamp}` | +| `--title TEXT` | PR title | `Terraform Upgrade: {directory}` | +| `--update` | Update existing PR if found | `False` | +| `--draft` | Create as draft PR | `False` | + +**Examples:** + +```bash +# Create PR +tf-upgrade github-pr /path/to/terraform/repo/module1 + +# Create PR with custom branch +tf-upgrade github-pr /path/to/terraform/repo/module1 --branch custom-branch-name + +# Create draft PR +tf-upgrade github-pr /path/to/terraform/repo/module1 --draft +``` + +**Output:** + +``` +Creating pull request for /path/to/terraform/repo/module1... +✓ Changes committed to branch tf-upgrade-module1-20230615 +✓ Pull request created: #145 +✓ PR linked to upgrade ticket +``` + +## Utility Commands + +### verify-tools + +Verify that required tools are installed and available. + +```bash +tf-upgrade verify-tools [OPTIONS] +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--install` | Attempt to install missing tools | `False` | + +**Examples:** + +```bash +# Verify tools +tf-upgrade verify-tools + +# Verify and try to install missing tools +tf-upgrade verify-tools --install +``` + +**Output:** + +``` +Checking required tools... +✓ terraform found: /usr/bin/terraform (v1.0.0) +✓ terraform-0.13 found: /usr/bin/terraform-0.13 (v0.13.7) +✓ terraform-0.14 found: /usr/bin/terraform-0.14 (v0.14.11) +✓ terraform-0.15 found: /usr/bin/terraform-0.15 (v0.15.5) +✓ git found: /usr/bin/git (v2.30.2) +✓ python3 found: /usr/bin/python3 (v3.9.5) +✓ All critical tools are available! +``` + +### detect-version + +Detect the Terraform version used in a configuration. + +```bash +tf-upgrade detect-version [OPTIONS] DIRECTORY +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--output FORMAT` | Output format (text, json) | `text` | + +**Examples:** + +```bash +# Detect version +tf-upgrade detect-version /path/to/terraform/repo/module1 + +# Output in JSON format +tf-upgrade detect-version /path/to/terraform/repo/module1 --output json +``` + +**Output:** + +``` +Directory: /path/to/terraform/repo/module1 +Detected Terraform version: 0.12.29 +Version constraints found in: versions.tf +Required upgrade path: 0.12 → 0.13 → 0.14 → 0.15 → 1.0 +``` + +### generate-graph + +Generate a dependency graph for Terraform configurations. + +```bash +tf-upgrade generate-graph [OPTIONS] DIRECTORY +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--output PATH` | Output file path | `dependency-graph.png` | +| `--format FORMAT` | Output format (png, svg, dot, pdf) | `png` | +| `--include-providers` | Include providers in graph | `False` | +| `--include-variables` | Include variables in graph | `False` | + +**Examples:** + +```bash +# Generate default graph +tf-upgrade generate-graph /path/to/terraform/repo/module1 + +# Generate SVG with providers +tf-upgrade generate-graph /path/to/terraform/repo/module1 --format svg --include-providers +``` + +**Output:** + +``` +Analyzing dependencies in /path/to/terraform/repo/module1... +✓ Found 12 resources and 3 modules +✓ Generated dependency graph: dependency-graph.png +``` + +### config + +View and modify tool configuration. + +```bash +tf-upgrade config [OPTIONS] [COMMAND] +``` + +**Commands:** + +- `get KEY`: Get configuration value +- `set KEY VALUE`: Set configuration value +- `list`: List all configuration values +- `reset`: Reset to default configuration + +**Examples:** + +```bash +# List all configuration +tf-upgrade config list + +# Get specific configuration value +tf-upgrade config get github.token + +# Set configuration value +tf-upgrade config set backup.keep_days 30 + +# Reset to defaults +tf-upgrade config reset +``` + +**Output:** + +``` +Configuration: +- backup.auto_create: true +- backup.keep_days: 14 +- github.organization: census-bureau +- github.project_name: terraform-upgrade +- github.token: +- logging.level: info +- logging.file: logs/tf-upgrade.log +- terraform.versions_path: /usr/local/bin +``` + +## Makefile Integration + +For convenience, all commands are available through the project's Makefile: + +```bash +# Scan directories +make scan DIR=/path/to/terraform/repo + +# Analyze directory +make analyze DIR=/path/to/terraform/repo/module1 + +# Dry run upgrade +make dry-run DIR=/path/to/terraform/repo/module1 + +# Perform upgrade +make upgrade DIR=/path/to/terraform/repo/module1 + +# Interactive upgrade +make step DIR=/path/to/terraform/repo/module1 + +# GitHub operations +make github-list +make github-claim DIR=/path/to/terraform/repo/module1 +make github-update DIR=/path/to/terraform/repo/module1 STATUS="In Review" +make github-pr DIR=/path/to/terraform/repo/module1 +``` + +## Environment Variables + +The following environment variables affect tool behavior: + +| Variable | Description | Default | +|----------|-------------|---------| +| `AWS_PROFILE` | AWS profile to use | `None` | +| `GITHUB_TOKEN` | GitHub personal access token | `None` | +| `TF_UPGRADE_CONFIG` | Path to config file | `~/.tf-upgrade/config.yaml` | +| `TF_UPGRADE_VERBOSE` | Enable verbose output | `0` | +| `TF_UPGRADE_NO_BACKUP` | Skip backups (not recommended) | `0` | +| `TF_UPGRADE_LOG_FILE` | Path to log file | `None` | + +## Exit Codes + +| Code | Meaning | +|------|---------| +| `0` | Success | +| `1` | General error | +| `2` | Configuration error | +| `3` | Environment error (missing tools) | +| `4` | Permission error | +| `5` | Terraform error | +| `6` | Validation error | +| `7` | Git error | +| `8` | GitHub API error | diff --git a/docs/github-workflow.md b/docs/github-workflow.md new file mode 100644 index 00000000..4e7b6a3e --- /dev/null +++ b/docs/github-workflow.md @@ -0,0 +1,268 @@ +# GitHub Workflow Integration + +This document outlines how the Terraform Upgrade Tool integrates with GitHub for managing the upgrade process across multiple Terraform directories. + +## Overview + +The GitHub integration allows teams to systematically track and manage upgrades across many Terraform configurations using a project board. It automates status updates, PR creation, and reporting to ensure consistency and visibility. + +## GitHub Project Board Integration + +### Project Board Structure + +The upgrade tool integrates with a GitHub Enterprise project board using the following structure: + +1. **Ticket States** + - **Todo**: Initial state for directories needing upgrade + - **In Progress**: Directories currently being upgraded + - **In Review**: Upgrade completed, awaiting validation + - **Done**: Successfully upgraded and verified + +### Authentication Methods + +The tool supports several authentication methods for GitHub integration: + +1. **Personal Access Token (PAT)** + - Configure via the `GITHUB_TOKEN` environment variable + - Set scope requirements for repository and project access + - Store in secure credential storage when not in use + +2. **GitHub App Authentication** + - More secure than PATs for organizational use + - Configure via app ID and private key + - Set appropriate repository and project permissions + +3. **Environment Variable Configuration** + - Use `GITHUB_TOKEN` environment variable + - Support for organization-specific variables + - Integration with secure credential providers + +4. **.netrc File Credentials** + - Read credentials from standard .netrc file + - Support for multiple GitHub Enterprise instances + - Compatible with other Git-based tools + +## Upgrade Workflow + +### Automated State Transitions + +The tool automates the workflow through GitHub project tickets: + +1. **Discovery Phase** + - `tf-upgrade github-scan` - Identifies directories and creates tickets if they don't exist + - Creates tickets with appropriate metadata (complexity, risk score) + - Links related configurations by dependency graph + +2. **Upgrade Phase** + - `tf-upgrade start-upgrade ` - Claims and moves ticket to "In Progress" + - Records upgrade start time and assignee + - Updates ticket with initial assessment information + +3. **Completion Phase** + - `tf-upgrade complete-upgrade ` - Moves ticket to "In Review" and adds results + - Creates pull request with upgrade changes + - Links PR to ticket and adds summary of changes + +4. **Verification Phase** + - `tf-upgrade verify-upgrade ` - Moves ticket to "Done" after validation + - Records verification steps completed + - Updates ticket with final validation results + +### CLI Commands + +The following CLI commands support the GitHub integration: + +```bash +# List all upgrade tickets and their status +tf-upgrade github-list + +# Claim a directory for upgrading (moves to "In Progress") +tf-upgrade github-claim DIR= + +# Update ticket status +tf-upgrade github-update DIR= STATUS= + +# Add comment to ticket +tf-upgrade github-comment DIR= MESSAGE="comment" + +# Attach upgrade report to ticket +tf-upgrade github-attach-report DIR= +``` + +### Makefile Integration + +For simpler usage, the following Make targets are available: + +```bash +# List all upgrade tickets and their status +make github-list + +# Claim a directory for upgrading (moves to "In Progress") +make github-claim DIR= + +# Update ticket status +make github-update DIR= STATUS= + +# Add comment to ticket +make github-comment DIR= MESSAGE="comment" + +# Attach upgrade report to ticket +make github-attach-report DIR= +``` + +## Pull Request Integration + +### PR Creation + +The tool creates pull requests for upgrade changes with the following features: + +1. **PR Content** + - Title formatted as "Terraform Upgrade: " + - Description includes summary of changes made + - Before/after comparison of key files + - Validation results from `terraform validate` + +2. **PR Linking** + - Links PR to project ticket + - Updates ticket with PR URL + - Adds PR status to ticket metadata + +3. **PR Templates** + - Uses repository's PR template if available + - Populates template fields automatically + - Includes test plan and upgrade approach + +### Review Process + +The PR approach facilitates the review process through: + +1. **Change Visibility** + - Clear display of what files were changed + - Syntax highlighting for terraform changes + - Separate commits for each upgrade step + +2. **Validation Evidence** + - Includes results of validation checks + - Highlights any warnings or potential issues + - Shows plan output comparison (before/after) + +3. **Review Guidance** + - Indicates areas needing manual review + - Lists potential risks based on complexity analysis + - Provides upgrade-specific context for reviewers + +## Reporting and Feedback + +### GitHub-Compatible Reports + +The tool generates reports designed for GitHub: + +1. **Markdown Reports** + - Compatible with GitHub comment formatting + - Uses collapsible sections for detailed information + - Includes formatted code blocks with syntax highlighting + +2. **Summary Reports** + - Concise overview for ticket descriptions + - Status badges for quick visual indication + - Links to detailed logs and reports + +3. **Before/After Comparisons** + - Side-by-side comparisons where appropriate + - Highlighted changes in GitHub-friendly format + - Resource count and structural changes + +### Project-Level Reporting + +The tool supports aggregate reporting across the project: + +1. **Progress Tracking** + - Dashboard of upgrade status across all configurations + - Percentage completion metrics + - Estimation of remaining work + +2. **Issue Tracking** + - Common patterns of problems encountered + - Statistics on issue frequency + - Links to example resolutions + +3. **Timeline Visualization** + - Visual representation of upgrade progress + - Milestone tracking + - Trend analysis for planning + +## Implementation Components + +The GitHub integration is implemented through the following components: + +1. **GitHub Client Module** + - Located in `tf_upgrade/integrations/github.py` + - Handles API communication with GitHub + - Manages authentication and rate limiting + +2. **Configuration Settings** + - GitHub-specific settings in config files + - Organization and repository mapping + - Project board configuration + +3. **CLI Commands** + - GitHub-specific commands in CLI + - Integration with existing scan and upgrade commands + - Helper commands for GitHub operations + +4. **Progress Reporting** + - Extensions to reporter system for GitHub formatting + - GitHub-specific progress updates + - Integration with existing reporting framework + +## Getting Started + +To begin using the GitHub integration: + +1. **Configure Authentication** + ```bash + # Set GitHub token (recommended to use environment variable) + export GITHUB_TOKEN=your_personal_access_token + + # Or configure in config file + tf-upgrade config set github.token "your_personal_access_token" + ``` + +2. **Set Project Board** + ```bash + # Configure project board + tf-upgrade config set github.organization "census-bureau" + tf-upgrade config set github.project "terraform-upgrade" + ``` + +3. **Initialize Repository Mapping** + ```bash + # Scan repositories and update project + tf-upgrade github-init + ``` + +4. **Start Using GitHub Workflow** + ```bash + # List tickets + make github-list + + # Claim a directory + make github-claim DIR=/path/to/terraform/config + ``` + +## Security Considerations + +1. **Token Security** + - Never commit tokens to repositories + - Use environment variables or secure credential storage + - Set appropriate token expiration + +2. **Permissions** + - Use minimal required permissions for tokens/apps + - Consider read-only tokens for reporting operations + - Audit access regularly + +3. **Information Exposure** + - Be mindful of sensitive information in reports + - Configure what information is included in PR descriptions + - Use private repositories for sensitive configurations diff --git a/docs/implementation-guide.md b/docs/implementation-guide.md new file mode 100644 index 00000000..53077bb5 --- /dev/null +++ b/docs/implementation-guide.md @@ -0,0 +1,223 @@ +# Terraform Upgrade Tool Implementation Guide + +## Python Implementation Overview + +The Terraform Upgrade Tool is implemented in Python to provide robust error handling, logging, reporting, and modular design. This approach offers several advantages over shell scripts: + +- **Better Error Handling**: Structured exception handling instead of complex exit code checking +- **Improved Logging**: Comprehensive logging with different verbosity levels +- **Configuration Management**: Better handling of complex configuration options +- **Modular Design**: Separate modules for each upgrade step +- **Testing**: Easier to write unit tests for Python code than Bash scripts +- **Reporting**: Generate detailed reports of upgrade status +- **Parallelization**: Option to process multiple directories concurrently where safe + +## Core Components + +### Environment Verification Module +- Tool verification and version checking +- AWS profile validation +- Git repository access validation + +### Terraform Configuration Parser +- Identify required upgrades based on syntax analysis +- Extract provider requirements and module dependencies +- Build dependency graph for optimal upgrade ordering + +### Upgrade Process Controller +- Orchestrate the step-by-step upgrade process +- Handle version-specific upgrade requirements +- Manage rollbacks if issues are encountered + +### Reporting Engine +- Generate pre-upgrade assessment reports +- Track progress during upgrades +- Produce final upgrade summary reports + +## Directory Structure + +``` +tf_upgrade/ +├── __init__.py +├── cli.py # Command-line interface +├── complexity_analyzer.py # Analyzes configuration complexity +├── config.py # Configuration management +├── dependency_graph.py # Module dependency analysis +├── env_validator.py # Environment validation +├── reporters/ # Output formatting +│ ├── __init__.py +│ ├── console.py # Terminal output +│ ├── file.py # File-based reporting +│ └── markdown.py # Markdown report generation +├── risk_assessment.py # Risk evaluation +├── scanner.py # Directory scanning +├── upgrade_controller.py # Upgrade orchestration +├── utils/ # Utility functions +│ ├── __init__.py +│ ├── file_manager.py # File operations and backups +│ ├── git.py # Git repository operations +│ ├── hcl_transformer.py # HCL syntax transformation +│ ├── parallel.py # Parallel processing +│ ├── provider_migration.py # Provider handling +│ ├── terraform.py # Terraform operations +│ ├── terraform_runner.py # Terraform command execution +│ └── validator.py # Configuration validation +├── version_detector.py # Version detection +└── version_upgraders/ # Version-specific upgraders + ├── __init__.py + ├── v0_13.py # 0.13 upgrade logic + ├── v0_14.py # 0.14 upgrade logic + ├── v0_15.py # 0.15 upgrade logic + └── v1_0.py # 1.0 upgrade logic +``` + +## Key Implementation Features + +### One-Time Operation Focus + +This upgrade tool is designed as a one-time operation to transition all existing Terraform resources from 0.12.x to 1.x. The implementation focuses on reliability and clarity rather than long-term maintenance or complex extensibility. + +### Pull Request-Based Validation + +- Each directory upgrade generates a separate pull request +- Enables human validation before applying changes +- PR description includes: + - Summary of changes made + - Before/after comparison of key files + - Validation results from `terraform validate` + - List of potential issues requiring manual review + +### Dry Run Implementation + +The dry run functionality is comprehensive: + +```bash +# Through Makefile +make dry-run DIR=/path/to/directory +``` + +Dry run output includes: +- Files that would be modified +- Syntax changes that would be applied +- Provider references that would be updated +- Potential issues or warnings +- Estimated success probability + +### Simplified Makefile Interface + +Enhanced Makefile for non-technical users: + +```makefile +# Basic operations +scan: ## Scan for directories needing upgrade + python3 -m tf_upgrade.scan $(DIR) + +dry-run: ## Show what changes would be made without applying them + python3 -m tf_upgrade.dry_run $(DIR) + +upgrade: ## Perform the actual upgrade + python3 -m tf_upgrade.upgrade $(DIR) + +generate-pr: ## Generate pull requests for upgrades + python3 -m tf_upgrade.generate_pr $(DIR) + +# Help target for self-documentation +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +.DEFAULT_GOAL := help +``` + +### Clear CLI Prompting + +For user input scenarios, we implement clear prompts with suggestions: +- Show current version and available target versions +- Provide sensible defaults (incremental next version) +- Handle invalid input gracefully +- Use color coding for better readability + +## Common Upgrade Utilities + +### File Management System +- Creates backups of different file types +- Provides utilities for modifying files with transformers +- Implements file restoration from backup capabilities +- Tracks backup information + +### Terraform Command Runner +- Methods for common terraform commands +- Error handling and cleanup routines +- Integration with logging system +- Version-specific command execution support + +### HCL Transformer System +- Regex pattern and callable transformers +- Rule sets for each version migration +- Directory processing with pattern matching +- Tracking of applied transformations + +### Provider Migration Utilities +- Provider mappings for hashicorp providers +- Required_providers block generation +- Support for complex provider references +- Version constraint handling + +### Validation Framework +- Terraform plan parsing to detect resource changes +- Pre and post upgrade validation comparisons +- Validation rules for specific terraform versions +- Detailed reporting of validation issues + +## Version-Specific Upgraders + +Each version upgrader implements specialized logic for its target version: + +### v0_13 Upgrader +- Provider source declaration handling +- Integration with 0.13upgrade command +- Required version constraint management + +### v0_14 Upgrader +- Dependency lock file generation +- Sensitive output handling +- State file change management + +### v0_15 Upgrader +- Deprecated function replacements +- Removed feature handling +- Provider configuration updates + +### v1_0 Upgrader +- Final validation routines +- 1.0-specific compatibility fixes +- Overall compatibility assurance + +## Upgrade Controller + +The UpgradeController orchestrates the version-specific upgraders: +- Version detection and upgrade path determination +- Step-by-step upgrade orchestration +- Progress reporting and tracking +- Interactive mode for guided upgrades +- Comprehensive result reporting + +## Rollback Procedure + +### Git-Based Rollback +- Use Git history to revert to pre-upgrade state if needed +- Use `git log` to identify pre-upgrade commits +- Use `git checkout` or `git revert` to restore previous state + +### State Management +- If state was corrupted, use terraform state management commands to restore +- Use `terraform state pull > state-backup.tfstate` before upgrades +- Execute `terraform init -reconfigure` to reset providers + +## Documentation Focus + +Documentation is comprehensive and clearly written for non-technical users: +- Step-by-step instructions with examples +- Troubleshooting section for common issues +- Visual indicators of success/failure +- Explanation of the PR review process +- Command reference with explanations diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..507b666b --- /dev/null +++ b/docs/index.md @@ -0,0 +1,202 @@ +# Terraform Upgrade Tool Documentation + +## Documentation Index + +### Getting Started + +1. [Overview](overview.md) + - Introduction and purpose + - Key features + - Prerequisites + - Quick start + - Timeline and risk management + +2. [Installation Guide](installation.md) + - System requirements + - Basic installation + - Installing required Terraform versions + - AWS configuration + - Git configuration + - Docker installation + - Troubleshooting installation issues + +3. [Quick Start Guide](quick-start.md) + - Prerequisites + - Install and configure + - Verify environment + - Scan directories + - Assess complexity + - Perform upgrades + - Validate results + +### User Documentation + +4. [User Guide](user-guide.md) + - Getting started + - Understanding the upgrade process + - Environment setup + - Workflow examples + - Interpreting results + - Advanced usage + - GitHub integration + +5. [Command Reference](command-reference.md) + - Core commands + - Utility commands + - GitHub integration commands + - Options and flags + - Environment variables + - Makefile integration + - Exit codes + +6. [Troubleshooting Guide](troubleshooting.md) + - Environment setup issues + - AWS profile problems + - Terraform version issues + - Common upgrade errors + - Git-related issues + - Recovery procedures + - Reporting bugs + +### Technical Documentation + +7. [Upgrade Path](upgrade-path.md) + - Upgrade sequence + - Assessment phase + - Version-specific upgrades + - Post-upgrade verification + - Common upgrade issues + - Command reference + +8. [Implementation Guide](implementation-guide.md) + - Python implementation overview + - Core components + - Directory structure + - Key implementation features + - Common upgrade utilities + - Version-specific upgraders + - Upgrade controller + - Rollback procedure + +9. [Census Integration](census-integration.md) + - Integration with Census Bureau tools + - Census Bureau-specific patterns + - AWS toolkit integration + - Census-specific module compatibility + - Infrastructure testing + - Implementation components + +10. [Census Examples](census-examples.md) + - Module migration best practices + - EDL-specific upgrade patterns + - Security implementation patterns + - AWS account structure handling + - Advanced configuration patterns + - Best practices for Census-specific upgrades + +11. [API Reference](api-reference.md) + - Core modules + - Utility modules + - Reporter modules + - Version upgraders + - Extension points + - Error handling + - Configuration management + - GitHub integration + - Command-line integration + +### Development & Testing + +12. [Testing Strategy](testing-strategy.md) + - Testing approach + - Built-in safeguards + - Testing components + - Test fixtures + - Pre-release verification + - Production readiness + - Census Bureau-specific testing + - Verification script + +13. [GitHub Workflow](github-workflow.md) + - GitHub project board integration + - Authentication methods + - Upgrade workflow + - CLI commands + - Pull request integration + - Reporting and feedback + - Implementation components + - Getting started + - Security considerations + +14. [Code Architecture](code-architecture.md) + - Design philosophy + - Architecture overview + - Component details + - Design patterns + - Code consolidation + - Dependency flow + - Configuration management + - Error handling + - Testing architecture + +### Project Management + +15. [Implementation Checklist](../checklist.md) + - Project phases and status + - Task tracking + - Future work + - Code consolidation plan + +16. [Test Plan](../testplan.md) + - Test environment setup + - Test cases + - AWS account management testing + - Census Bureau-specific pattern testing + - Edge cases and error handling + - AWS toolkit integration + +17. [Contributing Guide](../CONTRIBUTING.md) + - Code organization + - Design patterns + - Code style + - Testing + - Pull request process + - Consolidation roadmap + +## References + +- [HashiCorp Terraform Upgrade Guides](https://www.terraform.io/upgrade-guides) +- [Census Bureau Terraform Guidelines]() +- AWS Profile Documentation: `/apps/terraform/workspaces/morga471/terraform/docs/aws-profiles.md` +- Internal documentation and best practices + +## Command Quick Reference + +```bash +# Verify tools installation +make verify-tools + +# Check AWS profiles +aws configure list-profiles + +# Scan for directories needing upgrade +make scan DIR=/path/to/terraform + +# Perform dry run to see what would change +make dry-run DIR=/path/to/terraform + +# Upgrade specific directory +make upgrade-dir DIR=/path/to/terraform/config + +# Run step-by-step interactive upgrade +make step DIR=/path/to/terraform/config +``` + +## Tool Status + +Current implementation status: +- **Core functionality**: ✅ COMPLETE +- **Validation and testing**: ⏳ IN PROGRESS +- **Documentation**: ✅ COMPLETE +- **GitHub integration**: ⬜ NOT STARTED +- **Test case management**: ⬜ NOT STARTED diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 00000000..61bb87e0 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,229 @@ +# Installation Guide + +This guide provides detailed instructions for installing and configuring the Terraform Upgrade Tool in various environments. + +## System Requirements + +- **Operating System**: Linux, macOS, or Windows with WSL +- **Python**: Version 3.8 or newer +- **Terraform**: Access to multiple Terraform versions (0.12, 0.13, 0.14, 0.15, 1.0) +- **Git**: Version 2.0 or newer +- **AWS CLI**: Version 2.0 or newer (if working with AWS resources) + +## Basic Installation + +### 1. Clone the Repository + +```bash +git clone https://github.com/census-bureau/terraform-upgrade-tool.git +cd terraform-upgrade-tool +``` + +### 2. Create Python Virtual Environment (Recommended) + +```bash +python3 -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +``` + +### 3. Install the Tool + +```bash +pip install -e . +``` + +This installs the tool in development mode, allowing you to modify the code if needed. + +### 4. Verify Installation + +```bash +tf-upgrade --version +``` + +You should see the current version of the tool displayed. + +## Installing Required Terraform Versions + +The tool requires multiple Terraform versions to be available. Here are several ways to install them: + +### Method 1: Manual Installation + +```bash +# Create a directory for versions +mkdir -p ~/terraform-versions +cd ~/terraform-versions + +# Download and install specific versions +for version in "0.12.31" "0.13.7" "0.14.11" "0.15.5" "1.0.11"; do + wget "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_linux_amd64.zip" + unzip "terraform_${version}_linux_amd64.zip" -d "terraform-${version%.*}" + sudo mv "terraform-${version%.*}/terraform" "/usr/local/bin/terraform-${version%.*}" + rm -rf "terraform-${version%.*}" "terraform_${version}_linux_amd64.zip" +done +``` + +### Method 2: Using tfenv (Recommended) + +[tfenv](https://github.com/tfutils/tfenv) is a version manager for Terraform: + +```bash +# Install tfenv +git clone https://github.com/tfutils/tfenv.git ~/.tfenv +echo 'export PATH="$HOME/.tfenv/bin:$PATH"' >> ~/.bashrc +source ~/.bashrc + +# Install required Terraform versions +tfenv install 0.12.31 +tfenv install 0.13.7 +tfenv install 0.14.11 +tfenv install 0.15.5 +tfenv install 1.0.11 + +# Create symlinks (required for the tool) +sudo ln -s ~/.tfenv/versions/0.12.31/terraform /usr/local/bin/terraform-0.12 +sudo ln -s ~/.tfenv/versions/0.13.7/terraform /usr/local/bin/terraform-0.13 +sudo ln -s ~/.tfenv/versions/0.14.11/terraform /usr/local/bin/terraform-0.14 +sudo ln -s ~/.tfenv/versions/0.15.5/terraform /usr/local/bin/terraform-0.15 +sudo ln -s ~/.tfenv/versions/1.0.11/terraform /usr/local/bin/terraform-1.0 +``` + +### Method 3: Use the Built-in Installer (Coming Soon) + +The tool will eventually include a built-in installer for required Terraform versions: + +```bash +tf-upgrade install-terraform-versions +``` + +## Verifying Required Tools + +Once installed, verify that all required tools are available: + +```bash +tf-upgrade verify-tools +``` + +This will check for: +- All required Terraform versions +- Git +- Python + +## AWS Configuration + +If you're working with AWS resources, ensure your AWS CLI is properly configured: + +1. Check for existing profiles: + ```bash + aws configure list-profiles + ``` + +2. Create a new profile if needed: + ```bash + aws configure --profile your-profile-name + ``` + +3. For Census Bureau environments, import credentials from SSO: + ```bash + aws sso login --profile your-profile-name + ``` + +## Git Configuration + +Ensure Git is properly configured, especially if using the GitHub integration features: + +```bash +git config --global user.name "Your Name" +git config --global user.email "your.email@example.com" +``` + +If you're using GitHub integration, create and configure a personal access token: + +1. Create a token at GitHub (Settings > Developer settings > Personal access tokens) +2. Configure the token in the tool: + ```bash + export GITHUB_TOKEN=your_token_here + # Or store it in the tool's configuration: + tf-upgrade config set github.token "your_token_here" + ``` + +## Docker Installation (Alternative) + +For isolated environments, you can use Docker: + +```bash +# Build the Docker image +docker build -t terraform-upgrade-tool . + +# Run the tool +docker run -v $(pwd):/workspace terraform-upgrade-tool scan /workspace +``` + +## Troubleshooting Installation Issues + +### Missing Terraform Versions + +If `tf-upgrade verify-tools` indicates missing Terraform versions: + +1. Check the paths where the tool is looking: + ```bash + which terraform + which terraform-0.13 # Should exist if installation was successful + ``` + +2. Create symlinks if the binaries exist but aren't in the expected location: + ```bash + sudo ln -s /path/to/terraform-0.13 /usr/local/bin/terraform-0.13 + ``` + +### Python Package Issues + +If you encounter Python package errors: + +```bash +# Upgrade pip +pip install --upgrade pip + +# Install dependencies manually +pip install click pyyaml networkx +``` + +### Permission Issues + +If you encounter permission errors: + +```bash +# For local installation without sudo +pip install --user -e . + +# Update PATH to include user bin directory +export PATH=$PATH:$HOME/.local/bin +``` + +## Enterprise Installation + +For enterprise environments, consider these additional steps: + +1. **Central Installation**: + Install the tool on a shared server with appropriate permissions. + +2. **Configuration Management**: + Create a centralized configuration file for consistent settings. + +3. **CI/CD Integration**: + Integrate the tool into CI/CD pipelines for automated upgrades. + +4. **SSO Authentication**: + Configure AWS and GitHub authentication using enterprise SSO systems. + +## Next Steps + +After installation, continue with these steps: + +1. Read the [Quick Start Guide](quick-start.md) for basic usage +2. Configure your environment for optimal use +3. Run a basic scan to verify everything works correctly: + ```bash + tf-upgrade scan /path/to/sample/terraform + ``` + +For more detailed usage information, refer to the [User Guide](user-guide.md). diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 00000000..8962333a --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,90 @@ +# Terraform Upgrade Tool Overview + +## Introduction + +This document provides a high-level overview of the Terraform Upgrade Tool, which automates the process of upgrading Terraform configurations from version 0.12.x to 1.x following the Census Bureau's recommended upgrade process. + +## Purpose + +The Terraform Upgrade Tool is designed as a one-time operation to transition all existing Terraform resources from 0.12.x to 1.x in a systematic, safe manner. It guides users through a step-by-step upgrade process, providing safeguards and validation at each step. + +## Key Features + +- **Complete Upgrade Path**: Automated upgrades through each intermediate version (0.13, 0.14, 0.15, 1.x) +- **Built-in Safeguards**: Automatic backups, dry-run functionality, and interactive mode +- **Configuration Assessment**: Analysis of complexity and risk before upgrading +- **Detailed Reporting**: Comprehensive reports on changes made during upgrades +- **Census Bureau Integration**: Works with existing Census Bureau Terraform workflows +- **GitHub Workflow Support**: Manages upgrade tickets and pull requests (coming soon) + +## Prerequisites + +Before beginning the upgrade process, ensure you have: + +- ✅ Required tools installed and verified via `make verify-tools` +- ✅ AWS profiles configured in `~/.aws/config` +- ✅ Access to all Terraform repositories that need upgrading +- ✅ Testing environments to validate upgraded configurations +- ✅ Completed all pending infrastructure changes + +## Quick Start + +1. **Verify your environment**: + ```bash + make verify-tools + ``` + +2. **Scan your Terraform directories**: + ```bash + make scan DIR=/path/to/terraform + ``` + +3. **Try a dry run first**: + ```bash + make dry-run DIR=/path/to/terraform + ``` + +4. **Perform the upgrade**: + ```bash + make upgrade DIR=/path/to/terraform + ``` + +## Timeline + +| Stage | Timeframe | Key Deliverables | +|-------|-----------|------------------| +| Inventory & Assessment | Week 1-2 | Complete scan report, risk assessment | +| Pre-Upgrade Testing | Week 3 | Testing environments, dry-run reports | +| 0.12.x to 0.13.x | Week 4-5 | Upgraded configurations, validation report | +| 0.13.x to 0.14.x | Week 6-7 | Upgraded configurations, validation report | +| 0.14.x to 0.15.x | Week 8-9 | Upgraded configurations, validation report | +| 0.15.x to 1.x | Week 10-11 | Fully upgraded configurations | +| Verification & Documentation | Week 12 | Final validation reports, updated documentation | + +## Risk Management + +| Risk | Impact | Mitigation | +|------|--------|------------| +| State file corruption | High | Regular backups, testing in isolated environment | +| Provider incompatibilities | Medium | Thorough version compatibility research | +| Resource drift | Medium | Detailed planning and review of each `terraform plan` | +| Downtime during transition | Medium | Schedule upgrades during maintenance windows | +| Custom module failures | Medium | Test modules independently before integration | + +## Documentation Index + +For more detailed information, see the following documents: + +- [Upgrade Path](upgrade-path.md): Detailed information on version-specific upgrades +- [Implementation Guide](implementation-guide.md): Technical implementation details +- [Census Integration](census-integration.md): Census Bureau specific tools and patterns +- [Testing Strategy](testing-strategy.md): Testing approach and validation process +- [GitHub Workflow](github-workflow.md): Managing upgrades with GitHub +- [Code Architecture](code-architecture.md): Code organization and best practices + +## References + +- [HashiCorp Terraform Upgrade Guides](https://www.terraform.io/upgrade-guides) +- [Census Bureau Terraform Guidelines]() +- [Provider Version Compatibility Matrix]() +- AWS Profile Documentation: `/apps/terraform/workspaces/morga471/terraform/docs/aws-profiles.md` diff --git a/docs/quick-start.md b/docs/quick-start.md new file mode 100644 index 00000000..a139e7fe --- /dev/null +++ b/docs/quick-start.md @@ -0,0 +1,236 @@ +# Quick Start Guide + +This guide provides detailed step-by-step instructions for using the Terraform Upgrade Tool to upgrade your Terraform configurations from version 0.12.x to 1.x. + +## Prerequisites + +Before beginning, ensure you have: + +1. **Required Tools**: + - Python 3.8 or newer + - Terraform binaries for versions 0.12.x, 0.13.x, 0.14.x, 0.15.x, and 1.x + - Git + - AWS CLI (configured with appropriate profiles) + +2. **Environment Setup**: + - Proper AWS credentials with access to the necessary accounts + - Git repository access for the Terraform configurations + - `.tf-control` files correctly configured (if used) + +## Step 1: Install and Configure the Tool + +1. **Clone the repository**: + ```bash + git clone https://github.com/census-bureau/terraform-upgrade-tool.git + cd terraform-upgrade-tool + ``` + +2. **Install dependencies**: + ```bash + pip install -e . + ``` + +3. **Verify the installation**: + ```bash + tf-upgrade --version + ``` + +## Step 2: Verify Your Environment + +1. **Check for required tools**: + ```bash + make verify-tools + ``` + + You should see output similar to: + ``` + ✅ terraform found: /usr/bin/terraform + ✅ terraform-0.13 found: /usr/bin/terraform-0.13 + ✅ terraform-0.14 found: /usr/bin/terraform-0.14 + ✅ terraform-0.15 found: /usr/bin/terraform-0.15 + ✅ terraform-1.0 found: /usr/bin/terraform-1.0 + ✅ git found: /usr/bin/git + ✅ python3 found: /usr/bin/python3 + ✅ All critical tools are available! + ``` + + If any tools are missing, install them before proceeding. + +2. **Verify AWS profiles**: + ```bash + aws configure list-profiles + ``` + + Ensure you have the necessary profiles for your environments. + +3. **Export the appropriate AWS profile**: + ```bash + export AWS_PROFILE=your-profile-name + ``` + + Replace `your-profile-name` with the profile you want to use. + +## Step 3: Scan for Terraform Directories + +Scan your repository to identify directories containing Terraform configurations: + +```bash +make scan DIR=/path/to/your/terraform/repo +``` + +This will produce output similar to: + +``` +Scanning for Terraform configurations... +Found 15 Terraform configuration directories: + /path/to/your/terraform/repo/module1 + /path/to/your/terraform/repo/module2 + /path/to/your/terraform/repo/environments/dev + /path/to/your/terraform/repo/environments/prod + ... + +Analysis complete. See detailed report at: scan-report-20230615-123045.md +``` + +The report will include: +- Current Terraform version for each directory +- Required upgrade steps +- Risk assessment score +- Complexity analysis + +## Step 4: Assess a Specific Directory + +Before upgrading, perform a more detailed assessment of a specific directory: + +```bash +make analyze DIR=/path/to/your/terraform/repo/module1 +``` + +This will provide: +- Detailed complexity metrics +- Provider analysis +- Deprecated features detection +- Module dependencies + +## Step 5: Perform a Dry Run + +Before making any actual changes, perform a dry run to see what would change: + +```bash +make dry-run DIR=/path/to/your/terraform/repo/module1 +``` + +This will show: +- Files that would be modified +- Specific changes that would be made to each file +- Potential issues or warnings +- Success probability estimate + +Review the dry run output carefully to understand the changes that will be made. + +## Step 6: Upgrade a Directory + +When you're ready to perform the actual upgrade: + +```bash +make upgrade DIR=/path/to/your/terraform/repo/module1 +``` + +For a more controlled approach with confirmation at each step: + +```bash +make step DIR=/path/to/your/terraform/repo/module1 +``` + +The upgrade process: +1. Creates a backup of your files +2. Performs pre-upgrade validation +3. Upgrades to Terraform 0.13 +4. Validates the 0.13 configuration +5. Upgrades to Terraform 0.14 +6. Validates the 0.14 configuration +7. Upgrades to Terraform 0.15 +8. Validates the 0.15 configuration +9. Upgrades to Terraform 1.0 +10. Performs final validation +11. Generates a comprehensive report + +## Step 7: Validate the Upgraded Configuration + +After upgrading, validate the configuration with Terraform: + +```bash +cd /path/to/your/terraform/repo/module1 +terraform init +terraform validate +terraform plan +``` + +Ensure: +- No validation errors occur +- Plan output shows no unexpected changes +- Resource counts match pre-upgrade expectations + +## Step 8: Review the Upgrade Report + +Each upgrade generates a detailed report: + +```bash +cat upgrade-report-module1-20230615-124530.md +``` + +This report includes: +- Summary of changes made +- Files modified +- Before/after comparisons +- Validation results +- Issues encountered and resolutions +- Next steps + +## Common Workflows + +### Upgrading Multiple Directories with Common Patterns + +For repositories with multiple similar directories: + +1. **Start with a representative sample**: + ```bash + make upgrade DIR=/path/to/your/terraform/repo/environments/dev + ``` + +2. **Apply similar patterns to other directories**: + ```bash + make upgrade DIR=/path/to/your/terraform/repo/environments/test + make upgrade DIR=/path/to/your/terraform/repo/environments/prod + ``` + +### Working with Git Integration + +To create Git branches and PRs during upgrade: + +1. **Set up a branch for your changes**: + ```bash + make git-setup DIR=/path/to/your/terraform/repo/module1 + ``` + +2. **Perform the upgrade with Git integration**: + ```bash + make upgrade-with-git DIR=/path/to/your/terraform/repo/module1 + ``` + +3. **Generate a pull request** (if GitHub integration is enabled): + ```bash + make github-pr DIR=/path/to/your/terraform/repo/module1 + ``` + +## Next Steps + +After successfully upgrading a directory: + +1. Review all validation warnings and errors +2. Update any documentation referring to Terraform versions +3. Update CI/CD pipelines to use Terraform 1.x +4. Consider upgrading related modules that might depend on these configurations +5. Run a full deployment test in a non-production environment + +For more detailed information, see the [Implementation Guide](implementation-guide.md) and [Troubleshooting Guide](troubleshooting.md). diff --git a/docs/testing-strategy.md b/docs/testing-strategy.md new file mode 100644 index 00000000..5c646508 --- /dev/null +++ b/docs/testing-strategy.md @@ -0,0 +1,201 @@ +# Testing Strategy + +This document outlines the testing approach for the Terraform Upgrade Tool. + +## Testing Approach + +Given that this tool is designed for a one-time operation to upgrade Terraform configurations, we focus on practical validation and built-in safeguards rather than extensive automated test suites. + +### Built-in Safeguards + +1. **Automatic Backups** + - Every operation creates comprehensive backups before making changes + - Backup files are stored with timestamps for easy identification + - Restore capabilities are built into the FileManager class + +2. **Comprehensive Dry-Run Functionality** + - The `dry-run` command shows exactly what changes would be made + - Syntax transformations are previewed without altering files + - Built-in commands are listed with their expected effects + +3. **Step-by-Step Interactive Mode** + - The `-i` / `--interactive` flag allows confirming each step + - Users can see and approve each version upgrade individually + - Provides an opportunity to verify changes at each stage + +4. **Git-Based Workflow** + - Changes are made in new branches for easy comparison and rollback + - The original state is preserved in the main branch + - Changes can be reviewed through pull requests + +## Testing Components + +### Unit Tests + +Unit tests have been implemented for all core components: + +- **FileManager**: Tests backup, restore, and file transformation functionality +- **TerraformRunner**: Tests command execution with version-specific binaries +- **HCLTransformer**: Tests regex and callable transformations for HCL syntax +- **ProviderMigration**: Tests provider block conversion for 0.13+ compatibility +- **TerraformValidator**: Tests validation of configurations before and after upgrade + +Unit tests can be run with: +```bash +make test-unit +``` + +### Integration Tests + +Integration tests verify that components work together correctly: + +- Complete upgrade path (0.12 → 0.13 → 0.14 → 0.15 → 1.0) +- Step-by-step upgrades with validation at each stage +- Error handling and recovery during multi-step processes +- Configuration transformation and validation integration + +Integration tests can be run with: +```bash +make test-integration +``` + +### Validation Tests + +Validation tests ensure that upgrades produce correct results: + +- Simple configurations (single resource, minimal complexity) +- Medium configurations (multiple resources, basic modules) +- Complex configurations (multiple providers, count/for_each, complex modules) +- Verification that plans don't show unexpected resource changes + +Validation tests can be run with: +```bash +make test-validation +``` + +## Test Fixtures + +Test fixtures have been created for various complexity levels: + +### Simple Fixtures +- Basic provider and resource configuration +- Single module references +- Minimal dependencies + +### Medium Fixtures +- Multiple resources with interdependencies +- Basic module usage +- Simple variable references + +### Complex Fixtures +- Multiple providers +- Nested modules +- Dynamic block usage +- Complex expressions + +## Pre-Release Verification + +Before deploying the tool for widespread use, a structured verification process should be performed: + +### Controlled Verification Process + +1. **Select Representative Test Cases** + - Choose 3-5 Terraform configurations of varying complexity + - Include configurations using features from each Terraform version + +2. **Document the Baseline State** + - Run `terraform validate` and record output + - Run `terraform plan` and record resource counts + - Document any existing warnings + - Save copies of the original files for comparison + +3. **Verification Checklist** + - Create a verification checklist covering: + - Tool Installation & Dependencies + - Command Line Interface + - Analysis Functions + - Upgrade Functions + - Reporting Functions + - Error Handling & Recovery + +### Verification Matrix + +Create a verification matrix for each test configuration: + +| Feature | Simple Config | Medium Config | Complex Config | +|---------|---------------|---------------|----------------| +| Analysis scan | ✓/✗ | ✓/✗ | ✓/✗ | +| Version detection | ✓/✗ | ✓/✗ | ✓/✗ | +| Dry-run accuracy | ✓/✗ | ✓/✗ | ✓/✗ | +| Backup creation | ✓/✗ | ✓/✗ | ✓/✗ | +| Provider migration | ✓/✗ | ✓/✗ | ✓/✗ | +| Syntax transformation | ✓/✗ | ✓/✗ | ✓/✗ | +| Init/Apply success | ✓/✗ | ✓/✗ | ✓/✗ | +| Report generation | ✓/✗ | ✓/✗ | ✓/✗ | + +## Production Readiness + +Final checklist before authorizing production use: + +- ☐ All critical features working as expected +- ☐ Backups are created reliably +- ☐ Dry-run output matches actual changes +- ☐ Interactive mode functions properly +- ☐ Reports are generated with sufficient detail +- ☐ All error conditions are handled gracefully +- ☐ Documentation is clear and comprehensive +- ☐ Configuration examples are provided +- ☐ No critical bugs remain unfixed + +## Census Bureau-Specific Testing + +The [testplan.md](../testplan.md) document contains detailed Census Bureau-specific test cases, including: + +- AWS account management testing +- Census Bureau-specific pattern testing +- EDL workflow testing +- AWS provider with endpoints testing +- S3 backend with dynamic config testing +- Module reference upgrades +- Tag structure handling +- Infrastructure pattern testing + +Refer to this document for a comprehensive test strategy specific to the Census Bureau environment. + +## Verification Script + +A verification script has been created that automates parts of the testing process: + +```bash +#!/bin/bash +# Pre-release verification script for Terraform Upgrade Tool + +# Configuration +TEST_DIRS=("test-fixtures/simple" "test-fixtures/medium" "test-fixtures/complex") +TOOL_CMD="tf-upgrade" + +# Color setup +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "===== Terraform Upgrade Tool Verification =====" +echo + +# 1. Check dependencies +echo "Checking dependencies..." +declare -a DEPS=("terraform" "terraform-0.13" "terraform-0.14" "terraform-0.15" "terraform-1.0" "git" "python3") + +for dep in "${DEPS[@]}"; do + if command -v $dep &>/dev/null; then + echo -e "${GREEN}✓${NC} $dep found: $(command -v $dep)" + else + echo -e "${RED}✗${NC} $dep not found!" + fi +done + +# ...existing code... +``` + +This script should be run before each release to ensure basic functionality is working correctly. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 00000000..8d0e7cdb --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,361 @@ +# Troubleshooting Guide + +This guide helps diagnose and resolve common issues encountered when using the Terraform Upgrade Tool. + +## Table of Contents +- [Environment Setup Issues](#environment-setup-issues) +- [AWS Profile Problems](#aws-profile-problems) +- [Terraform Version Issues](#terraform-version-issues) +- [Common Upgrade Errors](#common-upgrade-errors) +- [Git-Related Issues](#git-related-issues) +- [Recovery Procedures](#recovery-procedures) +- [Reporting Bugs](#reporting-bugs) + +## Environment Setup Issues + +### Missing Dependencies + +**Problem:** The `make verify-tools` command shows missing tools. + +**Solution:** +1. Install missing Terraform versions: + ```bash + # Install Terraform 0.13.x + curl -LO https://releases.hashicorp.com/terraform/0.13.7/terraform_0.13.7_linux_amd64.zip + unzip terraform_0.13.7_linux_amd64.zip -d /tmp + sudo mv /tmp/terraform /usr/local/bin/terraform-0.13 + ``` + + Repeat for other versions (0.14.x, 0.15.x, 1.x) as needed. + +2. Verify Python version: + ```bash + python3 --version + ``` + + Ensure you have Python 3.8 or newer. If not, install it through your package manager. + +### Python Package Issues + +**Problem:** You encounter Python import errors or missing packages. + +**Solution:** +1. Reinstall the tool in development mode: + ```bash + pip install -e . + ``` + +2. Check for specific package errors and install them manually: + ```bash + pip install click pyyaml networkx pygraphviz + ``` + +### Permission Issues + +**Problem:** The tool fails with permission errors when reading or writing files. + +**Solution:** +1. Check file ownership and permissions: + ```bash + ls -la /path/to/your/terraform/repo + ``` + +2. Ensure you have write access to the directory: + ```bash + chmod -R u+w /path/to/your/terraform/repo + ``` + +## AWS Profile Problems + +### Profile Not Found + +**Problem:** You see "Profile not found" errors when the tool tries to use AWS profiles. + +**Solution:** +1. List available profiles: + ```bash + aws configure list-profiles + ``` + +2. Verify your AWS configuration files: + ```bash + cat ~/.aws/config + cat ~/.aws/credentials + ``` + +3. Export the profile explicitly: + ```bash + export AWS_PROFILE=your-profile-name + ``` + +### Expired Credentials + +**Problem:** You encounter AWS credential errors like "ExpiredToken" or "AccessDenied". + +**Solution:** +1. Refresh your credentials: + ```bash + aws sso login --profile your-profile-name + ``` + +2. For non-SSO profiles, update your credentials: + ```bash + aws configure --profile your-profile-name + ``` + +### Multiple Account Access Issues + +**Problem:** The tool fails when attempting to access resources across multiple AWS accounts. + +**Solution:** +1. Ensure you have the correct roles configured in your AWS config: + ``` + [profile account1] + role_arn = arn:aws:iam::123456789012:role/YourRole + source_profile = base-profile + + [profile account2] + role_arn = arn:aws:iam::210987654321:role/YourRole + source_profile = base-profile + ``` + +2. Test direct access to each account: + ```bash + aws sts get-caller-identity --profile account1 + aws sts get-caller-identity --profile account2 + ``` + +## Terraform Version Issues + +### Terraform Not Found in Path + +**Problem:** The tool can't find specific Terraform versions. + +**Solution:** +1. Check the paths to your Terraform binaries: + ```bash + which terraform + which terraform-0.13 + ``` + +2. Create symlinks if necessary: + ```bash + sudo ln -s /path/to/terraform-0.13 /usr/local/bin/terraform-0.13 + ``` + +3. Update your PATH variable: + ```bash + export PATH=$PATH:/path/to/terraform/binaries + ``` + +### Version Mismatch + +**Problem:** The tool detects a different Terraform version than what you expect. + +**Solution:** +1. Check the actual version of your Terraform binaries: + ```bash + terraform version + terraform-0.13 version + ``` + +2. Verify .tf-control files in your directories: + ```bash + cat /path/to/your/terraform/repo/.tf-control + ``` + +3. Check for version constraints in your Terraform configurations: + ```bash + grep -r "required_version" /path/to/your/terraform/repo + ``` + +## Common Upgrade Errors + +### Provider Source Declaration Issues + +**Problem:** The upgrade to 0.13.x fails with provider source declaration errors. + +**Solution:** +1. Check the provider blocks in your configuration: + ```bash + grep -r "provider" --include="*.tf" /path/to/your/terraform/repo + ``` + +2. Manually run the 0.13upgrade command to see specific errors: + ```bash + cd /path/to/your/terraform/repo/module1 + terraform-0.13 0.13upgrade -yes + ``` + +3. Verify the provider mappings in the tool are correct: + ```bash + cat /path/to/terraform-upgrade-tool/tf_upgrade/utils/provider_migration.py + ``` + +### Syntax Transformation Failures + +**Problem:** The tool fails while transforming HCL syntax. + +**Solution:** +1. Run with verbose logging to see detailed errors: + ```bash + make verbose upgrade DIR=/path/to/your/terraform/repo/module1 + ``` + +2. Check the backup files to compare with the original: + ```bash + diff /path/to/your/terraform/repo/module1/main.tf /path/to/your/terraform/repo/module1/.terraform-upgrade-backup-*/main.tf + ``` + +3. For complex syntax issues, try breaking the upgrade into smaller steps: + ```bash + # Manually upgrade to 0.13 first + make upgrade-version VERSION=0.13 DIR=/path/to/your/terraform/repo/module1 + + # Then continue to subsequent versions + make upgrade-version VERSION=0.14 DIR=/path/to/your/terraform/repo/module1 + ``` + +### Module Compatibility Issues + +**Problem:** Upgrades fail due to module compatibility problems. + +**Solution:** +1. Check the module sources in your configuration: + ```bash + grep -r "module" --include="*.tf" /path/to/your/terraform/repo + ``` + +2. Update module references to versions compatible with the target Terraform version: + ```terraform + module "example" { + source = "git::https://github.com/example/module.git?ref=v2.0.0" + } + ``` + +3. Use the Census Bureau's compatible module versions for common modules. + +## Git-Related Issues + +### Git Configuration Issues + +**Problem:** Git operations fail during the upgrade process. + +**Solution:** +1. Verify git is configured correctly: + ```bash + git config --list + ``` + +2. Set up your git identity if needed: + ```bash + git config --global user.name "Your Name" + git config --global user.email "your.email@example.com" + ``` + +3. Check repository permissions: + ```bash + cd /path/to/your/terraform/repo + git fetch + ``` + +### Branch Creation Failures + +**Problem:** The tool fails to create a new git branch. + +**Solution:** +1. Check if the branch already exists: + ```bash + git branch -a + ``` + +2. Create the branch manually: + ```bash + git checkout -b terraform-upgrade-$(date +%Y%m%d) + ``` + +3. Make sure you don't have uncommitted changes: + ```bash + git status + ``` + +## Recovery Procedures + +### Restoring from Backups + +**Problem:** You need to restore files from a backup after an unsuccessful upgrade. + +**Solution:** +1. Find the backup directory: + ```bash + ls -la /path/to/your/terraform/repo/module1/.terraform-upgrade-backup-* + ``` + +2. Restore all files from a backup: + ```bash + cp -r /path/to/your/terraform/repo/module1/.terraform-upgrade-backup-*/* /path/to/your/terraform/repo/module1/ + ``` + +3. Use the tool's restore functionality: + ```bash + make restore DIR=/path/to/your/terraform/repo/module1 BACKUP=20230615-124530 + ``` + +### Recovering from Failed Partial Upgrades + +**Problem:** The upgrade process was interrupted and left configurations in an inconsistent state. + +**Solution:** +1. Check the upgrade logs: + ```bash + cat logs/upgrade-module1-20230615-124530.log + ``` + +2. If an intermediary version was completed: + ```bash + # Resume from 0.14 if 0.13 was completed + make upgrade-from VERSION=0.14 DIR=/path/to/your/terraform/repo/module1 + ``` + +3. For severely corrupted configurations, restore from backup and restart: + ```bash + make restore DIR=/path/to/your/terraform/repo/module1 BACKUP=latest + make upgrade DIR=/path/to/your/terraform/repo/module1 + ``` + +### Git Reset + +**Problem:** You need to completely undo all upgrade changes in Git. + +**Solution:** +1. Discard all changes and return to original state: + ```bash + git reset --hard origin/main + git clean -fd + ``` + +2. For a specific directory: + ```bash + git checkout origin/main -- /path/to/your/terraform/repo/module1 + ``` + +## Reporting Bugs + +If you encounter an issue that isn't covered by this guide: + +1. Collect diagnostic information: + ```bash + make diagnostics DIR=/path/to/your/terraform/repo/module1 + ``` + +2. Review the generated diagnostic report: + ```bash + cat diagnostics-module1-20230615-124530.md + ``` + +3. Open an issue in the project repository with: + - A clear description of the problem + - Steps to reproduce + - The diagnostic report + - Error messages and logs + - Environment information (OS, tool versions) diff --git a/docs/upgrade-path.md b/docs/upgrade-path.md new file mode 100644 index 00000000..b0b27430 --- /dev/null +++ b/docs/upgrade-path.md @@ -0,0 +1,159 @@ +# Terraform Upgrade Path: 0.12.x to 1.x + +## Upgrade Sequence + +Following HashiCorp's recommended upgrade path and Census Bureau guidelines, we upgrade through each incremental version: + +``` +Terraform 0.12.x → 0.13.x → 0.14.x → 0.15.x → 1.x +``` + +Each step in the upgrade process addresses specific changes and ensures compatibility before proceeding to the next version. + +## Assessment Phase + +Before beginning any upgrades, we perform a comprehensive assessment: + +1. **Directory Scanning** + - Identify all Terraform configurations needing upgrades + - Determine current version for each configuration + - Generate inventory report with directory locations + +2. **Complexity Analysis** + - Analyze configuration complexity based on resources and patterns + - Identify usage of deprecated features + - Determine non-HCL syntax elements requiring attention + +3. **Risk Assessment** + - Calculate risk scores for each configuration + - Prioritize upgrades based on complexity and criticality + - Identify high-risk configurations needing extra attention + +## Version-Specific Upgrades + +### Terraform 0.12.x to 0.13.x + +#### Key Changes +- Provider source declarations required +- Module provider inheritance changes +- Count/for_each for modules + +#### Implementation +1. **Preparation** + - Review [0.13 upgrade guide](https://www.terraform.io/upgrade-guides/0-13.html) + - Update provider source declarations + - Address deprecated interpolation syntax + +2. **Execution** + - Run `terraform 0.13upgrade` command on each configuration + - Execute `make upgrade-dir DIR=[path]` with 0.13 target + - Validate state files and plan outputs + +### Terraform 0.13.x to 0.14.x + +#### Key Changes +- Dependency lock file (.terraform.lock.hcl) +- Sensitive output values +- Provider metadata requirements + +#### Implementation +1. **Preparation** + - Review [0.14 upgrade guide](https://www.terraform.io/upgrade-guides/0-14.html) + - Address provider version constraints + - Update sensitive output handling + +2. **Execution** + - Execute `make upgrade-dir DIR=[path]` with 0.14 target + - Validate configuration lock files + - Test with `terraform plan` and resolve discrepancies + +### Terraform 0.14.x to 0.15.x + +#### Key Changes +- Legacy function removal +- Type constraints enforcement +- Validation of variable values + +#### Implementation +1. **Preparation** + - Review [0.15 upgrade guide](https://www.terraform.io/upgrade-guides/0-15.html) + - Address deprecated function calls + - Update configuration for removed features + +2. **Execution** + - Execute `make upgrade-dir DIR=[path]` with 0.15 target + - Run `terraform validate` on all configurations + - Address any warnings that will become errors in 1.x + +### Terraform 0.15.x to 1.x + +#### Key Changes +- Stability and provider compatibility +- Stricter validation +- Performance improvements + +#### Implementation +1. **Preparation** + - Review [1.0 upgrade guide](https://www.terraform.io/upgrade-guides/1-0.html) + - Ensure compatibility with all used providers + - Update any custom modules + +2. **Execution** + - Execute `make upgrade-dir DIR=[path]` with 1.0 target + - Run comprehensive validation tests + - Apply configurations in test environment + +## Post-Upgrade Verification + +After each version upgrade, perform these verification steps: + +1. **Configuration Validation** + - Run `terraform validate` to check for syntax errors + - Address any warnings or errors before proceeding + +2. **Plan Validation** + - Run `terraform plan` to ensure no unexpected changes + - Verify resource counts match pre-upgrade state + +3. **Documentation Updates** + - Update READMEs with new version requirements + - Document any changes to module interfaces or variables + +## Common Upgrade Issues + +### Provider Compatibility +- Check provider version compatibility with each Terraform version +- Update provider constraints as needed +- Consider pinning provider versions during the upgrade process + +### State File Compatibility +- Use `terraform state pull > backup.tfstate` before upgrades +- Verify state compatibility after each version upgrade +- Use `terraform init -reconfigure` if state format changes + +### Module References +- Update module source references as needed +- Check for version constraints in module declarations +- Test modules independently before integration + +## Command Reference + +```bash +# Scan for directories needing upgrade +make scan + +# Analyze complexity and risk +make analyze DIR=/path/to/terraform + +# Perform dry run to see what would change +make dry-run DIR=/path/to/terraform + +# Upgrade specific directory +make upgrade-dir DIR=/path/to/terraform + +# Upgrade with interactive prompts +make step DIR=/path/to/terraform + +# Validate after upgrade +make validate DIR=/path/to/terraform +``` diff --git a/docs/user-guide.md b/docs/user-guide.md new file mode 100644 index 00000000..c4d9c98b --- /dev/null +++ b/docs/user-guide.md @@ -0,0 +1,445 @@ +# Terraform Upgrade Tool User Guide + +This comprehensive guide covers all aspects of using the Terraform Upgrade Tool for upgrading configurations from Terraform 0.12.x to 1.x. + +## Table of Contents + +- [Getting Started](#getting-started) +- [Understanding the Upgrade Process](#understanding-the-upgrade-process) +- [Environment Setup](#environment-setup) +- [Command Reference](#command-reference) +- [Workflow Examples](#workflow-examples) +- [Interpreting Results](#interpreting-results) +- [Troubleshooting](#troubleshooting) +- [Advanced Usage](#advanced-usage) +- [GitHub Integration](#github-integration) + +## Getting Started + +### Installation + +1. **Clone the repository**: + ```bash + git clone https://github.com/census-bureau/terraform-upgrade-tool.git + cd terraform-upgrade-tool + ``` + +2. **Install dependencies**: + ```bash + pip install -e . + ``` + +3. **Verify installation**: + ```bash + tf-upgrade --version + ``` + +4. **Check required tools**: + ```bash + tf-upgrade verify-tools + ``` + + The tool requires: + - Python 3.8+ + - Multiple Terraform binaries: 0.12, 0.13, 0.14, 0.15, and 1.0+ + - Git + - AWS CLI (if using AWS profiles) + +### Quick Start + +The basic workflow for upgrading a directory follows these steps: + +```bash +# 1. Verify environment +tf-upgrade verify-tools + +# 2. Scan for Terraform directories +tf-upgrade scan /path/to/repo + +# 3. Analyze a specific directory +tf-upgrade analyze /path/to/repo/module1 + +# 4. Perform a dry run to see what changes would be made +tf-upgrade dry-run /path/to/repo/module1 + +# 5. Upgrade the directory +tf-upgrade upgrade /path/to/repo/module1 + +# 6. Verify the results +cd /path/to/repo/module1 +terraform validate +terraform plan +``` + +## Understanding the Upgrade Process + +### Upgrade Phases + +The Terraform upgrade process consists of several phases: + +1. **Assessment Phase**: + - Scanning directories to identify Terraform configurations + - Analyzing complexity and risk of each configuration + - Building dependency graphs to determine upgrade order + +2. **Planning Phase**: + - Identifying version-specific changes needed + - Performing dry runs to preview changes + - Determining the optimal upgrade path + +3. **Execution Phase**: + - Creating backups of all files + - Step-by-step upgrade through each version (0.12 → 0.13 → 0.14 → 0.15 → 1.0) + - Validating after each step + +4. **Verification Phase**: + - Running `terraform validate` to check syntax + - Running `terraform plan` to verify resource changes + - Comparing before/after resource counts + +### Version-Specific Changes + +Each Terraform version upgrade involves specific changes: + +1. **0.12 to 0.13**: + - Provider source declarations + - Module provider inheritance changes + - Count/for_each for modules + +2. **0.13 to 0.14**: + - Dependency lock file creation + - Sensitive output updates + - Provider metadata requirements + +3. **0.14 to 0.15**: + - Deprecated function removal + - Type constraint enforcement + - Variable validation improvements + +4. **0.15 to 1.0**: + - Stability improvements + - Provider compatibility updates + - Performance enhancements + +## Environment Setup + +### AWS Configuration + +The tool integrates with AWS using profiles: + +1. **List available profiles**: + ```bash + aws configure list-profiles + ``` + +2. **Export a profile**: + ```bash + export AWS_PROFILE=your-profile-name + ``` + +3. **Test profile access**: + ```bash + aws sts get-caller-identity + ``` + +### Terraform Version Installation + +Ensure you have all required Terraform versions installed: + +```bash +# Create a directory for versions +mkdir -p ~/terraform-versions +cd ~/terraform-versions + +# Download and install specific versions +for version in "0.12.31" "0.13.7" "0.14.11" "0.15.5" "1.0.11"; do + curl -LO "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_linux_amd64.zip" + unzip "terraform_${version}_linux_amd64.zip" -d "terraform-${version%.*}" + mv "terraform-${version%.*}/terraform" "/usr/local/bin/terraform-${version%.*}" + rm -rf "terraform-${version%.*}" "terraform_${version}_linux_amd64.zip" +done + +# Verify installations +terraform-0.12 version +terraform-0.13 version +terraform-0.14 version +terraform-0.15 version +terraform-1.0 version +``` + +### Git Configuration + +Ensure Git is properly configured: + +```bash +git config --global user.name "Your Name" +git config --global user.email "your.email@example.com" +``` + +## Command Reference + +See the [Command Reference](command-reference.md) for detailed information on all available commands. + +## Workflow Examples + +### Basic Workflow + +```bash +# Scan repository for Terraform configs +tf-upgrade scan /path/to/repo + +# Analyze specific directory +tf-upgrade analyze /path/to/repo/module1 + +# Perform dry run +tf-upgrade dry-run /path/to/repo/module1 + +# Upgrade directory (default to Terraform 1.0) +tf-upgrade upgrade /path/to/repo/module1 +``` + +### Interactive Workflow + +```bash +# Run upgrade in interactive mode +tf-upgrade upgrade /path/to/repo/module1 --interactive +``` + +Interactive mode will: +- Confirm each version upgrade step +- Show changes before applying +- Allow aborting at any step + +### Git Integration Workflow + +```bash +# Upgrade and create a Git branch +tf-upgrade upgrade /path/to/repo/module1 --git-branch tf-upgrade-module1 + +# Review changes +cd /path/to/repo +git diff + +# Create pull request (if GitHub integration enabled) +tf-upgrade github-pr /path/to/repo/module1 +``` + +### Multi-Directory Workflow + +For repositories with multiple Terraform configurations: + +```bash +# Scan to identify all directories +tf-upgrade scan /path/to/repo + +# Generate dependency graph to determine upgrade order +tf-upgrade generate-graph /path/to/repo + +# Batch upgrade multiple directories +cat directories.txt | while read dir; do + tf-upgrade upgrade "$dir" +done +``` + +## Interpreting Results + +### Scan Results + +Scan results provide an overview of all Terraform configurations: + +``` +Found 15 Terraform configuration directories: + /path/to/repo/module1 (0.12.29, HIGH risk) + /path/to/repo/module2 (0.13.5, MEDIUM risk) + ... +``` + +Focus on directories with high risk scores first, as they may require more attention during upgrade. + +### Analysis Results + +Analysis results provide detailed information about a specific directory: + +``` +Complexity Analysis: +- Resources: 24 +- Data sources: 8 +- Modules: 3 +- Complexity score: 6.5/10 (HIGH) + +Risk Factors: +- Complex provider configurations +- Module dependencies +- Count/for_each usage +``` + +Use these results to anticipate potential issues during upgrades. + +### Upgrade Reports + +After an upgrade, review the generated report: + +``` +Upgrade Report: +- Successfully upgraded from 0.12.29 to 1.0.0 +- Modified 5 files +- Added required_providers block +- Updated 3 module references +- Created dependency lock file +``` + +The report also includes before/after comparisons and validation results. + +## Troubleshooting + +See the [Troubleshooting Guide](troubleshooting.md) for detailed help with common issues. + +### Common Issues + +1. **Missing Tools**: + ``` + Error: Terraform binary terraform-0.13 not found + ``` + Solution: Install the missing Terraform version. + +2. **AWS Profile Issues**: + ``` + Error: The configured profile 'profile-name' could not be found + ``` + Solution: Check your AWS configuration and ensure the profile exists. + +3. **Syntax Transformation Errors**: + ``` + Error: Failed to transform file main.tf: Unexpected syntax pattern + ``` + Solution: Run with verbose logging (`--verbose`) and check the problematic syntax. + +4. **Provider Source Issues**: + ``` + Error: Provider source could not be determined for provider 'aws' + ``` + Solution: Manually specify provider mappings or use the `--force-provider-map` option. + +### Recovery Options + +If an upgrade fails, you have several recovery options: + +1. **Restore from Backup**: + ```bash + tf-upgrade restore /path/to/repo/module1 + ``` + +2. **Git Reset**: + ```bash + git reset --hard HEAD + ``` + +3. **Resume Partial Upgrade**: + ```bash + # If 0.13 upgrade succeeded but 0.14 failed + tf-upgrade upgrade /path/to/repo/module1 --from-version 0.14 + ``` + +## Advanced Usage + +### Custom Configuration + +Create a custom configuration file: + +```bash +tf-upgrade config set backup.keep_days 30 +tf-upgrade config set logging.level debug +``` + +Use a custom configuration file: + +```bash +tf-upgrade --config-file my-config.yaml scan /path/to/repo +``` + +### Advanced Scanning Options + +```bash +# Deep scan with parallel processing +tf-upgrade scan /path/to/repo --depth 10 --parallel + +# Exclude certain patterns +tf-upgrade scan /path/to/repo --exclude "**/vendor/**" --exclude "**/.terraform/**" + +# Export results to JSON +tf-upgrade scan /path/to/repo --output json > scan-results.json +``` + +### Custom Transformation Rules + +For advanced users, you can create custom transformation rules: + +1. Create a transformation file (`transforms.json`): + ```json + { + "rules": [ + { + "name": "custom_tag_format", + "pattern": "tags\\s+=\\s+{([^}]*)}", + "replacement": "tags = local.standard_tags" + } + ] + } + ``` + +2. Use the custom rules: + ```bash + tf-upgrade upgrade /path/to/repo/module1 --transform-file transforms.json + ``` + +## GitHub Integration + +If you're using GitHub for repository management, you can integrate the upgrade process with GitHub projects: + +1. **Configure GitHub access**: + ```bash + export GITHUB_TOKEN=your_personal_access_token + tf-upgrade config set github.organization "your-org-name" + tf-upgrade config set github.project "terraform-upgrade" + ``` + +2. **List upgrade tickets**: + ```bash + tf-upgrade github-list + ``` + +3. **Claim a directory for upgrading**: + ```bash + tf-upgrade github-claim /path/to/repo/module1 + ``` + +4. **Update ticket status**: + ```bash + tf-upgrade github-update /path/to/repo/module1 --status "In Review" + ``` + +5. **Create a pull request**: + ```bash + tf-upgrade github-pr /path/to/repo/module1 + ``` + +For more details on GitHub integration, see the [GitHub Workflow Guide](github-workflow.md). + +## Census Bureau-Specific Guidelines + +When working with Census Bureau configurations, follow these additional guidelines: + +1. **Use GovCloud Profiles**: + Ensure you're using the appropriate GovCloud AWS profiles. + +2. **Module References**: + Update Census-specific module references to the appropriate versions. + See [Census Examples](census-examples.md) for details. + +3. **Tag Formats**: + Preserve Census Bureau tag formats (boc: prefixed tags, etc.). + +4. **Partition-Specific Resources**: + Handle GovCloud ARN formats correctly. + +For detailed Census-specific examples, see the [Census Examples](census-examples.md) document.