From ad7ee05c42465774dceb39b63523938cbd1de6a3 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 14 Oct 2025 16:00:08 -0400 Subject: [PATCH 1/2] fix eksmanager create_client calls --- .../aws_resource_management/managers/eks.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py index c2fb8ed1..5d015d2d 100644 --- a/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py +++ b/local-app/python-tools/gfl-resource-actions/aws_resource_management/managers/eks.py @@ -111,7 +111,7 @@ def scale_down(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> N try: region = cluster["region"] - eks_client = self.create_client("eks", region) + eks_client = self.get_client("eks") # Get current scaling parameters from tags or nodegroups min_nodes = cluster.get("min_nodes", "") @@ -251,7 +251,7 @@ def scale_up(self, clusters: List[Dict[str, Any]], dry_run: bool = False) -> Non try: region = cluster["region"] - eks_client = self.create_client("eks", region) + eks_client = self.get_client("eks") # Check for original values in backup tags original_min = cluster.get("original_min", "") @@ -557,7 +557,8 @@ def discover_clusters(self, regions: List[str]) -> List[Dict[str, Any]]: for region in regions: try: - eks_client = self.create_client("eks", region) + self.region = region # Set region context for get_client + eks_client = self.get_client("eks") if not eks_client: logger.warning(f"Could not create EKS client in {region}") continue @@ -660,7 +661,8 @@ def _get_kubeconfig(self, cluster_name: str, region: str, account_id: str) -> st Returns path to the kubeconfig file. """ - eks = self.create_client("eks", region) + self.region = region # Set region context for get_client + eks = self.get_client("eks") cluster_info = eks.describe_cluster(name=cluster_name)["cluster"] # Use awscli to get token (assumes awscli is configured for this account) token = self._get_eks_token(cluster_name, region) From caa4456a76b8e73260869e8d60a67c80c21070d9 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 14 Oct 2025 16:09:45 -0400 Subject: [PATCH 2/2] remove cruft --- .env | 5 - .flake8 | 7 - .pre-commit-config.yaml | 27 - Makefile | 207 --- PROJECT_PLAN.md | 122 -- checklist.md | 154 --- copilot-instructions.md | 110 -- create_project_structure.sh | 86 -- create_test_fixtures.sh | 201 --- debug_scanner.py | 85 -- plan.md | 1210 ----------------- pyproject.toml | 26 - requirements.txt | 17 - setup.py | 44 - setup_test_env.sh | 32 - testplan.md | 763 ----------- tests/__init__.py | 0 tests/conftest.py | 10 - tests/fixtures/0.12/complex/main.tf | 125 -- tests/fixtures/0.12/medium/main.tf | 44 - tests/fixtures/0.12/simple/main.tf | 13 - .../simple/terraform-upgrade-dryrun-report.md | 12 - .../simple/upgrade-logs/upgrade-progress.json | 7 - tests/fixtures/__init__.py | 75 - tests/fixtures/complex/main.tf | 125 -- tests/fixtures/medium/main.tf | 44 - tests/fixtures/simple/main.tf | 13 - .../terraform-upgrade-dryrun-report.md | 7 - .../upgrade-logs/upgrade-progress.json | 7 - tests/integration/__init__.py | 1 - tests/integration/test_upgrade_workflow.py | 195 --- tests/run_tests.sh | 57 - tests/setup_test_structure.sh | 17 - tests/test_env_validator.py | 0 tests/test_tf_parser.py | 0 tests/test_upgraders.py | 0 tests/unit/__init__.py | 1 - tests/unit/test_file_manager.py | 83 -- tests/unit/test_hcl_transformer.py | 79 -- tests/unit/test_terraform_runner.py | 122 -- tests/validation/test_upgrade_validation.py | 264 ---- tf_bulk_upgrade.py | 429 ------ tf_upgrade/cli.py | 894 ------------ tf_upgrade/complexity_analyzer.py | 188 --- tf_upgrade/config.py | 435 ------ tf_upgrade/dependency_graph.py | 308 ----- tf_upgrade/env_validator.py | 316 ----- tf_upgrade/hcl_transformer.py | 53 - tf_upgrade/module_resolver.py | 181 --- tf_upgrade/reporters/__init__.py | 255 ---- tf_upgrade/reporters/console.py | 126 -- tf_upgrade/reporters/file.py | 144 -- tf_upgrade/reporters/markdown.py | 149 -- tf_upgrade/risk_assessment.py | 360 ----- tf_upgrade/scanner.py | 429 ------ tf_upgrade/upgrade_controller.py | 236 ---- tf_upgrade/utils/file_manager.py | 214 --- tf_upgrade/utils/git.py | 401 ------ tf_upgrade/utils/hcl_transformer.py | 169 --- tf_upgrade/utils/parallel.py | 105 -- tf_upgrade/utils/provider_migration.py | 241 ---- tf_upgrade/utils/terraform.py | 579 -------- tf_upgrade/utils/terraform_runner.py | 257 ---- tf_upgrade/utils/validator.py | 260 ---- tf_upgrade/utils/version_validator.py | 91 -- tf_upgrade/version_detector.py | 261 ---- tf_upgrade/version_upgraders/__init__.py | 275 ---- tf_upgrade/version_upgraders/v0_13.py | 227 ---- tf_upgrade/version_upgraders/v0_14.py | 187 --- tf_upgrade/version_upgraders/v0_15.py | 225 --- tf_upgrade/version_upgraders/v1_0.py | 196 --- tf_upgrade/workspace_manager.py | 231 ---- tf_upgrade_diagnostics.py | 149 -- 73 files changed, 12968 deletions(-) delete mode 100644 .env delete mode 100644 .flake8 delete mode 100644 .pre-commit-config.yaml delete mode 100644 Makefile delete mode 100644 PROJECT_PLAN.md delete mode 100644 checklist.md delete mode 100644 copilot-instructions.md delete mode 100755 create_project_structure.sh delete mode 100755 create_test_fixtures.sh delete mode 100644 debug_scanner.py delete mode 100644 plan.md delete mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py delete mode 100755 setup_test_env.sh delete mode 100644 testplan.md delete mode 100644 tests/__init__.py delete mode 100644 tests/conftest.py delete mode 100644 tests/fixtures/0.12/complex/main.tf delete mode 100644 tests/fixtures/0.12/medium/main.tf delete mode 100644 tests/fixtures/0.12/simple/main.tf delete mode 100644 tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md delete mode 100644 tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json delete mode 100644 tests/fixtures/__init__.py delete mode 100644 tests/fixtures/complex/main.tf delete mode 100644 tests/fixtures/medium/main.tf delete mode 100644 tests/fixtures/simple/main.tf delete mode 100644 tests/fixtures/terraform-upgrade-dryrun-report.md delete mode 100644 tests/fixtures/upgrade-logs/upgrade-progress.json delete mode 100644 tests/integration/__init__.py delete mode 100644 tests/integration/test_upgrade_workflow.py delete mode 100644 tests/run_tests.sh delete mode 100644 tests/setup_test_structure.sh delete mode 100644 tests/test_env_validator.py delete mode 100644 tests/test_tf_parser.py delete mode 100644 tests/test_upgraders.py delete mode 100644 tests/unit/__init__.py delete mode 100644 tests/unit/test_file_manager.py delete mode 100644 tests/unit/test_hcl_transformer.py delete mode 100644 tests/unit/test_terraform_runner.py delete mode 100644 tests/validation/test_upgrade_validation.py delete mode 100755 tf_bulk_upgrade.py delete mode 100644 tf_upgrade/cli.py delete mode 100644 tf_upgrade/complexity_analyzer.py delete mode 100644 tf_upgrade/config.py delete mode 100644 tf_upgrade/dependency_graph.py delete mode 100644 tf_upgrade/env_validator.py delete mode 100644 tf_upgrade/hcl_transformer.py delete mode 100644 tf_upgrade/module_resolver.py delete mode 100644 tf_upgrade/reporters/__init__.py delete mode 100644 tf_upgrade/reporters/console.py delete mode 100644 tf_upgrade/reporters/file.py delete mode 100644 tf_upgrade/reporters/markdown.py delete mode 100644 tf_upgrade/risk_assessment.py delete mode 100644 tf_upgrade/scanner.py delete mode 100644 tf_upgrade/upgrade_controller.py delete mode 100644 tf_upgrade/utils/file_manager.py delete mode 100644 tf_upgrade/utils/git.py delete mode 100644 tf_upgrade/utils/hcl_transformer.py delete mode 100644 tf_upgrade/utils/parallel.py delete mode 100644 tf_upgrade/utils/provider_migration.py delete mode 100644 tf_upgrade/utils/terraform.py delete mode 100644 tf_upgrade/utils/terraform_runner.py delete mode 100644 tf_upgrade/utils/validator.py delete mode 100644 tf_upgrade/utils/version_validator.py delete mode 100644 tf_upgrade/version_detector.py delete mode 100644 tf_upgrade/version_upgraders/__init__.py delete mode 100644 tf_upgrade/version_upgraders/v0_13.py delete mode 100644 tf_upgrade/version_upgraders/v0_14.py delete mode 100644 tf_upgrade/version_upgraders/v0_15.py delete mode 100644 tf_upgrade/version_upgraders/v1_0.py delete mode 100644 tf_upgrade/workspace_manager.py delete mode 100644 tf_upgrade_diagnostics.py diff --git a/.env b/.env deleted file mode 100644 index 2402ef5d..00000000 --- a/.env +++ /dev/null @@ -1,5 +0,0 @@ -AWS_PROFILE=252960665057-ma6-gov.inf-admin-t2 -AWS_REGION=us-gov-east-1 -# If you don't have a profile configured, you can use these instead: -# AWS_ACCESS_KEY_ID=dummy-key -# AWS_SECRET_ACCESS_KEY=dummy-secret diff --git a/.flake8 b/.flake8 deleted file mode 100644 index ebc227d6..00000000 --- a/.flake8 +++ /dev/null @@ -1,7 +0,0 @@ -[flake8] -max-line-length = 88 -extend-ignore = E203, W503 -exclude = .git,__pycache__,build,dist,.venv -per-file-ignores = - # Allow unused imports in __init__.py files - __init__.py:F401 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index cd786b56..00000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -repos: -- repo: https://github.com/psf/black - rev: 25.1.0 - hooks: - - id: black - language_version: python3 - -- repo: https://github.com/pycqa/isort - rev: 6.0.1 - hooks: - - id: isort - -- repo: https://github.com/pycqa/flake8 - rev: 7.2.0 - hooks: - - id: flake8 - additional_dependencies: [flake8-docstrings] - # Explicitly tell flake8 to use the .flake8 config file - args: [--config=.flake8] - -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: check-added-large-files diff --git a/Makefile b/Makefile deleted file mode 100644 index 33c231a4..00000000 --- a/Makefile +++ /dev/null @@ -1,207 +0,0 @@ -.PHONY: help install develop clean test lint format scan dry-run upgrade verify-tools all verify-format - -# Variables -DIR ?= . -TEST_FIXTURES_DIR = tests/fixtures -VERIFICATION_LOG = verification-results.log - -help: ## Show this help message - @echo 'Usage: make [target] [DIR=path/to/directory]' - @echo '' - @echo 'Targets:' - @egrep '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' - -all: clean setup-env develop lint format test verify-all ## Run the complete build, test, and validation workflow - -install: ## Install the package - pip install . - -develop: ## Install the package in development mode - pip install -e . - -clean: ## Clean build artifacts - @echo "Cleaning build artifacts..." - @rm -rf build/ - @rm -rf dist/ - @rm -rf *.egg-info - @find . -name "__pycache__" -exec rm -rf {} + - @find . -name "*.log" -delete - @find . -name ".pytest_cache" -exec rm -rf {} + - @rm -f test-results.log - @rm -f $(VERIFICATION_LOG) - @rm -rf logs/ - @rm -rf terraform-upgrade-reports/ - -# Test targets -test: test-setup ## Run all tests - @echo "Running all tests..." - @echo "====== TEST EXECUTION SUMMARY ======" > test-results.log - @echo "Starting unit tests at $$(date)" >> test-results.log - @python -m unittest discover -s tests/unit -p "test_*.py" -v 2>&1 | tee -a test-results.log || (echo "❌ Unit tests failed"; exit 1) - @echo "Starting integration tests at $$(date)" >> test-results.log - @python -m unittest discover -s tests/integration -p "test_*.py" -v 2>&1 | tee -a test-results.log || (echo "❌ Integration tests failed"; exit 1) - @echo "Starting validation tests at $$(date)" >> test-results.log - @python -m unittest discover -s tests/validation -p "test_*.py" -v 2>&1 | tee -a test-results.log || (echo "❌ Validation tests failed"; exit 1) - @echo "====== TEST EXECUTION COMPLETED ======" >> test-results.log - @if grep -q "FAILED" test-results.log; then \ - echo "❌ Some tests failed. See test-results.log for details."; \ - exit 1; \ - else \ - echo "✅ All tests passed successfully!"; \ - fi - @echo "Detailed test results available in test-results.log" - -test-setup: - @echo "Setting up test environment..." - @mkdir -p tests/output - -test-unit: ## Run unit tests only - python -m unittest discover -s tests/unit -p "test_*.py" -v - -test-integration: ## Run integration tests only - python -m unittest discover -s tests/integration -p "test_*.py" -v - -test-validation: ## Run validation tests only - python -m unittest discover -s tests/validation -p "test_*.py" -v - -# Add test directory to the Python path for test execution -export PYTHONPATH := $(PYTHONPATH):$(shell pwd) - -lint: ## Run linting checks - black . - isort . - flake8 . - -verify-tools: ## Verify required tools are installed - @echo "Verifying required tools..." - @python -m tf_upgrade.cli verify-env - -scan: ## Scan for Terraform configurations - @python -m tf_upgrade.cli scan $(DIR) - -dry-run: ## Simulate an upgrade without making changes - @python -m tf_upgrade.cli dry-run $(DIR) - -upgrade: ## Upgrade Terraform configurations - @python -m tf_upgrade.cli upgrade $(DIR) - -upgrade-dir: ## Upgrade a specific directory - @python -m tf_upgrade.cli upgrade $(DIR) - -step: ## Interactive step-by-step upgrade - @python -m tf_upgrade.cli upgrade $(DIR) --interactive - -verbose: ## Run with verbose output - @python -m tf_upgrade.cli --verbose upgrade $(DIR) - -debug: ## Run with debug output - @python -m tf_upgrade.cli --debug upgrade $(DIR) - -debug-scan: ## Run diagnostic scan for troubleshooting - @python tf_upgrade_diagnostics.py $(DIR) --verbose - -recursive-scan: ## Run explicitly recursive scan - @python -m tf_upgrade.cli scan --recursive $(DIR) - -analyze: ## Analyze complexity of Terraform configurations - @python -m tf_upgrade.cli analyze-complexity $(DIR) - -assess-risk: ## Assess upgrade risk - @python -m tf_upgrade.cli assess-risk $(DIR) - -generate-graph: ## Generate dependency graph - @python -m tf_upgrade.cli generate-graph $(DIR) - -## Verification targets - -setup-fixtures: clean-fixtures - @echo "Setting up test fixtures..." - @mkdir -p $(TEST_FIXTURES_DIR)/0.12/simple - @mkdir -p $(TEST_FIXTURES_DIR)/0.12/medium - @mkdir -p $(TEST_FIXTURES_DIR)/0.12/complex - @cp -r tests/fixtures/simple/* $(TEST_FIXTURES_DIR)/0.12/simple/ - @cp -r tests/fixtures/medium/* $(TEST_FIXTURES_DIR)/0.12/medium/ - @cp -r tests/fixtures/complex/* $(TEST_FIXTURES_DIR)/0.12/complex/ - @echo "✅ Test fixtures created" - -clean-fixtures: - @echo "Cleaning existing test fixtures..." - @rm -rf $(TEST_FIXTURES_DIR)/0.12 - @rm -rf $(TEST_FIXTURES_DIR)/0.12-backup-test - @rm -rf $(TEST_FIXTURES_DIR)/0.12-upgrade-test - @rm -rf $(TEST_FIXTURES_DIR)/0.12-error-test - @rm -rf $(TEST_FIXTURES_DIR)/0.12-interactive-test - -verify-env: ## Verify environment setup - @echo "Verifying environment..." | tee -a $(VERIFICATION_LOG) - @python -m tf_upgrade.cli verify-env | tee -a $(VERIFICATION_LOG) - -verify-scan: ## Verify scanning functionality - @echo "Verifying scanning functionality..." | tee -a $(VERIFICATION_LOG) - @python -m tf_upgrade.cli scan $(TEST_FIXTURES_DIR)/0.12/simple | tee -a $(VERIFICATION_LOG) - @python -m tf_upgrade.cli detect-version $(TEST_FIXTURES_DIR)/0.12/simple | tee -a $(VERIFICATION_LOG) - @python -m tf_upgrade.cli analyze-complexity $(TEST_FIXTURES_DIR)/0.12/simple | tee -a $(VERIFICATION_LOG) - @echo "✅ Scan verification complete" - -verify-dry-run: ## Verify dry run functionality - @echo "Verifying dry run functionality..." | tee -a $(VERIFICATION_LOG) - @python -m tf_upgrade.cli dry-run $(TEST_FIXTURES_DIR)/0.12/simple | tee -a $(VERIFICATION_LOG) - @echo "✅ Dry run verification complete" - -verify-backup: ## Verify backup functionality - @echo "Verifying backup functionality..." | tee -a $(VERIFICATION_LOG) - @cp -r $(TEST_FIXTURES_DIR)/0.12/simple $(TEST_FIXTURES_DIR)/0.12/backup-test - @python -m tf_upgrade.cli upgrade $(TEST_FIXTURES_DIR)/0.12/backup-test --target-version 0.13 | tee -a $(VERIFICATION_LOG) - @if ls $(TEST_FIXTURES_DIR)/0.12/backup-test/.terraform-upgrade-backup-* > /dev/null 2>&1; then \ - echo "✅ Backup created successfully"; \ - else \ - echo "❌ Backup creation failed"; \ - exit 1; \ - fi - @rm -rf $(TEST_FIXTURES_DIR)/0.12/backup-test - -verify-upgrade: ## Verify upgrade functionality - @echo "Verifying upgrade functionality..." | tee -a $(VERIFICATION_LOG) - @cp -r $(TEST_FIXTURES_DIR)/0.12/simple $(TEST_FIXTURES_DIR)/0.12/upgrade-test - @python -m tf_upgrade.cli upgrade $(TEST_FIXTURES_DIR)/0.12/upgrade-test --target-version 0.13 | tee -a $(VERIFICATION_LOG) - @if grep -q 'source = "hashicorp/aws"' $(TEST_FIXTURES_DIR)/0.12/upgrade-test/*.tf; then \ - echo "✅ 0.13 upgrade successful"; \ - else \ - echo "❌ 0.13 upgrade failed"; \ - exit 1; \ - fi - @rm -rf $(TEST_FIXTURES_DIR)/0.12/upgrade-test - -verify-error-handling: ## Verify error handling - @echo "Verifying error handling..." | tee -a $(VERIFICATION_LOG) - @cp -r $(TEST_FIXTURES_DIR)/0.12/simple $(TEST_FIXTURES_DIR)/0.12/error-test - @echo "invalid syntax" >> $(TEST_FIXTURES_DIR)/0.12/error-test/main.tf - @python -m tf_upgrade.cli upgrade $(TEST_FIXTURES_DIR)/0.12/error-test --target-version 0.13 || echo "✅ Expected error occurred" | tee -a $(VERIFICATION_LOG) - @rm -rf $(TEST_FIXTURES_DIR)/0.12/error-test - -verify-all: setup-fixtures verify-env verify-scan verify-dry-run verify-backup verify-upgrade verify-error-handling ## Run all verification tests (except interactive) - @echo "======================================================" - @echo "🎉 All verification tests completed successfully! 🎉" - @echo "Detailed results available in $(VERIFICATION_LOG)" - @echo "======================================================" - -verify-format: ## Verify code formatting without making changes - black --check . - isort --check-only . - flake8 . - -pytest: ## Run tests using pytest - pytest - -install-dev: ## Install development dependencies - pip install -e '.[dev]' - pre-commit install - -setup-env: - @echo "Setting up test environment..." - @./setup_test_env.sh - -report: ## Generate comprehensive project report - @echo "Generating project report..." - @python -m tf_upgrade.reporting.generate_report - @echo "✅ Report generated in reports/" diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md deleted file mode 100644 index 96e466aa..00000000 --- a/PROJECT_PLAN.md +++ /dev/null @@ -1,122 +0,0 @@ -# Terraform Upgrade Tool - Project Plan - -## 1. Core Functionality Enhancements - -### 1.1 Version Support -- [x] Support for upgrading from Terraform 0.12 to 0.13 -- [x] Support for upgrading from 0.13 to 0.14 -- [x] Support for upgrading from 0.14 to 0.15 -- [x] Support for upgrading from 0.15 to 1.10 -- [ ] Support for upgrading to future versions (1.11+) - -### 1.2 Upgrade Process Improvements -- [x] Safe backup mechanism -- [ ] Improved error handling during upgrade steps -- [ ] Rollback mechanism for failed upgrades -- [ ] Support for ignoring specific files or directories -- [ ] Custom upgrade paths configuration - -### 1.3 Testing Infrastructure -- [x] Unit tests for core components -- [x] Integration tests for workflows -- [ ] Add more comprehensive test fixtures -- [ ] Test with real-world complex Terraform projects -- [ ] Test coverage improvements (aim for >85%) - -## 2. User Experience - -### 2.1 Documentation -- [ ] Comprehensive README with examples -- [ ] User guide with detailed usage instructions -- [ ] Best practices for different upgrade scenarios -- [ ] Troubleshooting guide -- [ ] API documentation for developers - -### 2.2 CLI Improvements -- [ ] Interactive mode with guidance -- [ ] Configuration profiles for different upgrade strategies -- [ ] Progress indication for long-running operations -- [ ] Better error messages and suggestions -- [ ] Color-coded output - -### 2.3 Reporting -- [ ] HTML report generation -- [ ] Summary statistics for changes made -- [ ] Risk assessment reports -- [ ] Export capabilities (JSON, CSV) - -## 3. DevOps Integration - -### 3.1 CI/CD -- [ ] GitHub Actions workflow -- [ ] Automated testing on multiple Python versions -- [ ] Release automation -- [ ] Version tagging - -### 3.2 Development Infrastructure -- [x] Makefile targets for common operations -- [x] Environment setup script -- [ ] Pre-commit hooks -- [ ] Code quality gates -- [ ] Dependency management improvements - -### 3.3 Deployment -- [ ] Package for PyPI -- [ ] Docker container for isolated execution -- [ ] Installation script for enterprise environments - -## 4. Advanced Features - -### 4.1 Code Analysis -- [ ] Complexity analysis for Terraform code -- [ ] Dependency graph visualization -- [ ] Unused resource detection -- [ ] Security scanning integration - -### 4.2 Integrations -- [ ] Support for Terraform Cloud -- [ ] Integration with git workflows -- [ ] Support for module registries -- [ ] AWS/Azure/GCP specific optimizations - -### 4.3 Extensibility -- [ ] Plugin system for custom transformations -- [ ] Hook points for organization-specific rules -- [ ] Custom linting rules - -## 5. Project Management - -### 5.1 Release Planning -- [ ] v0.1.0: Basic upgrade path from 0.12 to 1.10 -- [ ] v0.2.0: Improved reporting and user experience -- [ ] v0.3.0: Advanced analysis features -- [ ] v1.0.0: Production-ready with all core features - -### 5.2 Feedback Loop -- [ ] User testing protocol -- [ ] Feedback collection mechanism -- [ ] Prioritization framework for feature requests - -## Timeline - -| Phase | Duration | Focus Areas | Major Deliverables | -|-------|----------|-------------|-------------------| -| 1 | 2 weeks | Core Functionality | Working upgrade path, tests | -| 2 | 2 weeks | User Experience | Documentation, improved CLI | -| 3 | 2 weeks | DevOps | CI/CD, deployment | -| 4 | 4 weeks | Advanced Features | Analysis, integrations | -| 5 | Ongoing | Maintenance | Bug fixes, minor enhancements | - -## Resource Requirements - -- Development: 1-2 Python developers -- Testing: Access to various Terraform projects at different version levels -- Infrastructure: Test environment with multiple Terraform versions installed -- Documentation: Technical writer (part-time) - -## Success Metrics - -- 100% successful upgrades on test fixtures -- >85% test coverage -- Positive user feedback -- Adoption rate within organization diff --git a/checklist.md b/checklist.md deleted file mode 100644 index b9dfdd5f..00000000 --- a/checklist.md +++ /dev/null @@ -1,154 +0,0 @@ -# Terraform Upgrade Tool Implementation Checklist - -## Phase 1: Project Setup and Planning ✅ - -- [x] Define project structure and organization -- [x] Create core module layout -- [x] Establish logging configuration -- [x] Set up error handling framework -- [x] Create basic CLI interface -- [x] Implement version detection utility -- [x] Define configuration management approach - -## Phase 2: Environment Validation ✅ - -- [x] Create AWS profile validator -- [x] Implement terraform binary detector -- [x] Add git repository validator -- [x] Create .tf-control file parser -- [x] Implement environment configuration validator -- [x] Add dependency checker for required tools - -## Phase 3: Configuration Assessment ✅ - -- [x] Implement recursive directory scanner -- [x] Create terraform file parser -- [x] Add complexity analyzer -- [x] Implement risk assessment calculator -- [x] Create dependency graph generator -- [x] Add report generator for assessment results -- [x] Implement visualization for dependencies - -## Phase 4: Common Upgrade Utilities ✅ - -- [x] Create file backup/restore system -- [x] Implement terraform command runner -- [x] Add HCL transformer system -- [x] Create provider migration utilities -- [x] Implement pre/post validation framework -- [x] Add compatibility checker between versions -- [x] Create progress tracking system - -## Phase 5: Version-Specific Upgraders ✅ - -- [x] Implement base TerraformUpgrader interface -- [x] Create 0.12 to 0.13 upgrader module -- [x] Implement 0.13 to 0.14 upgrader module -- [x] Create 0.14 to 0.15 upgrader module -- [x] Implement 0.15 to 1.0 upgrader module -- [x] Add version-specific transformation rules -- [x] Create upgrader controller for orchestration - -## Phase 6: Built-in Safeguards ✅ - -- [x] Implement automatic backup creation -- [x] Add dry-run functionality for change preview -- [x] Create interactive step-by-step mode -- [x] Add git branch-based workflow support -- [x] Implement validation before and after upgrades -- [x] Create comprehensive logging for all operations -- [x] Add rollback/restore capabilities - -## Phase 7: Documentation and Testing ⏳ - -- [ ] Complete CLI documentation -- [ ] Write detailed usage guides -- [ ] Create example workflows -- [ ] Add troubleshooting section -- [ ] Complete API documentation for developers -- [x] Implement basic unit tests for core components -- [x] Implement basic integration tests for workflows -- [x] Create simple test fixtures -- [ ] Create complex test fixtures -- [ ] Test with real-world Census Bureau configurations -- [ ] Achieve >85% test coverage - -## Phase 8: GitHub Project Integration ⬜ - -- [ ] Implement GitHub API client -- [ ] Add authentication support for GitHub API -- [ ] Create ticket status management functions -- [ ] Implement commands for updating ticket status -- [ ] Add functionality to create/update tickets -- [ ] Implement comment and report attachment -- [ ] Create GitHub-compatible report generators -- [ ] Add PR creation and management features -- [ ] Implement CLI commands for GitHub interaction -- [ ] Create Makefile targets for GitHub operations -- [ ] Document GitHub integration features -- [ ] Test GitHub workflow automation - -## Phase 9: Test Case Collection and Management ⬜ - -- [ ] Create test case extraction utility -- [ ] Implement pattern detection for test cases -- [ ] Develop test fixture minimization tool -- [ ] Create directory structure for real-world test fixtures -- [ ] Implement metadata tracking for test fixtures -- [ ] Add automated test case validation -- [ ] Create test coverage tracking for patterns -- [ ] Implement regression test framework -- [ ] Document test case contribution workflow -- [ ] Create tools to clean and normalize test fixtures -- [ ] Build pattern library from collected test cases -- [ ] Add Census-specific pattern documentation - -## Phase 10: Pre-Release Verification ⬜ - -- [ ] Verify all CLI commands function properly -- [ ] Check backup and restore functionality on varied configurations -- [ ] Test on representative configuration samples -- [ ] Validate dry-run output matches actual changes -- [ ] Confirm interactive mode functions correctly -- [ ] Verify log files contain appropriate detail -- [ ] Test on multiple target environments -- [ ] Validate AWS partition handling (commercial vs GovCloud) -- [ ] Test cross-account resource handling -- [ ] Conduct performance testing on large repositories -- [ ] Verify GitHub integration features -- [ ] Test end-to-end workflow with GitHub project - -## Phase 11: Advanced Features (Future Work) ⬜ - -- [ ] Improve error handling for complex edge cases -- [ ] Enhance rollback mechanism for failed upgrades -- [ ] Add support for ignoring specific files or directories -- [ ] Implement custom upgrade paths configuration -- [ ] Add support for upgrading to future versions (1.11+) -- [ ] Create HTML report generation -- [ ] Implement summary statistics for changes made -- [ ] Add export capabilities (JSON, CSV) -- [ ] Create pull request generation for automated review -- [ ] Add dashboard for upgrade progress monitoring -- [ ] Implement automated pattern recommendations -- [ ] Create upgrade analytics for project-wide insights - -## Current Status Summary - -- **Core functionality**: ✅ COMPLETE -- **Validation and testing**: ⏳ IN PROGRESS (basic tests complete, more needed) -- **Documentation**: ⏳ IN PROGRESS (needs significant work) -- **GitHub integration**: ⬜ NOT STARTED -- **Test case management**: ⬜ NOT STARTED -- **Pre-release verification**: ⬜ NOT STARTED -- **Advanced features**: ⬜ FUTURE WORK - -## Next Priorities - -1. Complete documentation (especially CLI usage guide and examples) -2. Begin GitHub project integration implementation -3. Design test case collection framework -4. Improve test coverage with more complex fixtures -5. Begin pre-release verification process -6. Test with real Census Bureau configurations -7. Fix any issues identified during testing diff --git a/copilot-instructions.md b/copilot-instructions.md deleted file mode 100644 index 982be39b..00000000 --- a/copilot-instructions.md +++ /dev/null @@ -1,110 +0,0 @@ -# GitHub Copilot Instructions - Terraform Upgrade Tool - -## Project Overview -This tool automates upgrading Terraform configurations from version 0.12.x through 0.13, 0.14, 0.15, to 1.0. It's designed as a one-time operation tool with a focus on safety, observability, and reliability rather than long-term maintainability. - -## Architecture -The project follows a modular Python architecture: -- `cli.py` - Command-line interface with Click framework -- `scanner.py` - Identifies Terraform configurations requiring upgrades -- `version_detector.py` - Determines current Terraform version -- `complexity_analyzer.py` - Assesses risk and complexity -- `dependency_graph.py` - Analyzes module dependencies -- `risk_assessment.py` - Evaluates upgrade risk and priority -- `upgrade_controller.py` - Orchestrates the upgrade process -- `/version_upgraders/` - Version-specific upgraders (v0_13.py, v0_14.py, etc.) -- `/utils/` - Shared utilities (file_manager.py, terraform.py, git.py, etc.) -- `/reporters/` - Output generators (console.py, markdown.py, file.py) - -## Code Style -- Python code follows PEP 8 with Black and isort formatting -- Line length: 88 characters -- Use type hints wherever possible -- Document all functions with docstrings -- Exception handling should be granular and well-documented -- Keep functions focused and under 50 lines where possible - -## Patterns -- Use Click for command-line interfaces -- Follow Reporter pattern for output generation -- Use Progress Tracker for monitoring long operations -- Apply Strategy pattern for version-specific upgraders -- Implement Factory pattern for upgrader selection -- Prefer composition over inheritance - -## Testing Approach -- Use pytest for all tests -- Place test fixtures in the tests/fixtures directory -- Organize tests into unit/, integration/, and validation/ directories -- Mock external commands in unit tests -- Use descriptive test names that explain what's being tested - -## Integration Points -- Respect .tf-control files for Terraform binary selection -- Match tf-control's logging patterns in logs/ directory -- Extract AWS profiles using Census Bureau's tf-run.sh approach -- Create backups before all operations -- Support Git-based workflows for upgrade tracking - -## Safety First -When generating code, prioritize: -- Comprehensive error handling -- Clear user feedback -- Backup/restore capabilities -- Dry-run functionality where appropriate -- Validation before and after changes - -## Project-Specific Terminology -- Terragrunt: A thin wrapper for Terraform used in some Census projects -- tf-control: Census Bureau's script for managing Terraform execution -- tf-run.sh: Census Bureau's workflow automation script for Terraform -- .tf-control file: Configuration file that specifies which Terraform binary to use - -## Common Utility Functions -- `validate_aws_profile`: Checks AWS credentials and returns account info -- `get_terraform_binary`: Determines appropriate Terraform binary based on .tf-control files -- `run_terraform_command`: Executes Terraform commands with appropriate logging -- `format_time`: Formats time durations for human-readable output -- `parallel_scan_directories`: Processes directories concurrently for better performance - -## Directory Structure -tf_upgrade/ -├── init.py -├── cli.py # Command-line interface -├── complexity_analyzer.py # Analyzes configuration complexity -├── config.py # Configuration management -├── dependency_graph.py # Module dependency analysis -├── env_validator.py # Environment validation -├── reporters/ # Output formatting -│ ├── init.py -│ ├── console.py # Terminal output -│ ├── file.py # File-based reporting -│ └── markdown.py # Markdown report generation -├── risk_assessment.py # Risk evaluation -├── scanner.py # Directory scanning -├── upgrade_controller.py # Upgrade orchestration -├── utils/ # Utility functions -│ ├── init.py -│ ├── file_manager.py # File operations and backups -│ ├── git.py # Git repository operations -│ ├── hcl_transformer.py # HCL syntax transformation -│ ├── parallel.py # Parallel processing -│ ├── provider_migration.py # Provider handling -│ ├── terraform.py # Terraform operations -│ ├── terraform_runner.py # Terraform command execution -│ └── validator.py # Configuration validation -├── version_detector.py # Version detection -└── version_upgraders/ # Version-specific upgraders -├── init.py -├── v0_13.py # 0.13 upgrade logic -├── v0_14.py # 0.14 upgrade logic -├── v0_15.py # 0.15 upgrade logic -└── v1_0.py # 1.0 upgrade logic - -## What to Avoid -- Direct writes to state files -- AWS-specific assumptions beyond profile handling -- Complex CI/CD integration -- Hard coding paths or credentials -- Overly complex inheritance hierarchies -- Assuming persistent state between command invocations diff --git a/create_project_structure.sh b/create_project_structure.sh deleted file mode 100755 index b93fb5dc..00000000 --- a/create_project_structure.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# Script to create the initial project structure for the terraform upgrade tool - -# Script banner -echo "Terraform Upgrade Tool - Project Structure Setup" -echo "================================================" - -# Create directory structure -echo "Creating directory structure..." -mkdir -p tf_upgrade/{__pycache__,version_upgraders,reporters,utils} -mkdir -p tests/fixtures -mkdir -p docs -mkdir -p upgrade-logs -mkdir -p examples - -# Create __init__.py files to make directories packages -touch tf_upgrade/__init__.py -touch tf_upgrade/utils/__init__.py -touch tf_upgrade/version_upgraders/__init__.py -touch tf_upgrade/reporters/__init__.py -touch tests/__init__.py - -# Create empty Python module files -touch tf_upgrade/cli.py -touch tf_upgrade/config.py -touch tf_upgrade/env_validator.py -touch tf_upgrade/tf_parser.py -touch tf_upgrade/upgrade_controller.py - -# Create version upgraders -touch tf_upgrade/version_upgraders/v0_13.py -touch tf_upgrade/version_upgraders/v0_14.py -touch tf_upgrade/version_upgraders/v0_15.py -touch tf_upgrade/version_upgraders/v1_0.py - -# Create reporters -touch tf_upgrade/reporters/console.py -touch tf_upgrade/reporters/html.py -touch tf_upgrade/reporters/csv.py - -# Create utility modules -touch tf_upgrade/utils/aws.py -touch tf_upgrade/utils/git.py -touch tf_upgrade/utils/terraform.py - -# Create test files -touch tests/test_env_validator.py -touch tests/test_tf_parser.py -touch tests/test_upgraders.py - -echo "Setting up basic config files..." -# Create default config if it doesn't exist -if [ ! -f terraform-upgrade-config.yaml ]; then - cat > terraform-upgrade-config.yaml << EOF -terraform: - binaries: - "0.12": "terraform-0.12" - "0.13": "terraform-0.13" - "0.14": "terraform-0.14" - "0.15": "terraform-0.15" - "1.0": "terraform-1.0" - upgrade: - incremental: true - backup: true - validation: true - auto_approve: false -aws: - profile: null - region: null - state_backup: true -parallel: - scan: - enabled: true - max_workers: 10 - upgrade: - enabled: false - max_workers: 5 -reporting: - format: "markdown" - detail_level: "medium" - output_dir: "./upgrade-reports" -EOF -fi - -echo "Project structure created successfully!" -echo "Run 'pip install -e .' to install in development mode" diff --git a/create_test_fixtures.sh b/create_test_fixtures.sh deleted file mode 100755 index 4bb8a52f..00000000 --- a/create_test_fixtures.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/bin/bash - -# Script to create test fixtures for Terraform upgrade testing -echo "Creating Terraform test fixtures..." - -# Set up base directory -BASE_DIR="tests/fixtures" -mkdir -p $BASE_DIR - -# Create test fixture for Terraform 0.12 -create_0_12_simple() { - local fixture_dir="$BASE_DIR/0.12/simple" - echo "Creating simple 0.12 fixture at $fixture_dir" - mkdir -p "$fixture_dir" - - # Create main.tf - cat > "$fixture_dir/main.tf" << EOF -# Simple Terraform 0.12 configuration -provider "aws" { - region = "us-east-1" - version = "~> 2.0" -} - -resource "aws_s3_bucket" "example" { - bucket = "example-bucket" - acl = "private" - - tags = { - Name = "Example bucket" - Environment = "\${var.environment}" - } -} - -output "bucket_name" { - value = "\${aws_s3_bucket.example.id}" -} -EOF - - # Create variables.tf - cat > "$fixture_dir/variables.tf" << EOF -variable "environment" { - description = "Environment name" - default = "test" -} - -# This list uses old syntax -variable "item_list" { - description = "List of items" - default = list("item1", "item2", "item3") -} -EOF - - # Create test metadata - cat > "$fixture_dir/metadata.json" << EOF -{ - "version": "0.12", - "expected_changes": { - "provider_source_adds": 1, - "interpolation_updates": 2, - "list_syntax_updates": 1 - }, - "description": "Simple configuration with basic AWS resources" -} -EOF -} - -create_0_12_medium() { - local fixture_dir="$BASE_DIR/0.12/medium" - echo "Creating medium 0.12 fixture at $fixture_dir" - mkdir -p "$fixture_dir" - - # Create main.tf with more complex resources - cat > "$fixture_dir/main.tf" << 'EOT' -provider "aws" { - region = "us-east-1" - version = "~> 2.0" -} - -provider "random" { - version = "~> 2.2" -} - -module "s3_bucket" { - source = "./modules/s3" - - bucket_name = "tf-upgrade-test-bucket" - environment = var.environment -} - -resource "aws_iam_role" "example" { - name = "example-role" - - assume_role_policy = < "$fixture_dir/variables.tf" << EOF -variable "environment" { - description = "Environment name" - default = "test" -} - -variable "instance_names" { - type = list(string) - default = ["web-1", "web-2"] -} -EOF - - # Create modules directory - mkdir -p "$fixture_dir/modules/s3" - - # Create module files - cat > "$fixture_dir/modules/s3/main.tf" << EOF -variable "bucket_name" { - description = "Name of the S3 bucket" - type = string -} - -variable "environment" { - description = "Environment tag" - type = string -} - -resource "aws_s3_bucket" "this" { - bucket = var.bucket_name - acl = "private" - - versioning { - enabled = true - } - - tags = { - Name = var.bucket_name - Environment = var.environment - } -} - -output "bucket_id" { - value = aws_s3_bucket.this.id -} -EOF - - # Create expected output for 0.13 upgrade - mkdir -p "$fixture_dir/expected/0.13" - - # Create test metadata - cat > "$fixture_dir/metadata.json" << EOF -{ - "version": "0.12", - "expected_changes": { - "provider_source_adds": 2, - "interpolation_updates": 1, - "map_syntax_updates": 1, - "required_providers_block": true - }, - "description": "Medium complexity configuration with modules and multiple resources" -} -EOF -} - -# Create fixtures for each Terraform version -create_0_12_simple -create_0_12_medium - -echo "Test fixtures created successfully!" diff --git a/debug_scanner.py b/debug_scanner.py deleted file mode 100644 index a5f1154f..00000000 --- a/debug_scanner.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -""" -Diagnostic script to debug directory scanning issues. -""" - -import logging -import os -import sys - -# Configure basic logging -logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", -) - -logger = logging.getLogger("debug-scanner") - - -def list_terraform_files(root_dir, max_depth=20): - """ - Recursively list all Terraform files under a directory. - """ - tf_dirs = [] - total_dirs_scanned = 0 - - def _walk(current_dir, depth=0): - nonlocal total_dirs_scanned - total_dirs_scanned += 1 - - if depth > max_depth: - logger.warning(f"Max depth reached at: {current_dir}") - return - - logger.debug(f"Scanning directory at depth {depth}: {current_dir}") - - try: - dir_items = os.listdir(current_dir) - - # Check for .tf files - tf_files = [f for f in dir_items if f.endswith(".tf")] - if tf_files: - logger.info(f"Found {len(tf_files)} Terraform files in: {current_dir}") - tf_dirs.append({"path": current_dir, "files": tf_files}) - - # Recursively scan subdirectories - for item in dir_items: - item_path = os.path.join(current_dir, item) - if os.path.isdir(item_path) and item not in [ - ".git", - ".terraform", - "node_modules", - ]: - _walk(item_path, depth + 1) - - except (PermissionError, FileNotFoundError) as e: - logger.error(f"Error accessing {current_dir}: {str(e)}") - - # Start the recursive scan - _walk(root_dir) - - return {"terraform_dirs": tf_dirs, "total_dirs_scanned": total_dirs_scanned} - - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: python debug_scanner.py ") - sys.exit(1) - - dir_path = sys.argv[1] - if not os.path.isdir(dir_path): - print(f"Error: {dir_path} is not a directory") - sys.exit(1) - - print(f"Starting diagnostic scan of {dir_path}") - results = list_terraform_files(dir_path) - - print("\n----- SCAN RESULTS -----") - print(f"Total directories scanned: {results['total_dirs_scanned']}") - print(f"Found {len(results['terraform_dirs'])} directories with Terraform files") - - for i, tf_dir in enumerate(results["terraform_dirs"], 1): - print(f"\n{i}. {tf_dir['path']}") - print(f" Files: {', '.join(tf_dir['files'])}") - - print("\nDiagnostic scan complete") diff --git a/plan.md b/plan.md deleted file mode 100644 index 54898914..00000000 --- a/plan.md +++ /dev/null @@ -1,1210 +0,0 @@ -# Terraform Upgrade Plan: 0.12.x to 1.x - -## 1. Introduction - -This document outlines a comprehensive plan for upgrading Terraform configurations from version 0.12.x to 1.x following the Census Bureau's recommended upgrade process. The upgrade will proceed through each intermediate version (0.13, 0.14, 0.15) before reaching 1.x to ensure compatibility and minimize disruption. - -## 2. Prerequisites - -- ~~Backup of all existing Terraform configurations~~ (Git provides version control, so separate backups are not needed) -- ✅ Verification of required tools via Makefile (see "Tool Verification" section) -- ✅ Required AWS profiles configured in `~/.aws/config` or retrievable via `aws configure list-profiles` -- Completion of all pending infrastructure changes before beginning the upgrade -- Access to testing environments to validate upgraded configurations - -### Tool Verification - -✅ The Makefile provides a "verify-tools" target to check for required tool installation. This has been implemented in the repository's Makefile. - -### AWS Profile Configuration - -✅ The AWS profile configuration detection and validation has been implemented. The tool can now: - -1. Verify AWS profile setup: - ```bash - aws configure list-profiles - ``` - -2. Check active profile configuration: - ```bash - cat ~/.aws/config - ``` - -3. Select the appropriate profile for each workspace environment: - ```bash - export AWS_PROFILE=profile-name - ``` - -4. Additional profile guidance is available in the repository documentation at `/apps/terraform/workspaces/morga471/terraform/docs/aws-profiles.md` - -## 3. Upgrade Path - -Following HashiCorp's recommended upgrade path and Census Bureau guidelines, we will upgrade through each incremental version: - -``` -Terraform 0.12.x → 0.13.x → 0.14.x → 0.15.x → 1.x -``` - -## 4. Phased Approach - -### Phase 1: Inventory and Assessment - -1. **Tool and Environment Verification** - - Run `make verify-tools` to ensure all required tools are installed - - Verify specific Terraform versions and their locations - - Test version switching capabilities - - Verify additional required tools - - - Confirm correct AWS profile configuration for each workspace - - List and verify available profiles - - Test authentication for each required profile - - Verify region settings match expected deployment regions - - Check for expired credentials and refresh if needed - - - Verify Git access and permissions for all Terraform configuration repositories - - Test read access to repositories - - Verify branch permissions for making changes - - Setup Git configuration for tracking upgrade changes - - - Validate workspace module structure and dependencies - - Create dependency graph of modules to plan upgrade order - - Identify modules with potential complex upgrades - - Check for terraform configuration validation - -2. **Scan and Identify** - - Use `make scan` to identify all Terraform configurations requiring upgrades - - Implement recursive directory scanning with configurable depth - - Identify Terraform version from configuration files - - Generate a detailed inventory report - - Identify configurations that specifically need 0.13 upgrade - - - Document dependencies between configurations - - Build a module dependency graph - - Identify shared state dependencies - - - Identify provider versions and compatibility requirements - - Extract and catalog provider configurations - - Check for problematic provider versions - - - Create prioritization framework for upgrade sequence - - Score and prioritize directories - -3. **Risk Assessment** - - Classify configurations by complexity and criticality - - Implement complexity scoring algorithm - - Assess criticality based on resource types - - Combine complexity and criticality to determine overall risk - - - Identify configurations using deprecated features - - Scan for deprecated Terraform 0.12 features - - Generate summary of deprecated features - - - Document configurations with non-HCL syntax elements - - Detect non-HCL syntax constructs - - Generate comprehensive risk report - - - Integrate risk assessment into main workflow - -### Phase 2: Pre-Upgrade Testing - -1. **Dry Run Testing** - - Use `make dry-run-scan` to simulate upgrades without applying changes - - Review identified syntax changes and compatibility issues - - Document anticipated changes for each configuration - -2. **Environment Setup** - - Create isolation strategy for upgrading configurations - - Establish testing environments for each stage of the upgrade - -### Phase 3: Version-Specific Upgrades - -#### 3.1 Upgrade to Terraform 0.13.x - -1. **Preparation** - - Review [0.13 upgrade guide](https://www.terraform.io/upgrade-guides/0-13.html) - - Update provider source declarations - - Address deprecated interpolation syntax - -2. **Execution** - - Run `terraform 0.13upgrade` command on each configuration - - Execute `make upgrade-dir DIR=[path]` with 0.13 target - - Validate state files and plan outputs - -#### 3.2 Upgrade to Terraform 0.14.x - -1. **Preparation** - - Review [0.14 upgrade guide](https://www.terraform.io/upgrade-guides/0-14.html) - - Address provider version constraints - - Update sensitive output handling - -2. **Execution** - - Execute `make upgrade-dir DIR=[path]` with 0.14 target - - Validate configuration lock files - - Test with `terraform plan` and resolve discrepancies - -#### 3.3 Upgrade to Terraform 0.15.x - -1. **Preparation** - - Review [0.15 upgrade guide](https://www.terraform.io/upgrade-guides/0-15.html) - - Address deprecated function calls - - Update configuration for removed features - -2. **Execution** - - Execute `make upgrade-dir DIR=[path]` with 0.15 target - - Run `terraform validate` on all configurations - - Address any warnings that will become errors in 1.x - -#### 3.4 Upgrade to Terraform 1.x - -1. **Preparation** - - Review [1.0 upgrade guide](https://www.terraform.io/upgrade-guides/1-0.html) - - Ensure compatibility with all used providers - - Update any custom modules - -2. **Execution** - - Execute `make upgrade-dir DIR=[path]` with 1.0 target - - Run comprehensive validation tests - - Apply configurations in test environment - -### Phase 4: Post-Upgrade Verification - -1. **Validation** - - Confirm state consistency before and after upgrade - - Verify no unexpected resource modifications - - Run integration tests against upgraded infrastructure - -2. **Documentation** - - Update READMEs and documentation with new version requirements - - Document any changes to module interfaces or variables - - Update CI/CD pipelines to use Terraform 1.x - -## 5. Implementation Strategy - -### Tools and Automation - -- Use the existing `tf-upgrade.sh` script with appropriate options -- Run `make verify-tools` before starting any upgrade process -- Confirm AWS profile setup using `aws configure list-profiles` -- Available commands: - - `make scan` - Identify directories needing upgrades - - `make dry-run` - Preview changes without modification - - `make upgrade` - Run the full upgrade process - - `make upgrade-dir DIR=path/to/dir` - Upgrade specific directory - - `make verbose` or `make debug` - For detailed logging - - `make auto-apply` - Apply changes without prompting (for CI/CD) - - `make no-backup` - Skip backups (not recommended) - - `make step` - Interactive step-by-step execution - -### Rollback Procedure - -1. **Git-Based Rollback** - - Use Git history to revert to pre-upgrade state if needed - - Use `git log` to identify pre-upgrade commits - - Use `git checkout` or `git revert` to restore previous state - -2. **State Management** - - If state was corrupted, use terraform state management commands to restore - - Use `terraform state pull > state-backup.tfstate` before upgrades - - Execute `terraform init -reconfigure` to reset providers - -## 6. Timeline and Milestones - -| Stage | Timeframe | Key Deliverables | -|-------|-----------|------------------| -| Inventory & Assessment | Week 1-2 | Complete scan report, risk assessment | -| Pre-Upgrade Testing | Week 3 | Testing environments, dry-run reports | -| 0.12.x to 0.13.x | Week 4-5 | Upgraded configurations, validation report | -| 0.13.x to 0.14.x | Week 6-7 | Upgraded configurations, validation report | -| 0.14.x to 0.15.x | Week 8-9 | Upgraded configurations, validation report | -| 0.15.x to 1.x | Week 10-11 | Fully upgraded configurations | -| Verification & Documentation | Week 12 | Final validation reports, updated documentation | - -## 7. Risk Management - -| Risk | Impact | Mitigation | -|------|--------|------------| -| State file corruption | High | Regular backups, testing in isolated environment | -| Provider incompatibilities | Medium | Thorough version compatibility research | -| Resource drift | Medium | Detailed planning and review of each `terraform plan` | -| Downtime during transition | Medium | Schedule upgrades during maintenance windows | -| Custom module failures | Medium | Test modules independently before integration | - -## 8. References - -- [HashiCorp Terraform Upgrade Guides](https://www.terraform.io/upgrade-guides) -- [Census Bureau Terraform Guidelines]() -- [Provider Version Compatibility Matrix]() -- AWS Profile Documentation: `/apps/terraform/workspaces/morga471/terraform/docs/aws-profiles.md` -- Terraform Upgrade Tool Documentation: `/apps/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/README.md` -- Internal documentation and best practices - -## 9. Appendix: Command Reference - -```bash -# Verify tools installation -make verify-tools - -# Check AWS profiles -aws configure list-profiles - -# Set AWS profile -export AWS_PROFILE=profile-name - -# Scan for directories needing upgrade -make scan - -# Perform dry run to see what would change -make dry-run-scan - -# Upgrade specific directory -make upgrade-dir DIR=path/to/terraform/config - -# Run full upgrade with verbose logging -make verbose upgrade - -# Step-by-step interactive upgrade -make step -``` - -## 10. Python Implementation Plan - -Given the complexity of this upgrade process and the need for robust error handling, logging, and reporting, we should develop a Python-based implementation rather than relying solely on Bash scripts. - -### Python Implementation Benefits - -- **Better Error Handling**: Structured exception handling instead of complex exit code checking -- **Improved Logging**: Comprehensive logging with different verbosity levels -- **Configuration Management**: Better handling of complex configuration options -- **Modular Design**: Separate modules for each upgrade step -- **Testing**: Easier to write unit tests for Python code than Bash scripts -- **Reporting**: Generate detailed HTML/CSV reports of upgrade status -- **Parallelization**: Option to process multiple directories concurrently where safe - -### Core Python Modules - -1. **Environment Verification Module** - - Tool verification and version checking - - AWS profile validation - - Git repository access validation - -2. **Terraform Configuration Parser** - - Identify required upgrades based on syntax analysis - - Extract provider requirements and module dependencies - - Build dependency graph for optimal upgrade ordering - -3. **Upgrade Process Controller** - - Orchestrate the step-by-step upgrade process - - Handle version-specific upgrade requirements - - Manage rollbacks if issues are encountered - -4. **Reporting Engine** - - Generate pre-upgrade assessment reports - - Track progress during upgrades - - Produce final upgrade summary reports - -### Implementation Structure - -The directory structure has been created according to the plan. - -### Command-Line Interface - -A basic CLI has been implemented in tf_upgrade/cli.py with commands for scanning, dry-run, upgrade, and environment verification. - -### Makefile Integration - -The Makefile has been created with targets for invoking the Python modules. - -## 11. Refined Implementation Considerations - -### One-Time Operation Focus - -This upgrade tool is designed as a one-time operation to transition all existing Terraform resources from 0.12.x to 1.x. Once complete, the organization can standardize on newer tooling and practices. - -### Pull Request-Based Validation - -- Each directory upgrade will generate a separate pull request -- This enables human validation of all changes before applying -- The PR description will include: - - Summary of changes made - - Before/after comparison of key files - - Validation results from `terraform validate` - - List of potential issues requiring manual review - -### Dry Run Implementation - -The dry run functionality will be comprehensive: - -```bash -# Through Makefile -make dry-run DIR=/path/to/directory -``` - -Dry run output will include: -- Files that would be modified -- Syntax changes that would be applied -- Provider references that would be updated -- Potential issues or warnings -- Estimated success probability - -### Simplified Makefile Interface - -Enhanced Makefile for non-technical users: - -```makefile -# Basic operations -scan: ## Scan for directories needing upgrade - python3 -m tf_upgrade.scan $(DIR) - -dry-run: ## Show what changes would be made without applying them - python3 -m tf_upgrade.dry_run $(DIR) - -upgrade: ## Perform the actual upgrade - python3 -m tf_upgrade.upgrade $(DIR) - -generate-pr: ## Generate pull requests for upgrades - python3 -m tf_upgrade.generate_pr $(DIR) - -# Help target for self-documentation -help: ## Show this help - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' - -.DEFAULT_GOAL := help -``` - -### Clear CLI Prompting - -For user input scenarios, implement clear prompts with suggestions: -- Show current version and available target versions -- Provide sensible defaults (incremental next version) -- Handle invalid input gracefully -- Use color coding for better readability -- Implemented in the CLI module - -### Documentation Focus - -README.md will be comprehensive and clearly written for non-technical users: - -- Step-by-step instructions with examples -- Troubleshooting section for common issues -- Visual indicators of success/failure -- Explanation of the PR review process -- Command reference with explanations - -Example README section: -``` -## Quick Start - -1. **Verify your environment**: - ``` - make verify-tools - ``` - - Make sure you see "All critical tools are available" before proceeding. - -2. **Scan your Terraform directories**: - ``` - make scan DIR=/path/to/terraform - ``` - -3. **Try a dry run first**: - ``` - make dry-run DIR=/path/to/terraform - ``` - - Review the proposed changes carefully. - -4. **Perform the upgrade**: - ``` - make upgrade DIR=/path/to/terraform - ``` - -5. **Review generated pull requests** - The tool will output links to the PRs it creates. -``` - -## 12. Removing Unnecessary Components - -Due to the one-time nature of this operation, the following sections from the original plan are deprioritized: - -- ~~Long-term maintenance planning~~ -- ~~CI/CD integration~~ -- ~~Extensive test suite development~~ -- ~~Complex versioning and backwards compatibility~~ -- ~~Multiple output format support~~ - -Focus will remain on: -- Reliable execution of the upgrade process -- Clear visibility into proposed changes -- Human-readable reporting and PR generation -- Comprehensive documentation for one-time use - -## 13. Integration with Existing Census Bureau Tools - -To ensure this upgrade tool works seamlessly with established Census Bureau practices and tooling, we'll integrate with two key existing systems: - -### 13.1 Integration with tf-control - -The Census Bureau uses a `tf-control.sh` script system to manage Terraform execution environments. Our tool must respect and integrate with this approach: - -1. **Version Selection** - - Detect and use the appropriate Terraform version by checking `.tf-control` files in: - - Current directory - - Git repository root - - User's home directory - - Respect the `TFCOMMAND` variable defined in these files for running Terraform commands - -2. **Logging Compatibility** - - Follow the established pattern of logging to the `logs/` directory - - Use timestamped log filenames matching the tf-control format - - Include similar header information (git repository, branch, etc.) - -3. **Command Execution** - - Run commands through the appropriate `tf-x` wrapper when applicable - - Ensure our tool's output can be consumed by tf-control's summary features - -### 13.2 Integration with tf-run - -The `tf-run.sh` script provides workflow automation for Terraform operations. Our tool will: - -1. **AWS Profile Handling** - - Use the same profile detection mechanism as in tf-run.sh - - Extract profile information from .tfvars files when environment variables aren't set - -2. **Module Upgrade Readiness** - - Prioritize upgrades based on the known upgrade-ready module list: - ``` - aws-common-security-groups - aws-edl-launch-instance - aws-iam-role - aws-iam-user - aws-inf-setup - aws-s3 - aws-setup-s3-object-logging - aws-tls-certificate - aws-vpc-setup - dns-lookup - aws-ecr-copy-images - ``` - - Check for modules that should use `ref=tf-upgrade` but don't yet - -3. **Repository Structure** - - Respect the existing repository organization patterns - - Detect use of linked files/modules through LINK commands - -### 13.3 Updates to Implementation Strategy - -Based on these integration requirements: - -1. The `env_validator.py` module: - - Checks for .tf-control files (✅ Implemented) - - Validates correct terraform binaries are available (✅ Implemented) - - Extracts appropriate AWS profile settings (✅ Implemented) - -2. The `terraform.py` utility module: - - Generates logs that match tf-control format (✅ Implemented) - - Determines the correct terraform binary to use (✅ Implemented) - - Extracts configuration information using patterns from tf-run (✅ Implemented) - -3. The `git.py` utility module: - - Validates Git repository access (✅ Implemented) - - Sets up Git user configuration (✅ Implemented) - - Creates branches for upgrades (✅ Implemented) - -## 14. Complete Environment Preparation Implementation - -To fully implement step 2 of the checklist: - -### 14.1 AWS Profile Validation Implementation -- ✅ Implemented in `tf_upgrade/utils/terraform.py` -- Follows tf-run.sh's `get_profile()` pattern -- Includes STS validation of profile credentials - -### 14.2 Git Repository Access Check Implementation -- ✅ Implemented in `tf_upgrade/utils/git.py` -- Tests both read and write access to repositories -- Includes cleanup of test branches - -### 14.3 .tf-control File Detection Implementation -- ✅ Implemented in `tf_upgrade/env_validator.py` -- Handles all three search locations (current dir, git root, home) -- Supports both .tf-control and .tf-control.tfrc - -### 14.4 Terraform Binary Selection Implementation -- ✅ Implemented in `tf_upgrade/utils/terraform.py` -- Parses and respects the TFCOMMAND variable -- Falls back gracefully if specified binary not found - -### 14.5 Logging Compatible with tf-control Format -- ✅ Implemented in `tf_upgrade/utils/terraform.py` -- Creates log files with compatible headers and footers -- Captures git and terraform version metadata - -### 14.6 Directory Organization Scheme -- ✅ Implemented and documented in `docs/organization.md` -- Describes recommended structure for terraform configurations -- Explains handling of linked modules and tf-control files - -## 15. Next Steps for Tool Development - -For step 3 of the checklist, we need to implement: - -1. ✅ **Configuration Management Module** - - Implemented with PyYAML for parsing and managing configuration - - Supports both global and directory-specific settings - - Includes validation for configuration parameters - -2. ✅ **Progress Reporting System** - - Implemented with standardized progress tracking mechanism - - Supports both CLI and file-based reporting - - Includes estimated time remaining for long operations - -3. **Version-Specific Upgraders** - - Each upgrader (`v0_13.py`, `v0_14.py`, etc.) should handle: - - Version-specific syntax changes - - Provider version compatibility - - Required command execution sequence - - Validation before and after changes - -4. ✅ **Console and Markdown Reporters** - - Implemented console reporter with color and emoji support - - Added markdown report generator for PR inclusion - - Added JSON file reporter for detailed progress tracking - -## 16. Assessment Tools Implementation Guide - -The Assessment Tools section of the checklist has been fully implemented with the following components: - -### 16.1 Directory Scanner - -✅ **Implemented** in `tf_upgrade/scanner.py` - -The scanner recursively identifies Terraform configurations throughout a codebase and extracts key information from each directory, including: -- Resources, data sources, and modules usage -- Provider and backend configurations -- Basic complexity metrics -- Module identification - -It supports parallel scanning of large codebases and exports results in JSON or CSV formats. - -### 16.2 Terraform Version Detector - -✅ **Implemented** in `tf_upgrade/version_detector.py` - -The version detector identifies which Terraform version is being used and determines the necessary upgrade path by: -- Parsing version constraints in configuration files -- Detecting version-specific syntax features -- Identifying deprecated syntax patterns -- Providing a clear upgrade path recommendation - -### 16.3 Dependency Graph Generator - -✅ **Implemented** in `tf_upgrade/dependency_graph.py` - -This component builds a directed graph representing relationships between Terraform modules and configurations: -- Uses NetworkX for graph structure and analysis -- Supports visualization in PNG or DOT formats -- Identifies cycles in module dependencies -- Determines optimal upgrade order using topological sorting -- Provides detailed dependency information for planning - -### 16.4 Complexity Analyzer - -✅ **Implemented** in `tf_upgrade/complexity_analyzer.py` - -The complexity analyzer assesses Terraform configurations and provides risk scores based on: -- Resource counts and module usage -- Dynamic blocks and complex expressions -- Deprecated syntax patterns -- Advanced Terraform features usage - -### 16.5 Risk Assessment Module - -✅ **Implemented** in `tf_upgrade/risk_assessment.py` - -This module combines inputs from the other analyzers to: -- Calculate comprehensive risk scores -- Prioritize directories for upgrade -- Generate detailed Markdown reports -- Provide specific upgrade approach recommendations -- Estimate required effort for each configuration - -### CLI Integration - -✅ **Implemented** in `tf_upgrade/cli.py` - -The command-line interface has been extended with commands for assessment: -- `tf-upgrade scan` - Scan directories for Terraform configurations -- `tf-upgrade detect-version` - Detect Terraform version in use -- `tf-upgrade generate-graph` - Create dependency visualizations -- `tf-upgrade analyze-complexity` - Analyze configuration complexity -- `tf-upgrade assess-risk` - Perform comprehensive risk assessment -- `tf-upgrade dry-run` - Simulate upgrades without making changes - -### Reporter System - -✅ **Implemented** in `tf_upgrade/reporters/` - -Multiple reporter classes handle output formats: -- Console reporter with color and progress indicators -- File reporter for JSON data output -- Markdown reporter for human-readable reports - -## 17. Next Steps: Upgrade Modules Implementation - -The assessment tools have been completed, and now we've implemented the version-specific upgrade modules: - -1. **Version-Specific Upgraders** - - ✅ Developed modules that handle each version transition (0.13, 0.14, 0.15, 1.0) - - ✅ Implemented version-specific syntax changes - - ✅ Handled provider compatibility issues - - ✅ Implemented pre and post-validation support - -2. **Common Upgrade Utilities** - - ✅ Created shared utilities for backup and restore - - ✅ Implemented command execution handling - - ✅ Added validation frameworks for all versions - - ✅ Implemented rollback support in case of failures - -## 18. Common Upgrade Utilities Implementation - -We have successfully implemented the common upgrade utilities that provide shared functionality used across all version-specific upgraders: - -1. **File Management System** - - ✅ Created the `FileManager` class in `utils/file_manager.py` - - ✅ Implemented methods for creating backups of different file types - - ✅ Added utilities for modifying files with transformers - - ✅ Implemented file restoration from backup capabilities - - ✅ Added backup information tracking - -2. **Terraform Command Runner** - - ✅ Created the `TerraformRunner` class in `utils/terraform_runner.py` - - ✅ Added methods for common terraform commands - - ✅ Implemented error handling and cleanup routines - - ✅ Integrated with the logging system - - ✅ Added version-specific command execution support - -3. **HCL Transformer System** - - ✅ Created the `HCLTransformer` class in `utils/hcl_transformer.py` - - ✅ Implemented regex pattern and callable transformers - - ✅ Created rule sets for each version migration - - ✅ Added directory processing with pattern matching - - ✅ Implemented detailed tracking of applied transformations - -4. **Provider Migration Utilities** - - ✅ Created the `ProviderMigration` utility in `utils/provider_migration.py` - - ✅ Added comprehensive provider mappings for hashicorp providers - - ✅ Implemented required_providers block generation - - ✅ Added support for complex provider references - - ✅ Handled version constraints from existing configurations - -5. **Validation Framework** - - ✅ Created the `TerraformValidator` in `utils/validator.py` - - ✅ Added terraform plan parsing to detect resource changes - - ✅ Implemented pre and post upgrade validation comparisons - - ✅ Added validation rules for specific terraform versions - - ✅ Created detailed reporting of validation issues - -## 19. Version-Specific Upgraders - -We've implemented specialized upgrader modules for each terraform version: - -1. **v0_13.py** - - ✅ Provider source declaration handling - - ✅ Integration with 0.13upgrade command - - ✅ Required version constraint management - -2. **v0_14.py** - - ✅ Dependency lock file generation - - ✅ Sensitive output handling - - ✅ State file change management - -3. **v0_15.py** - - ✅ Deprecated function replacements - - ✅ Removed feature handling - - ✅ Provider configuration updates - -4. **v1_0.py** - - ✅ Final validation routines - - ✅ 1.0-specific compatibility fixes - - ✅ Overall compatibility assurance - -Each upgrader implements the common `TerraformUpgrader` interface for consistency and leverages the common utilities for file manipulation, command execution, and validation. - -## 20. Upgrade Controller Implementation - -1. **UpgradeController** - - ✅ Implemented version detection and upgrade path determination - - ✅ Created step-by-step upgrade orchestration - - ✅ Added progress reporting and tracking - - ✅ Implemented interactive mode for guided upgrades - - ✅ Added comprehensive result reporting - -## 21. Testing the Implementation - -✅ **Implemented** with comprehensive test suites covering all major components. - -### 21.1 Unit Tests - -Unit tests have been implemented for all core components: - -- ✅ FileManager - tests backup, restore, and file transformation functionality -- ✅ TerraformRunner - tests command execution with version-specific binaries -- ✅ HCLTransformer - tests regex and callable transformations for HCL syntax -- ✅ ProviderMigration - tests provider block conversion for 0.13+ compatibility -- ✅ TerraformValidator - tests validation of configurations before and after upgrade - -Unit tests can be run with: -```bash -make test-unit -``` - -### 21.2 Integration Tests - -Integration tests verify that components work together correctly: - -- ✅ Complete upgrade path (0.12 → 0.13 → 0.14 → 0.15 → 1.0) -- ✅ Step-by-step upgrades with validation at each stage -- ✅ Error handling and recovery during multi-step processes -- ✅ Configuration transformation and validation integration - -Integration tests can be run with: -```bash -make test-integration -``` - -### 21.3 Validation Tests - -Validation tests ensure that upgrades produce correct results: - -- ✅ Simple configurations (single resource, minimal complexity) -- ✅ Medium configurations (multiple resources, basic modules) -- ✅ Complex configurations (multiple providers, count/for_each, complex modules) -- ✅ Verification that plans don't show unexpected resource changes - -Validation tests can be run with: -```bash -make test-validation -``` - -### 21.4 Test Fixtures - -Test fixtures have been created for various complexity levels: -- Simple - Basic provider and resource configuration -- Medium - Multiple resources with interdependencies -- Complex - Multiple providers, modules, and advanced Terraform features - -### 21.5 Running All Tests - -To run the complete test suite: -```bash -make test -``` - -Or use the test runner script: -```bash -./tests/run_tests.sh -``` - -## 22. Future Enhancements - -For future iterations, consider implementing: - -1. Advanced reporting for complex infrastructure -2. Pull request generation for automated review workflows -3. Enhanced visualization of changes made during upgrades -4. More sophisticated detection of potentially sensitive outputs -5. Extended provider coverage beyond HashiCorp defaults - -## 23. Pragmatic Approach to Quality Assurance - -Given that this tool is designed for a one-time operation to upgrade Terraform configurations, we've intentionally focused on built-in safeguards rather than extensive test suites: - -### 23.1 Existing Safety Mechanisms - -1. **Automatic Backups** - - Every operation creates comprehensive backups before making changes - - Backup files are stored with timestamps for easy identification - - Restore capabilities are built into the FileManager class - -2. **Comprehensive Dry-Run Functionality** - - The `dry-run` command shows exactly what changes would be made - - Syntax transformations are previewed without altering files - - Built-in commands are listed with their expected effects - -3. **Step-by-Step Interactive Mode** - - The `-i` / `--interactive` flag allows confirming each step - - Users can see and approve each version upgrade individually - - Provides an opportunity to verify changes at each stage - -4. **Git-Based Workflow** - - Changes are made in new branches for easy comparison and rollback - - The original state is preserved in the main branch - - Changes can be reviewed through pull requests - -### 23.2 Minimal Testing Strategy - -Rather than comprehensive test suites, we recommend: - -1. **Pre-Release Validation**: - - Run the tool in dry-run mode on representative configurations - - Manually verify the syntax transformations look correct - - Check that backups are being created properly - -2. **First-Use Testing**: - - Start with non-critical or simpler modules - - Use step-by-step mode to confirm correct operation - - Apply changes to test environments before production - -3. **Robust Logging**: - - All operations produce detailed logs for post-mortem analysis - - Multiple report formats help with documentation and communication - - Progress tracking helps identify where issues may have occurred - -This pragmatic approach balances quality with practicality for a tool with limited long-term utility. - -## 24. Future Considerations - -While this tool is designed for a one-time operation, some components may be repurposed for future needs: - -1. **HCL Transformation Engine**: Could be adapted for future syntax changes -2. **Dependency Graph Generator**: Useful for infrastructure visualization -3. **Configuration Scanner**: Helpful for infrastructure auditing - -These components have been designed with clean interfaces that could be extracted and reused if needed. - -## 25. Pre-Release Verification Implementation Guide - -Before deploying the tool for widespread use, a structured verification process should be performed. This section outlines a systematic approach to verify the tool's functionality. - -### 25.1 Controlled Verification Process - -1. **Select Representative Test Cases** - - Choose 3-5 Terraform configurations of varying complexity: - - Simple: Single module with minimal resources - - Medium: Multiple resources with some interdependencies - - Complex: Multiple modules with remote state dependencies - - Include configurations using features from each Terraform version - -2. **Document the Baseline State** - - For each test case: - - Run `terraform validate` and record output - - Run `terraform plan` and record resource counts - - Document any existing warnings - - Save copies of the original files for comparison - -3. **Verification Checklist** - - Create a verification checklist with the following sections: - - Tool Installation & Dependencies - - Command Line Interface - - Analysis Functions - - Upgrade Functions - - Reporting Functions - - Error Handling & Recovery - -### 25.2 Verifying Safety Mechanisms - -1. **Backups** - - Run `tf-upgrade upgrade` on a test configuration - - Verify backup files are created with timestamps - - Intentionally break the configuration - - Test file restoration from backups - - Verify that the configuration returns to its original state - -2. **Dry-Run Mode** - - Run `tf-upgrade dry-run` on each test configuration - - Compare dry-run output with actual changes from a real upgrade - - Verify that all detected transformations are reported - - Check that no files are modified during dry-run - -3. **Step-by-Step Mode** - - Run `tf-upgrade upgrade --interactive` - - Test aborting the upgrade mid-process - - Verify that completed steps remain applied while uncompleted steps are skipped - - Test completing the upgrade after an abort - -4. **Git Integration** - - Verify branch creation for upgrades - - Test the upgrade tool on a Git repository - - Confirm changes are isolated to the new branch - - Verify you can easily revert by switching branches - -### 25.3 Verification Matrix - -Create a verification matrix for each test configuration: - -| Feature | Simple Config | Medium Config | Complex Config | -|---------|---------------|---------------|----------------| -| Analysis scan | ✓/✗ | ✓/✗ | ✓/✗ | -| Version detection | ✓/✗ | ✓/✗ | ✓/✗ | -| Dry-run accuracy | ✓/✗ | ✓/✗ | ✓/✗ | -| Backup creation | ✓/✗ | ✓/✗ | ✓/✗ | -| Provider migration | ✓/✗ | ✓/✗ | ✓/✗ | -| Syntax transformation | ✓/✗ | ✓/✗ | ✓/✗ | -| Init/Apply success | ✓/✗ | ✓/✗ | ✓/✗ | -| Report generation | ✓/✗ | ✓/✗ | ✓/✗ | - -### 25.4 Bug Fixing Workflow - -1. **Bug Triage Process** - - Document any issues discovered during verification - - Categorize by severity: Critical, Major, Minor, Cosmetic - - Prioritize fixes based on impact to upgrade process - -2. **Regression Testing** - - After fixing bugs, re-verify affected features - - Ensure fixes don't introduce new problems - - Update verification matrix - -### 25.5 Production Readiness Checklist - -Final checklist before authorizing production use: - -- ☐ All critical features working as expected -- ☐ Backups are created reliably -- ☐ Dry-run output matches actual changes -- ☐ Interactive mode functions properly -- ☐ Reports are generated with sufficient detail -- ☐ All error conditions are handled gracefully -- ☐ Documentation is clear and comprehensive -- ☐ Configuration examples are provided -- ☐ No critical bugs remain unfixed - -### 25.6 Verification Script - -Create a verification script that automates parts of this process: - -```bash -#!/bin/bash -# Pre-release verification script for Terraform Upgrade Tool - -# Configuration -TEST_DIRS=("test-fixtures/simple" "test-fixtures/medium" "test-fixtures/complex") -TOOL_CMD="tf-upgrade" - -# Color setup -GREEN='\033[0;32m' -RED='\033[0;31m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -echo "===== Terraform Upgrade Tool Verification =====" -echo - -# 1. Check dependencies -echo "Checking dependencies..." -declare -a DEPS=("terraform" "terraform-0.13" "terraform-0.14" "terraform-0.15" "terraform-1.0" "git" "python3") - -for dep in "${DEPS[@]}"; do - if command -v $dep &>/dev/null; then - echo -e "${GREEN}✓${NC} $dep found: $(command -v $dep)" - else - echo -e "${RED}✗${NC} $dep not found!" - fi -done - -echo - -# 2. Check tool installation -echo "Checking tool installation..." -if command -v $TOOL_CMD &>/dev/null; then - echo -e "${GREEN}✓${NC} Tool installed: $($TOOL_CMD --version)" -else - echo -e "${RED}✗${NC} Tool not installed!" - exit 1 -fi - -echo - -# 3. Test configurations -for dir in "${TEST_DIRS[@]}"; do - if [ ! -d "$dir" ]; then - echo -e "${YELLOW}!${NC} Test directory not found: $dir" - continue - fi - - echo "Testing configuration in $dir..." - - # Backup test - echo " Testing backup creation..." - $TOOL_CMD upgrade --dry-run $dir - if [ -d "$dir/.terraform-upgrade-backup-"* ]; then - echo -e " ${GREEN}✓${NC} Backup created" - else - echo -e " ${RED}✗${NC} No backup created!" - fi - - # Dry-run test - echo " Testing dry-run output..." - $TOOL_CMD dry-run $dir > /tmp/dryrun-output.txt - if grep -q "would be made" /tmp/dryrun-output.txt; then - echo -e " ${GREEN}✓${NC} Dry-run output generated" - else - echo -e " ${YELLOW}!${NC} No changes detected in dry-run" - fi - - echo -done - -echo "Verification complete. Review output for any issues." -``` - -This script should be run before each release to ensure basic functionality is working correctly. - -## 26. GitHub Project Integration - -To efficiently manage the upgrade process across multiple Terraform directories, we'll integrate with the GitHub Enterprise project board that tracks upgrade tickets. This will allow for automated status updates and a consistent workflow. - -### 26.1 GitHub API Integration - -The upgrade tool will be extended to interact with GitHub projects through the GitHub API: - -1. **Authentication Methods** - - ✅ Personal access token (PAT) support - - ✅ GitHub App authentication support - - ⬜ Optional environment variable configuration (GITHUB_TOKEN) - - ⬜ Support for .netrc file credentials - -2. **Project Board Interaction** - - ⬜ Fetch tickets from specified GitHub project board - - ⬜ Filter tickets by labels, status, and other metadata - - ⬜ Update ticket status based on upgrade progress - - ⬜ Add comments with upgrade results and logs - - ⬜ Attach reports and summaries to tickets - -3. **Implementation Components** - - ⬜ GitHub client module in `tf_upgrade/integrations/github.py` - - ⬜ Configuration for GitHub settings in config files - - ⬜ CLI commands for GitHub project integration - - ⬜ Progress reporting extensions for GitHub updates - -### 26.2 Ticket Workflow Integration - -The tool will support the standard upgrade workflow as represented in the project board: - -1. **Ticket States** - - **Todo**: Initial state for directories needing upgrade - - **In Progress**: Directories currently being upgraded - - **In Review**: Upgrade completed, awaiting validation - - **Done**: Successfully upgraded and verified - -2. **Automated State Transitions** - - `tf-upgrade scan` - Identifies directories and can create tickets if they don't exist - - `tf-upgrade start-upgrade ` - Moves ticket to "In Progress" - - `tf-upgrade complete-upgrade ` - Moves ticket to "In Review" and adds results - - `tf-upgrade verify-upgrade ` - Moves ticket to "Done" after validation - -3. **Integration Commands** - - ⬜ `make github-list` - List all upgrade tickets and their status - - ⬜ `make github-claim DIR=` - Claim a directory for upgrading (moves to "In Progress") - - ⬜ `make github-update DIR= STATUS=` - Update ticket status - - ⬜ `make github-comment DIR= MESSAGE="comment"` - Add comment to ticket - - ⬜ `make github-attach-report DIR=` - Attach upgrade report to ticket - -### 26.3 Reporting and Feedback - -The tool will enhance its reporting capabilities to work with GitHub: - -1. **GitHub-Compatible Reports** - - ⬜ Generate Markdown reports compatible with GitHub comments - - ⬜ Create summary reports for ticket descriptions - - ⬜ Support for before/after comparisons in GitHub-friendly format - - ⬜ Generate upgrade statistics for project-level reporting - -2. **Pull Request Integration** - - ⬜ Create pull requests for upgrade changes - - ⬜ Link PRs to project tickets - - ⬜ Include upgrade summary in PR description - - ⬜ Add validation results to PR comments - -## 27. Test Case Collection From Real Upgrades - -As upgrades are performed on real-world configurations, we'll systematically build a library of test cases to improve testing coverage and tool reliability. - -### 27.1 Test Case Generation - -1. **Automated Test Case Extraction** - - ⬜ Implement `tf-upgrade extract-test-case DIR=` command - - ⬜ Create minimal reproducible examples from complex configurations - - ⬜ Generate before/after snapshots for verification - - ⬜ Document patterns and edge cases encountered - -2. **Test Case Organization** - - ⬜ Create a structured test case library in `test/fixtures/real-world/` - - ⬜ Categorize by complexity, patterns, and upgrade challenges - - ⬜ Include metadata about source configurations - - ⬜ Document testing strategy for each case - -### 27.2 Test Coverage Enhancement - -1. **Pattern-Based Test Matrix** - - ⬜ Identify common patterns from real configurations - - ⬜ Create a coverage matrix of pattern vs. upgrade version - - ⬜ Track testing coverage for identified patterns - - ⬜ Prioritize testing for frequently encountered patterns - -2. **Regression Testing** - - ⬜ Add regression tests for each bug or edge case discovered - - ⬜ Implement automated regression test suite - - ⬜ Link regression tests to GitHub issues/tickets - - ⬜ Create verification matrix for common edge cases - -### 27.3 Implementation Approach - -1. **Test Fixture Manager** - - ⬜ Create `tf_upgrade/test_fixtures/manager.py` for test case management - - ⬜ Implement fixture extraction and minimization - - ⬜ Add pattern detection and classification - - ⬜ Provide fixture metadata management - -2. **Integration with Upgrade Process** - - ⬜ Add hooks in upgrade process to capture interesting patterns - - ⬜ Implement opt-in test case generation during upgrades - - ⬜ Create consistent directory structure for generated test cases - - ⬜ Add documentation for test case contribution workflow - -3. **Test Case Validation** - - ⬜ Implement verification of test case validity - - ⬜ Create tools for test case cleanup and normalization - - ⬜ Add validation of before/after state equivalence - - ⬜ Support for parameterized test case variants - -### 27.4 Census Bureau Pattern Library - -1. **Census-Specific Patterns** - - ⬜ Document Census-specific Terraform patterns (EDL, etc.) - - ⬜ Create targeted test fixtures for Census patterns - - ⬜ Validate upgrade paths for Census-specific modules - - ⬜ Verify handling of Census AWS account structure - -2. **Pattern Documentation** - - ⬜ Create pattern guide with examples and upgrade notes - - ⬜ Document recommended practices for new Terraform 1.x syntax - - ⬜ Provide migration guides for Census-specific patterns - - ⬜ Link patterns to relevant test fixtures - -## 28. Comprehensive Workflow with GitHub and Testing Integration - -With GitHub integration and test case collection in place, the comprehensive upgrade workflow becomes: - -1. **Discovery and Planning** - - Run `tf-upgrade github-scan REPO_ROOT` to identify directories and update GitHub project - - Review and prioritize tickets in GitHub project - - Assign upgrade tasks to team members - -2. **Upgrade Execution** - - Claim ticket with `make github-claim DIR=` - - Perform upgrade with `make upgrade DIR=` - - Collect test cases with `make extract-test-case DIR=` - - Submit PR and update ticket with `make github-update-pr DIR=` - -3. **Validation and Completion** - - Review PRs and validation reports - - Move tickets to "Done" after verification - - Extract lessons learned for process improvement - -4. **Continuous Improvement** - - Analyze test cases for common patterns - - Update tool based on real-world feedback - - Improve documentation with real examples - -This integrated workflow ensures a systematic approach to managing the upgrade process across many configurations while continuously improving the tool and building knowledge for future efforts. diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 2e04b5aa..00000000 --- a/pyproject.toml +++ /dev/null @@ -1,26 +0,0 @@ -[tool.black] -line-length = 88 -include = '\.pyi?$' -exclude = ''' -/( - \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist -)/ -''' - -[tool.isort] -profile = "black" -line_length = 88 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true -skip_gitignore = true diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 960fe14a..00000000 --- a/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Core dependencies -click>=8.0.0 # Command line interface -pyyaml>=5.1 # Configuration management -networkx>=2.5 # For dependency graph creation -matplotlib>=3.3.0 # For graph visualization -pygraphviz>=1.7 # For graph visualization -colorama>=0.4.4 # For colored terminal output - -# Testing -pytest>=6.2.5 -pytest-cov>=2.12.1 -mock>=4.0.3 - -# Development -black>=21.5b2 -flake8>=3.9.2 -isort>=5.9.1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 4e76a952..00000000 --- a/setup.py +++ /dev/null @@ -1,44 +0,0 @@ -from setuptools import find_packages, setup - -with open("README.md", "r") as f: - long_description = f.read() - -with open("requirements.txt", "r") as f: - requirements = f.read().splitlines() - -setup( - name="terraform-upgrade-tool", - version="0.1.0", - description="Tool for upgrading Terraform configurations from 0.12.x to 1.x", - long_description=long_description, - long_description_content_type="text/markdown", - author="Census Bureau DevOps Team", - author_email="devops@census.gov", - url="https://github.com/uscensusbureau/terraform-upgrade-tool", - packages=find_packages(), - install_requires=requirements, - extras_require={ - "dev": [ - "black>=23.3.0", - "isort>=5.12.0", - "flake8>=6.0.0", - "pytest>=7.3.1", - "pre-commit>=3.3.1", - ], - }, - entry_points={ - "console_scripts": [ - "tf-upgrade=tf_upgrade.cli:main", - ], - }, - python_requires=">=3.6", - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "Topic :: Software Development :: Build Tools", - ], -) diff --git a/setup_test_env.sh b/setup_test_env.sh deleted file mode 100755 index 6bb63705..00000000 --- a/setup_test_env.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Source environment variables from .env file -if [ -f .env ]; then - echo "Loading environment variables from .env file" - export $(grep -v '^#' .env | xargs) -else - echo "Warning: .env file not found, using default values" -fi - -# Initialize Git repository if not already initialized -if [ ! -d .git ]; then - git init - git config --local user.name "Test User" - git config --local user.email "test@example.com" - git add . - git commit -m "Initial commit for testing" -fi - -# Export AWS environment variables if not set from .env file -if [ -z "$AWS_PROFILE" ]; then - export AWS_PROFILE=252960665057-ma6-gov.inf-admin-t2 - export AWS_REGION=us-gov-east-1 - # Uncomment if needed: - # export AWS_ACCESS_KEY_ID=dummy-key - # export AWS_SECRET_ACCESS_KEY=dummy-secret -fi - -echo "Test environment setup complete!" -echo "AWS Profile: $AWS_PROFILE" -echo "AWS Region: $AWS_REGION" -echo "Git repository initialized." diff --git a/testplan.md b/testplan.md deleted file mode 100644 index a0fd1700..00000000 --- a/testplan.md +++ /dev/null @@ -1,763 +0,0 @@ -# Test Plan for Terraform Upgrade Tool - -## 1. Overview - -This test plan outlines the strategy for verifying that the Terraform Upgrade Tool correctly handles a variety of real-world Census Bureau Terraform configurations. It focuses especially on ensuring proper AWS account management and configuration pattern detection. - -## 2. Test Environment Setup - -### 2.1 Prerequisites - -- AWS credentials with access to multiple accounts for testing -- Multiple AWS profiles configured in ~/.aws/config -- Terraform 0.12.x, 0.13.x, 0.14.x, 0.15.x, and 1.x binaries available -- Python 3.8+ with required dependencies -- Git repository access - -### 2.2 Test Repository Structure - -Create a test fixtures directory with the following structure: - -test-fixtures/ -├── simple/ -│ ├── basic-0.12-config/ # Basic 0.12 configuration -│ ├── 0.13-compatible/ # Already 0.13 compatible -│ └── 1.0-compatible/ # Already 1.0 compatible -├── account-patterns/ -│ ├── single-account/ # Configuration targeting a single AWS account -│ ├── multi-account/ # Cross-account resource configuration -│ └── profile-in-provider/ # AWS profile specified in provider block -├── census-patterns/ -│ ├── edl-workflow/ # EDL-specific workflow -│ ├── aws-blueprint/ # Census Bureau blueprint patterns -│ ├── sensitive-outputs/ # Configurations with sensitive outputs -│ └── dynamic-backends/ # Dynamic S3 backend configurations -└── complex-modules/ -├── nested-modules/ # Configuration with nested module dependencies -├── external-sources/ # Configuration with external module sources -├── terragrunt-config/ # Configuration using Terragrunt -└── deprecated-features/ # Configuration with multiple deprecated features - -## 3. Test Cases - -### 3.1 Account Management Testing - -#### TC-1: AWS Account Discovery - -**Objective:** Verify the tool can discover AWS accounts from Organizations API and fall back to profile scanning - -**Steps:** -1. Create configurations that reference specific accounts -2. Run tool with `--verify-env` flag -3. Run tool with account discovery features - -**Expected Outcome:** -- Tool lists available AWS accounts -- Tool links configurations with the correct AWS accounts -- When Organizations API access fails, tool successfully falls back to profile scanning - -#### TC-2: Profile-to-Account Matching - -**Objective:** Verify the tool matches AWS profiles to the correct accounts - -**Steps:** -1. Create configurations using multiple AWS profiles -2. Set up AWS config with various profile naming patterns -3. Test profile detection capability -4. Verify account-ID to profile matching - -**Expected Outcome:** -- Tool identifies the correct profile for each account ID -- Tool handles various profile naming patterns (e.g., `accountid.AdministratorAccess`, `accountid-gov.administratoraccess`) -- Tool properly prioritizes profiles with admin permissions over other profiles - -#### TC-3: Cross-Account Module References - -**Objective:** Ensure tool correctly identifies and handles cross-account dependencies - -**Steps:** -1. Create configurations that reference resources in other accounts -2. Test upgrade of configurations with cross-account data sources -3. Test with assumable roles vs. direct profile access - -**Expected Outcome:** -- Tool identifies cross-account references -- Tool maintains correct cross-account references throughout upgrades -- Proper handling of role assumption vs. direct profile access - -### 3.2 Census Bureau-Specific Pattern Testing - -#### TC-4: EDL Workflow Testing - -**Objective:** Verify tool handles EDL workflow patterns properly - -**Steps:** -1. Create test configurations with EDL module references -2. Test with both 0.12 and newer versions of EDL workflows -3. Verify module references are correctly upgraded - -**Expected Outcome:** -- `edl_*` module references are properly identified -- Tool suggests appropriate module reference versions for upgraded Terraform -- Correct handling of EDL-specific configuration patterns - -#### TC-5: AWS Provider with Endpoints - -**Objective:** Test handling of AWS provider configurations with custom endpoint settings - -**Steps:** -1. Create test configurations with custom endpoint settings -2. Test upgrade with focus on provider block transformation -3. Verify endpoints property is preserved through upgrades - -**Expected Outcome:** -- Custom endpoint configurations are preserved during upgrade -- Provider block is correctly restructured with required_providers -- No duplication or loss of endpoint configuration - -#### TC-6: S3 Backend with Dynamic Config - -**Objective:** Test upgrade of S3 backends with dynamic configuration - -**Steps:** -1. Create test configurations with dynamic S3 backends -2. Test upgrade to verify interpolation handling -3. Verify complex key paths are maintained - -**Expected Outcome:** -- S3 backend configurations with interpolation are properly maintained -- Dynamic backend key paths are preserved -- No syntax errors or unexpected changes in backend configuration - -### 3.3 General Terraform Upgrade Testing - -#### TC-7: Complete Upgrade Path - -**Objective:** Verify full upgrade path from 0.12 through 1.0 - -**Steps:** -1. Start with known 0.12 configuration -2. Run complete upgrade to 1.0 -3. Verify each intermediate version upgrade - -**Expected Outcome:** -- Successful upgrade through all versions -- Appropriate transformations applied at each step -- Final configuration is 1.0 compatible and passes validation - -#### TC-8: Stepwise Interactive Upgrade - -**Objective:** Test interactive step-by-step upgrade process - -**Steps:** -1. Run the upgrade tool in interactive mode -2. Test pausing and resuming at each version step -3. Verify user can abort and restart process - -**Expected Outcome:** -- Tool allows users to confirm each version upgrade -- Progress is saved between versions -- Aborted upgrades can be resumed - -#### TC-9: Provider Migration - -**Objective:** Verify provider source declarations are properly migrated - -**Steps:** -1. Test with various provider declarations -2. Include third-party providers -3. Verify provider version constraints are maintained - -**Expected Outcome:** -- Provider blocks are correctly restructured -- Provider source declarations match expected format -- Version constraints are properly migrated to required_providers block - -#### TC-10: HCL Transformer Validation - -**Objective:** Validate HCL transformation patterns on complex configurations - -**Steps:** -1. Test complex HCL transformations including nested blocks -2. Verify syntax preservation in complex expressions -3. Test with real-world configuration patterns - -**Expected Outcome:** -- Complex HCL structures are correctly transformed -- Syntax validity is maintained -- Transformed code passes `terraform validate` - -### 3.4 Census Bureau Module Compatibility Testing - -#### TC-11: Module Reference Upgrades - -**Objective:** Verify module references are upgraded to compatible versions - -**Steps:** -1. Test with common Census module references -2. Verify proper ref/tag suggestions for each Terraform version -3. Test auto-upgrade of module references - -**Expected Outcome:** -- Tool suggests appropriate module version for target Terraform version -- Update of module references happens correctly -- Compatible modules are selected for each version - -#### TC-12: Module Dependency Resolution - -**Objective:** Test dependency resolution for module upgrades - -**Steps:** -1. Create complex module dependency chains -2. Test upgrade ordering logic -3. Verify circular dependencies are detected - -**Expected Outcome:** -- Module dependencies are properly mapped -- Modules are upgraded in correct order -- Circular dependencies are reported with helpful information - -## 4. Edge Cases and Error Handling - -### TC-13: Error Recovery - -**Objective:** Verify tool handles and recovers from errors gracefully - -**Steps:** -1. Create malformed Terraform configurations -2. Introduce permission errors -3. Test with inaccessible AWS accounts - -**Expected Outcome:** -- Clear error messages displayed -- Failed operations provide useful diagnostics -- Partial progress is maintained where possible - -### TC-14: Large Repository Support - -**Objective:** Test performance with large repositories - -**Steps:** -1. Create/find a large Terraform repository (100+ configuration files) -2. Test scanning and assessment performance -3. Verify upgrade process works with large codebases - -**Expected Outcome:** -- Tool performs acceptably with large repositories -- Parallel processing options improve performance -- Memory usage remains reasonable - -## 5. Integration Testing - -### TC-15: Git Integration - -**Objective:** Test Git-based workflow support - -**Steps:** -1. Set up test Git repository -2. Test branch creation and PR generation -3. Verify changes can be committed - -**Expected Outcome:** -- Tool creates branches for upgrades -- Changes are properly committed -- PR generation includes appropriate description - -### TC-16: CI/CD Pipeline Testing - -**Objective:** Verify tool works in automated environments - -**Steps:** -1. Set up CI pipeline for automated testing -2. Test tool in non-interactive mode -3. Verify automation-friendly features - -**Expected Outcome:** -- Tool runs successfully in CI environment -- Non-interactive mode works as expected -- Exit codes properly reflect success/failure - -## 6. AWS Toolkit Integration - -### TC-17: AWS Profile Configuration - -**Objective:** Test integration with existing AWS profiles - -**Steps:** -1. Configure various AWS profile types -2. Test detection and validation of profiles -3. Verify credential caching works properly - -**Expected Outcome:** -- Tool detects and uses existing AWS profiles -- Profile validation works correctly -- Credentials are appropriately cached for better performance - -### TC-18: Cross-Partition Support - -**Objective:** Test support for AWS partitions (commercial, GovCloud) - -**Steps:** -1. Create configurations for both commercial and GovCloud -2. Test partition detection logic -3. Verify appropriate region handling - -**Expected Outcome:** -- Tool correctly detects and handles different AWS partitions -- GovCloud regions and commercial regions are properly distinguished -- ARNs are correctly formatted for each partition - -## 7. Test Data Requirements - -### 7.1 AWS Account Requirements - -- Access to at least 2-3 AWS accounts -- Both commercial and GovCloud account access -- Appropriate SSO profiles configured - -### 7.2 Terraform Configuration Samples - -Collect representative samples of: -- Standard Census Bureau module usage patterns -- EDL workflow configurations -- Complex provider configurations -- Dynamic and static backend configurations -- Multi-account resource configurations - -## 8. Test Execution Strategy - -1. **Unit Tests:** Focus on individual components (80% coverage goal) -2. **Integration Tests:** Test component interaction (20 key scenarios) -3. **End-to-End Tests:** Complete workflows with real-world configurations (5 key workflows) -4. **Manual Testing:** Complex edge cases and UX validation - -## 9. Reporting and Analysis - -1. Document test results in standardized format -2. Track issues found during testing with severity ratings -3. Identify patterns in failures for systematic fixes -4. Generate coverage reports for code and configuration patterns - -## 10. Success Criteria - -The Terraform Upgrade Tool is ready for release when: - -1. All automated tests pass with 90%+ success rate -2. Tool successfully upgrades 5 diverse real-world repositories -3. AWS account and profile management functions work reliably -4. Census Bureau-specific patterns are correctly handled -5. Performance is acceptable on large repositories (100+ files) - -## 11. GitHub Project Integration Testing - -### TC-19: GitHub Authentication and Connection - -**Objective:** Verify the tool can authenticate and connect to GitHub Enterprise instance - -**Steps:** -1. Configure various authentication methods (PAT, GitHub App, environment variables) -2. Test connection to GitHub API with each auth method -3. Verify proper error handling for authentication failures -4. Test access to repositories and project boards - -**Expected Outcome:** -- Tool successfully authenticates with all supported methods -- Connection to GitHub API is established -- Appropriate error messages are shown for invalid credentials -- Access to required repositories and project boards is confirmed - -### TC-20: Project Board Ticket Management - -**Objective:** Verify the tool can interact with project board tickets - -**Steps:** -1. Create test project board with sample tickets -2. Run tool with commands to fetch and update tickets -3. Test ticket status transitions -4. Verify comment and attachment capabilities - -**Expected Outcome:** -- Tool correctly retrieves tickets from project board -- Ticket status is updated appropriately -- Comments are added with proper formatting -- Reports and logs are attached to tickets - -### TC-21: Directory-to-Ticket Mapping - -**Objective:** Verify the tool correctly maps Terraform directories to GitHub tickets - -**Steps:** -1. Run `tf-upgrade github-scan` on a directory with multiple Terraform configurations -2. Verify tickets are created for each directory needing upgrade -3. Test directory path normalization and matching -4. Verify existing tickets are updated rather than duplicated - -**Expected Outcome:** -- Each directory maps to exactly one ticket -- Directory paths are normalized consistently -- Existing tickets are found and updated -- New tickets are created only when necessary - -### TC-22: Workflow Automation - -**Objective:** Test end-to-end workflow with GitHub integration - -**Steps:** -1. Start with a new directory requiring upgrade -2. Create ticket with `make github-create DIR=` -3. Claim ticket with `make github-claim DIR=` -4. Perform upgrade with `make upgrade DIR=` -5. Update ticket with results using `make github-update DIR=` -6. Generate PR with `make github-pr DIR=` - -**Expected Outcome:** -- Complete workflow succeeds end-to-end -- Ticket transitions through expected states -- PR is created with appropriate content -- Ticket and PR are properly linked - -## 12. Test Case Collection and Management - -### TC-23: Test Case Extraction - -**Objective:** Verify automatic extraction of test cases from real configurations - -**Steps:** -1. Run `make extract-test-case DIR=` on a complex configuration -2. Check generated test fixture structure and content -3. Verify key patterns are preserved in the minimal example -4. Test the extracted fixture with the upgrade process - -**Expected Outcome:** -- Test case is extracted with minimal required elements -- Important upgrade patterns are preserved -- Test fixture can be run independently -- Fixture includes before/after states for validation - -### TC-24: Pattern Detection - -**Objective:** Test pattern detection capability for test case categorization - -**Steps:** -1. Run pattern analysis on diverse test configurations -2. Check automatic categorization of configurations -3. Test pattern matching against known patterns -4. Verify coverage tracking for patterns - -**Expected Outcome:** -- Common patterns are correctly identified -- Configurations are assigned to appropriate categories -- Pattern matching works consistently -- Coverage reporting shows accurate statistics - -### TC-25: Test Fixture Management - -**Objective:** Verify organization and management of collected test cases - -**Steps:** -1. Populate test fixture library with multiple fixtures -2. Test search and filtering of fixtures by pattern -3. Verify metadata storage and retrieval -4. Check normalization and cleaning of fixtures - -**Expected Outcome:** -- Fixtures are organized in logical structure -- Search returns appropriate fixtures by criteria -- Metadata accurately describes fixture contents -- Normalized fixtures maintain essential patterns - -### TC-26: Regression Test Generation - -**Objective:** Test generation of regression tests from identified issues - -**Steps:** -1. Document a specific upgrade issue or bug -2. Create regression test using test case framework -3. Verify the test fails when issue is present -4. Fix the issue and verify test now passes - -**Expected Outcome:** -- Regression test accurately represents the issue -- Test fails when issue exists in the code -- Test passes when issue is fixed -- Test can be integrated into automated test suite - -## 13. Census Bureau Specific Testing - -### TC-27: Census Bureau Pattern Library - -**Objective:** Test handling of Census Bureau-specific Terraform patterns - -**Steps:** -1. Create test fixtures for Census-specific patterns -2. Run pattern detection on Census configurations -3. Verify documentation of Census patterns -4. Test upgrade paths for Census-specific modules - -**Expected Outcome:** -- Census-specific patterns are correctly identified -- Census module references are properly handled -- Documentation covers Census-specific recommendations -- Test fixtures represent true Census patterns - -### TC-28: Cross-Team Collaboration - -**Objective:** Test workflow for multi-team collaboration on upgrades - -**Steps:** -1. Set up test environment with multiple user roles -2. Configure GitHub permissions to match team structure -3. Test assignment and handoff of tickets between teams -4. Verify notification and status update flows - -**Expected Outcome:** -- Multi-team workflow functions correctly -- Assignment and handoff work as expected -- Status updates are visible to all stakeholders -- Notifications are sent to appropriate teams - -## 14. End-to-End Testing with GitHub and Test Case Collection - -### TC-29: Integrated Workflow Testing - -**Objective:** Verify complete end-to-end workflow with all features enabled - -**Steps:** -1. Start with repository containing multiple terraform configurations -2. Run full workflow from scanning to completion with GitHub integration -3. Extract test cases during the upgrade process -4. Complete upgrade and verify all components worked together - -**Expected Outcome:** -- Complete workflow succeeds end-to-end -- GitHub project board reflects accurate state -- Test cases are collected and categorized -- Upgraded configurations pass validation - -### TC-30: Performance Testing with Large Repositories - -**Objective:** Test tool performance with large repositories and many tickets - -**Steps:** -1. Set up test environment with 50+ terraform directories -2. Configure GitHub project board with matching tickets -3. Run full workflow on complete repository -4. Measure performance metrics - -**Expected Outcome:** -- Tool handles large repository without excessive memory usage -- GitHub API rate limits are respected -- Performance remains acceptable with many tickets -- Parallel processing options improve performance - -## 15. Census Bureau Infrastructure Pattern Testing - -### TC-31: VPC Upgrade Script Compatibility - -**Objective:** Test upgrade tool compatibility with VPC upgrade scripts - -**Steps:** -1. Locate and analyze `upgrade.vpc-code.sh` scripts -2. Test tool's handling of curl-based file downloads in scripts -3. Verify module reference checking (git::https to git@ format) -4. Test analysis of ref=tf-upgrade references - -**Expected Outcome:** -- Tool correctly identifies scripts that modify Terraform files -- Upgrade recommendations don't conflict with script-based upgrades -- Tool can properly analyze files before and after script execution -- Proper handling of alternative upgrade mechanisms - -### TC-32: EC2 Keypair Generation Patterns - -**Objective:** Verify handling of EC2 keypair generation patterns during upgrade - -**Steps:** -1. Identify configurations with null_resource keypair generation -2. Test upgrade of different keypair generation patterns -3. Verify DSA to RSA key migration recommendations -4. Test handling of different key sizes (1024, 2048) - -**Expected Outcome:** -- Correct transformation of provisioner blocks in null_resource -- Proper handling of depends_on references after upgrade -- Key size recommendations align with current security best practices -- SSH-keygen command formats are preserved through upgrade - -### TC-33: Module Version Reference Patterns - -**Objective:** Test upgrade of various module reference patterns - -**Steps:** -1. Create test fixtures from repository patterns of module references -2. Test different git reference formats (git::https://, git@github) -3. Verify handling of ref= parameters in module sources -4. Test branch and tag reference recommendations - -**Expected Outcome:** -- Module references are correctly upgraded with proper syntax -- Git URL formats are consistently handled -- Version constraints are preserved or upgraded appropriately -- Census-specific module branches (tf-upgrade) are correctly recognized - -### TC-34: DNS Integration Handling - -**Objective:** Test Terraform DNS provider configuration upgrades - -**Steps:** -1. Create test fixtures with dns_a_record_set and dns_ptr_record resources -2. Test upgrade of TSIG key configurations -3. Verify preservation of DNS update server configurations -4. Test record type transformations - -**Expected Outcome:** -- DNS provider configurations are correctly upgraded -- TSIG key authentications are preserved -- Record set configurations maintain their structure -- DNS validation logic is preserved through upgrade - -### TC-35: DynamoDB Table Attribute Patterns - -**Objective:** Test upgrade of DynamoDB table definitions - -**Steps:** -1. Create test fixtures with various DynamoDB table configurations -2. Test attribute definitions with different types (S, N, B) -3. Verify handling of global secondary indexes -4. Test multiline item declarations with heredoc syntax - -**Expected Outcome:** -- Table attribute definitions are correctly preserved -- Global secondary index configurations are properly upgraded -- Heredoc syntax for item declarations is correctly transformed -- Proper handling of JSON document structure in heredocs - -### TC-36: Container Service Definitions - -**Objective:** Test upgrade of ECS/Fargate task definitions - -**Steps:** -1. Create test fixtures with container definitions and port mappings -2. Test jsonencode function usage in container definitions -3. Verify upgrade of log configuration blocks -4. Test complex nested block structures in task definitions - -**Expected Outcome:** -- Task definition structures remain valid after upgrade -- jsonencode functions are properly upgraded -- Nested container configuration blocks are correctly transformed -- Array syntax in container definitions is preserved - -### TC-37: Infrastructure Setup Files - -**Objective:** Test handling of Census-specific infrastructure setup files - -**Steps:** -1. Analyze tf-run.data.bkp files and similar bootstrapping configurations -2. Test upgrade tool's handling of LINKTOP commands -3. Verify COMMAND, COMMENT pattern recognition -4. Test integration with tf-directory-setup.py references - -**Expected Outcome:** -- Tool correctly recognizes and preserves infrastructure setup patterns -- LINKTOP references are properly maintained or updated -- COMMAND directives are preserved during upgrade -- Integration with existing Census Bureau tooling is maintained - -### TC-38: Tag Structure Transformation - -**Objective:** Test census-specific tag structure upgrades - -**Steps:** -1. Create test fixtures with various Census-specific tag patterns -2. Test upgrades of boc: prefixed tags -3. Verify CostAllocation tag format preservation -4. Test special tag handling (boc:dns:name, boc:dns:zone) - -**Expected Outcome:** -- Census-specific tag structures are preserved -- Tag format standardization is applied where appropriate -- Special tag prefixes are recognized and handled correctly -- Tag value format (string interpolation) is correctly upgraded - -### TC-39: Pull Request Template Integration - -**Objective:** Test integration with Census Bureau PR templates - -**Steps:** -1. Analyze existing PR template format (.github/PULL_REQUEST_TEMPLATE.md) -2. Test generated PR content against template requirements -3. Verify tool can populate template sections -4. Test Plan integration into PR description - -**Expected Outcome:** -- Generated PRs follow the Census Bureau template format -- Required sections of PR template are populated -- Plan output is correctly formatted for template -- Tool helps users meet PR requirements - -### TC-40: Remote State Reference Handling - -**Objective:** Test upgrade of remote state references - -**Steps:** -1. Create test fixtures with terraform_remote_state data sources -2. Test cross-account state references -3. Verify workspace-specific state path formats -4. Test handling of output references from remote state - -**Expected Outcome:** -- Remote state references are preserved or correctly upgraded -- Output references using remote state are correctly transformed -- Workspace-specific state paths are correctly handled -- Cross-account references are identified and properly managed - -## 16. Census-Specific AWS Architecture Patterns - -### TC-41: VPC Organization Structure - -**Objective:** Test upgrade tool's handling of Census VPC directory structure - -**Steps:** -1. Analyze repository structure of account/vpc/region/vpc# patterns -2. Test scanning and file relationship detection across structure -3. Verify output variable references between parent/child directories -4. Test handling of linked files and modules across directory structure - -**Expected Outcome:** -- Tool correctly identifies related configurations across directory structure -- Upgrade maintains relationships between parent/child directories -- Output references between dirs are correctly transformed -- Tool respects organizational boundaries for upgrade units - -### TC-42: Multi-Environment Module References - -**Objective:** Test upgrade of modules used across different environments - -**Steps:** -1. Identify modules used in multiple env contexts (dev, prod, etc) -2. Test upgrade consistency across different environment references -3. Verify version constraint recommendations align across environments -4. Test handling of environment-specific module configuration - -**Expected Outcome:** -- Consistent upgrade recommendations across environments -- Environment-specific configurations are preserved -- Version constraints are handled appropriately for each environment -- Proper handling of environment-conditional blocks - -### TC-43: Terraform Workflow Script Integration - -**Objective:** Test integration with Census-specific Terraform workflow scripts - -**Steps:** -1. Analyze tf-control, tf-run, and related scripts -2. Test upgrade tool's compatibility with these workflow patterns -3. Verify proper handling of .tf-control.tfrc files -4. Test log format compatibility with existing scripts - -**Expected Outcome:** -- Tool integrates properly with existing workflow scripts -- Command execution follows expected patterns -- Log formats are compatible with existing processing -- Tool respects configuration from .tf-control files diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index a5dc254b..00000000 --- a/tests/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Pytest configuration for terraform-upgrade-tool tests. -This file ensures the package is in the Python path. -""" - -import os -import sys - -# Add the parent directory to sys.path to make the package importable -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) diff --git a/tests/fixtures/0.12/complex/main.tf b/tests/fixtures/0.12/complex/main.tf deleted file mode 100644 index a2bc11fa..00000000 --- a/tests/fixtures/0.12/complex/main.tf +++ /dev/null @@ -1,125 +0,0 @@ -terraform { - required_version = "~> 0.12.0" - - backend "s3" { - bucket = "terraform-state" - key = "complex/terraform.tfstate" - region = "us-west-2" - } -} - -provider "aws" { - version = "~> 2.0" - region = "us-west-2" - alias = "main" -} - -provider "aws" { - version = "~> 2.0" - region = "us-east-1" - alias = "east" -} - -provider "google" { - version = "~> 3.0" - project = "my-project" - region = "us-central1" -} - -locals { - common_tags = { - Project = "TerraformUpgrade" - Environment = var.environment - } - - instance_count = { - "dev" = 1 - "test" = 2 - "prod" = 3 - } -} - -resource "aws_vpc" "main" { - provider = aws.main - cidr_block = "10.0.0.0/16" - - tags = merge(local.common_tags, { - Name = "main-vpc" - }) -} - -resource "aws_subnet" "app_subnets" { - provider = aws.main - count = 3 - vpc_id = aws_vpc.main.id - cidr_block = "10.0.${count.index + 1}.0/24" - - tags = merge(local.common_tags, { - Name = "app-subnet-${count.index + 1}" - }) -} - -resource "aws_instance" "app" { - provider = aws.main - count = lookup(local.instance_count, var.environment, 1) - ami = "ami-123456" - instance_type = "t2.micro" - subnet_id = "${element(aws_subnet.app_subnets.*.id, count.index)}" - - tags = merge(local.common_tags, { - Name = "app-server-${count.index + 1}" - }) -} - -resource "aws_s3_bucket" "logs" { - provider = aws.east - bucket = "my-log-bucket-${var.environment}" - acl = "private" - - versioning { - enabled = true - } - - lifecycle_rule { - enabled = true - prefix = "logs/" - - transition { - days = 30 - storage_class = "STANDARD_IA" - } - - transition { - days = 90 - storage_class = "GLACIER" - } - - expiration { - days = 365 - } - } - - tags = local.common_tags -} - -variable "environment" { - description = "Deployment environment (dev, test, prod)" - type = string - default = "dev" -} - -output "vpc_id" { - value = aws_vpc.main.id -} - -output "subnet_ids" { - value = "${join(",", aws_subnet.app_subnets.*.id)}" -} - -output "instance_ips" { - value = "${aws_instance.app.*.private_ip}" -} - -output "bucket_name" { - value = "${aws_s3_bucket.logs.id}" -} diff --git a/tests/fixtures/0.12/medium/main.tf b/tests/fixtures/0.12/medium/main.tf deleted file mode 100644 index 82b91364..00000000 --- a/tests/fixtures/0.12/medium/main.tf +++ /dev/null @@ -1,44 +0,0 @@ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -resource "aws_vpc" "main" { - cidr_block = "10.0.0.0/16" - - tags = { - Name = "main-vpc" - } -} - -resource "aws_subnet" "primary" { - vpc_id = "${aws_vpc.main.id}" - cidr_block = "10.0.1.0/24" - - tags = { - Name = "primary-subnet" - } -} - -resource "aws_instance" "web" { - ami = "ami-123456" - instance_type = "t2.micro" - subnet_id = "${aws_subnet.primary.id}" - - tags = { - Name = "web-server" - Environment = "${var.environment}" - } -} - -variable "environment" { - default = "dev" -} - -output "instance_ip" { - value = "${aws_instance.web.private_ip}" -} - -output "vpc_id" { - value = "${aws_vpc.main.id}" -} diff --git a/tests/fixtures/0.12/simple/main.tf b/tests/fixtures/0.12/simple/main.tf deleted file mode 100644 index 4d50d6a8..00000000 --- a/tests/fixtures/0.12/simple/main.tf +++ /dev/null @@ -1,13 +0,0 @@ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -resource "aws_s3_bucket" "simple" { - bucket = "my-simple-bucket" - acl = "private" -} - -output "bucket_id" { - value = "${aws_s3_bucket.simple.id}" -} diff --git a/tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md b/tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md deleted file mode 100644 index ae9bf5f1..00000000 --- a/tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md +++ /dev/null @@ -1,12 +0,0 @@ - -## Summary - -- Current Version: 0.12 -- Upgrade Path: 0.13 → 0.14 → 0.15 → 1.0 -- Complexity Level: Low -- Complexity Score: 8.5 - -### Deprecated Syntax - -- interpolation_only: `"${aws_s3_bucket.simple.id}"` in main.tf -- quoted_interpolation: `"${aws_s3_bucket.simple.id}"` in main.tf diff --git a/tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json b/tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json deleted file mode 100644 index 79c787f6..00000000 --- a/tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "started_at": "2025-04-01T18:46:37.696653", - "steps": [], - "completed": 0, - "total": 0, - "status": "in_progress" -} diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py deleted file mode 100644 index b822e3a9..00000000 --- a/tests/fixtures/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -Test fixtures for Terraform upgrade tool. -""" - -import os - - -def create_terraform_test_files(directory, version): - """ - Create mock Terraform output files based on version. - - Args: - directory: The test directory - version: The target version (e.g., "0.14.0") - """ - # Create a lock file for version 0.14.0 or higher - if ( - version.startswith("0.14") - or version.startswith("0.15") - or version.startswith("1.") - ): - lock_file_path = os.path.join(directory, ".terraform.lock.hcl") - with open(lock_file_path, "w") as f: - f.write( - """# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "3.0.0" - constraints = "~> 3.0.0" - hashes = [ - "h1:UyKRcHE2W0ahi+7XM+FX+c+7xUKBjr1uHDWS2mo4sX4=", - ] -} -""" - ) - - -def transform_provider_version(directory): - """ - Transform provider version for 0.13+ compatibility. - - Args: - directory: The test directory - """ - main_tf = os.path.join(directory, "main.tf") - - if os.path.exists(main_tf): - with open(main_tf, "r") as f: - content = f.read() - - # Replace provider block with required_providers syntax - if 'provider "aws"' in content and "version =" in content: - # Fix the string replacement - the previous version had mismatched quotes - content = content.replace( - """provider "aws" { - version = "~> 2.0" - region = "us-west-2" - }""", - """terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 2.0" - } - } - } - - provider "aws" { - region = "us-west-2" - }""", - ) - - with open(main_tf, "w") as f: - f.write(content) diff --git a/tests/fixtures/complex/main.tf b/tests/fixtures/complex/main.tf deleted file mode 100644 index a2bc11fa..00000000 --- a/tests/fixtures/complex/main.tf +++ /dev/null @@ -1,125 +0,0 @@ -terraform { - required_version = "~> 0.12.0" - - backend "s3" { - bucket = "terraform-state" - key = "complex/terraform.tfstate" - region = "us-west-2" - } -} - -provider "aws" { - version = "~> 2.0" - region = "us-west-2" - alias = "main" -} - -provider "aws" { - version = "~> 2.0" - region = "us-east-1" - alias = "east" -} - -provider "google" { - version = "~> 3.0" - project = "my-project" - region = "us-central1" -} - -locals { - common_tags = { - Project = "TerraformUpgrade" - Environment = var.environment - } - - instance_count = { - "dev" = 1 - "test" = 2 - "prod" = 3 - } -} - -resource "aws_vpc" "main" { - provider = aws.main - cidr_block = "10.0.0.0/16" - - tags = merge(local.common_tags, { - Name = "main-vpc" - }) -} - -resource "aws_subnet" "app_subnets" { - provider = aws.main - count = 3 - vpc_id = aws_vpc.main.id - cidr_block = "10.0.${count.index + 1}.0/24" - - tags = merge(local.common_tags, { - Name = "app-subnet-${count.index + 1}" - }) -} - -resource "aws_instance" "app" { - provider = aws.main - count = lookup(local.instance_count, var.environment, 1) - ami = "ami-123456" - instance_type = "t2.micro" - subnet_id = "${element(aws_subnet.app_subnets.*.id, count.index)}" - - tags = merge(local.common_tags, { - Name = "app-server-${count.index + 1}" - }) -} - -resource "aws_s3_bucket" "logs" { - provider = aws.east - bucket = "my-log-bucket-${var.environment}" - acl = "private" - - versioning { - enabled = true - } - - lifecycle_rule { - enabled = true - prefix = "logs/" - - transition { - days = 30 - storage_class = "STANDARD_IA" - } - - transition { - days = 90 - storage_class = "GLACIER" - } - - expiration { - days = 365 - } - } - - tags = local.common_tags -} - -variable "environment" { - description = "Deployment environment (dev, test, prod)" - type = string - default = "dev" -} - -output "vpc_id" { - value = aws_vpc.main.id -} - -output "subnet_ids" { - value = "${join(",", aws_subnet.app_subnets.*.id)}" -} - -output "instance_ips" { - value = "${aws_instance.app.*.private_ip}" -} - -output "bucket_name" { - value = "${aws_s3_bucket.logs.id}" -} diff --git a/tests/fixtures/medium/main.tf b/tests/fixtures/medium/main.tf deleted file mode 100644 index 82b91364..00000000 --- a/tests/fixtures/medium/main.tf +++ /dev/null @@ -1,44 +0,0 @@ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -resource "aws_vpc" "main" { - cidr_block = "10.0.0.0/16" - - tags = { - Name = "main-vpc" - } -} - -resource "aws_subnet" "primary" { - vpc_id = "${aws_vpc.main.id}" - cidr_block = "10.0.1.0/24" - - tags = { - Name = "primary-subnet" - } -} - -resource "aws_instance" "web" { - ami = "ami-123456" - instance_type = "t2.micro" - subnet_id = "${aws_subnet.primary.id}" - - tags = { - Name = "web-server" - Environment = "${var.environment}" - } -} - -variable "environment" { - default = "dev" -} - -output "instance_ip" { - value = "${aws_instance.web.private_ip}" -} - -output "vpc_id" { - value = "${aws_vpc.main.id}" -} diff --git a/tests/fixtures/simple/main.tf b/tests/fixtures/simple/main.tf deleted file mode 100644 index 4d50d6a8..00000000 --- a/tests/fixtures/simple/main.tf +++ /dev/null @@ -1,13 +0,0 @@ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -resource "aws_s3_bucket" "simple" { - bucket = "my-simple-bucket" - acl = "private" -} - -output "bucket_id" { - value = "${aws_s3_bucket.simple.id}" -} diff --git a/tests/fixtures/terraform-upgrade-dryrun-report.md b/tests/fixtures/terraform-upgrade-dryrun-report.md deleted file mode 100644 index c36ca036..00000000 --- a/tests/fixtures/terraform-upgrade-dryrun-report.md +++ /dev/null @@ -1,7 +0,0 @@ - -## Summary - -- Current Version: 0.12 -- Upgrade Path: 0.13 → 0.14 → 0.15 → 1.0 -- Complexity Level: Low -- Complexity Score: 0.0 diff --git a/tests/fixtures/upgrade-logs/upgrade-progress.json b/tests/fixtures/upgrade-logs/upgrade-progress.json deleted file mode 100644 index 38185bba..00000000 --- a/tests/fixtures/upgrade-logs/upgrade-progress.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "started_at": "2025-03-27T00:11:02.421694", - "steps": [], - "completed": 0, - "total": 0, - "status": "in_progress" -} diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py deleted file mode 100644 index 452ae761..00000000 --- a/tests/integration/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Integration tests for terraform-upgrade-tool.""" diff --git a/tests/integration/test_upgrade_workflow.py b/tests/integration/test_upgrade_workflow.py deleted file mode 100644 index 7162d8e3..00000000 --- a/tests/integration/test_upgrade_workflow.py +++ /dev/null @@ -1,195 +0,0 @@ -import os -import shutil -import tempfile -import unittest -from unittest.mock import MagicMock, patch - -# Importing the main upgrader components -from tf_upgrade.upgrade_controller import UpgradeController -from tf_upgrade.utils.file_manager import FileManager -from tf_upgrade.utils.terraform_runner import TerraformRunner - -# Create a test fixture for 0.12 style Terraform configuration -TERRAFORM_0_12_FIXTURE = """ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -resource "aws_instance" "example" { - ami = "ami-123456" - instance_type = "t2.micro" -} - -output "instance_id" { - value = "${aws_instance.example.id}" -} -""" - - -class TestUpgradeWorkflow(unittest.TestCase): - def setUp(self): - """Set up test environment with pre-configured mocks.""" - # Create patchers - self.patchers = [ - patch("shutil.which"), - patch("tf_upgrade.utils.git.os.path.exists"), - patch("tf_upgrade.utils.terraform.subprocess.check_output"), - patch("subprocess.check_output"), - patch("subprocess.run"), - patch("tf_upgrade.utils.git.subprocess.check_output"), - patch("tf_upgrade.utils.git.subprocess.run"), - # Add patches for config loading and environment checks - patch("tf_upgrade.config.os.path.exists", return_value=False), - patch( - "tf_upgrade.utils.terraform.get_terraform_binary", - return_value="/usr/bin/terraform", - ), - ] - - # Start all the patchers and get the mocks - self.mock_which = self.patchers[0].start() - self.mock_git_exists = self.patchers[1].start() - self.mock_tf_check_output = self.patchers[2].start() - self.mock_check_output = self.patchers[3].start() - self.mock_run = self.patchers[4].start() - self.mock_git_check_output = self.patchers[5].start() - self.mock_git_run = self.patchers[6].start() - - # Create a temporary directory for the test - self.test_dir = tempfile.mkdtemp() - - # Configure mocks to handle git commands - self.mock_which.return_value = "/usr/bin/git" - self.mock_git_exists.return_value = True # Simulate that .git directory exists - self.mock_git_check_output.return_value = b"main" - self.mock_check_output.return_value = b"dummy/git/path" - self.mock_tf_check_output.return_value = b"terraform version" - self.mock_run.return_value = MagicMock(returncode=0, stdout=b"Success") - self.mock_git_run.return_value = MagicMock(returncode=0, stdout=b"Success") - - # Create a test Terraform configuration file - self.config_file = os.path.join(self.test_dir, "main.tf") - with open(self.config_file, "w") as f: - f.write(TERRAFORM_0_12_FIXTURE) - - # Initialize the components we need - with patch( - "tf_upgrade.config.load_config", return_value={} - ): # Mock config loading - self.file_manager = FileManager(self.test_dir) - self.terraform_runner = TerraformRunner(self.test_dir) - self.controller = UpgradeController(self.test_dir) - - def tearDown(self): - # Stop all the patchers - for patcher in self.patchers: - patcher.stop() - - # Clean up the temporary directory - shutil.rmtree(self.test_dir) - - @patch("tf_upgrade.version_detector.VersionDetector.analyze_directory") - @patch("tf_upgrade.utils.terraform.get_available_terraform_versions") - def test_complete_upgrade_workflow(self, mock_get_versions, mock_analyze): - """Test complete upgrade workflow with mocked git commands.""" - # Mock version detection - mock_analyze.return_value = { - "final_version": "0.12", - "upgrade_path": ["0.13", "0.14", "0.15", "1.0"], - "requires_upgrade": True, - } - - # Mock available Terraform versions - mock_get_versions.return_value = [ - "0.12.31", - "0.13.7", - "0.14.11", - "0.15.5", - "1.0.11", - "1.10.5", - ] - - # Replace the upgrade method with a mock that always succeeds - with patch.object( - self.controller, - "upgrade_to_version", - return_value={"success": True, "message": "Successful upgrade"}, - ): - # Run the upgrade process - result = self.controller.upgrade(target_version="1.10", backup=True) - - # Verify the result - self.assertTrue(result["success"]) - - @patch("tf_upgrade.version_detector.VersionDetector.analyze_directory") - @patch("tf_upgrade.utils.terraform.get_available_terraform_versions") - def test_step_by_step_upgrade(self, mock_get_versions, mock_analyze): - """Test step-by-step upgrade with mocked git commands.""" - # Mock version detection for each version - mock_analyze.side_effect = [ - { - "final_version": "0.12", - "upgrade_path": ["0.13"], - "requires_upgrade": True, - }, - { - "final_version": "0.13", - "upgrade_path": ["0.14"], - "requires_upgrade": True, - }, - { - "final_version": "0.14", - "upgrade_path": ["0.15"], - "requires_upgrade": True, - }, - { - "final_version": "0.15", - "upgrade_path": ["1.10"], - "requires_upgrade": True, - }, - ] - - # Mock available Terraform versions - mock_get_versions.return_value = [ - "0.12.31", - "0.13.7", - "0.14.11", - "0.15.5", - "1.0.11", - "1.10.5", - ] - - # Replace the upgrade_to_version method to always succeed - with patch.object( - self.controller, - "upgrade_to_version", - return_value={"success": True, "message": "Successful upgrade"}, - ): - # Run individual upgrades - result_0_13 = self.controller.upgrade(target_version="0.13") - self.assertTrue(result_0_13["success"], "Upgrade to 0.13 failed") - - result_0_14 = self.controller.upgrade(target_version="0.14") - self.assertTrue(result_0_14["success"], "Upgrade to 0.14 failed") - - result_0_15 = self.controller.upgrade(target_version="0.15") - self.assertTrue(result_0_15["success"], "Upgrade to 0.15 failed") - - result_1_10 = self.controller.upgrade(target_version="1.10") - self.assertTrue(result_1_10["success"], "Upgrade to 1.10 failed") - - def _configure_mocks(self): - """Configure the mocks with common behavior.""" - # Set up mock return values (in case they need to be reset) - self.mock_which.return_value = "/usr/bin/git" - self.mock_git_exists.return_value = True - self.mock_git_check_output.return_value = b"main" - self.mock_tf_check_output.return_value = b"terraform version" - self.mock_check_output.return_value = b"dummy/git/path" - self.mock_run.return_value = MagicMock(returncode=0, stdout=b"Success") - self.mock_git_run.return_value = MagicMock(returncode=0, stdout=b"Success") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/run_tests.sh b/tests/run_tests.sh deleted file mode 100644 index 89d769a1..00000000 --- a/tests/run_tests.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# Colors for output -GREEN='\033[0;32m' -RED='\033[0;31m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -echo -e "${YELLOW}===== Running Terraform Upgrade Tool Tests =====${NC}" - -# Set working directory to project root -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "$SCRIPT_DIR/.." - -# Run unit tests -echo -e "\n${YELLOW}Running Unit Tests${NC}" -python -m unittest discover -s tests/unit -p "test_*.py" -v -UNIT_RESULT=$? - -# Run integration tests -echo -e "\n${YELLOW}Running Integration Tests${NC}" -python -m unittest discover -s tests/integration -p "test_*.py" -v -INTEGRATION_RESULT=$? - -# Run validation tests -echo -e "\n${YELLOW}Running Validation Tests${NC}" -python -m unittest discover -s tests/validation -p "test_*.py" -v -VALIDATION_RESULT=$? - -# Summarize results -echo -e "\n${YELLOW}===== Test Results =====${NC}" -if [ $UNIT_RESULT -eq 0 ]; then - echo -e "${GREEN}✓${NC} Unit Tests: PASSED" -else - echo -e "${RED}✗${NC} Unit Tests: FAILED" -fi - -if [ $INTEGRATION_RESULT -eq 0 ]; then - echo -e "${GREEN}✓${NC} Integration Tests: PASSED" -else - echo -e "${RED}✗${NC} Integration Tests: FAILED" -fi - -if [ $VALIDATION_RESULT -eq 0 ]; then - echo -e "${GREEN}✓${NC} Validation Tests: PASSED" -else - echo -e "${RED}✗${NC} Validation Tests: FAILED" -fi - -# Overall result -if [ $UNIT_RESULT -eq 0 ] && [ $INTEGRATION_RESULT -eq 0 ] && [ $VALIDATION_RESULT -eq 0 ]; then - echo -e "\n${GREEN}All tests passed!${NC}" - exit 0 -else - echo -e "\n${RED}Some tests failed!${NC}" - exit 1 -fi diff --git a/tests/setup_test_structure.sh b/tests/setup_test_structure.sh deleted file mode 100644 index 57831133..00000000 --- a/tests/setup_test_structure.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Create main test directories -mkdir -p tests/unit -mkdir -p tests/integration -mkdir -p tests/validation -mkdir -p tests/fixtures/simple -mkdir -p tests/fixtures/medium -mkdir -p tests/fixtures/complex - -# Create __init__.py files to make directories importable -touch tests/__init__.py -touch tests/unit/__init__.py -touch tests/integration/__init__.py -touch tests/validation/__init__.py - -echo "Test directory structure created successfully." diff --git a/tests/test_env_validator.py b/tests/test_env_validator.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_tf_parser.py b/tests/test_tf_parser.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_upgraders.py b/tests/test_upgraders.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py deleted file mode 100644 index 18a93346..00000000 --- a/tests/unit/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Unit tests for terraform-upgrade-tool.""" diff --git a/tests/unit/test_file_manager.py b/tests/unit/test_file_manager.py deleted file mode 100644 index 83f4b787..00000000 --- a/tests/unit/test_file_manager.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -import shutil -import tempfile -import unittest - -# Import the module to test -from tf_upgrade.utils.file_manager import FileManager - - -class TestFileManager(unittest.TestCase): - def setUp(self): - # Create a temporary directory for testing - self.test_dir = tempfile.mkdtemp() - self.file_manager = FileManager(self.test_dir) - - # Create some test files - self.test_file1 = os.path.join(self.test_dir, "test1.tf") - self.test_file2 = os.path.join(self.test_dir, "test2.tf") - - with open(self.test_file1, "w") as f: - f.write( - """resource "aws_instance" "example" { - ami = "ami-123456" - instance_type = "t2.micro" - }""" - ) - - with open(self.test_file2, "w") as f: - f.write('variable "region" {\n default = "us-west-2"\n}') - - def tearDown(self): - # Clean up the temporary directory - shutil.rmtree(self.test_dir) - - def test_create_backup(self): - # Test the backup creation functionality - backup_info = self.file_manager.create_backup() - backup_dir = backup_info["backup_dir"] - self.assertTrue(os.path.exists(backup_dir)) - self.assertTrue(os.path.exists(os.path.join(backup_dir, "test1.tf"))) - self.assertTrue(os.path.exists(os.path.join(backup_dir, "test2.tf"))) - - with open(os.path.join(backup_dir, "test1.tf"), "r") as f: - content = f.read() - self.assertEqual( - content, - """resource "aws_instance" "example" { - ami = "ami-123456" - instance_type = "t2.micro" - }""", - ) - - def test_restore_backup(self): - # Test the backup restore functionality - backup_info = self.file_manager.create_backup() - backup_dir = backup_info["backup_dir"] - with open(self.test_file1, "w") as f: - f.write("modified content") - - self.file_manager.restore_backup(backup_dir) - with open(self.test_file1, "r") as f: - content = f.read() - self.assertEqual( - content, - """resource "aws_instance" "example" { - ami = "ami-123456" - instance_type = "t2.micro" - }""", - ) - - def test_transform_file(self): - # Test file transformation - def transformer(content): - return content.replace("ami", "transformed_ami") - - self.file_manager.transform_file(self.test_file1, transformer) - with open(self.test_file1, "r") as f: - content = f.read() - self.assertIn("transformed_ami", content) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/test_hcl_transformer.py b/tests/unit/test_hcl_transformer.py deleted file mode 100644 index 79954a79..00000000 --- a/tests/unit/test_hcl_transformer.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import shutil -import tempfile -import unittest - -# Import the correct module -from tf_upgrade.hcl_transformer import HCLTransformer - - -class TestHCLTransformer(unittest.TestCase): - def setUp(self): - self.test_dir = tempfile.mkdtemp() - - # Create test files with HCL content - self.test_file = os.path.join(self.test_dir, "main.tf") - with open(self.test_file, "w") as f: - f.write( - """ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -resource "aws_instance" "example" { - ami = "ami-123456" - instance_type = "t2.micro" -} -""" - ) - - def tearDown(self): - shutil.rmtree(self.test_dir) - - def test_regex_transform(self): - # Test regex transformation - transformer = HCLTransformer() - transformer.add_regex_transformation( - r"version = \"~> 2.0\"", 'version = "~> 3.0"' - ) - transformer.transform_file(self.test_file) - - with open(self.test_file, "r") as f: - content = f.read() - self.assertIn('version = "~> 3.0"', content) - - def test_callable_transform(self): - # Test callable transformation - def transform_func(content): - return content.replace('region = "us-west-2"', 'region = "us-east-1"') - - transformer = HCLTransformer() - transformer.add_callable_transformation(transform_func) - transformer.transform_file(self.test_file) - - with open(self.test_file, "r") as f: - content = f.read() - self.assertIn('region = "us-east-1"', content) - - def test_multiple_transformations(self): - # Test multiple transformations - transformer = HCLTransformer() - transformer.add_regex_transformation( - r"version = \"~> 2.0\"", 'version = "~> 3.0"' - ) - - def transform_func(content): - return content.replace('region = "us-west-2"', 'region = "us-east-1"') - - transformer.add_callable_transformation(transform_func) - transformer.transform_file(self.test_file) - - with open(self.test_file, "r") as f: - content = f.read() - self.assertIn('version = "~> 3.0"', content) - self.assertIn('region = "us-east-1"', content) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/test_terraform_runner.py b/tests/unit/test_terraform_runner.py deleted file mode 100644 index c09153af..00000000 --- a/tests/unit/test_terraform_runner.py +++ /dev/null @@ -1,122 +0,0 @@ -import shutil -import tempfile -import unittest -from unittest.mock import MagicMock, patch - -from tf_upgrade.utils.terraform_runner import TerraformRunner - - -class TestTerraformRunner(unittest.TestCase): - def setUp(self): - self.test_dir = tempfile.mkdtemp() - # Mock git-related operations first - with patch("subprocess.run") as mock_subprocess: - # Make subprocess.run return success for git commands - mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - self.runner = TerraformRunner(self.test_dir) - - def tearDown(self): - shutil.rmtree(self.test_dir) - - @patch.object(TerraformRunner, "run_command") - @patch("subprocess.run") # Add this to mock any Git subprocess calls - def test_run_command(self, mock_subprocess, mock_run_command): - # Mock subprocess to avoid Git errors - mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - - # Mock the run_command method directly - mock_run_command.return_value = (True, "Success output", "log_file.log") - - # Call the method (which is now mocked) - success, output, log_file = self.runner.run_command("version") - - # Assert the mock was called with the right arguments - mock_run_command.assert_called_once_with("version") - - # Assert success - self.assertTrue(success) - self.assertEqual(output, "Success output") - self.assertEqual(log_file, "log_file.log") - - @patch.object(TerraformRunner, "run_command") - @patch("subprocess.run") # Add this to mock any Git subprocess calls - def test_init(self, mock_subprocess, mock_run_command): - # Mock subprocess to avoid Git errors - mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - - # Mock for init command - mock_run_command.return_value = (True, "Terraform initialized", "init_log.log") - - # Run the method to test - success, output, log_file = self.runner.init() - - # Assert the mock was called with the expected arguments - mock_run_command.assert_called_once() - self.assertEqual(mock_run_command.call_args[0][0], "init") - - # Assert success - self.assertTrue(success) - self.assertEqual(output, "Terraform initialized") - self.assertEqual(log_file, "init_log.log") - - @patch.object(TerraformRunner, "run_command") - @patch("subprocess.run") # Add this to mock any Git subprocess calls - def test_validate(self, mock_subprocess, mock_run_command): - # Mock subprocess to avoid Git errors - mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - - # Mock for validate command - mock_run_command.return_value = ( - True, - "Success! Configuration is valid", - "validate_log.log", - ) - - # Run the method to test - success, output, log_file = self.runner.validate() - - # Assert success - self.assertTrue(success) - self.assertEqual(output, "Success! Configuration is valid") - self.assertEqual(log_file, "validate_log.log") - - @patch.object(TerraformRunner, "run_command") - @patch("subprocess.run") # Add this to mock any Git subprocess calls - def test_plan(self, mock_subprocess, mock_run_command): - # Mock subprocess to avoid Git errors - mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - - # Mock for plan command - mock_run_command.return_value = (True, "No changes needed", "plan_log.log") - - # Run the method to test - success, output, log_file = self.runner.plan() - - # Assert success - self.assertTrue(success) - self.assertEqual(output, "No changes needed") - self.assertEqual(log_file, "plan_log.log") - - @patch.object(TerraformRunner, "run_command") - @patch("subprocess.run") # Add this to mock any Git subprocess calls - def test_specific_version(self, mock_subprocess, mock_run_command): - # Mock subprocess to avoid Git errors - mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - - # Mock for specific version - mock_run_command.return_value = (True, "Terraform v0.13.7", "version_log.log") - - # Create a TerraformRunner instance with a specific version - runner = TerraformRunner(self.test_dir, terraform_version="0.13") - - # Run the method (which uses the mocked run_command) - success, output, log_file = runner.run_command("version") - - # Verify the result - self.assertTrue(success) - self.assertEqual(output, "Terraform v0.13.7") - self.assertEqual(log_file, "version_log.log") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/validation/test_upgrade_validation.py b/tests/validation/test_upgrade_validation.py deleted file mode 100644 index c82e0bbe..00000000 --- a/tests/validation/test_upgrade_validation.py +++ /dev/null @@ -1,264 +0,0 @@ -import os -import shutil -import tempfile -import unittest -from unittest.mock import MagicMock, patch - -from tf_upgrade.upgrade_controller import UpgradeController -from tf_upgrade.utils.terraform_runner import TerraformRunner - -# Test fixtures -SIMPLE_FIXTURE = """ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -resource "aws_s3_bucket" "simple" { - bucket = "my-simple-bucket" - acl = "private" -} -""" - -MEDIUM_FIXTURE = """ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -module "vpc" { - source = "./modules/vpc" - cidr_block = "10.0.0.0/16" -} - -resource "aws_instance" "web" { - ami = "ami-123456" - instance_type = "t2.micro" - subnet_id = "${module.vpc.subnet_id}" -} - -output "instance_ip" { - value = "${aws_instance.web.private_ip}" -} -""" - -COMPLEX_FIXTURE = """ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -provider "google" { - version = "~> 3.0" - project = "my-project" - region = "us-central1" -} - -module "network" { - source = "./modules/network" - vpc_cidr = "10.0.0.0/16" -} - -resource "aws_instance" "app" { - count = 3 - ami = "ami-123456" - instance_type = "t2.micro" - subnet_id = "${element(module.network.subnet_ids, count.index)}" - - tags = { - Name = "app-${count.index}" - } -} - -resource "aws_s3_bucket" "logs" { - bucket = "my-log-bucket" - acl = "private" - - versioning { - enabled = true - } - - lifecycle_rule { - enabled = true - prefix = "logs/" - - transition { - days = 30 - storage_class = "STANDARD_IA" - } - - transition { - days = 90 - storage_class = "GLACIER" - } - } -} - -output "instance_ips" { - value = "${join(",", aws_instance.app.*.private_ip)}" -} - -output "bucket_name" { - value = "${aws_s3_bucket.logs.id}" -} -""" - - -class TestUpgradeValidation(unittest.TestCase): - def setUp(self): - # Create temporary directories for test fixtures - self.simple_dir = tempfile.mkdtemp() - self.medium_dir = tempfile.mkdtemp() - self.complex_dir = tempfile.mkdtemp() - - # Create test Terraform configuration files - with open(os.path.join(self.simple_dir, "main.tf"), "w") as f: - f.write(SIMPLE_FIXTURE) - - with open(os.path.join(self.medium_dir, "main.tf"), "w") as f: - f.write(MEDIUM_FIXTURE) - - with open(os.path.join(self.complex_dir, "main.tf"), "w") as f: - f.write(COMPLEX_FIXTURE) - - # Create modules directory for medium fixture - os.makedirs(os.path.join(self.medium_dir, "modules/vpc")) - with open(os.path.join(self.medium_dir, "modules/vpc/main.tf"), "w") as f: - f.write( - """ -resource "aws_vpc" "main" { - cidr_block = var.cidr_block -} - -resource "aws_subnet" "main" { - vpc_id = aws_vpc.main.id - cidr_block = cidrsubnet(var.cidr_block, 8, 1) -} - -output "subnet_id" { - value = aws_subnet.main.id -} -""" - ) - - with open(os.path.join(self.medium_dir, "modules/vpc/variables.tf"), "w") as f: - f.write( - """ -variable "cidr_block" { - description = "CIDR block for the VPC" - type = string -} -""" - ) - - # Create modules directory for complex fixture - os.makedirs(os.path.join(self.complex_dir, "modules/network")) - with open(os.path.join(self.complex_dir, "modules/network/main.tf"), "w") as f: - f.write( - """ -resource "aws_vpc" "main" { - cidr_block = var.vpc_cidr -} - -resource "aws_subnet" "main" { - count = 3 - vpc_id = aws_vpc.main.id - cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index) -} - -output "subnet_ids" { - value = aws_subnet.main.*.id -} -""" - ) - - with open( - os.path.join(self.complex_dir, "modules/network/variables.tf"), "w" - ) as f: - f.write( - """ -variable "vpc_cidr" { - description = "CIDR block for the VPC" - type = string -} -""" - ) - - def tearDown(self): - # Clean up temporary directories - shutil.rmtree(self.simple_dir) - shutil.rmtree(self.medium_dir) - shutil.rmtree(self.complex_dir) - - @patch("tf_upgrade.utils.terraform.find_terraform_config_files") - @patch("subprocess.run") - @patch.object(TerraformRunner, "run_command") - def test_simple_configuration_upgrade( - self, mock_run_command, mock_subprocess_run, mock_find_config - ): - # Mock find_terraform_config_files to avoid git repository detection - mock_find_config.return_value = { - "tf_files": [os.path.join(self.simple_dir, "main.tf")], - "tfvars_files": [], - "tf_control": None, - } - - # Mock subprocess calls - mock_subprocess_run.return_value = MagicMock(returncode=0, stdout=b"Success") - mock_run_command.return_value = (True, "Success", "log_file.log") - - # Initialize controller with mocked upgrade method - controller = UpgradeController(self.simple_dir) - with patch.object(controller, "upgrade", return_value={"success": True}): - result = controller.upgrade(target_version="1.0.0") - self.assertTrue(result["success"]) - - @patch("tf_upgrade.utils.terraform.find_terraform_config_files") - @patch("subprocess.run") - @patch.object(TerraformRunner, "run_command") - def test_medium_configuration_upgrade( - self, mock_run_command, mock_subprocess_run, mock_find_config - ): - # Mock find_terraform_config_files to avoid git repository detection - mock_find_config.return_value = { - "tf_files": [os.path.join(self.medium_dir, "main.tf")], - "tfvars_files": [], - "tf_control": None, - } - - # Mock subprocess calls - mock_subprocess_run.return_value = MagicMock(returncode=0, stdout=b"Success") - mock_run_command.return_value = (True, "Success", "log_file.log") - - # Initialize controller with mocked upgrade method - controller = UpgradeController(self.medium_dir) - with patch.object(controller, "upgrade", return_value={"success": True}): - result = controller.upgrade(target_version="1.0.0") - self.assertTrue(result["success"]) - - @patch("tf_upgrade.utils.terraform.find_terraform_config_files") - @patch("subprocess.run") - @patch.object(TerraformRunner, "run_command") - def test_complex_configuration_upgrade( - self, mock_run_command, mock_subprocess_run, mock_find_config - ): - # Mock find_terraform_config_files to avoid git repository detection - mock_find_config.return_value = { - "tf_files": [os.path.join(self.complex_dir, "main.tf")], - "tfvars_files": [], - "tf_control": None, - } - - # Mock subprocess calls - mock_subprocess_run.return_value = MagicMock(returncode=0, stdout=b"Success") - mock_run_command.return_value = (True, "Success", "log_file.log") - - # Initialize controller with mocked upgrade method - controller = UpgradeController(self.complex_dir) - with patch.object(controller, "upgrade", return_value={"success": True}): - result = controller.upgrade(target_version="1.0.0") - self.assertTrue(result["success"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tf_bulk_upgrade.py b/tf_bulk_upgrade.py deleted file mode 100755 index b6258969..00000000 --- a/tf_bulk_upgrade.py +++ /dev/null @@ -1,429 +0,0 @@ -#!/usr/bin/env python3 -""" -Terraform Bulk Upgrade Tool - -This script enables bulk upgrades of Terraform configurations -across an entire repository, creating separate Git branches -and PRs for each directory where changes are made. -""" - -import argparse -import logging -import os -import subprocess -import sys - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", -) -logger = logging.getLogger("tf_bulk_upgrade") - - -def git_cmd(args, cwd, check=True): - """Run a git command and return the output.""" - cmd = ["git"] + args - try: - result = subprocess.run( - cmd, - cwd=cwd, - check=check, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - return result.stdout.strip(), result.stderr.strip(), result.returncode == 0 - except subprocess.CalledProcessError as e: - logger.error(f"Git command failed: {e}") - return None, str(e), False - - -def is_git_repo(path): - """Check if a directory is a git repository.""" - return os.path.isdir(os.path.join(path, ".git")) - - -def get_repo_root(path): - """Get the git repository root directory.""" - output, error, success = git_cmd(["rev-parse", "--show-toplevel"], path) - if success: - return output - return None - - -def create_branch(repo_path, branch_name): - """Create a new git branch.""" - # Check if branch already exists - output, _, success = git_cmd(["branch", "--list", branch_name], repo_path) - if output: - logger.info(f"Branch {branch_name} already exists") - # Checkout the branch - _, _, success = git_cmd(["checkout", branch_name], repo_path) - return success - - # Create and checkout new branch from current HEAD - _, _, success = git_cmd(["checkout", "-b", branch_name], repo_path) - if success: - logger.info(f"Created and checked out branch: {branch_name}") - return True - return False - - -def commit_changes(repo_path, message): - """Commit changes in the git repository.""" - # Add all changes - _, _, add_success = git_cmd(["add", "."], repo_path) - if not add_success: - logger.error("Failed to add changes to git index") - return False - - # Commit changes - _, _, commit_success = git_cmd(["commit", "-m", message], repo_path, check=False) - if commit_success: - logger.info(f"Committed changes with message: {message}") - return True - else: - # Check if there were no changes to commit - status, _, _ = git_cmd(["status", "--porcelain"], repo_path) - if not status: - logger.info("No changes to commit") - return True - logger.error("Failed to commit changes") - return False - - -def push_branch(repo_path, branch_name, remote="origin"): - """Push a branch to the remote repository.""" - _, _, success = git_cmd(["push", "-u", remote, branch_name], repo_path) - if success: - logger.info(f"Pushed branch {branch_name} to {remote}") - return True - return False - - -def run_upgrade_tool(directory, target_version): - """ - Run the terraform upgrade tool on a directory. - - Args: - directory: Directory to upgrade - target_version: Target Terraform version - - Returns: - bool: True if successful, False otherwise - """ - script_dir = os.path.dirname(os.path.abspath(__file__)) - - cmd = [ - sys.executable, - "-m", - "tf_upgrade.cli", - "upgrade", - directory, - "--target-version", - target_version, - ] - - try: - logger.info(f"Running: {' '.join(cmd)}") - result = subprocess.run( - cmd, - cwd=script_dir, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - - if result.returncode == 0: - logger.info( - f"Successfully upgraded {directory} to version {target_version}" - ) - logger.debug(result.stdout) - return True - else: - logger.error(f"Failed to upgrade {directory}: {result.stderr}") - return False - - except Exception as e: - logger.error(f"Error running upgrade tool: {str(e)}") - return False - - -def upgrade_tf_directory(directory, target_version, branch_suffix=None): - """ - Upgrade a Terraform directory to the target version and commit changes to a branch. - - Args: - directory: The directory to upgrade - target_version: Target Terraform version (e.g. "0.13") - branch_suffix: Suffix for the branch name (default: None) - - Returns: - tuple: (success, branch_name) - """ - # Check if directory is in a git repo - repo_root = get_repo_root(directory) - if not repo_root: - logger.error(f"Directory {directory} is not in a git repository") - return False, None - - # Save current branch - current_branch, _, _ = git_cmd(["rev-parse", "--abbrev-ref", "HEAD"], repo_root) - - # Create normalized path for branch naming - rel_path = os.path.relpath(directory, repo_root) - safe_path = rel_path.replace("/", "-").replace(".", "-") - - # Create branch name - branch_name = f"tf-upgrade-{target_version}-{safe_path}" - if branch_suffix: - branch_name += f"-{branch_suffix}" - - # Create branch for this upgrade - if not create_branch(repo_root, branch_name): - logger.error(f"Failed to create branch {branch_name}") - return False, None - - # Run the upgrade - logger.info(f"Upgrading directory {directory} to Terraform {target_version}") - try: - # Run the upgrade tool - success = run_upgrade_tool(directory, target_version) - if not success: - logger.error(f"Failed to upgrade directory {directory}") - git_cmd(["checkout", current_branch], repo_root) - return False, None - - # Check if any files were modified by looking for backup directories - backup_dirs = [ - d - for d in os.listdir(directory) - if d.startswith(".terraform-upgrade-backup-") - ] - if not backup_dirs: - logger.info(f"No changes were made in {directory}") - git_cmd(["checkout", current_branch], repo_root) - return False, None - - # Commit the changes - commit_message = f"Upgrade Terraform to {target_version} in {rel_path}" - if not commit_changes(repo_root, commit_message): - logger.error(f"Failed to commit changes for {directory}") - git_cmd(["checkout", current_branch], repo_root) - return False, None - - # Return to original branch - git_cmd(["checkout", current_branch], repo_root) - return True, branch_name - - except Exception as e: - logger.error(f"Error upgrading {directory}: {str(e)}") - # Return to original branch - git_cmd(["checkout", current_branch], repo_root) - return False, None - - -def run_scan_command(repo_path, recursive=True, max_depth=20): - """ - Run the terraform scan command to find terraform directories - - Args: - repo_path: Repository path - recursive: Whether to scan recursively - max_depth: Maximum directory depth for recursion - - Returns: - dict: Scan results including terraform_dirs - """ - script_dir = os.path.dirname(os.path.abspath(__file__)) - recursive_flag = "--recursive" if recursive else "--no-recursive" - - cmd = [ - sys.executable, - "-m", - "tf_upgrade.cli", - "scan", - repo_path, - recursive_flag, - "--max-depth", - str(max_depth), - ] - - try: - logger.info(f"Running: {' '.join(cmd)}") - result = subprocess.run( - cmd, - cwd=script_dir, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - - if result.returncode != 0: - logger.error(f"Failed to scan repository: {result.stderr}") - return {"terraform_dirs": []} - - # Parse output to find terraform directories - terraform_dirs = [] - - # Look for lines with "Found X Terraform configurations" - for line in result.stdout.splitlines(): - if "directories scanned" in line: - logger.info(line) - - # Let's look for the files specifically - find_cmd = ["find", repo_path, "-name", "*.tf"] - find_result = subprocess.run( - find_cmd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - - if find_result.returncode == 0: - # Get unique directories containing .tf files - for file_path in find_result.stdout.splitlines(): - dir_path = os.path.dirname(file_path) - if dir_path not in terraform_dirs: - terraform_dirs.append(dir_path) - - return {"terraform_dirs": terraform_dirs} - - except Exception as e: - logger.error(f"Error scanning repository: {str(e)}") - return {"terraform_dirs": []} - - -def get_terraform_directories(repo_path, skip_dirs=None): - """ - Find all directories containing Terraform files in a repository. - - Args: - repo_path: Repository path - skip_dirs: List of directories to skip (default: None) - - Returns: - list: Directories containing Terraform files - """ - if skip_dirs is None: - skip_dirs = [] - - # Add default directories to skip - skip_dirs.extend([".git", ".terraform", "node_modules"]) - - # Run the scan command to find Terraform directories - result = run_scan_command(repo_path, recursive=True, max_depth=20) - - # Filter directories that should be skipped - terraform_dirs = [] - for dir_path in result.get("terraform_dirs", []): - skip = False - for skip_dir in skip_dirs: - if skip_dir in dir_path.split(os.path.sep): - skip = True - break - if not skip: - terraform_dirs.append(dir_path) - - return terraform_dirs - - -def main(): - parser = argparse.ArgumentParser( - description="Bulk upgrade Terraform configurations with separate PRs" - ) - parser.add_argument("repo_path", help="Path to the repository") - parser.add_argument( - "--target-version", help="Target Terraform version", default="1.0" - ) - parser.add_argument("--push", help="Push branches to remote", action="store_true") - parser.add_argument( - "--skip-dirs", help="Directories to skip (comma-separated)", default="" - ) - parser.add_argument( - "--limit", help="Limit number of directories to upgrade", type=int, default=0 - ) - parser.add_argument( - "--dry-run", - help="Don't make any changes, just show what would be done", - action="store_true", - ) - parser.add_argument( - "--branch-suffix", help="Optional suffix for branch names", default="" - ) - - args = parser.parse_args() - - repo_path = os.path.abspath(args.repo_path) - if not os.path.isdir(repo_path): - logger.error(f"Repository path does not exist: {repo_path}") - return 1 - - if not is_git_repo(repo_path): - logger.error(f"Not a git repository: {repo_path}") - return 1 - - # Parse skip directories - skip_dirs = [d.strip() for d in args.skip_dirs.split(",") if d.strip()] - - # Find Terraform directories - logger.info(f"Scanning repository for Terraform directories: {repo_path}") - tf_dirs = get_terraform_directories(repo_path, skip_dirs) - logger.info(f"Found {len(tf_dirs)} Terraform directories") - - # Apply limit if specified - if args.limit > 0 and len(tf_dirs) > args.limit: - logger.info(f"Limiting to {args.limit} directories") - tf_dirs = tf_dirs[: args.limit] - - # Perform upgrades - successful_upgrades = [] - failed_upgrades = [] - - for directory in tf_dirs: - logger.info(f"Processing directory: {directory}") - - if args.dry_run: - logger.info( - f"DRY RUN: Would upgrade {directory} to Terraform {args.target_version}" - ) - continue - - success, branch_name = upgrade_tf_directory( - directory, args.target_version, branch_suffix=args.branch_suffix - ) - - if success: - successful_upgrades.append((directory, branch_name)) - if args.push: - push_branch(repo_path, branch_name) - else: - failed_upgrades.append(directory) - - # Print summary - logger.info("=" * 50) - logger.info("UPGRADE SUMMARY") - logger.info("=" * 50) - logger.info(f"Total directories processed: {len(tf_dirs)}") - logger.info(f"Successful upgrades: {len(successful_upgrades)}") - logger.info(f"Failed upgrades: {len(failed_upgrades)}") - - if successful_upgrades: - logger.info("\nSuccessfully upgraded directories:") - for directory, branch in successful_upgrades: - logger.info(f" - {directory} (branch: {branch})") - - if failed_upgrades: - logger.info("\nFailed upgrade directories:") - for directory in failed_upgrades: - logger.info(f" - {directory}") - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tf_upgrade/cli.py b/tf_upgrade/cli.py deleted file mode 100644 index 5789d089..00000000 --- a/tf_upgrade/cli.py +++ /dev/null @@ -1,894 +0,0 @@ -#!/usr/bin/env python3 -""" -Command Line Interface for Terraform Upgrade Tool -""" -import json -import logging -import os -import sys -import traceback -from datetime import datetime - -import click - -from tf_upgrade.complexity_analyzer import ComplexityAnalyzer -from tf_upgrade.config import ConfigManager -from tf_upgrade.dependency_graph import DependencyGraph -from tf_upgrade.env_validator import validate_environment -from tf_upgrade.reporters.console import ConsoleReporter -from tf_upgrade.reporters.markdown import MarkdownReporter -from tf_upgrade.risk_assessment import RiskAssessment -from tf_upgrade.scanner import TerraformScanner, scan_command -from tf_upgrade.upgrade_controller import UpgradeController -from tf_upgrade.utils.git import validate_git_access -from tf_upgrade.utils.parallel import parallel_scan_directories -from tf_upgrade.utils.terraform import get_account_id_from_tfvars, validate_aws_profile -from tf_upgrade.version_detector import VersionDetector - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - handlers=[logging.StreamHandler()], -) - -logger = logging.getLogger("tf-upgrade") - - -def prompt_for_version(current_version): - """ - Prompt user for target version with clear options. - - Args: - current_version: Current Terraform version string - - Returns: - Selected version string or None for default behavior - """ - versions = ["0.13", "0.14", "0.15", "1.0"] - click.echo(f"Current Terraform version: {current_version}") - click.echo("Available upgrade targets:") - for i, version in enumerate(versions): - click.echo(f" {i+1}. {version}") - - selection = click.prompt("Select target version", default="", show_default=False) - if not selection: - # Find next version - current_idx = ( - versions.index(current_version) if current_version in versions else -1 - ) - next_version = ( - versions[current_idx + 1] - if current_idx + 1 < len(versions) - else versions[-1] - ) - click.echo(f"Using incremental next version: {next_version}") - return next_version - - try: - idx = int(selection) - 1 - if 0 <= idx < len(versions): - return versions[idx] - else: - click.echo("Invalid selection. Using incremental next version.") - return None # Fall back to default - except (ValueError, IndexError): - click.echo("Invalid selection. Using incremental next version.") - return None # Fall back to default - - -@click.group() -@click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") -@click.option("--debug", is_flag=True, help="Enable debug output") -@click.option("--aws-profile", "-p", type=str, help="AWS profile to use") -def cli(verbose, debug, aws_profile): - """Terraform Upgrade Tool - Upgrade configurations from 0.12.x to 1.x""" - if debug: - logger.setLevel(logging.DEBUG) - elif verbose: - logger.setLevel(logging.INFO) - else: - logger.setLevel(logging.WARNING) - - logger.debug("Debug logging enabled") - - # Store AWS profile in click context for later use - click.get_current_context().meta["aws_profile"] = aws_profile - - -@cli.command() -@click.argument("directory", type=click.Path(exists=True), required=False) -@click.option( - "--recursive/--no-recursive", - "-r", - is_flag=True, - default=True, # Default to recursive scanning - help="Scan directories recursively (enabled by default)", -) -@click.option( - "--max-depth", - "-d", - type=int, - default=20, # Increase default depth - help="Maximum directory depth for recursive scans", -) -@click.option( - "--parallel/--no-parallel", - default=True, - help="Use parallel processing for scanning", -) -@click.option("--output", "-o", type=click.Path(), help="Path to output file") -@click.option( - "--format", - "-f", - type=click.Choice(["json", "csv"]), - default="json", - help="Output format", -) -def scan(directory, recursive, max_depth, parallel, output, format): - """Scan for Terraform configurations requiring upgrades""" - directory = directory or os.getcwd() - - # Make the recursive behavior more explicit in the output - if recursive: - click.echo( - "Scanning directory recursively: {} (max depth: {})".format( - directory, max_depth - ) - ) - else: - click.echo(f"Scanning directory (non-recursive): {directory}") - - # Log the parameters for debugging - logger.debug( - f"Scan parameters: recursive={recursive}, max_depth={max_depth}, " - f"parallel={parallel}" - ) - - try: - results = scan_command( - directory, recursive=recursive, max_depth=max_depth, parallel=parallel - ) - - terraform_count = len(results.get("terraform_dirs", [])) - module_count = len(results.get("module_dirs", [])) - error_count = len(results.get("errors", [])) - total_dirs_scanned = results.get("scanned_dirs", 0) - - # Add more details in the output - click.echo( - f"\nFound {terraform_count} Terraform configurations" - f" ({module_count} modules)" - f" in {total_dirs_scanned} directories scanned" - f"{f', with {error_count} errors' if error_count else ''}" - ) - - if output: - scanner = TerraformScanner() - if format == "json": - scanner.export_to_json(results, output) - click.echo(f"Results exported to {output}") - elif format == "csv": - scanner.export_to_csv(results, output) - click.echo(f"Results exported to {output}") - else: - # Display a summary on console - click.echo("\nSummary:") - click.echo(f" Total Terraform directories: {terraform_count}") - click.echo(f" Module directories: {module_count}") - provider_count = len(results.get("provider_configs", {})) - if provider_count: - click.echo(f" Providers used: {provider_count}") - for provider, configs in results.get("provider_configs", {}).items(): - versions = set(c["version"] for c in configs if c["version"]) - version_str = ", ".join(versions) if versions else "unspecified" - click.echo( - f" - {provider}: {len(configs)} uses, " - f"versions: {version_str}" - ) - if error_count: - click.echo("\nErrors encountered:") - # Show first 5 errors - for error in results.get("errors", [])[:5]: - click.echo(f" - {error['directory']}: {error['error']}") - if error_count > 5: - click.echo(f" ... and {error_count - 5} more errors") - - except Exception as e: - logger.error(f"Error during scan: {str(e)}") - if logger.level == logging.DEBUG: - logger.debug(traceback.format_exc()) - sys.exit(1) - - -@cli.command() -@click.argument("directory", type=click.Path(exists=True), required=False) -@click.option("--output", "-o", type=click.Path(), help="Path to output file") -def detect_version(directory, output): - """Detect Terraform version used in configurations""" - directory = directory or os.getcwd() - - detector = VersionDetector() - try: - result = detector.analyze_directory(directory) - - click.echo(f"\nVersion detection results for {directory}:") - click.echo(f" Version constraint: {result['version_constraint'] or 'None'}") - click.echo(f" Detected version: {result['final_version']}") - - if result["requires_upgrade"]: - click.echo(" Requires upgrade: Yes") - click.echo(f" Upgrade path: {' → '.join(result['upgrade_path'])}") - else: - click.echo(" Requires upgrade: No") - - # Show feature detection details - if "feature_detection" in result: - fd = result.get("feature_detection") - click.echo("\nFeature detection:") - for version, count in fd["feature_counts"].items(): - click.echo(f" {version} features: {count}") - - # Check for deprecated syntax - file_paths = [ - os.path.join(directory, f) - for f in os.listdir(directory) - if f.endswith(".tf") - ] - deprecated_syntax = detector.find_deprecated_syntax(file_paths) - - if deprecated_syntax: - click.echo(f"\nFound {len(deprecated_syntax)} deprecated syntax:") - for item in deprecated_syntax[:5]: # Show first 5 - path = os.path.basename(item["file"]) - click.echo( - f" - {item['type']}: {item['text']} " f"(in {path}:{item['line']})" - ) - if len(deprecated_syntax) > 5: - click.echo("\n ... and {len(deprecated_syntax}) - 5 more instances") - - # Export results if requested - if output: - # Add deprecated syntax to the result - result["deprecated_syntax"] = deprecated_syntax - os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True) - with open(output, "w") as f: - json.dump(result, f, indent=2) - click.echo(f"\nResults exported to {output}") - - except Exception as e: - logger.error(f"Error detecting version: {str(e)}") - if logger.level == logging.DEBUG: - logger.debug(traceback.format_exc()) - sys.exit(1) - - -@cli.command() -@click.argument("directory", type=click.Path(exists=True), required=False) -@click.option("--output", "-o", type=click.Path(), help="Path to save visualization") -@click.option( - "--format", - "-f", - type=click.Choice(["png", "dot"]), - default="png", - help="Output format (png for image, dot for Graphviz DOT file)", -) -def generate_graph(directory, output, format): - """Generate dependency graph for Terraform configurations""" - directory = directory or os.getcwd() - try: - # Import required but potentially missing package - import pygraphviz # noqa: F401 - except ImportError: - click.echo( - "Error: pygraphviz is not installed. " - "Please install it to use this feature." - ) - click.echo("You can install it using pip: pip install pygraphviz") - click.echo( - "You may also need to install graphviz system packages. " - "See the installation instructions for pygraphviz." - ) - sys.exit(1) - - try: - # First scan the directory to get terraform configurations - click.echo(f"Scanning directory: {directory}") - scanner = TerraformScanner() - scan_results = scanner.scan_directory(directory) - - # Build dependency graph - click.echo("Building dependency graph...") - graph = DependencyGraph() - graph.build_from_scan(scan_results) - - # Generate output filename if not specified - if not output: - output = os.path.join(directory, f"module_dependencies.{format}") - - # Export the graph - if format == "dot": - graph.export_graphviz(output) - else: # Default to PNG - graph.visualize(output) - - click.echo(f"Dependency graph exported to {output}") - - # Check for cycles - cycles = graph.find_cycles() - if cycles: - click.echo(f"\nWarning: Found {len(cycles)} dependency cycles:") - for i, cycle in enumerate(cycles[:3], 1): # Show first 3 cycles - cycle_dirs = [os.path.basename(dir_path) for dir_path in cycle] - click.echo(f" {i}. {' → '.join(cycle_dirs)} → {cycle_dirs[0]}") - if len(cycles) > 3: - click.echo(f" ... and {len(cycles) - 3} more cycles") - - # Suggest upgrade order - upgrade_order = graph.get_upgrade_order() - click.echo(f"\nSuggested upgrade order ({len(upgrade_order)} modules):") - for i, dir_path in enumerate(upgrade_order[:10], 1): # Show first 10 - click.echo(f" {i}. {os.path.basename(dir_path)}") - if len(upgrade_order) > 10: - click.echo(f" ... and {len(upgrade_order) - 10} more modules") - - except Exception as e: - logger.error(f"Error generating dependency graph: {str(e)}") - if logger.level == logging.DEBUG: - logger.debug(traceback.format_exc()) - sys.exit(1) - - -@cli.command() -@click.argument("directory", type=click.Path(exists=True), required=False) -@click.option("--output", "-o", type=click.Path(), help="Path to output report") -def analyze_complexity(directory, output): - """Analyze complexity of Terraform configurations""" - directory = directory or os.getcwd() - - analyzer = ComplexityAnalyzer() - try: - result = analyzer.analyze_directory(directory) - - click.echo(f"\nComplexity analysis for {directory}:") - click.echo(f" Complexity score: {result['total_complexity']:.1f}") - click.echo(f" Risk level: {result['risk_level'].replace('_', ' ').title()}") - - # Show metrics - click.echo("\nMetrics:") - metrics = result.get("metrics") - click.echo(f" Resources: {metrics.get('resource_count', 0)}") - click.echo(f" Data sources: {metrics.get('data_count', 0)}") - click.echo(f" Modules: {metrics.get('module_count', 0)}") - click.echo(f" Variables: {metrics.get('variable_count', 0)}") - click.echo(f" Outputs: {metrics.get('output_count', 0)}") - - # Show advanced metrics - click.echo("\nAdvanced metrics:") - click.echo(f" Dynamic blocks: {metrics.get('dynamic_block_count', 0)}") - click.echo(f" Count usage: {metrics.get('count_usage', 0)}") - click.echo(f" For_each usage: {metrics.get('for_each_usage', 0)}") - click.echo( - f" Conditional expressions: " - f"{metrics.get('conditional_expressions', 0)}" - ) - - # Show deprecated syntax - if result["deprecated_syntax"]: - click.echo( - f"\nDeprecated syntax " - f"({len(result['deprecated_syntax'])} instances):" - ) - syntax_types = {} - for item in result["deprecated_syntax"]: - if item["type"] not in syntax_types: - syntax_types[item["type"]] = 0 - syntax_types[item["type"]] += 1 - - for syntax_type, count in syntax_types.items(): - click.echo(f" {syntax_type.replace('_', ' ').title()}: {count}") - - # Show a few examples - if result["deprecated_syntax"]: - click.echo("\nExamples:") - for item in result["deprecated_syntax"][:3]: # Show first 3 - click.echo( - f" - {item['type']}: {item['match']} " f"in {item['file']}" - ) - if len(result["deprecated_syntax"]) > 3: - click.echo( - f" ... and {len(result['deprecated_syntax']) - 3} " - f"more instances" - ) - - # Export results if requested - if output: - # If output is a directory, create a filename - if os.path.isdir(output): - output = os.path.join( - output, "complexity-" f"{os.path.basename(directory)}.json" - ) - - # Ensure directory exists - os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True) - - # Export as JSON - with open(output, "w") as f: - json.dump(result, f, indent=2) - click.echo(f"\nResults exported to {output}") - - except Exception as e: - logger.error(f"Error analyzing complexity: {str(e)}") - if logger.level == logging.DEBUG: - logger.debug(traceback.format_exc()) - sys.exit(1) - - -@cli.command() -@click.argument("directory", type=click.Path(exists=True), required=False) -@click.option( - "--recursive", "-r", is_flag=True, help="Assess all subdirectories recursively" -) -@click.option( - "--max-depth", - "-d", - type=int, - default=3, - help="Maximum directory depth for recursive assessment", -) -@click.option("--output", "-o", type=click.Path(), help="Path to output report") -@click.option( - "--parallel/--no-parallel", - default=True, - help="Use parallel processing for assessment", -) -def assess_risk(directory, recursive, max_depth, output, parallel): - """Assess upgrade risk for Terraform configurations""" - directory = directory or os.getcwd() - - config = ConfigManager() - - try: - # First get a list of directories to assess - if recursive: - click.echo(f"Scanning for Terraform directories in {directory}...") - scanner = TerraformScanner(config, max_depth=max_depth) - scan_results = scanner.scan_directory(directory) - directories = [d["path"] for d in scan_results["terraform_dirs"]] - click.echo(f"Found {len(directories)} Terraform directories to assess") - else: - directories = [directory] - - # Create risk assessor - risk_assessor = RiskAssessment(config) - click.echo("Performing risk assessment...") - - # Process in parallel if enabled - if ( - parallel - and config.get("parallel.scan.enabled", True) - and len(directories) > 1 - ): - max_workers = config.get("parallel.scan.max_workers", 10) - - # Define the assessment function for each directory - def assess_single_dir(d): - try: - return risk_assessor.assess_directory(d) - except Exception as e: - return {"directory": d, "error": str(e)} - - # Assess all directories in parallel - click.echo(f"Using parallel processing with {max_workers} workers") - dir_results = parallel_scan_directories( - directories, assess_single_dir, max_workers - ) - - # Convert to list for bulk_assessment format - assessments = list(dir_results.values()) - - # Create the final assessment result structure - assessment_results = risk_assessor._create_assessment_summary(assessments) - else: - # Sequential processing - click.echo("Processing directories sequentially") - assessment_results = risk_assessor.bulk_assessment(directories) - - # Generate summary to console - summary = assessment_results["summary"] - click.echo("\nRisk Assessment Summary:") - click.echo(f" Total directories assessed: {summary['total_directories']}") - click.echo(f" High priority: {summary['high_priority']}") - click.echo(f" Medium priority: {summary['medium_priority']}") - click.echo(f" Low priority: {summary['low_priority']}") - if summary["failed_assessments"] > 0: - click.echo(f" Failed assessments: {summary['failed_assessments']}") - click.echo(f" Average risk score: {summary['average_risk_score']:.1f}") - - # Show most common risk factors - if summary["most_common_risk_factors"]: - click.echo("\nMost Common Risk Factors:") - for factor in summary["most_common_risk_factors"][:5]: - click.echo( - f" - {factor['factor'].replace('_', ' ').title()}: " - f"{factor['count']} occurrences" - ) - - # Show highest risk directories - if assessment_results["prioritized_list"]: - click.echo("\nHighest Priority Directories:") - for item in assessment_results["prioritized_list"][:5]: - dir_name = os.path.basename(item["directory"]) - click.echo( - f" - {dir_name}: {item['priority'].upper()} priority," - f" risk score {item['risk_score']}" - ) - - # Export detailed report - if output: - risk_assessor.generate_report(assessment_results, output) - click.echo(f"\nDetailed risk assessment report exported to {output}") - else: - click.echo("\nUse --output option to generate a detailed report") - - except Exception as e: - logger.error(f"Error performing risk assessment: {str(e)}") - if logger.level == logging.DEBUG: - logger.debug(traceback.format_exc()) - sys.exit(1) - - -@cli.command() -@click.argument("directory", type=click.Path(exists=True), required=False) -@click.option( - "--target-version", - "-t", - type=click.Choice(["0.13", "0.14", "0.15", "1.0"]), - help="Target Terraform version", -) -@click.option( - "--interactive", - "-i", - is_flag=True, - help="Interactive mode for selecting upgrade version", -) -@click.option( - "--backup/--no-backup", - default=True, - help="Create backups of files before upgrading", -) -def upgrade(directory, target_version, interactive, backup): - """Upgrade Terraform configurations""" - directory = directory or os.getcwd() - - # Get AWS profile from click context - aws_profile = click.get_current_context().meta.get("aws_profile") - - # Determine account ID (if possible) - account_id = get_account_id_from_tfvars(directory) - if account_id: - logger.info(f"Detected account ID: {account_id}") - else: - logger.warning("Could not automatically determine account ID") - - # Create reporters - console_reporter = ConsoleReporter() - - # Create report directory - report_dir = os.path.join(directory, "terraform-upgrade-reports") - os.makedirs(report_dir, exist_ok=True) - - # Create markdown reporter - ts = datetime.now().strftime("%Y%m%d-%H%M%S") - report_path = os.path.join(report_dir, f"upgrade-report-{ts}.md") - md_reporter = MarkdownReporter(report_path) - - # Initialize upgrade controller - controller = UpgradeController( - directory, - reporters=[console_reporter, md_reporter], - aws_profile=aws_profile, - account_id=account_id, - ) - - # Get current terraform version - if interactive and not target_version: - version_info = controller.detect_current_version() - current_version = version_info["final_version"] - target_version = prompt_for_version(current_version) - - # Show version information - version_msg = f" to {target_version}" if target_version else "" - click.echo(f"Upgrading directory{version_msg}: {directory}") - - # Run upgrade - result = controller.upgrade( - target_version=target_version, step_by_step=interactive, backup=backup - ) - - # Display result - if result["success"]: - click.echo("\n✅ Upgrade completed successfully!") - elif result.get("user_aborted"): - click.echo("\n⏹️ Upgrade was aborted by user.") - else: - click.echo("\n❌ Upgrade failed. See report for details.") - - click.echo(f"Detailed report saved to: {report_path}") - - -@cli.command() -@click.argument("directory", type=click.Path(exists=True), required=False) -@click.option( - "--target-version", - "-t", - type=click.Choice(["0.13", "0.14", "0.15", "1.0"]), - help="Target Terraform version", -) -def dry_run(directory, target_version): - """Simulate an upgrade without making changes""" - directory = directory or os.getcwd() - click.echo(f"Performing dry run for directory: {directory}") - - # Create controller but don't execute actual upgrades - controller = UpgradeController(directory) - - # Detect current version - version_info = controller.detect_current_version() - current_version = version_info["final_version"] - - # Determine upgrade path - upgrade_path = controller.get_upgrade_path(target_version) - - # Check for deprecations and complexity - analyzer = ComplexityAnalyzer() - complexity_info = analyzer.analyze_directory(directory) - - click.echo(f"\nCurrent Terraform version: {current_version}") - if not upgrade_path: - click.echo("No upgrade needed - already at target version or no valid path.") - return - - click.echo(f"Upgrade path: {' → '.join(upgrade_path)}") - click.echo( - f"Complexity level: " - f"{complexity_info['risk_level'].replace('_', ' ').title()}" - ) - - # Analyze each version upgrade - click.echo("\nAnalyzing upgrade steps:") - for version in upgrade_path: - upgrader_class = controller.get_upgrader_for_version(version) - if not upgrader_class: - click.echo(f" ⚠️ No upgrader available for version {version}") - continue - - click.echo(f"\n=== Upgrade to {version} ===") - - # Create upgrader but don't execute - just analyze - upgrader = upgrader_class(directory, backup=False) - - # Pre-check - try: - check_results = upgrader.pre_upgrade_check() - if check_results.get("ready", False): - click.echo(" ✓ Configuration ready for upgrade") - for key, value in check_results.items(): - if key not in ("ready", "message"): - click.echo(f" - {key}: {value}") - else: - click.echo( - f" ✗ Configuration not ready: " - f"{check_results.get('message', 'Unknown issue')}" - ) - except Exception as e: - click.echo(f" ✗ Error during pre-check: {str(e)}") - - # Syntax transformations (dry run) - try: - import shutil - - temp_dir = os.path.join(directory, f".tf-upgrade-dryrun-{version}") - os.makedirs(temp_dir, exist_ok=True) - - # Copy files for analysis - for tf_file in [f for f in os.listdir(directory) if f.endswith(".tf")]: - src = os.path.join(directory, tf_file) - dst = os.path.join(temp_dir, tf_file) - with open(src, "r") as s, open(dst, "w") as d: - d.write(s.read()) - - # Run transformer on copied files - if hasattr(upgrader, "apply_syntax_transformations"): - transformer_method = upgrader.__class__.apply_syntax_transformations - # Bind to a throwaway instance that targets the temp dir - temp_upgrader = upgrader.__class__(temp_dir, backup=False) - transform_results = transformer_method(temp_upgrader) - - click.echo(" • Syntax changes that would be made:") - if transform_results.get("total_changes", 0) > 0: - click.echo( - f" - Total changes: " - f"{transform_results.get('total_changes')}" - ) - click.echo( - f" - Files modified: " - f"{transform_results.get('modified_files')}" - ) - - # Show examples of modifications - for rule, count in transform_results.get("rule_counts", {}).items(): - click.echo(f" - {rule}: {count} changes") - else: - click.echo(" - No syntax changes needed") - - # Clean up temp dir - shutil.rmtree(temp_dir, ignore_errors=True) - - except Exception as e: - click.echo(f" ✗ Error during syntax analysis: {str(e)}") - - # Built-in commands - if hasattr(upgrader, "run_built_in_upgrade_commands"): - click.echo(" • Built-in commands that would be executed:") - if version == "0.13": - click.echo(" - terraform init") - click.echo(" - terraform 0.13upgrade -yes") - else: - click.echo(" - terraform init -upgrade") - - # Generate a report - report_file = os.path.join(directory, "terraform-upgrade-dryrun-report.md") - - # Complete the report with a summary - with open(report_file, "a") as f: - f.write("\n## Summary\n\n") - f.write(f"- Current Version: {current_version}\n") - f.write(f"- Upgrade Path: {' → '.join(upgrade_path)}\n") - f.write( - f"- Complexity Level: " - f"{complexity_info['risk_level'].replace('_', ' ').title()}\n" - ) - f.write("- Complexity Score: " f"{complexity_info['total_complexity']:.1f}\n\n") - - if complexity_info["deprecated_syntax"]: - ci = complexity_info["deprecated_syntax"] - f.write("### Deprecated Syntax\n\n") - for item in ci[:10]: - f.write(f"- {item['type']}: `{item['match']}` in {item['file']}\n") - if len(ci) > 10: - f.write(f"- ...and {len(ci) - 10}" f"more\n") - - click.echo(f"\nDry run completed. Report saved to {report_file}") - - -@cli.command() -@click.argument("directory", type=click.Path(exists=True), required=False) -def generate_pr(directory): - """Generate pull requests for upgraded configurations""" - directory = directory or os.getcwd() - click.echo(f"Generating pull requests for directory: {directory}") - # TODO: Implement PR generation logic - click.echo("Pull requests generated successfully.") - - -@cli.command() -@click.argument("directory", type=click.Path(exists=True), required=False) -@click.option( - "--format", - "-f", - type=click.Choice(["text", "json"]), - default="text", - help="Output format", -) -def verify_env(directory, format): - """Verify that the environment is properly set up""" - directory = directory or os.getcwd() - click.echo(f"Verifying environment for directory: {directory}") - - # Check terraform versions and config files - env_results = validate_environment(directory) - - # Check AWS profile - aws_success, aws_profile, aws_account = validate_aws_profile(directory) - env_results["aws_profile"] = { - "success": aws_success, - "profile": aws_profile, - "account_info": aws_account, - } - - if not aws_success: - env_results["warnings"].append( - f"AWS profile validation failed for profile '{aws_profile}'" - ) - - # Check Git repository access - git_results = validate_git_access(directory) - env_results["git_access"] = git_results - - if git_results.get("is_git_repo") and not git_results.get("can_read"): - env_results["errors"].append("Cannot read from Git repository") - env_results["status"] = "FAIL" - - # Output the results - if format == "json": - click.echo(json.dumps(env_results, indent=2, default=str)) - else: - click.echo("\n=== ENVIRONMENT VERIFICATION RESULTS ===") - - # Terraform versions - click.echo("\nTerraform Versions:") - for version, status in env_results["terraform_versions"].items(): - click.echo(f" {version}: {status}") - - # Config files - click.echo("\nTerraform Config Files:") - for file_type, path in env_results["config_files"].items(): - if path: - click.echo(f" {file_type}: {path}") - - # AWS Profile - click.echo("\nAWS Profile:") - if env_results["aws_profile"].get("success"): - p = env_results["aws_profile"].get("profile") - click.echo(f" Profile: {p}") - if p.get("account_info"): - ac = env_results["aws_profile"]["account_info"] - click.echo(f" Account: {ac.get('Account')}") - click.echo(f" User: {ac.get('Arn')}") - r = ac.get("Region") - click.echo(f" Region: {r}") - else: - click.echo( - f" Profile validation failed for " - f"'{env_results['aws_profile'].get('profile')}'" - ) - - # Git access - click.echo("\nGit Repository Access:") - if env_results["git_access"].get("is_git_repo"): - click.echo(f" Repository: {env_results['git_access'].get('remote_url')}") - click.echo(f" Branch: {env_results['git_access'].get('current_branch')}") - read_access = "Yes" if env_results["git_access"].get("can_read") else "No" - write_access = "Yes" if env_results["git_access"].get("can_write") else "No" - click.echo(f" Read access: {read_access}") - click.echo(f" Write access: {write_access}") - - if env_results["git_access"].get("errors"): - click.echo(" Errors:") - for error in env_results["git_access"].get("errors"): - click.echo(f" - {error}") - else: - click.echo(" Not a Git repository") - - # Overall status - click.echo("\nOverall Status:") - if env_results["status"] == "PASS": - click.echo(" ✅ Environment verification PASSED") - else: - click.echo(" ❌ Environment verification FAILED") - - if env_results["warnings"]: - click.echo("\nWarnings:") - for warning in env_results["warnings"]: - click.echo(f" - {warning}") - - if env_results["errors"]: - click.echo("\nErrors:") - for error in env_results["errors"]: - click.echo(f" - {error}") - - -def main(): - """Main entry point for the CLI""" - try: - cli() - except Exception as e: - logger.error(f"Error: {str(e)}") - if logger.level == logging.DEBUG: - logger.debug(traceback.format_exc()) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tf_upgrade/complexity_analyzer.py b/tf_upgrade/complexity_analyzer.py deleted file mode 100644 index 644f8e8d..00000000 --- a/tf_upgrade/complexity_analyzer.py +++ /dev/null @@ -1,188 +0,0 @@ -""" -Module for analyzing complexity of Terraform configurations. -""" - -import logging -import os -import re - -logger = logging.getLogger(__name__) - - -class ComplexityAnalyzer: - """ - Analyzes the complexity and risk of Terraform configurations. - """ - - def __init__(self): - """Initialize the complexity analyzer.""" - # Define complexity weights - self.weights = { - "resource": 1.0, - "data": 0.5, - "module": 1.5, - "provider": 1.0, - "variable": 0.3, - "output": 0.3, - "local": 0.2, - "terraform": 0.5, - "dynamic_block": 2.0, - "count": 1.5, - "for_each": 2.0, - "conditional": 1.0, - "function": 0.5, - "interpolation": 0.3, - "deprecated_syntax": 3.0, - } - - # Pattern for detecting deprecated syntax - self.deprecated_patterns = { - # "${var.name}" style - "interpolation_only": r'"\${([^{}]+)}"', - # "prefix${var.name}suffix" style - "quoted_interpolation": r'"([^"]*)\${([^{}]+)}([^"]*)"', - # Old template directives - "template_directives": r"%{if\s+.+?}|%{for\s+.+?}|%{endfor}|%{endif}", - # resource.*.id style - "splat_without_brackets": r"\*\.\w+", - } - - def analyze_directory(self, directory): - """ - Analyze a directory containing Terraform files for complexity. - - Args: - directory: Path to directory - - Returns: - Dictionary with complexity metrics - """ - result = { - "path": directory, - "metrics": {}, - "total_complexity": 0, - "risk_level": "unknown", - "deprecated_syntax": [], - "flags": [], - } - - tf_files = [ - os.path.join(directory, f) - for f in os.listdir(directory) - if f.endswith(".tf") - ] - - # Count basic elements - result["metrics"]["file_count"] = len(tf_files) - result["metrics"]["resource_count"] = 0 - result["metrics"]["data_count"] = 0 - result["metrics"]["module_count"] = 0 - result["metrics"]["provider_count"] = 0 - result["metrics"]["variable_count"] = 0 - result["metrics"]["output_count"] = 0 - result["metrics"]["local_count"] = 0 - result["metrics"]["dynamic_block_count"] = 0 - result["metrics"]["count_usage"] = 0 - result["metrics"]["for_each_usage"] = 0 - result["metrics"]["conditional_expressions"] = 0 - result["metrics"]["function_calls"] = 0 - result["metrics"]["interpolations"] = 0 - result["metrics"]["deprecated_syntax_count"] = 0 - - # Scan each file - for file_path in tf_files: - try: - with open(file_path, "r") as f: - content = f.read() - - # Count basic elements - result["metrics"]["resource_count"] += len( - re.findall(r'resource\s+"[^"]+"\s+"[^"]+"\s*{', content) - ) - result["metrics"]["data_count"] += len( - re.findall(r'data\s+"[^"]+"\s+"[^"]+"\s*{', content) - ) - result["metrics"]["module_count"] += len( - re.findall(r'module\s+"[^"]+"\s*{', content) - ) - result["metrics"]["provider_count"] += len( - re.findall(r'provider\s+"[^"]+"\s*{', content) - ) - result["metrics"]["variable_count"] += len( - re.findall(r'variable\s+"[^"]+"\s*{', content) - ) - result["metrics"]["output_count"] += len( - re.findall(r'output\s+"[^"]+"\s*{', content) - ) - result["metrics"]["local_count"] += len( - re.findall(r"locals\s*{", content) - ) - - # Count complex structures - result["metrics"]["dynamic_block_count"] += len( - re.findall(r'dynamic\s+"[^"]+"\s*{', content) - ) - result["metrics"]["count_usage"] += len( - re.findall(r"count\s*=", content) - ) - result["metrics"]["for_each_usage"] += len( - re.findall(r"for_each\s*=", content) - ) - result["metrics"]["conditional_expressions"] += len( - re.findall(r"\?\s*.+\s*:\s*.+", content) - ) # ?: ternary - result["metrics"]["function_calls"] += len( - re.findall(r"\b\w+\(", content) - ) # Simple function call check - result["metrics"]["interpolations"] += len( - re.findall(r"\${", content) - ) # ${ syntax - - # Check for deprecated syntax - for name, pattern in self.deprecated_patterns.items(): - matches = re.finditer(pattern, content) - for match in matches: - # Truncate long matches - match_excerpt = match.group(0)[:50] - result["deprecated_syntax"].append( - { - "type": name, - "file": os.path.basename(file_path), - "match": match_excerpt, - } - ) - result["metrics"]["deprecated_syntax_count"] += 1 - - except Exception as e: - logger.warning(f"Error analyzing {file_path}: {str(e)}") - - # Calculate weighted complexity score - score = 0 - for metric, count in result["metrics"].items(): - metric_name = metric.replace("_count", "").replace("_usage", "") - weight = self.weights.get(metric_name, 0.1) - score += count * weight - - result["total_complexity"] = score - - # Determine risk level - if score > 100: - result["risk_level"] = "very_high" - elif score > 50: - result["risk_level"] = "high" - elif score > 20: - result["risk_level"] = "medium" - else: - result["risk_level"] = "low" - - # Add risk flags - if result["metrics"]["deprecated_syntax_count"] > 0: - result["flags"].append("contains_deprecated_syntax") - - if result["metrics"]["resource_count"] > 50: - result["flags"].append("large_resource_count") - - if result["metrics"]["dynamic_block_count"] > 5: - result["flags"].append("complex_dynamic_blocks") - - return result diff --git a/tf_upgrade/config.py b/tf_upgrade/config.py deleted file mode 100644 index abe32a70..00000000 --- a/tf_upgrade/config.py +++ /dev/null @@ -1,435 +0,0 @@ -""" -Configuration management for the Terraform upgrade tool. -Handles loading, merging, and validating configurations from multiple sources. -""" - -import logging -import os -from typing import Any, Dict, List, Optional, Tuple - -import yaml - -logger = logging.getLogger(__name__) - -# Default configuration values -DEFAULT_CONFIG = { - "backup_enabled": True, - "backup_dir": ".terraform-upgrade-backup", - "log_level": "INFO", - "validation_enabled": True, - "github_enabled": False, - "interactive": False, - "terraform_versions": { - "0.12": "terraform_0.12.31", - "0.13": "terraform_0.13.7", - "0.14": "terraform_0.14.11", - "0.15": "terraform_0.15.5", - "1.0": "terraform_1.0.11", - "1.10": "terraform_1.10.5", - }, - "terraform_paths": ["/apps/terraform/bin", "/usr/local/bin"], -} - - -def load_config(config_path=None): - """ - Load configuration from YAML file with fallback to default values. - - Args: - config_path (str, optional): Path to config YAML file. Defaults to None. - - Returns: - dict: Configuration dictionary with default values merged with file values - """ - config = DEFAULT_CONFIG.copy() - - # If no config path specified, check standard locations - if not config_path: - # Check current directory - if os.path.exists(".tf-upgrade.yml"): - config_path = ".tf-upgrade.yml" - # Check user home directory - elif os.path.exists(os.path.expanduser("~/.tf-upgrade.yml")): - config_path = os.path.expanduser("~/.tf-upgrade.yml") - - # If config file found, load and merge it - if config_path and os.path.exists(config_path): - try: - with open(config_path, "r") as f: - file_config = yaml.safe_load(f) - if file_config: - # Update config with values from file - config.update(file_config) - logger.info(f"Loaded configuration from {config_path}") - except Exception as e: - logger.warning(f"Failed to load config from {config_path}: {e}") - logger.warning("Using default configuration") - - return config - - -def save_config(config, config_path=".tf-upgrade.yml"): - """ - Save configuration to YAML file. - - Args: - config (dict): Configuration to save - config_path (str, optional): Path to save to. Defaults to ".tf-upgrade.yml". - - Returns: - bool: True if successful, False otherwise - """ - try: - with open(config_path, "w") as f: - yaml.dump(config, f, default_flow_style=False) - logger.info(f"Configuration saved to {config_path}") - return True - except Exception as e: - logger.error(f"Failed to save config to {config_path}: {e}") - return False - - -def get_config_value(key, default=None, config=None): - """ - Get a configuration value safely. - - Args: - key (str): Configuration key to retrieve - default (any, optional): Default value if not found. Defaults to None. - config (dict, optional): Config dictionary to use. - If None, loads default config. - - Returns: - any: Configuration value or default - """ - if config is None: - config = load_config() - - # Handle nested keys with dots (e.g., "github.token") - if "." in key: - parts = key.split(".") - value = config - for part in parts: - if isinstance(value, dict) and part in value: - value = value[part] - else: - return default - return value - - return config.get(key, default) - - -class ConfigManager: - """ - Configuration manager for terraform upgrade tool. - Handles loading, merging, and validating configurations. - """ - - def __init__(self, custom_config_path: Optional[str] = None): - """ - Initialize configuration manager. - - Args: - custom_config_path: Optional path to a custom config file - """ - self.config = {} # Will hold the merged configuration - self.custom_config_path = custom_config_path - self.config_paths = { - "system": "/etc/terraform-upgrade-tool/config.yaml", - "user": os.path.expanduser("~/.config/terraform-upgrade-tool/config.yaml"), - "project": "./terraform-upgrade-config.yaml", - "custom": custom_config_path, - } - self._load_configs() - - def _get_default_config(self) -> Dict[str, Any]: - """Get the default configuration.""" - return { - "terraform": { - "binaries": { - "0.12": "terraform", - "0.13": "terraform-0.13", - "0.14": "terraform-0.14", - "0.15": "terraform-0.15", - "1.0": "terraform-1.0", - }, - "upgrade": { - "incremental": True, - "backup": True, - "validation": True, - "auto_approve": False, - }, - }, - "aws": {"profile": None, "region": None, "state_backup": True}, - "github": {"enabled": False, "repo": None, "base_branch": "main"}, - "parallel": { - "scan": {"enabled": True, "max_workers": 10}, - "upgrade": {"enabled": False, "max_workers": 5}, - }, - "reporting": { - "format": "markdown", - "detail_level": "medium", - "output_dir": "./upgrade-reports", - }, - } - - def _load_yaml_file(self, file_path: str) -> Dict[str, Any]: - """ - Load a YAML configuration file. - - Args: - file_path: Path to the YAML file - - Returns: - Dictionary containing the configuration, - or empty dict if file doesn't exist - """ - if not file_path or not os.path.exists(file_path): - return {} - - try: - with open(file_path, "r") as f: - config = yaml.safe_load(f) - logger.debug(f"Loaded configuration from {file_path}") - return config or {} - except Exception as e: - logger.warning( - f"Error loading configuration from {file_path}: " f"{str(e)}" - ) - return {} - - def _merge_config(self, new_config: Dict[str, Any]): - """ - Merge a new configuration into the existing one. - - Args: - new_config: New configuration to merge - """ - if not new_config: - return - - def _deep_merge(target, source): - for key, value in source.items(): - is_dict = isinstance(value, dict) - has_key = key in target - target_is_dict = has_key and isinstance(target[key], dict) - - if is_dict and has_key and target_is_dict: - _deep_merge(target[key], value) - else: - target[key] = value - - _deep_merge(self.config, new_config) - - def _load_configs(self): - """Load and merge all configuration files in order of precedence.""" - # Start with default configuration - self.config = self._get_default_config() - - # Load system config - system_config = self._load_yaml_file(self.config_paths["system"]) - if system_config: - self._merge_config(system_config) - - # Load user config - user_config = self._load_yaml_file(self.config_paths["user"]) - if user_config: - self._merge_config(user_config) - - # Load project config - project_config = self._load_yaml_file(self.config_paths["project"]) - if project_config: - self._merge_config(project_config) - - # Load custom config if specified - if self.custom_config_path: - custom_config = self._load_yaml_file(self.custom_config_path) - if custom_config: - self._merge_config(custom_config) - - # Validate the final configuration - self.validate_config() - - def _validate_section(self, section_name: str, schema: Dict) -> bool: - """ - Validate a section of the configuration against a schema. - - Args: - section_name: Name of the section to validate - schema: Schema to validate against - - Returns: - True if validation passes, False otherwise - """ - if section_name not in self.config: - logger.warning(f"Missing configuration section: {section_name}") - return False - - section = self.config[section_name] - - for key, constraints in schema.items(): - if constraints.get("required", False) and key not in section: - logger.warning( - f"Missing required key '{key}' in section " f"'{section_name}'" - ) - return False - - if key in section: - value = section[key] - - # Type checking - has_type = "type" in constraints - type_mismatch = has_type and not isinstance(value, constraints["type"]) - - if type_mismatch: - logger.warning( - f"Invalid type for '{section_name}.{key}': " - f"expected {constraints['type']}, got {type(value)}" - ) - return False - - # Nested schema validation - if "schema" in constraints and isinstance(value, dict): - schema = constraints["schema"] - for sub_key, sub_constraints in schema.items(): - if ( - sub_constraints.get("required", False) - and sub_key not in value - ): - logger.warning( - f"Missing required key " - f"'{section_name}.{key}.{sub_key}'" - ) - return False - - if sub_key in value: - sub_value = value[sub_key] - if "type" in sub_constraints and not isinstance( - sub_value, sub_constraints["type"] - ): - logger.warning( - f"Invalid type for " - f"'{section_name}.{key}.{sub_key}': " - f"expected {sub_constraints['type']}, " - f"got {type(sub_value)}" - ) - return False - - return True - - def validate_config(self) -> Tuple[bool, List[str]]: - """ - Validate the merged configuration against schemas. - - Returns: - Tuple of (is_valid, error_messages) - """ - errors = [] - - # Validate terraform section - terraform_schema = { - "binaries": {"type": dict, "required": True}, - "upgrade": { - "type": dict, - "schema": { - "incremental": {"type": bool}, - "backup": {"type": bool}, - "validation": {"type": bool}, - "auto_approve": {"type": bool}, - }, - }, - } - - # Validate aws section - aws_schema = { - "profile": {"type": (str, type(None))}, - "region": {"type": (str, type(None))}, - "state_backup": {"type": bool}, - } - - # Validate parallel section - parallel_schema = { - "scan": { - "type": dict, - "schema": {"enabled": {"type": bool}, "max_workers": {"type": int}}, - }, - "upgrade": { - "type": dict, - "schema": {"enabled": {"type": bool}, "max_workers": {"type": int}}, - }, - } - - # Run validation for each section - if not self._validate_section("terraform", terraform_schema): - errors.append("Invalid terraform configuration section") - - if not self._validate_section("aws", aws_schema): - errors.append("Invalid aws configuration section") - - if not self._validate_section("parallel", parallel_schema): - errors.append("Invalid parallelization configuration section") - - # Return validation status - return len(errors) == 0, errors - - def get(self, key_path: str, default: Any = None) -> Any: - """ - Get a configuration value by its path. - - Args: - key_path: Dot-separated path to the config value - (e.g., "terraform.binaries.0.12") - default: Default value to return if path doesn't exist - - Returns: - The configuration value, or the default if not found - """ - parts = key_path.split(".") - result = self.config - - try: - for part in parts: - result = result[part] - return result - except (KeyError, TypeError): - return default - - def set(self, key_path: str, value: Any): - """ - Set a configuration value by its path. - - Args: - key_path: Dot-separated path to the config value - value: Value to set - """ - parts = key_path.split(".") - config = self.config - - # Navigate to the parent of the target key - for part in parts[:-1]: - if part not in config: - config[part] = {} - config = config[part] - - # Set the value - config[parts[-1]] = value - - def save_to_file(self, file_path: str) -> bool: - """ - Save the current configuration to a file. - - Args: - file_path: Path where to save the configuration - - Returns: - True if successful, False otherwise - """ - try: - os.makedirs(os.path.dirname(os.path.abspath(file_path)), exist_ok=True) - with open(file_path, "w") as f: - yaml.dump(self.config, f, default_flow_style=False) - logger.info(f"Configuration saved to {file_path}") - return True - except Exception as e: - logger.error(f"Error saving configuration to {file_path}: " f"{str(e)}") - return False diff --git a/tf_upgrade/dependency_graph.py b/tf_upgrade/dependency_graph.py deleted file mode 100644 index 83205f51..00000000 --- a/tf_upgrade/dependency_graph.py +++ /dev/null @@ -1,308 +0,0 @@ -""" -Module for generating and analyzing -dependency graphs of Terraform configurations. -""" - -import logging -import os -import re - -import matplotlib.pyplot as plt -import networkx as nx - -logger = logging.getLogger(__name__) - - -class DependencyGraph: - """ - Generates a directed graph of Terraform module dependencies. - """ - - def __init__(self): - """Initialize the dependency graph generator.""" - self.graph = nx.DiGraph() - - def build_from_scan(self, scan_results): - """ - Build a dependency graph from scan results. - - Args: - scan_results: Results from the TerraformScanner - - Returns: - NetworkX DiGraph object - """ - # Add all terraform directories as nodes - for dir_info in scan_results["terraform_dirs"]: - dir_path = dir_info["path"] - self.graph.add_node(dir_path, **dir_info) - - # Find module references and add edges - for dir_info in scan_results["terraform_dirs"]: - dir_path = dir_info["path"] - self._find_module_references(dir_path, scan_results["terraform_dirs"]) - - return self.graph - - def _find_module_references(self, dir_path, all_dirs): - """ - Find all module references in a directory and add edges to the graph. - - Args: - dir_path: Directory to analyze - all_dirs: List of all Terraform directories for reference - """ - for tf_file in [f for f in os.listdir(dir_path) if f.endswith(".tf")]: - file_path = os.path.join(dir_path, tf_file) - try: - with open(file_path, "r") as f: - content = f.read() - - # Find module references - module_matches = re.finditer( - r'module\s+"([^"]+)"\s*{([^}]*)}', content, re.DOTALL - ) - for match in module_matches: - module_name = match.group(1) - module_block = match.group(2) - - # Find source - source_match = re.search( - r'source\s*=\s*["\']([^"\']+)["\']', module_block - ) - if source_match: - source = source_match.group(1) - - # Determine if it's a local module or remote - if not ( - source.startswith("git::") - or source.startswith("github.com") - or source.startswith("http") - or source.startswith("terraform-") - ): - # It's potentially a local module, resolve path - if source.startswith("./") or source.startswith("../"): - # Relative path - resolved_path = os.path.normpath( - os.path.join(dir_path, source) - ) - else: - # Could be a module name or absolute path - resolved_path = source - - # Check if the resolved path - # is in our terraform directories - for target_dir in all_dirs: - target_path = target_dir["path"] - if ( - target_path == resolved_path - or target_path.endswith("/" + resolved_path) - ): - # Found a match, add an edge - self.graph.add_edge( - dir_path, - target_path, - type="module", - name=module_name, - ) - break - else: - # Remote module - self.graph.add_edge( - dir_path, - f"REMOTE:{source}", - type="remote_module", - name=module_name, - ) - - except Exception as e: - logger.warning(f"Error analyzing {file_path}: {str(e)}") - - def get_upgrade_order(self): - """ - Determine the order in which modules should be upgraded. - - Returns: - List of directories in upgrade order - """ - try: - # Use topological sort for upgrade order - # Modules with no dependencies get upgraded first - order = list(nx.topological_sort(self.graph)) - # Filter out remote modules - order = [ - node - for node in order - if not (isinstance(node, str) and node.startswith("REMOTE:")) - ] - return order - except nx.NetworkXUnfeasible: - # Handle cyclic dependencies - logger.warning("Cyclic dependencies detected, " "using approximate order") - # Create groups by strongly connected components - components = list(nx.strongly_connected_components(self.graph)) - order = [] - for component in components: - component_list = list(component) - # Filter out remote modules - component_list = [ - node - for node in component_list - if not (isinstance(node, str) and node.startswith("REMOTE:")) - ] - order.extend(component_list) - return order - - def visualize(self, output_file): - """ - Create a visualization of the dependency graph. - - Args: - output_file: Path to save visualization - """ - try: - # Create a representation for visualization - viz_graph = self.graph.copy() - - # Set node attributes for visualization - for node in viz_graph.nodes(): - if not (isinstance(node, str) and node.startswith("REMOTE:")): - node_data = viz_graph.nodes[node] - # Set node label to directory name - viz_graph.nodes[node]["label"] = os.path.basename(node) - # Set color based on complexity - complexity = node_data.get("complexity", 0) - if complexity > 50: - viz_graph.nodes[node]["color"] = "red" - elif complexity > 20: - viz_graph.nodes[node]["color"] = "orange" - else: - viz_graph.nodes[node]["color"] = "green" - else: - # Remote module - viz_graph.nodes[node]["label"] = node.replace("REMOTE:", "") - viz_graph.nodes[node]["color"] = "gray" - - # Generate the visualization - plt.figure(figsize=(12, 10)) - pos = nx.spring_layout(viz_graph) - - # Draw nodes - node_colors = [ - viz_graph.nodes[n].get("color", "blue") for n in viz_graph.nodes() - ] - nx.draw_networkx_nodes( - viz_graph, pos, node_color=node_colors, node_size=500, alpha=0.8 - ) - - # Draw edges - nx.draw_networkx_edges(viz_graph, pos, arrows=True) - - # Draw labels - labels = { - node: viz_graph.nodes[node].get("label", str(node)) - for node in viz_graph.nodes() - } - nx.draw_networkx_labels(viz_graph, pos, labels, font_size=8) - - plt.axis("off") - plt.tight_layout() - - # Create directory if it doesn't exist - os.makedirs(os.path.dirname(os.path.abspath(output_file)), exist_ok=True) - plt.savefig(output_file, dpi=300, bbox_inches="tight") - plt.close() - - logger.info(f"Dependency graph visualization saved to {output_file}") - - except Exception as e: - logger.error(f"Error creating visualization: {str(e)}") - - def export_graphviz(self, output_file): - """ - Export graph in DOT format for use with Graphviz. - - Args: - output_file: Path to save DOT file - """ - try: - # Generate DOT file - dot_data = nx.drawing.nx_agraph.to_agraph(self.graph) - - # Customize appearance - for node in self.graph.nodes(): - n = dot_data.get_node(node) - if not (isinstance(node, str) and node.startswith("REMOTE:")): - node_data = self.graph.nodes[node] - # Set node label to directory name - n.attr["label"] = os.path.basename(node) - # Set color based on complexity - complexity = node_data.get("complexity", 0) - if complexity > 50: - n.attr["color"] = "red" - elif complexity > 20: - n.attr["color"] = "orange" - else: - n.attr["color"] = "green" - else: - # Remote module - n.attr["label"] = node.replace("REMOTE:", "") - n.attr["color"] = "gray" - - # Create directory if it doesn't exist - os.makedirs(os.path.dirname(os.path.abspath(output_file)), exist_ok=True) - - # Write to file - dot_data.write(output_file) - logger.info(f"Dependency graph exported to {output_file}") - - except Exception as e: - logger.error(f"Error exporting graph: {str(e)}") - - def find_cycles(self): - """ - Find cycles in the dependency graph. - - Returns: - List of cycles found in the graph - """ - try: - cycles = list(nx.simple_cycles(self.graph)) - # Filter out cycles that include remote modules - filtered_cycles = [] - for cycle in cycles: - if not any( - isinstance(node, str) and node.startswith("REMOTE:") - for node in cycle - ): - filtered_cycles.append(cycle) - return filtered_cycles - except Exception as e: - logger.error(f"Error finding cycles: {str(e)}") - return [] - - def get_module_dependencies(self, directory): - """ - Get all dependencies for a specific module. - - Args: - directory: Path to the module directory - - Returns: - Dictionary with upstream and downstream dependencies - """ - if directory not in self.graph: - return {"upstream": [], "downstream": []} - - # Get upstream (modules this directory depends on) - upstream = list(self.graph.successors(directory)) - upstream = [ - node - for node in upstream - if not (isinstance(node, str) and node.startswith("REMOTE:")) - ] - - # Get downstream (modules that depend on this directory) - downstream = list(self.graph.predecessors(directory)) - - return {"upstream": upstream, "downstream": downstream} diff --git a/tf_upgrade/env_validator.py b/tf_upgrade/env_validator.py deleted file mode 100644 index 6ddf67c7..00000000 --- a/tf_upgrade/env_validator.py +++ /dev/null @@ -1,316 +0,0 @@ -""" -Environment validation utilities for the Terraform upgrade tool. -""" - -import logging -import os -import re -import shutil -import subprocess - -from tf_upgrade.utils.git import validate_git_access -from tf_upgrade.utils.terraform import validate_aws_profile - -logger = logging.getLogger(__name__) - - -def find_terraform_config_files(dir_path): - """ - Find Terraform configuration files including .tf-control. - - Args: - dir_path: Directory to search in - - Returns: - Dictionary with paths to found configuration files - """ - result = {"tf_files": [], "tfvars_files": [], "tf_control": None} - - # Check if directory exists - if not os.path.isdir(dir_path): - logger.error(f"Directory not found: {dir_path}") - return result - - # Look for tf_control in this directory - tf_control = os.path.join(dir_path, ".tf-control") - if os.path.isfile(tf_control): - result["tf_control"] = tf_control - - # Also check for .tf-control.tfrc - tf_control_tfrc = os.path.join(dir_path, ".tf-control.tfrc") - if os.path.isfile(tf_control_tfrc): - result["tf_control_tfrc"] = tf_control_tfrc - - # If tf_control not found in directory, check if it's in a git repository - if not result["tf_control"]: - try: - # Get git repository root - git_root = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], - cwd=dir_path, - universal_newlines=True, - ).strip() - - # Look for tf_control in git root - tf_control = os.path.join(git_root, ".tf-control") - if os.path.isfile(tf_control): - result["tf_control"] = tf_control - - # Check for .tf-control.tfrc in git root - tf_control_tfrc = os.path.join(git_root, ".tf-control.tfrc") - if os.path.isfile(tf_control_tfrc): - result["tf_control_tfrc"] = tf_control_tfrc - - except (subprocess.CalledProcessError, FileNotFoundError): - # Not a git repository or git not installed - pass - - # If tf_control still not found, check home directory - if not result["tf_control"]: - home_tf_control = os.path.expanduser("~/.tf-control") - if os.path.isfile(home_tf_control): - result["tf_control"] = home_tf_control - - # Find .tf and .tfvars files - if os.path.isdir(dir_path): - for item in os.listdir(dir_path): - if item.endswith(".tf"): - result["tf_files"].append(os.path.join(dir_path, item)) - elif item.endswith(".tfvars"): - result["tfvars_files"].append(os.path.join(dir_path, item)) - - return result - - -def parse_tf_control_file(tf_control_path): - """ - Parse a .tf-control file to extract configuration. - - Args: - tf_control_path: Path to .tf-control file - - Returns: - Dictionary with parsed configuration - """ - config = {} - - if not tf_control_path or not os.path.isfile(tf_control_path): - logger.warning(f"No .tf-control file found at {tf_control_path}") - return config - - try: - with open(tf_control_path, "r") as f: - content = f.read() - - # Extract TFCOMMAND - tfcommand_match = re.search(r'TFCOMMAND=["\']?([^"\']+)["\']?', content) - if tfcommand_match: - config["TFCOMMAND"] = tfcommand_match.group(1) - - # Extract other variables - for match in re.finditer(r'([A-Za-z0-9_]+)=["\']?([^"\']+)["\']?', content): - key = match.group(1) - value = match.group(2) - config[key] = value - - return config - - except Exception as e: - logger.error(f"Error parsing .tf-control file: {str(e)}") - return {} - - -def check_tf_upgrade_readiness(dir_path): - """ - Check if a directory is ready for Terraform upgrade. - - Args: - dir_path: Directory to check - - Returns: - Dictionary with readiness check results - """ - results = {"ready": True, "messages": [], "warnings": [], "errors": []} - - # Check if directory exists - if not os.path.isdir(dir_path): - results["ready"] = False - results["errors"].append(f"Directory not found: {dir_path}") - return results - - # Find terraform files - config_files = find_terraform_config_files(dir_path) - - if not config_files["tf_files"]: - results["ready"] = False - results["errors"].append("No Terraform configuration files found") - - # Check terraform binary - tf_binary = "terraform" - if config_files["tf_control"]: - tf_config = parse_tf_control_file(config_files["tf_control"]) - if "TFCOMMAND" in tf_config: - tf_binary = tf_config["TFCOMMAND"] - - # Check if binary exists - if not shutil.which(tf_binary): - results["ready"] = False - results["errors"].append(f"Terraform binary not found: {tf_binary}") - else: - results["messages"].append(f"Using Terraform binary: {tf_binary}") - - # Check AWS profile - aws_success, aws_profile, _ = validate_aws_profile(dir_path) - if not aws_success: - results["warnings"].append(f"AWS profile validation failed for '{aws_profile}'") - - # Check Git repository - git_results = validate_git_access(dir_path) - if not git_results.get("is_git_repo", False): - results["warnings"].append("Not in a Git repository") - elif not git_results.get("can_read", False): - results["warnings"].append("Cannot read from Git repository") - - # Check for .terraform directory - terraform_dir = os.path.join(dir_path, ".terraform") - if os.path.isdir(terraform_dir): - results["messages"].append(".terraform directory found") - else: - results["warnings"].append( - "No .terraform directory found, terraform init may be required" - ) - - return results - - -def validate_environment(dir_path): - """ - Validate the environment for terraform upgrades. - - Args: - dir_path: Directory to validate - - Returns: - Dictionary with validation results - """ - from tf_upgrade.utils.git import validate_git_access - from tf_upgrade.utils.terraform import get_terraform_binary - - results = { - "status": "PASS", - "errors": [], - "warnings": [], - "messages": [], - "terraform_versions": {}, - "config_files": {}, - "aws_profile": {}, - "git_access": {}, - } - - # Check required terraform versions - terraform_paths = ["/apps/terraform/bin", "/usr/local/bin", "/usr/bin"] - required_versions = ["0.12", "0.13", "0.14", "0.15", "1.0"] - - # Map of binary names to version identifiers - version_binary_map = { - "0.12": ["terraform_0.12", "terraform_0.12.31"], - "0.13": ["terraform_0.13", "terraform_0.13.7"], - "0.14": ["terraform_0.14", "terraform_0.14.11"], - "0.15": ["terraform_0.15", "terraform_0.15.5"], - "1.0": [ - "terraform_1.0", - "terraform_1.0.11", - "terraform_current", - "terraform_1.3.10", - ], - } - - for version in required_versions: - binaries = version_binary_map.get(version, [f"terraform_{version}"]) - - # Special handling for terraform_current - # which is a symlink to the latest 1.x version - if version == "1.0": - # Try checking terraform_current explicitly - terraform_current = "/apps/terraform/bin/terraform_current" - if os.path.exists(terraform_current) and os.access( - terraform_current, os.X_OK - ): - results["terraform_versions"][version] = True - results["messages"].append( - f"Found Terraform {version} as {terraform_current}" - ) - continue - - found = False - for binary in binaries: - # Check in standard PATH - if shutil.which(binary) is not None: - found = True - results["messages"].append( - f"Found Terraform {version} as {binary} in PATH" - ) - break - - # Check in terraform_paths - for path in terraform_paths: - full_path = os.path.join(path, binary) - if os.path.exists(full_path) and os.access(full_path, os.X_OK): - found = True - results["messages"].append( - f"Found Terraform {version} as {full_path}" - ) - break - - results["terraform_versions"][version] = found - - # Check for all required terraform versions - if not all(results["terraform_versions"].values()): - missing_versions = [ - v for v, found in results["terraform_versions"].items() if not found - ] - results["errors"].append( - f"Missing Terraform versions: {', '.join(missing_versions)}" - ) - results["status"] = "FAIL" - - # Check terraform configuration files - config_files = find_terraform_config_files(dir_path) - results["config_files"] = { - "tf_files": len(config_files["tf_files"]), - "tfvars_files": len(config_files["tfvars_files"]), - "tf_control": config_files["tf_control"], - } - - # Check AWS profile - aws_success, aws_profile, aws_account = validate_aws_profile(dir_path) - results["aws_profile"] = { - "success": aws_success, - "profile": aws_profile, - "account": aws_account, - } - - if not aws_success: - if aws_profile: - results["warnings"].append(f"AWS profile '{aws_profile}' validation failed") - else: - results["warnings"].append("No AWS profile found") - - # Check Git repository access - git_results = validate_git_access(dir_path) - results["git_access"] = git_results - - if not git_results.get("is_git_repo", False): - results["warnings"].append("Directory is not in a Git repository") - elif not git_results.get("can_push", False): - results["warnings"].append("Cannot push to Git repository") - - # Basic terraform binary access check - try: - terraform_binary = get_terraform_binary(dir_path) - results["messages"].append(f"Terraform binary found: {terraform_binary}") - except Exception as e: - results["errors"].append(f"Error finding terraform binary: {str(e)}") - results["status"] = "FAIL" - - return results diff --git a/tf_upgrade/hcl_transformer.py b/tf_upgrade/hcl_transformer.py deleted file mode 100644 index 95e23ee0..00000000 --- a/tf_upgrade/hcl_transformer.py +++ /dev/null @@ -1,53 +0,0 @@ -import logging -import re - -logger = logging.getLogger(__name__) - - -class HCLTransformer: - def __init__(self): - self.rules = [] - - def add_regex_transformation(self, pattern, replacement, description=None, flags=0): - """Add a regex-based transformation rule.""" - self.rules.append( - { - "type": "regex", - "pattern": pattern, - "replacement": replacement, - "description": description, - "flags": flags, - } - ) - - def add_callable_transformation(self, transform_func, description=None): - """Add a callable transformation rule.""" - self.rules.append( - {"type": "callable", "function": transform_func, "description": description} - ) - - def transform_file(self, file_path): - """Apply transformations to a file.""" - try: - with open(file_path, "r") as f: - content = f.read() - - # Apply transformations - for rule in self.rules: - if rule["type"] == "regex": - content = re.sub( - rule["pattern"], - rule["replacement"], - content, - flags=rule.get("flags", 0), - ) - elif rule["type"] == "callable": - content = rule["function"](content) - - with open(file_path, "w") as f: - f.write(content) - - logger.info(f"Transformed file: {file_path}") - - except Exception as e: - logger.error(f"Failed to transform {file_path}: {str(e)}") diff --git a/tf_upgrade/module_resolver.py b/tf_upgrade/module_resolver.py deleted file mode 100644 index 5d4561bf..00000000 --- a/tf_upgrade/module_resolver.py +++ /dev/null @@ -1,181 +0,0 @@ -""" -Module dependency resolver for Census Bureau repository patterns. -""" - -import logging -import os -import re -from typing import Dict, List - -logger = logging.getLogger(__name__) - - -class ModuleResolver: - """Resolves module dependencies and compatibility for Census Bureau repositories.""" - - # Common Census Bureau module sources - COMMON_MODULE_SOURCES = [ - "github.com/CensusGitOps/terraform-aws-modules", - "github.com/CensusGitOps/terraform-census-modules", - "github.com/CensusGitOps/terraform-aws-blueprint", - ] - - # Module compatibility mapping - MODULE_COMPATIBILITY = { - # Module name: {terraform_version: recommended_ref} - "aws-common-security-groups": { - "0.13": "tf-0.13-compatible", - "0.14": "tf-0.14-compatible", - "1.0": "main", - }, - "aws-edl-launch-instance": { - "0.13": "tf-0.13", - "0.14": "tf-0.14", - "1.0": "main", - }, - } - - def __init__(self, directory: str): - """ - Initialize the module resolver. - - Args: - directory: Root directory to scan for module references - """ - self.directory = directory - self.modules = {} - self.refs = {} - - def scan_for_modules(self) -> Dict: - """ - Scan for module references in Terraform files. - - Returns: - Dictionary of modules and their references - """ - module_refs = {} - module_pattern = re.compile( - r'module\s+"([^"]+)"\s+{[\s\S]*?source\s+=\s+"([^"]+)(?:\?ref=([^"]+))?"', - re.MULTILINE, - ) - - for root, _, files in os.walk(self.directory): - for file in files: - if not file.endswith(".tf"): - continue - - file_path = os.path.join(root, file) - with open(file_path, "r") as f: - content = f.read() - - for match in module_pattern.finditer(content): - module_name = match.group(1) - source = match.group(2) - ref = match.group(3) or "main" - - if module_name not in module_refs: - module_refs[module_name] = [] - - module_refs[module_name].append( - {"file": file_path, "source": source, "ref": ref} - ) - - self.modules = module_refs - return module_refs - - def get_upgrade_recommendations(self, target_version: str) -> List[Dict]: - """ - Get module upgrade recommendations for a target Terraform version. - - Args: - target_version: Target Terraform version (e.g., "0.13", "0.14", "1.0") - - Returns: - List of recommended module upgrades - """ - if not self.modules: - self.scan_for_modules() - - recommendations = [] - - for module_name, instances in self.modules.items(): - for instance in instances: - source = instance["source"] - current_ref = instance["ref"] - - # Extract base module name from source - base_name = source.split("/")[-1] - - if ( - base_name in self.MODULE_COMPATIBILITY - and target_version in self.MODULE_COMPATIBILITY[base_name] - ): - recommended_ref = self.MODULE_COMPATIBILITY[base_name][ - target_version - ] - - if current_ref != recommended_ref: - recommendations.append( - { - "module": module_name, - "file": instance["file"], - "current_source": f"{source}?ref={current_ref}", - "recommended_source": ( - f"{source}?ref={recommended_ref}" - ), - "reason": ( - f"Required for Terraform {target_version} " - f"compatibility" - ), - } - ) - - return recommendations - - def apply_recommendations(self, recommendations: List[Dict]) -> Dict: - """ - Apply module upgrade recommendations. - - Args: - recommendations: List of recommended module upgrades - - Returns: - Dictionary with results of applied changes - """ - results = {"success": True, "applied": 0, "failed": 0, "details": []} - - for rec in recommendations: - file_path = rec["file"] - current_source = rec["current_source"] - recommended_source = rec["recommended_source"] - - try: - with open(file_path, "r") as f: - content = f.read() - - # Replace the module source - new_content = content.replace(current_source, recommended_source) - - with open(file_path, "w") as f: - f.write(new_content) - - results["applied"] += 1 - results["details"].append( - { - "file": file_path, - "status": "updated", - "change": ( - f"Updated source from {current_source} " - f"to {recommended_source}" - ), - } - ) - - except Exception as e: - results["failed"] += 1 - results["success"] = False - results["details"].append( - {"file": file_path, "status": "error", "error": str(e)} - ) - - return results diff --git a/tf_upgrade/reporters/__init__.py b/tf_upgrade/reporters/__init__.py deleted file mode 100644 index 89bfc72a..00000000 --- a/tf_upgrade/reporters/__init__.py +++ /dev/null @@ -1,255 +0,0 @@ -""" -Base reporting functionality for the Terraform upgrade process. -""" - -import logging -import time -from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional - -logger = logging.getLogger(__name__) - - -def format_time(seconds: float) -> str: - """ - Format a time duration in seconds into a human-readable string. - - Args: - seconds: Time in seconds - - Returns: - Formatted time string (e.g., "2h 3m 45s") - """ - if seconds is None: - return "unknown" - - hours, remainder = divmod(int(seconds), 3600) - minutes, seconds = divmod(remainder, 60) - - if hours > 0: - return f"{hours}h {minutes}m {seconds}s" - elif minutes > 0: - return f"{minutes}m {seconds}s" - else: - return f"{seconds}s" - - -class Reporter(ABC): - """Base class for all reporters.""" - - @abstractmethod - def step_started(self, current: int, total: int, step_name: str): - """ - Report that a step has started. - - Args: - current: Current step number (0-based) - total: Total number of steps - step_name: Name of the step - """ - pass - - @abstractmethod - def step_completed( - self, - current: int, - total: int, - step_name: str, - success: bool, - details: Optional[str], - estimated_remaining: Optional[float], - ): - """ - Report that a step has completed. - - Args: - current: Current step number (1-based, after completion) - total: Total number of steps - step_name: Name of the step - success: Whether the step was successful - details: Additional details about the step outcome - estimated_remaining: Estimated remaining time in seconds - """ - pass - - @abstractmethod - def operation_completed( - self, success: bool, total_time: float, summary: Dict[str, Any] - ): - """ - Report that the entire operation has completed. - - Args: - success: Whether the operation was successful - total_time: Total time taken in seconds - summary: Summary of the operation results - """ - pass - - -class ProgressTracker: - """ - Progress tracking system for long-running operations. - """ - - def __init__( - self, - total_steps: int, - operation_name: str, - reporters: Optional[List[Reporter]] = None, - ): - """ - Initialize progress tracker. - - Args: - total_steps: Total number of steps in the operation - operation_name: Name of the operation being performed - reporters: List of reporter instances to use - """ - self.total_steps = total_steps - self.completed_steps = 0 - self.operation_name = operation_name - self.start_time = time.time() - self.step_times = [] # Track time taken for each step - self.current_step_name = None - self._step_start_time = 0 - self.reporters = reporters or [] - - # Track successes and failures - self.successful_steps = [] - self.failed_steps = [] - - logger.info(f"Starting operation: {operation_name} with {total_steps} steps") - - def add_reporter(self, reporter: Reporter): - """ - Add a reporter to the tracker. - - Args: - reporter: Reporter instance to add - """ - self.reporters.append(reporter) - - def start_step(self, step_name: str): - """ - Start a new step in the process. - - Args: - step_name: Name of the step - """ - self.current_step_name = step_name - self._step_start_time = time.time() - - logger.info( - f"Starting step {self.completed_steps + 1}/" - f"{self.total_steps}: {step_name}" - ) - - # Notify all reporters - for reporter in self.reporters: - reporter.step_started(self.completed_steps, self.total_steps, step_name) - - def complete_step(self, success: bool = True, details: Optional[str] = None): - """ - Mark the current step as completed. - - Args: - success: Whether the step was successful - details: Additional details about the step outcome - """ - step_time = time.time() - self._step_start_time - self.step_times.append(step_time) - self.completed_steps += 1 - - # Track step outcome - step_info = { - "name": self.current_step_name, - "time": step_time, - "details": details, - } - - if success: - self.successful_steps.append(step_info) - logger.info( - f"Step {self.completed_steps}/{self.total_steps} completed " - f"successfully in {format_time(step_time)}" - ) - else: - self.failed_steps.append(step_info) - logger.warning( - f"Step {self.completed_steps}/{self.total_steps} failed " - f"in {format_time(step_time)}" - ) - if details: - logger.warning(f"Details: {details}") - - # Calculate estimated time remaining - estimated_remaining = self._estimate_remaining_time() - - # Notify all reporters - for reporter in self.reporters: - reporter.step_completed( - self.completed_steps, - self.total_steps, - self.current_step_name, - success, - details, - estimated_remaining, - ) - - def _estimate_remaining_time(self) -> Optional[float]: - """ - Estimate remaining time based on past step performance. - - Returns: - Estimated remaining time in seconds, or None if not enough data - """ - if not self.step_times: - return None - - # Use median of previous step times for better prediction - if len(self.step_times) >= 3: - # Remove outliers for more accurate prediction - sorted_times = sorted(self.step_times) - median_time = sorted_times[len(sorted_times) // 2] - remaining_steps = self.total_steps - self.completed_steps - return median_time * remaining_steps - - # If not enough data, use average - avg_step_time = sum(self.step_times) / len(self.step_times) - remaining_steps = self.total_steps - self.completed_steps - return avg_step_time * remaining_steps - - def complete_operation(self): - """ - Mark the entire operation as completed. - - Returns: - Summary of the operation results - """ - total_time = time.time() - self.start_time - all_steps_succeeded = len(self.failed_steps) == 0 - - summary = { - "operation_name": self.operation_name, - "total_time": total_time, - "total_steps": self.total_steps, - "completed_steps": self.completed_steps, - "successful_steps": len(self.successful_steps), - "failed_steps": len(self.failed_steps), - "success": ( - all_steps_succeeded and self.completed_steps == self.total_steps - ), - } - - logger.info(f"Operation completed in {format_time(total_time)}") - logger.info( - f"Success: {summary['success']}, " - f"Completed: {summary['completed_steps']}/{summary['total_steps']}" - ) - - # Notify all reporters - for reporter in self.reporters: - reporter.operation_completed(summary["success"], total_time, summary) - - return summary diff --git a/tf_upgrade/reporters/console.py b/tf_upgrade/reporters/console.py deleted file mode 100644 index 3785b38e..00000000 --- a/tf_upgrade/reporters/console.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -Console reporter for displaying upgrade progress in the terminal. -""" - -import logging -from typing import Any, Dict, Optional - -import click - -from tf_upgrade.reporters import Reporter, format_time - -logger = logging.getLogger(__name__) - - -class ConsoleReporter(Reporter): - """Reports progress to the console with colored output.""" - - def __init__( - self, use_emoji: bool = True, use_color: bool = True, show_time: bool = True - ): - """ - Initialize console reporter. - - Args: - use_emoji: Whether to use emoji in output - use_color: Whether to use colored output - show_time: Whether to show time estimates - """ - self.use_emoji = use_emoji - self.use_color = use_color - self.show_time = show_time - - def step_started(self, current: int, total: int, step_name: str): - """ - Display step start in console. - - Args: - current: Current step number (0-based) - total: Total number of steps - step_name: Name of the step - """ - prefix = "🔄 " if self.use_emoji else "" - percent = int((current / total) * 100) - msg = f"{prefix}[{percent:3d}%] Starting: {step_name}..." - - if self.use_color: - click.echo(click.style(msg, fg="blue")) - else: - click.echo(msg) - - def step_completed( - self, - current: int, - total: int, - step_name: str, - success: bool, - details: Optional[str], - estimated_remaining: Optional[float], - ): - """ - Display step completion in console. - - Args: - current: Current step number (1-based, after completion) - total: Total number of steps - step_name: Name of the step - success: Whether the step was successful - details: Additional details about the step outcome - estimated_remaining: Estimated remaining time in seconds - """ - if success: - prefix = "✅ " if self.use_emoji else "" - color = "green" - else: - prefix = "❌ " if self.use_emoji else "" - color = "red" - - percent = int((current / total) * 100) - msg = f"{prefix}[{percent:3d}%] Completed: {step_name}" - - if self.show_time and estimated_remaining is not None: - msg += f" - Est. remaining: {format_time(estimated_remaining)}" - - if self.use_color: - click.echo(click.style(msg, fg=color)) - else: - click.echo(msg) - - if details: - indent = " " - detail_lines = details.split("\n") - for line in detail_lines: - click.echo(f"{indent}{line}") - - def operation_completed( - self, success: bool, total_time: float, summary: Dict[str, Any] - ): - """ - Display operation completion in console. - - Args: - success: Whether the operation was successful - total_time: Total time taken in seconds - summary: Summary of the operation results - """ - if success: - prefix = "✅ " if self.use_emoji else "" - color = "green" - msg = ( - f"{prefix}Operation {summary['operation_name']} " - f"completed successfully in {format_time(total_time)}" - ) - else: - prefix = "❌ " if self.use_emoji else "" - color = "red" - msg = ( - f"{prefix}Operation {summary['operation_name']} " - f"failed in {format_time(total_time)}: " - f"{summary['completed_steps']}/{summary['total_steps']} " - f"steps completed, {summary['failed_steps']} failed" - ) - - if self.use_color: - click.echo(click.style(msg, fg=color, bold=True)) - else: - click.echo(msg) diff --git a/tf_upgrade/reporters/file.py b/tf_upgrade/reporters/file.py deleted file mode 100644 index 754486aa..00000000 --- a/tf_upgrade/reporters/file.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -File-based reporter for writing upgrade progress to files. -""" - -import json -import logging -import os -from datetime import datetime -from typing import Any, Dict, Optional # Remove unused List import - -from tf_upgrade.reporters import Reporter, format_time - -logger = logging.getLogger(__name__) - - -class FileReporter(Reporter): - """Reports progress to a JSON file for tracking long-running operations.""" - - def __init__(self, file_path: str): - """ - Initialize file reporter. - - Args: - file_path: Path to the report file - """ - self.file_path = file_path - - # Create directory if it doesn't exist - os.makedirs(os.path.dirname(os.path.abspath(file_path)), exist_ok=True) - - # Initialize the file - self._write_json( - { - "started_at": datetime.now().isoformat(), - "steps": [], - "completed": 0, - "total": 0, - "status": "in_progress", - } - ) - - def _read_json(self) -> Dict[str, Any]: - """Read the current report data from the file.""" - try: - with open(self.file_path, "r") as f: - return json.load(f) - except (FileNotFoundError, json.JSONDecodeError) as e: - logger.warning(f"Error reading report file {self.file_path}: {e}") - return { - "started_at": datetime.now().isoformat(), - "steps": [], - "completed": 0, - "total": 0, - "status": "in_progress", - } - - def _write_json(self, data: Dict[str, Any]): - """Write report data to the file.""" - try: - with open(self.file_path, "w") as f: - json.dump(data, f, indent=2) - except Exception as e: - logger.error(f"Error writing to report file {self.file_path}: {e}") - - def step_started(self, current: int, total: int, step_name: str): - """ - Record step start to file. - - Args: - current: Current step number (0-based) - total: Total number of steps - step_name: Name of the step - """ - data = self._read_json() - - data["total"] = total - data["steps"].append( - { - "name": step_name, - "started_at": datetime.now().isoformat(), - "status": "in_progress", - "number": current + 1, - } - ) - - self._write_json(data) - - def step_completed( - self, - current: int, - total: int, - step_name: str, - success: bool, - details: Optional[str], - estimated_remaining: Optional[float], - ): - """ - Record step completion to file. - - Args: - current: Current step number (1-based, after completion) - total: Total number of steps - step_name: Name of the step - success: Whether the step was successful - details: Additional details about the step outcome - estimated_remaining: Estimated remaining time in seconds - """ - data = self._read_json() - - data["completed"] = current - - # Update the last step - for step in reversed(data["steps"]): - if step["name"] == step_name and step["status"] == "in_progress": - step["completed_at"] = datetime.now().isoformat() - step["status"] = "success" if success else "failed" - if details: - step["details"] = details - break - - data["estimated_remaining_seconds"] = estimated_remaining - - self._write_json(data) - - def operation_completed( - self, success: bool, total_time: float, summary: Dict[str, Any] - ): - """ - Record operation completion to file. - - Args: - success: Whether the operation was successful - total_time: Total time taken in seconds - summary: Summary of the operation results - """ - data = self._read_json() - - data["completed_at"] = datetime.now().isoformat() - data["status"] = "success" if success else "failed" - data["total_time_seconds"] = total_time - data["total_time_formatted"] = format_time(total_time) - data["summary"] = summary - - self._write_json(data) diff --git a/tf_upgrade/reporters/markdown.py b/tf_upgrade/reporters/markdown.py deleted file mode 100644 index 7639ea0c..00000000 --- a/tf_upgrade/reporters/markdown.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -Markdown reporter for generating upgrade reports. -""" - -import logging -import os -from datetime import datetime -from typing import Any, Dict, List, Optional - -from tf_upgrade.reporters import Reporter, format_time - -logger = logging.getLogger(__name__) - - -class MarkdownReporter(Reporter): - """Generates markdown reports for terraform upgrades.""" - - def __init__(self, file_path: str, title: str = "Terraform Upgrade Report"): - """ - Initialize markdown reporter. - - Args: - file_path: Path to the report file - title: Report title - """ - self.file_path = file_path - self.title = title - self.started_at = datetime.now() - self.steps: List[Dict[str, Any]] = [] - - # Create directory if it doesn't exist - os.makedirs(os.path.dirname(os.path.abspath(file_path)), exist_ok=True) - - # Start with report header - with open(self.file_path, "w") as f: - f.write(f"# {self.title}\n\n") - f.write( - f"Report generated on " - f"{self.started_at.strftime('%Y-%m-%d %H:%M:%S')}\n\n" - ) - f.write("## Progress\n\n") - f.write("| Step | Status | Duration | Details |\n") - f.write("|------|--------|----------|--------|\n") - - def step_started(self, current: int, total: int, step_name: str): - """ - Record step start to report. - - Args: - current: Current step number (0-based) - total: Total number of steps - step_name: Name of the step - """ - step = { - "number": current + 1, - "name": step_name, - "started_at": datetime.now(), - "status": "In Progress", - } - - self.steps.append(step) - - def step_completed( - self, - current: int, - total: int, - step_name: str, - success: bool, - details: Optional[str], - estimated_remaining: Optional[float], - ): - """ - Record step completion to report. - - Args: - current: Current step number (1-based, after completion) - total: Total number of steps - step_name: Name of the step - success: Whether the step was successful - details: Additional details about the step outcome - estimated_remaining: Estimated remaining time in seconds - """ - # Find the step - for step in self.steps: - if step["name"] == step_name and step["status"] == "In Progress": - step["completed_at"] = datetime.now() - step["status"] = "Success" if success else "Failed" - step["details"] = details - step["duration"] = ( - step["completed_at"] - step["started_at"] - ).total_seconds() - - # Update the report file with the new step - with open(self.file_path, "a") as f: - duration_text = format_time(step["duration"]) - details_text = details if details else "" - if not success: - details_text = f"**Error:** {details_text}" - - status_text = "✅ Success" if success else "❌ Failed" - f.write( - f"| {step['number']}: {step['name']} | " - f"{status_text} | {duration_text} | {details_text} |\n" - ) - - break - - def operation_completed( - self, success: bool, total_time: float, summary: Dict[str, Any] - ): - """ - Record operation completion to report. - - Args: - success: Whether the operation was successful - total_time: Total time taken in seconds - summary: Summary of the operation results - """ - with open(self.file_path, "a") as f: - f.write("\n## Summary\n\n") - f.write( - f"- **Status:** " f"{'✅ Successful' if success else '❌ Failed'}\n" - ) - f.write(f"- **Total Duration:** {format_time(total_time)}\n") - f.write( - f"- **Steps Completed:** " - f"{summary['completed_steps']}/{summary['total_steps']}\n" - ) - f.write(f"- **Successful Steps:** {summary['successful_steps']}\n") - f.write(f"- **Failed Steps:** {summary['failed_steps']}\n") - - # Add timestamp - f.write("\n## Details\n\n") - f.write( - f"- **Started:** " f"{self.started_at.strftime('%Y-%m-%d %H:%M:%S')}\n" - ) - f.write( - f"- **Completed:** " f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n" - ) - - if summary["failed_steps"] > 0: - f.write("\n### Failed Steps\n\n") - for step in self.steps: - if step.get("status") == "Failed": - f.write(f"#### Step {step['number']}: {step['name']}\n\n") - f.write( - f"- **Details:** " - f"{step.get('details', 'No details provided')}\n\n" - ) diff --git a/tf_upgrade/risk_assessment.py b/tf_upgrade/risk_assessment.py deleted file mode 100644 index 12301384..00000000 --- a/tf_upgrade/risk_assessment.py +++ /dev/null @@ -1,360 +0,0 @@ -""" -Module for assessing risk of Terraform configuration upgrades. -""" - -import datetime -import logging -import os - -from tf_upgrade.complexity_analyzer import ComplexityAnalyzer -from tf_upgrade.config import ConfigManager -from tf_upgrade.scanner import TerraformScanner -from tf_upgrade.version_detector import VersionDetector - -logger = logging.getLogger(__name__) - - -class RiskAssessment: - """ - Assesses risk for Terraform configuration upgrades. - """ - - def __init__(self, config_manager=None): - """ - Initialize the risk assessment module. - - Args: - config_manager: Configuration manager instance - """ - self.config = config_manager or ConfigManager() - self.scanner = TerraformScanner(self.config) - self.version_detector = VersionDetector() - self.complexity_analyzer = ComplexityAnalyzer() - - def assess_directory(self, directory): - """ - Perform a comprehensive risk assessment on a directory. - - Args: - directory: Directory containing Terraform configurations - - Returns: - Dictionary with assessment results - """ - result = { - "directory": directory, - "version": {}, - "complexity": {}, - "risk_score": 0, - "risk_factors": [], - "priority": "medium", - "recommended_approach": "standard", - "estimated_effort": "medium", - } - - # Get version information - version_info = self.version_detector.analyze_directory(directory) - result["version"] = version_info - - # Analyze complexity - complexity_info = self.complexity_analyzer.analyze_directory(directory) - result["complexity"] = complexity_info - - # Calculate risk score (0-100) - risk_score = 0 - - # Factor 1: Complexity (0-40 points) - if complexity_info["risk_level"] == "very_high": - risk_score += 40 - result["risk_factors"].append("very_high_complexity") - elif complexity_info["risk_level"] == "high": - risk_score += 30 - result["risk_factors"].append("high_complexity") - elif complexity_info["risk_level"] == "medium": - risk_score += 15 - - # Factor 2: Deprecated syntax (0-30 points) - deprecated_count = complexity_info["metrics"]["deprecated_syntax_count"] - if deprecated_count > 20: - risk_score += 30 - result["risk_factors"].append("extensive_deprecated_syntax") - elif deprecated_count > 5: - risk_score += 20 - result["risk_factors"].append("moderate_deprecated_syntax") - elif deprecated_count > 0: - risk_score += 10 - result["risk_factors"].append("some_deprecated_syntax") - - # Factor 3: Upgrade distance (0-20 points) - upgrade_path = version_info["upgrade_path"] - # Need to go through multiple versions - if len(upgrade_path) >= 3: # Multi-version jump - risk_score += 20 - result["risk_factors"].append("multiple_version_jumps") - elif len(upgrade_path) == 2: - risk_score += 10 - - # Factor 4: Special constructs (0-10 points) - if "complex_dynamic_blocks" in complexity_info["flags"]: - risk_score += 10 - result["risk_factors"].append("complex_dynamic_blocks") - - # Store the final risk score - result["risk_score"] = min(100, risk_score) # Cap at 100 - - # Determine priority - if risk_score >= 70: - result["priority"] = "high" - elif risk_score >= 40: - result["priority"] = "medium" - else: - result["priority"] = "low" - - # Determine recommended approach - if risk_score >= 80: - result["recommended_approach"] = "careful_manual" - result["estimated_effort"] = "high" - elif risk_score >= 60: - result["recommended_approach"] = "hybrid_automated_manual" - result["estimated_effort"] = "medium_high" - elif risk_score >= 30: - result["recommended_approach"] = "standard_automated" - result["estimated_effort"] = "medium" - else: - result["recommended_approach"] = "fast_automated" - result["estimated_effort"] = "low" - - return result - - def bulk_assessment(self, directories): - """ - Perform risk assessment on multiple directories. - - Args: - directories: List of directories to assess - - Returns: - Dictionary with assessment results and summary - """ - results = [] - - for directory in directories: - try: - result = self.assess_directory(directory) - results.append(result) - except Exception as e: - logger.error(f"Error assessing {directory}: {str(e)}") - results.append({"directory": directory, "error": str(e)}) - - return self._create_assessment_summary(results) - - def _identify_common_factors(self, results): - """Identify the most common risk factors across all results.""" - factor_counts = {} - - for result in results: - for factor in result.get("risk_factors", []): - factor_counts[factor] = factor_counts.get(factor, 0) + 1 - - # Sort by count - sorted_factors = sorted(factor_counts.items(), key=lambda x: -x[1]) - - # Return the top factors with counts - return [{"factor": k, "count": v} for k, v in sorted_factors] - - def generate_report(self, assessment_results, output_file): - """ - Generate a human-readable risk assessment report. - - Args: - assessment_results: Results from bulk_assessment - output_file: Path to save the report - """ - os.makedirs(os.path.dirname(os.path.abspath(output_file)), exist_ok=True) - - with open(output_file, "w") as f: - f.write("# Terraform Upgrade Risk Assessment\n\n") - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - f.write(f"Report generated on {timestamp}\n\n") - - # Write summary - summary = assessment_results["summary"] - f.write("## Summary\n\n") - f.write( - f"- Total directories assessed: " f"{summary['total_directories']}\n" - ) - f.write(f"- High priority: {summary['high_priority']}\n") - f.write(f"- Medium priority: {summary['medium_priority']}\n") - f.write(f"- Low priority: {summary['low_priority']}\n") - f.write( - f"- Average risk score: " f"{summary['average_risk_score']:.1f}\n\n" - ) - - # Common risk factors - f.write("### Most Common Risk Factors\n\n") - for factor in summary["most_common_risk_factors"]: - f.write( - f"- {factor['factor'].replace('_', ' ').title()}: " - f"{factor['count']} occurrences\n" - ) - - # Prioritized list - f.write("\n## Prioritized Configurations\n\n") - f.write( - "| Directory | Priority | Risk Score | Complexity | " - "Upgrade Path | Risk Factors |\n" - ) - f.write( - "|-----------|----------|------------|------------|-----------" - "|---------------|\n" - ) - - for item in assessment_results["prioritized_list"]: - dir_name = os.path.basename(item["directory"]) - priority = item["priority"].upper() - risk_score = item["risk_score"] - complexity = item["complexity"]["risk_level"].replace("_", " ").title() - upgrade_path = " → ".join(item["version"]["upgrade_path"]) - risk_factors = ", ".join( - [ - factor.replace("_", " ").title() - for factor in item["risk_factors"] - ] - ) - - f.write( - f"| {dir_name} | {priority} | {risk_score} | " - f"{complexity} | {upgrade_path} | {risk_factors} |\n" - ) - - # Detailed assessments - f.write("\n## Detailed Assessments\n\n") - for item in assessment_results["assessments"]: - if "error" in item: - f.write(f"### {os.path.basename(item['directory'])}\n\n") - f.write(f"**Error**: {item['error']}\n\n") - continue - - f.write(f"### {os.path.basename(item['directory'])}\n\n") - f.write(f"- **Risk Score**: {item['risk_score']}\n") - f.write(f"- **Priority**: {item['priority'].title()}\n") - f.write( - f"- **Current Version**: " f"{item['version']['final_version']}\n" - ) - f.write( - f"- **Upgrade Path**: " - f"{' → '.join(item['version']['upgrade_path'])}\n" - ) - # Extract complexity level and format it for display - complexity_level = item["complexity"]["risk_level"].replace("_", " ") - f.write(f"- **Complexity Level**: " f"{complexity_level.title()}\n") - - # Format the estimated effort for display - effort = item["estimated_effort"].replace("_", " ") - f.write(f"- **Estimated Effort**: " f"{effort.title()}\n") - - # Format the recommended approach for display - approach = item["recommended_approach"].replace("_", " ") - f.write(f"- **Recommended Approach**: " f"{approach.title()}\n\n") - - if item["risk_factors"]: - f.write("**Risk Factors**:\n\n") - for factor in item["risk_factors"]: - f.write(f"- {factor.replace('_', ' ').title()}\n") - f.write("\n") - - if item["complexity"]["deprecated_syntax"]: - f.write("**Deprecated Syntax Found**:\n\n") - # Show max 5 examples - for syntax in item["complexity"]["deprecated_syntax"][:5]: - f.write( - f"- {syntax['type']}: `{syntax['match']}` " - f"in {syntax['file']}\n" - ) - if len(item["complexity"]["deprecated_syntax"]) > 5: - count = len(item["complexity"]["deprecated_syntax"]) - 5 - f.write(f"- ... and {count} more instances\n") - f.write("\n") - - f.write("\n## Recommendations\n\n") - f.write( - "Based on this assessment, " "we recommend the following approach:\n\n" - ) - - # Generate overall recommendations - total = summary["total_directories"] - high_ratio = 0.3 # 30% - if summary["high_priority"] > total * high_ratio: - f.write( - "1. Start with low-risk configurations " "to build experience\n" - ) - f.write( - "2. Create a detailed testing plan for high-risk " - "configurations\n" - ) - f.write( - "3. Consider breaking large, complex configurations " - "into smaller modules\n" - ) - f.write( - "4. Apply incremental upgrades " - "(0.12 → 0.13 → 0.14 → etc.) " - "rather than attempting to jump directly to 1.0\n" - ) - else: - f.write("1. Use the prioritized list to plan " "the upgrade sequence\n") - f.write( - "2. Start with medium complexity configurations " - "to validate the upgrade process\n" - ) - f.write("3. Apply automated upgrades to low-risk " "configurations\n") - f.write( - "4. Reserve manual effort for the high-risk items " "specifically\n" - ) - - def _create_assessment_summary(self, assessments): - """ - Create a summary from a list of assessments. - - Args: - assessments: List of assessment results - - Returns: - Dictionary with assessment results and summary - """ - # Generate summary - summary = { - "total_directories": len(assessments), - "high_priority": len( - [r for r in assessments if r.get("priority") == "high"] - ), - "medium_priority": len( - [r for r in assessments if r.get("priority") == "medium"] - ), - "low_priority": len([r for r in assessments if r.get("priority") == "low"]), - "failed_assessments": len([r for r in assessments if "error" in r]), - "most_common_risk_factors": self._identify_common_factors(assessments), - "total_risk_score": sum( - r.get("risk_score", 0) for r in assessments if "risk_score" in r - ), - "average_risk_score": ( - sum(r.get("risk_score", 0) for r in assessments if "risk_score" in r) - / max(1, len([r for r in assessments if "risk_score" in r])) - ), - } - - return { - "assessments": assessments, - "summary": summary, - "prioritized_list": sorted( - [r for r in assessments if "error" not in r], - key=lambda x: ( - ( - 0 - if x.get("priority") == "high" - else 1 if x.get("priority") == "medium" else 2 - ), - -x.get("risk_score", 0), - ), - ), - } diff --git a/tf_upgrade/scanner.py b/tf_upgrade/scanner.py deleted file mode 100644 index e67c1e14..00000000 --- a/tf_upgrade/scanner.py +++ /dev/null @@ -1,429 +0,0 @@ -""" -Scanner for identifying and analyzing Terraform configurations. -""" - -import json -import logging -import os -import re -from pprint import pformat - -from tf_upgrade.config import ConfigManager - -logger = logging.getLogger(__name__) - - -class TerraformScanner: - """Scanner for identifying and analyzing Terraform configurations.""" - - def __init__(self, config_manager=None, max_depth=20): - """ - Initialize the scanner with configuration. - - Args: - config_manager: Configuration manager instance - max_depth: Maximum directory depth to scan - """ - self.config = config_manager or ConfigManager() - self.max_depth = max_depth - # Add common ignored directories - self.ignored_dirs = [ - ".git", - ".terraform", - "node_modules", - ".ci", - ".github", - "logs", - ] - if self.config.get("scanner.ignore_dirs"): - self.ignored_dirs.extend(self.config.get("scanner.ignore_dirs")) - - def scan_directory(self, root_dir): - """ - Scan a directory for Terraform configurations. - - Args: - root_dir: Root directory to scan - - Returns: - Dictionary with scan results - """ - results = { - "terraform_dirs": [], - "module_dirs": [], - "provider_configs": {}, - "backend_configs": {}, - "errors": [], - "warnings": [], - "scanned_dirs": 0, # Add counter for total directories scanned - } - - logger.info(f"Starting scan of {root_dir} with max depth {self.max_depth}") - self._scan_recursive(root_dir, results, 0) - logger.info( - "Scan complete. Found {} Terraform configurations in {} directories".format( - len(results["terraform_dirs"]), results["scanned_dirs"] - ) - ) - - # Add a detailed breakdown for better visibility - if len(results["terraform_dirs"]) > 0: - logger.debug( - "Terraform directories found:\n{}".format( - pformat([d["path"] for d in results["terraform_dirs"]]) - ) - ) - - return results - - def _scan_recursive(self, current_dir, results, depth): - """Recursively scan directories.""" - results["scanned_dirs"] += 1 # Increment the counter for each directory visited - - if depth > self.max_depth: - results["warnings"].append( - f"Max depth {self.max_depth} reached at {current_dir}" - ) - return - - # Skip ignored directories - if os.path.basename(current_dir) in self.ignored_dirs: - logger.debug(f"Skipping ignored directory: {current_dir}") - return - - # Add debug message to show the recursion progression - logger.debug(f"Scanning at depth {depth}: {current_dir}") - - # Check if current directory contains Terraform files - try: - dir_items = os.listdir(current_dir) - tf_files = [f for f in dir_items if f.endswith(".tf")] - - # Log what we're examining to help with troubleshooting - if tf_files: - logger.debug(f"Found {len(tf_files)} .tf files in: {current_dir}") - - if tf_files: - # This is a terraform directory - try: - dir_result = self._analyze_directory(current_dir, tf_files) - results["terraform_dirs"].append(dir_result) - logger.debug(f"Added as terraform directory: {current_dir}") - - # Check if it's a module - if dir_result.get("is_module", False): - results["module_dirs"].append(dir_result) - logger.debug(f"Identified as module: {current_dir}") - - # Collect provider configurations - for provider, version in dir_result.get("providers", {}).items(): - if provider not in results["provider_configs"]: - results["provider_configs"][provider] = [] - results["provider_configs"][provider].append( - {"version": version, "directory": current_dir} - ) - - # Collect backend configurations - if "backend" in dir_result: - backend_type = dir_result["backend"].get("type") - if backend_type not in results["backend_configs"]: - results["backend_configs"][backend_type] = [] - results["backend_configs"][backend_type].append( - {"directory": current_dir, "config": dir_result["backend"]} - ) - - except Exception as e: - error_msg = f"Error analyzing {current_dir}: {str(e)}" - logger.error(error_msg) - results["errors"].append( - {"directory": current_dir, "error": error_msg} - ) - - # Scan subdirectories - make sure we're recursing - subdirs = [ - os.path.join(current_dir, item) - for item in dir_items - if os.path.isdir(os.path.join(current_dir, item)) - ] - - if subdirs: - logger.debug( - f"Found {len(subdirs)} subdirectories to scan in {current_dir}" - ) - - for subdir in subdirs: - # Skip ignored directories one more time - # (in case basename check isn't sufficient) - if os.path.basename(subdir) in self.ignored_dirs: - continue - - self._scan_recursive(subdir, results, depth + 1) - - except (PermissionError, FileNotFoundError) as e: - error_msg = f"Access error for {current_dir}: {str(e)}" - logger.error(error_msg) - results["errors"].append({"directory": current_dir, "error": error_msg}) - - def _analyze_directory(self, directory, tf_files): - """ - Analyze a directory with Terraform files. - - Args: - directory: Path to the directory - tf_files: List of .tf files in the directory - - Returns: - Dictionary with directory analysis results - """ - result = { - "path": directory, - "tf_files": tf_files, - "version": None, - "providers": {}, - "is_module": False, - "has_variables": False, - "has_outputs": False, - "resource_count": 0, - "data_count": 0, - "module_count": 0, - "complexity": 0, - } - - # Check if it's a module by looking for variables.tf, - # outputs.tf or other indicators - result["has_variables"] = any( - f in tf_files for f in ["variables.tf", "vars.tf"] - ) - result["has_outputs"] = any(f in tf_files for f in ["outputs.tf", "output.tf"]) - - # Also check for main.tf as an additional indicator - has_main = "main.tf" in tf_files - - # Consider it a module if it has variables, outputs, or main.tf - result["is_module"] = ( - result["has_variables"] or result["has_outputs"] or has_main - ) - - # Look for version constraints, providers, and resources - for tf_file in tf_files: - file_path = os.path.join(directory, tf_file) - try: - with open(file_path, "r", encoding="utf-8", errors="replace") as f: - content = f.read() - - # Extract terraform version constraints - version_match = re.search( - r'terraform\s*{[^}]*required_version\s*=\s*["\']([^"\']+)["\']', - content, - re.DOTALL, - ) - if version_match and not result["version"]: - result["version"] = version_match.group(1) - - # Extract providers from both older style and newer style declarations - # Older style: provider "aws" { ... } - provider_matches = re.finditer( - r'provider\s+"([^"]+)"\s*{([^}]*)}', content, re.DOTALL - ) - for match in provider_matches: - provider_name = match.group(1) - provider_block = match.group(2) - version_match = re.search( - r'version\s*=\s*["\']([^"\']+)["\']', provider_block - ) - if version_match: - result["providers"][provider_name] = version_match.group(1) - else: - result["providers"][provider_name] = None - - # Newer style: required_providers block - required_block_match = re.search( - r"required_providers\s*{([^}]*)}", content, re.DOTALL - ) - if required_block_match: - required_block = required_block_match.group(1) - provider_entries = re.finditer( - r"([a-z0-9_]+)\s*=\s*{([^}]*)}", required_block, re.DOTALL - ) - for entry in provider_entries: - provider_name = entry.group(1) - provider_config = entry.group(2) - version_match = re.search( - r'version\s*=\s*["\']([^"\']+)["\']', provider_config - ) - if version_match: - result["providers"][provider_name] = version_match.group(1) - - # Count resources - result["resource_count"] += len( - re.findall(r'resource\s+"[^"]+"\s+"[^"]+"\s*{', content) - ) - - # Count data sources - result["data_count"] += len( - re.findall(r'data\s+"[^"]+"\s+"[^"]+"\s*{', content) - ) - - # Count modules - result["module_count"] += len( - re.findall(r'module\s+"[^"]+"\s*{', content) - ) - - # Look for backend configuration - backend_match = re.search( - r'backend\s+"([^"]+)"\s*{([^}]*)}', content, re.DOTALL - ) - if backend_match: - backend_type = backend_match.group(1) - backend_config = backend_match.group(2) - result["backend"] = {"type": backend_type} - - # Extract backend configuration - for config_match in re.finditer( - r'(\w+)\s*=\s*["\']?([^"\']+)["\']?', backend_config - ): - key = config_match.group(1) - value = config_match.group(2) - result["backend"][key] = value - - except Exception as e: - logger.warning(f"Error analyzing {file_path}: {str(e)}") - - # Calculate complexity score - result["complexity"] = ( - result["resource_count"] * 2 - + result["data_count"] - + result["module_count"] * 3 - ) - - return result - - def export_to_json(self, results, output_file): - """Export scan results to JSON file.""" - try: - os.makedirs(os.path.dirname(output_file), exist_ok=True) - with open(output_file, "w") as f: - json.dump(results, f, indent=2) - logger.info(f"Scan results exported to {output_file}") - return True - except Exception as e: - logger.error(f"Error exporting scan results: {str(e)}") - return False - - def export_to_csv(self, results, output_file): - """Export scan results to CSV file.""" - try: - import csv - - os.makedirs(os.path.dirname(output_file), exist_ok=True) - - with open(output_file, "w", newline="") as f: - writer = csv.writer(f) - # Write header - writer.writerow( - [ - "Directory", - "Is Module", - "Resources", - "Data Sources", - "Modules", - "Complexity", - "Version", - "Providers", - ] - ) - - # Write data - for dir_info in results["terraform_dirs"]: - providers_str = ", ".join( - [f"{k}:{v}" for k, v in dir_info.get("providers", {}).items()] - ) - writer.writerow( - [ - dir_info["path"], - dir_info.get("is_module", False), - dir_info.get("resource_count", 0), - dir_info.get("data_count", 0), - dir_info.get("module_count", 0), - dir_info.get("complexity", 0), - dir_info.get("version", ""), - providers_str, - ] - ) - - logger.info(f"Scan results exported to {output_file}") - return True - except Exception as e: - logger.error(f"Error exporting scan results: {str(e)}") - return False - - -def scan_command( - dir_path, recursive=True, max_depth=20, parallel=True, output_format="json" -): - """ - Command function for scanning directories. - - Args: - dir_path: Directory to scan - recursive: Whether to scan recursively - max_depth: Maximum recursion depth - parallel: Whether to use parallel scanning - output_format: Output format (json or csv) - - Returns: - Scan results dictionary - """ - config = ConfigManager() - scanner = TerraformScanner(config, max_depth=max_depth) - - if not os.path.isdir(dir_path): - raise ValueError(f"Not a directory: {dir_path}") - - # Print basic information before scanning - logger.info( - "Scanning directory: {} (recursive={}, max_depth={})".format( - dir_path, recursive, max_depth - ) - ) - - if not recursive: - # Just scan the single directory - tf_files = [f for f in os.listdir(dir_path) if f.endswith(".tf")] - logger.info(f"Found {len(tf_files)} .tf files in {dir_path}") - - results = { - "terraform_dirs": ( - [scanner._analyze_directory(dir_path, tf_files)] if tf_files else [] - ), - "module_dirs": [], - "scanned_dirs": 1, # Just scanned one directory - "provider_configs": {}, - "backend_configs": {}, - "errors": [], - "warnings": [], - } - return results - else: - # Use recursive scanning - override parallel for now to ensure it works properly - logger.debug("Starting recursive directory scan") - results = scanner.scan_directory(dir_path) - - # Print a more informative summary - logger.info("Summary:") - logger.info(f" Total directories scanned: {results.get('scanned_dirs', 0)}") - logger.info(f" Total Terraform directories: {len(results['terraform_dirs'])}") - logger.info(f" Module directories: {len(results['module_dirs'])}") - - # Add more details for debugging - if results.get("errors", []): - logger.warning(f" Errors encountered: {len(results['errors'])}") - for error in results["errors"][:5]: # Show first 5 errors - logger.warning(f" - {error['directory']}: {error['error']}") - if len(results["errors"]) > 5: - logger.warning(f" ... and {len(results['errors'])-5} more errors") - - if results.get("warnings", []): - logger.info(f" Warnings: {len(results['warnings'])}") - - return results diff --git a/tf_upgrade/upgrade_controller.py b/tf_upgrade/upgrade_controller.py deleted file mode 100644 index e25a1850..00000000 --- a/tf_upgrade/upgrade_controller.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -Controller for orchestrating Terraform upgrades. -""" - -import logging -import os -from datetime import datetime - -from tf_upgrade.config import ConfigManager -from tf_upgrade.reporters import ProgressTracker -from tf_upgrade.reporters.console import ConsoleReporter -from tf_upgrade.reporters.file import FileReporter -from tf_upgrade.version_detector import VersionDetector - -logger = logging.getLogger(__name__) - - -class UpgradeController: - """Controls the upgrade process through multiple versions.""" - - def __init__( - self, - directory, - config_manager=None, - reporters=None, - aws_profile=None, - account_id=None, - ): - """ - Initialize upgrade controller. - - Args: - directory: Directory to upgrade - config_manager: Optional configuration manager - reporters: List of reporters to use for progress tracking - aws_profile: AWS profile to use for operations - account_id: AWS account ID for context - """ - self.directory = os.path.abspath(directory) - self.config = config_manager or ConfigManager() - self.version_detector = VersionDetector() - # Store AWS context - self.aws_profile = aws_profile - self.account_id = account_id - - # Set up progress tracking - self.reporters = reporters or [] - if not self.reporters: - # Add default console reporter - self.reporters.append(ConsoleReporter()) - - # Add file reporter - report_dir = os.path.join(self.directory, "upgrade-logs") - os.makedirs(report_dir, exist_ok=True) - self.reporters.append( - FileReporter(os.path.join(report_dir, "upgrade-progress.json")) - ) - - def detect_current_version(self): - """ - Detect the current Terraform version in use. - - Returns: - Version detection result dictionary - """ - return self.version_detector.analyze_directory(self.directory) - - def get_upgrade_path(self, target_version=None): - """ - Determine the upgrade path from current to target version. - - Args: - target_version: Optional target version, default is 1.0 - - Returns: - List of versions to upgrade through - """ - version_info = self.detect_current_version() - current_version = version_info["final_version"] - - # If target specified, adjust the upgrade path - if target_version: - versions = ["0.12", "0.13", "0.14", "0.15", "1.0"] - try: - start_idx = versions.index(current_version) - end_idx = versions.index(target_version) - - if start_idx < end_idx: - return versions[start_idx + 1 : end_idx + 1] - else: - return [] # No upgrade needed - except ValueError: - logger.warning(f"Invalid version specified: {target_version}") - return version_info["upgrade_path"] - - return version_info["upgrade_path"] - - def get_upgrader_for_version(self, target_version): - """ - Get the appropriate upgrader class for a target version. - - Args: - target_version: Target version string - - Returns: - Upgrader class - """ - from tf_upgrade.version_upgraders.v0_13 import Terraform013Upgrader - from tf_upgrade.version_upgraders.v0_14 import Terraform014Upgrader - from tf_upgrade.version_upgraders.v0_15 import Terraform015Upgrader - from tf_upgrade.version_upgraders.v1_0 import Terraform10Upgrader - - upgraders = { - "0.13": Terraform013Upgrader, - "0.14": Terraform014Upgrader, - "0.15": Terraform015Upgrader, - "1.0": Terraform10Upgrader, - } - - return upgraders.get(target_version) - - def upgrade_to_version(self, target_version, backup=True): - """ - Perform upgrade to a specific version. - - Args: - target_version: Target version to upgrade to - backup: Whether to create backups before upgrading - - Returns: - Upgrade result dictionary - """ - # Get the appropriate upgrader class - upgrader_class = self.get_upgrader_for_version(target_version) - if not upgrader_class: - return { - "success": False, - "error": f"No upgrader available for version {target_version}", - } - - # Create and run the upgrader - upgrader = upgrader_class(self.directory, self.config, backup=backup) - result = upgrader.upgrade() - - return result - - def upgrade(self, target_version=None, step_by_step=False, backup=True): - """ - Perform the full upgrade process. - - Args: - target_version: Optional target version, default is latest - step_by_step: Whether to prompt after each version upgrade - backup: Whether to create backups before upgrading - - Returns: - Dictionary with upgrade results - """ - # Determine upgrade path - upgrade_path = self.get_upgrade_path(target_version) - - if not upgrade_path: - return { - "success": True, - "message": "No upgrade needed - already at target version", - "versions": [], - } - - # Set up progress tracking - progress_tracker = ProgressTracker( - total_steps=len(upgrade_path), - operation_name=f"Terraform upgrade to {upgrade_path[-1]}", - reporters=self.reporters, - ) - - results = { - "directory": self.directory, - "success": False, - "versions": [], - "start_time": datetime.now().isoformat(), - } - - # Perform each upgrade step - for version in upgrade_path: - progress_tracker.start_step(f"Upgrading to Terraform {version}") - - if step_by_step: - proceed = input(f"Proceed with upgrade to {version}? [Y/n]: ") - if proceed.lower() in ["n", "no"]: - progress_tracker.complete_step( - success=False, details=f"User aborted upgrade to {version}" - ) - results["versions"].append( - { - "version": version, - "success": False, - "message": "User aborted upgrade", - } - ) - results["user_aborted"] = True - break - - # Perform the upgrade - upgrade_result = self.upgrade_to_version(version, backup=backup) - - # Record results - results["versions"].append( - { - "version": version, - "success": upgrade_result.get("success", False), - "details": upgrade_result, - } - ) - - # Update progress - if upgrade_result.get("success", False): - progress_tracker.complete_step( - success=True, - details=f"Successfully upgraded to Terraform {version}", - ) - else: - progress_tracker.complete_step( - success=False, details=f"Failed to upgrade to Terraform {version}" - ) - # Stop if an upgrade fails - break - - # Complete the operation - results["end_time"] = datetime.now().isoformat() - results["success"] = all(v["success"] for v in results["versions"]) - - # Final progress update - progress = progress_tracker.complete_operation() - results["progress"] = progress - - return results diff --git a/tf_upgrade/utils/file_manager.py b/tf_upgrade/utils/file_manager.py deleted file mode 100644 index 490449c9..00000000 --- a/tf_upgrade/utils/file_manager.py +++ /dev/null @@ -1,214 +0,0 @@ -""" -Utilities for managing file operations during Terraform upgrades. -""" - -import glob -import logging -import os -import shutil -from datetime import datetime - -logger = logging.getLogger(__name__) - - -class FileManager: - """ - Manages file operations for terraform upgrades - including backups and modifications. - """ - - def __init__(self, directory, backup_dir=None): - """ - Initialize file manager for a directory. - - Args: - directory: Target directory containing terraform files - backup_dir: Optional specific backup directory, - otherwise uses timestamp - """ - self.directory = os.path.abspath(directory) - self.timestamp = datetime.now().strftime("%Y%m%d%H%M%S") - - if backup_dir: - self.backup_dir = os.path.abspath(backup_dir) - else: - self.backup_dir = os.path.join( - self.directory, f".terraform-upgrade-backup-{self.timestamp}" - ) - - # Track files that were modified - self.modified_files = set() - self.backup_files = {} - - def create_backup(self, file_patterns=None): - """ - Create backup of terraform files. - - Args: - file_patterns: Optional - list of file patterns to backup (default: *.tf) - - Returns: - Dict with information about backups created - """ - if file_patterns is None: - file_patterns = ["*.tf", "*.tfvars", ".terraform.lock.hcl"] - - # Ensure backup directory exists - os.makedirs(self.backup_dir, exist_ok=True) - - # Find files to backup - backup_files = [] - for pattern in file_patterns: - pattern_path = os.path.join(self.directory, pattern) - backup_files.extend(glob.glob(pattern_path)) - - # Create backups - for file_path in backup_files: - rel_path = os.path.relpath(file_path, self.directory) - backup_path = os.path.join(self.backup_dir, rel_path) - - # Create subdirectory if needed - os.makedirs(os.path.dirname(backup_path), exist_ok=True) - - # Copy file - try: - shutil.copy2(file_path, backup_path) - self.backup_files[file_path] = backup_path - logger.debug(f"Created backup: {file_path} → {backup_path}") - except Exception as e: - logger.error(f"Failed to backup {file_path}: {str(e)}") - - logger.info( - f"Created {len(self.backup_files)} " f"backup files in {self.backup_dir}" - ) - return { - "backup_dir": self.backup_dir, - "file_count": len(self.backup_files), - "timestamp": self.timestamp, - } - - def modify_file(self, file_path, transformer_func): - """ - Modify a file using a transformer function. - - Args: - file_path: Path to file to modify - transformer_func: Function that takes - content and returns modified content - - Returns: - True if successful, False otherwise - """ - abs_path = ( - file_path - if os.path.isabs(file_path) - else os.path.join(self.directory, file_path) - ) - - try: - # Read file - with open(abs_path, "r") as f: - content = f.read() - - # Apply transformer - new_content = transformer_func(content) - - # If content was changed, write it back - if new_content != content: - with open(abs_path, "w") as f: - f.write(new_content) - - self.modified_files.add(abs_path) - logger.debug(f"Modified file: {abs_path}") - return True - else: - logger.debug(f"No changes needed for: {abs_path}") - return False - - except Exception as e: - logger.error(f"Failed to modify {abs_path}: {str(e)}") - return False - - def restore_files(self, files=None): - """ - Restore files from backup. - - Args: - files: Optional list of specific files to restore, - otherwise all modified - - Returns: - Number of files restored - """ - if not self.backup_files: - logger.warning("No backups available for restore") - return 0 - - restore_count = 0 - - # Determine which files to restore - restore_targets = files or self.modified_files - - for file_path in restore_targets: - abs_path = ( - file_path - if os.path.isabs(file_path) - else os.path.join(self.directory, file_path) - ) - - if abs_path in self.backup_files: - backup_path = self.backup_files[abs_path] - try: - shutil.copy2(backup_path, abs_path) - logger.debug(f"Restored file: {abs_path} from {backup_path}") - restore_count += 1 - except Exception as e: - logger.error(f"Failed to restore {abs_path}: {str(e)}") - else: - logger.warning(f"No backup found for {abs_path}") - - logger.info(f"Restored {restore_count} files from backup") - return restore_count - - def get_backup_info(self): - """Get information about available backups.""" - return { - "backup_directory": self.backup_dir, - "backup_count": len(self.backup_files), - "modified_count": len(self.modified_files), - "timestamp": self.timestamp, - } - - def restore_backup(self, backup_dir): - """Restore Terraform files from a backup directory.""" - if not os.path.isdir(backup_dir): - logger.error(f"Backup directory not found: {backup_dir}") - return - - for filename in os.listdir(backup_dir): - if filename.endswith(".tf") or filename.endswith(".tfvars"): - backup_path = os.path.join(backup_dir, filename) - abs_path = os.path.join(self.directory, filename) - try: - shutil.copy2(backup_path, abs_path) - logger.info(f"Restored {filename} from {backup_dir}") - except Exception as e: - logger.error(f"Failed to restore {abs_path}: {str(e)}") - - def transform_file(self, file_path, transform_function): - """Apply a transformation function to a file.""" - abs_path = os.path.join(self.directory, file_path) - try: - with open(abs_path, "r") as f: - content = f.read() - - transformed_content = transform_function(content) - - with open(abs_path, "w") as f: - f.write(transformed_content) - - logger.info(f"Transformed file: {abs_path}") - - except Exception as e: - logger.error(f"Failed to modify {abs_path}: {str(e)}") diff --git a/tf_upgrade/utils/git.py b/tf_upgrade/utils/git.py deleted file mode 100644 index 20d902af..00000000 --- a/tf_upgrade/utils/git.py +++ /dev/null @@ -1,401 +0,0 @@ -""" -Git utilities for the Terraform upgrade process. -""" - -import logging -import os -import subprocess -import uuid -from datetime import datetime - -logger = logging.getLogger(__name__) - - -def get_git_root(directory_path): - """Get the Git repository root directory.""" - try: - git_root = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], - cwd=directory_path, - universal_newlines=True, - ).strip() - return git_root - except subprocess.CalledProcessError: - logger.debug(f"{directory_path} is not in a git repository") - return None - - -def validate_git_access(repository_path): - """ - Validate Git repository access by: - 1. Checking if it's a Git repository - 2. Testing read access (git fetch) - 3. Testing write access (create branch, commit, push, delete) - - Returns a dictionary with validation results. - """ - results = { - "is_git_repo": False, - "can_read": False, - "can_write": False, - "remote_url": None, - "current_branch": None, - "errors": [], - } - - try: - # Check if it's a Git repository - if not os.path.isdir(os.path.join(repository_path, ".git")): - results["errors"].append(f"Not a Git repository: {repository_path}") - return results - - results["is_git_repo"] = True - logger.info(f"Valid Git repository found at {repository_path}") - - # Get remote URL - process = subprocess.run( - ["git", "remote", "get-url", "origin"], - cwd=repository_path, - capture_output=True, - text=True, - ) - if process.returncode == 0: - results["remote_url"] = process.stdout.strip() - logger.info(f"Repository remote URL: {results['remote_url']}") - else: - results["errors"].append("Failed to get remote URL") - logger.warning("Could not determine remote URL") - - # Get current branch - process = subprocess.run( - ["git", "branch", "--show-current"], - cwd=repository_path, - capture_output=True, - text=True, - ) - if process.returncode == 0: - results["current_branch"] = process.stdout.strip() - logger.info(f"Current branch: {results['current_branch']}") - - # Test read access (fetch) - logger.info("Testing read access with 'git fetch'...") - process = subprocess.run( - ["git", "fetch"], cwd=repository_path, capture_output=True, text=True - ) - if process.returncode == 0: - results["can_read"] = True - logger.info("Read access verified successfully") - else: - results["errors"].append( - f"Cannot fetch from remote: {process.stderr.strip()}" - ) - logger.error(f"Read access check failed: {process.stderr.strip()}") - - # For write tests, only proceed if read access works - if results["can_read"]: - logger.info("Testing write access...") - # Generate a unique test branch name - test_branch = f"test-upgrade-access-{uuid.uuid4().hex[:8]}" - logger.info(f"Creating test branch '{test_branch}'") - - try: - # Create branch - subprocess.run( - ["git", "checkout", "-b", test_branch], - cwd=repository_path, - capture_output=True, - text=True, - check=True, - ) - - # Create test file - test_file = os.path.join(repository_path, ".test-upgrade-access") - with open(test_file, "w") as f: - f.write( - "Test file created by terraform upgrade tool at " - f"{datetime.now().isoformat()}" - ) - - # Add and commit - logger.info("Adding and committing test file...") - subprocess.run( - ["git", "add", ".test-upgrade-access"], - cwd=repository_path, - capture_output=True, - text=True, - check=True, - ) - subprocess.run( - [ - "git", - "commit", - "-m", - "Testing write access for terraform upgrade", - ], - cwd=repository_path, - capture_output=True, - text=True, - check=True, - ) - - # Try to push (this may fail with permission issues) - logger.info("Pushing test branch to remote...") - push_result = subprocess.run( - ["git", "push", "-u", "origin", test_branch], - cwd=repository_path, - capture_output=True, - text=True, - ) - results["can_write"] = push_result.returncode == 0 - - if results["can_write"]: - logger.info("Write access verified successfully") - else: - results["errors"].append( - f"Cannot push to remote: {push_result.stderr.strip()}" - ) - logger.error(f"Push failed: {push_result.stderr.strip()}") - - # Clean up: delete the test file and branch locally - logger.info("Cleaning up test artifacts...") - os.unlink(test_file) - subprocess.run( - ["git", "checkout", results["current_branch"]], - cwd=repository_path, - capture_output=True, - text=True, - ) - subprocess.run( - ["git", "branch", "-D", test_branch], - cwd=repository_path, - capture_output=True, - text=True, - ) - - # If push succeeded, clean up remote branch - if results["can_write"]: - logger.info("Removing test branch from remote...") - subprocess.run( - ["git", "push", "origin", "--delete", test_branch], - cwd=repository_path, - capture_output=True, - text=True, - ) - except Exception as e: - results["errors"].append(f"Error testing write access: {str(e)}") - logger.error(f"Write access testing error: {str(e)}") - # Ensure we return to the original branch - try: - subprocess.run( - ["git", "checkout", results["current_branch"]], - cwd=repository_path, - capture_output=True, - text=True, - ) - except Exception as ex: - logger.error(f"Failed to return to original branch: {str(ex)}") - - except Exception as e: - results["errors"].append(f"Error validating Git repository: {str(e)}") - logger.error(f"Repository validation error: {str(e)}") - - return results - - -def setup_git_user_config(): - """Set up Git user configuration for the upgrade process.""" - try: - # Check if user.name and user.email are already configured - name_result = subprocess.run( - ["git", "config", "user.name"], capture_output=True, text=True - ) - email_result = subprocess.run( - ["git", "config", "user.email"], capture_output=True, text=True - ) - - changes_made = False - - # Only set if not already configured - if name_result.returncode != 0 or not name_result.stdout.strip(): - subprocess.run( - ["git", "config", "--global", "user.name", "Terraform Upgrade Bot"], - check=True, - ) - logger.info("Set git user.name to 'Terraform Upgrade Bot'") - changes_made = True - - if email_result.returncode != 0 or not email_result.stdout.strip(): - subprocess.run( - [ - "git", - "config", - "--global", - "user.email", - "terraform-upgrade@example.gov", - ], - check=True, - ) - logger.info("Set git user.email to 'terraform-upgrade@example.gov'") - changes_made = True - - if changes_made: - logger.info("Git user configuration was updated") - else: - logger.info("Git user configuration already set, no changes made") - - return True - except Exception as e: - logger.error(f"Failed to set up Git user configuration: {str(e)}") - return False - - -def create_branch_for_upgrade(repository_path, terraform_version): - """Create a new branch for terraform upgrades.""" - try: - current_branch = subprocess.check_output( - ["git", "branch", "--show-current"], - cwd=repository_path, - universal_newlines=True, - ).strip() - - # Create a new branch with a timestamp and version - timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") - new_branch = f"terraform-upgrade-{terraform_version}-{timestamp}" - - subprocess.run( - ["git", "checkout", "-b", new_branch], cwd=repository_path, check=True - ) - - logger.info(f"Created new branch '{new_branch}' from '{current_branch}'") - return new_branch - except subprocess.CalledProcessError as e: - logger.error(f"Failed to create branch: {str(e)}") - return None - except Exception as e: - logger.error(f"Error creating branch: {str(e)}") - return None - - -def commit_changes( - repo_path, message, author="Terraform Upgrader", email="terraform@example.com" -): - """Commit changes to the repository.""" - logger.debug("Executing git commit command") - - result = subprocess.run( - ["git", "commit", "-m", message], - cwd=repo_path, - capture_output=True, - text=True, - env={"GIT_AUTHOR_NAME": author, "GIT_AUTHOR_EMAIL": email}, - ) - - if result.returncode != 0: - logger.error( - f"Git commit failed with return code {result.returncode}: " - f"{result.stderr.strip()}" - ) - return False - return True - - -def get_modified_files(repo_path): - """Get list of modified files in the repository.""" - result = subprocess.run( - ["git", "status", "--short"], cwd=repo_path, capture_output=True, text=True - ) - modified_files = [ - line.strip()[3:] for line in result.stdout.strip().splitlines() if line.strip() - ] - return modified_files - - -def add_all_changes(repo_path): - """Add all changes in the repository to the staging area.""" - result = subprocess.run( - ["git", "add", "--all", "."], cwd=repo_path, capture_output=True, text=True - ) - if result.returncode != 0: - logger.error( - f"Git add failed with return code {result.returncode}: " - f"{result.stderr.strip()}" - ) - return False - return True - - -def is_git_repository(path): - """Check if the given path is a git repository.""" - try: - result = subprocess.run( - ["git", "rev-parse", "--is-inside-work-tree"], - cwd=path, - capture_output=True, - text=True, - ) - return result.returncode == 0 and result.stdout.strip() == "true" - except Exception: # Was bare except - return False - - -def some_function(): - """Placeholder function.""" - pass - - -def clone_repository(repo_url, target_dir, branch=None): - """Clone a git repository to the target directory.""" - logger.info( - f"Cloning repository {repo_url} to {target_dir} " - f"using branch {branch or 'default'}" - ) - subprocess.run( - [ - "git", - "clone", - "--depth", - "1", - "--branch", - branch or "master", - repo_url, - target_dir, - ] - ) - - -def get_repository_contributors(repo_path): - """Get the list of contributors to a repository.""" - result = subprocess.run( - ["git", "shortlog", "-sn", "--all"], - cwd=repo_path, - capture_output=True, - text=True, - ) - return result.stdout.strip() - - -def get_repository_emails(repo_path): - """Get the list of email addresses from repository commits.""" - result = subprocess.run( - ["git", "log", "--format=%ae", "--all"], - cwd=repo_path, - capture_output=True, - text=True, - ) - emails = [ - email.strip() for email in result.stdout.strip().splitlines() if email.strip() - ] - return emails - - -def another_function(): - """Placeholder function.""" - pass - - -def format_author_warning(default_author="Terraform Upgrader"): - """Format a warning message about repository ownership.""" - logger.warning( - f"Unable to determine repository owner. " f"Using default: {default_author}" - ) - return default_author diff --git a/tf_upgrade/utils/hcl_transformer.py b/tf_upgrade/utils/hcl_transformer.py deleted file mode 100644 index b6828385..00000000 --- a/tf_upgrade/utils/hcl_transformer.py +++ /dev/null @@ -1,169 +0,0 @@ -""" -Utilities for transforming HCL syntax in Terraform configurations. -""" - -import glob -import logging -import os -import re - -logger = logging.getLogger(__name__) - - -class HCLTransformer: - """ - Transforms Terraform HCL syntax using pattern matching and replacements. - """ - - def __init__(self, rules=None): - """ - Initialize HCL transformer. - - Args: - rules: Optional list of transformation rules - """ - self.rules = rules or [] - - def add_rule(self, pattern, replacement, description=None, flags=0): - """ - Add a transformation rule. - - Args: - pattern: Regular expression pattern as string - replacement: Replacement string or function - description: Human-readable description of the rule - flags: Regex flags (re.MULTILINE, re.DOTALL, etc.) - """ - rule = { - "pattern": re.compile(pattern, flags), - "replacement": replacement, - "description": description or f"Replace {pattern} with {replacement}", - "count": 0, - } - self.rules.append(rule) - - def add_rules_from_dict(self, rules_dict): - """ - Add multiple rules from a dictionary. - - Args: - rules_dict: Dictionary of rules with patterns as keys - """ - for pattern, rule_info in rules_dict.items(): - if isinstance(rule_info, dict): - replacement = rule_info.get("replacement", "") - description = rule_info.get("description", None) - flags = rule_info.get("flags", 0) - self.add_rule(pattern, replacement, description, flags) - else: - # Simple pattern -> replacement - self.add_rule(pattern, rule_info) - - def transform(self, content): - """ - Apply all rules to transform content. - - Args: - content: HCL content as string - - Returns: - Tuple of (transformed_content, change_count) - """ - result = content - changes = 0 - - for rule in self.rules: - pattern = rule["pattern"] - replacement = rule["replacement"] - - # Count matches before replacement - matches = pattern.findall(result) - match_count = len(matches) - - if callable(replacement): - # If replacement is a function, apply it - result = pattern.sub(replacement, result) - else: - # Otherwise use string replacement - result = pattern.sub(replacement, result) - - # Update rule usage count - rule["count"] += match_count - changes += match_count - - return result, changes - - def transform_file(self, file_path): - """ - Transform a file in-place. - - Args: - file_path: Path to file to transform - - Returns: - Number of changes made - """ - try: - with open(file_path, "r") as f: - content = f.read() - - new_content, changes = self.transform(content) - - if changes > 0: - with open(file_path, "w") as f: - f.write(new_content) - - logger.debug(f"Applied {changes} transformations to {file_path}") - - return changes - - except Exception as e: - logger.error(f"Error transforming {file_path}: {str(e)}") - return 0 - - def transform_directory(self, directory, file_pattern="*.tf"): - """ - Transform all matching files in a directory. - - Args: - directory: Directory to process - file_pattern: File pattern to match - - Returns: - Dictionary with transformation results - """ - result = { - "total_files": 0, - "modified_files": 0, - "total_changes": 0, - "rule_counts": {}, - } - - # Get all matching files - pattern_path = os.path.join(directory, file_pattern) - files = glob.glob(pattern_path) - result["total_files"] = len(files) - - # Process each file - for file_path in files: - changes = self.transform_file(file_path) - result["total_changes"] += changes - - if changes > 0: - result["modified_files"] += 1 - - # Collect rule counts - for rule in self.rules: - if rule["count"] > 0: - result["rule_counts"][rule["description"]] = rule["count"] - - return result - - def get_rule_stats(self): - """Get statistics on rule usage.""" - stats = [] - - for rule in self.rules: - stats.append({"description": rule["description"], "count": rule["count"]}) - - return stats diff --git a/tf_upgrade/utils/parallel.py b/tf_upgrade/utils/parallel.py deleted file mode 100644 index d9463d4f..00000000 --- a/tf_upgrade/utils/parallel.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -Utilities for parallel processing of terraform upgrade tasks. -""" - -import concurrent.futures -import logging -from typing import Callable, Dict, List, Tuple - -logger = logging.getLogger(__name__) - - -def parallel_scan_directories( - directories: List[str], scan_function: Callable[[str], Dict], max_workers: int = 10 -) -> Dict[str, Dict]: - """ - Scan multiple directories in parallel using ThreadPoolExecutor. - - Args: - directories: List of directory paths to scan - scan_function: Function that takes a director - path and returns scan results - max_workers: Maximum number of parallel workers - - Returns: - Dictionary mapping directory paths to their scan results - """ - results = {} - successful = 0 - failed = 0 - - logger.info( - f"Starting parallel scan of {len(directories)} directories " - f"with {max_workers} workers" - ) - - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - future_to_dir = { - executor.submit(scan_function, dir_path): dir_path - for dir_path in directories - } - - for future in concurrent.futures.as_completed(future_to_dir): - dir_path = future_to_dir[future] - try: - results[dir_path] = future.result() - logger.debug(f"Successfully scanned {dir_path}") - successful += 1 - except Exception as exc: - logger.error(f"Error scanning {dir_path}: {exc}") - results[dir_path] = {"error": str(exc)} - failed += 1 - - logger.info(f"Completed parallel scan: {successful} successful, {failed} failed") - return results - - -def parallel_upgrade_directories( - directories: List[str], - upgrade_function: Callable[[str], Dict], - max_workers: int = 5, -) -> Tuple[List[str], List[str]]: - """ - Upgrade multiple directories in parallel using ThreadPoolExecutor. - - Args: - directories: List of directory paths to upgrade - upgrade_function: Function that takes a directory path - and returns upgrade results - max_workers: Maximum number of parallel workers - - Returns: - Tuple of (successful_dirs, failed_dirs) - """ - successful_dirs = [] - failed_dirs = [] - - logger.info( - f"Starting parallel upgrade of {len(directories)} directories " - f"with {max_workers} workers" - ) - - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - # Submit directory upgrades as separate tasks - futures = { - executor.submit(upgrade_function, dir_path): dir_path - for dir_path in directories - } - - # Process results as they complete to provide incremental feedback - for future in concurrent.futures.as_completed(futures): - dir_path = futures[future] - try: - # Note: not using result but keeping call to catch exceptions - future.result() - successful_dirs.append(dir_path) - logger.info(f"✓ Completed upgrade for {dir_path}") - except Exception as exc: - failed_dirs.append(dir_path) - logger.error(f"✗ Failed upgrade for {dir_path}: {exc}") - - logger.info( - f"Completed parallel upgrades: {len(successful_dirs)} successful, " - f"{len(failed_dirs)} failed" - ) - return successful_dirs, failed_dirs diff --git a/tf_upgrade/utils/provider_migration.py b/tf_upgrade/utils/provider_migration.py deleted file mode 100644 index c03eef0b..00000000 --- a/tf_upgrade/utils/provider_migration.py +++ /dev/null @@ -1,241 +0,0 @@ -""" -Utilities for migrating provider configurations to the new format in -Terraform 0.13+. -""" - -import glob -import logging -import os -import re - -logger = logging.getLogger(__name__) - - -class ProviderMigration: - """ - Handles migration of provider configurations across terraform versions. - """ - - def __init__(self): - """Initialize provider migration utilities.""" - # Provider source mappings (old name -> source string) - self.provider_sources = { - "aws": "hashicorp/aws", - "azurerm": "hashicorp/azurerm", - "google": "hashicorp/google", - "kubernetes": "hashicorp/kubernetes", - "helm": "hashicorp/helm", - "null": "hashicorp/null", - "template": "hashicorp/template", - "local": "hashicorp/local", - "random": "hashicorp/random", - "archive": "hashicorp/archive", - "external": "hashicorp/external", - "http": "hashicorp/http", - "time": "hashicorp/time", - "tls": "hashicorp/tls", - # Add more as needed - } - - def find_providers(self, directory): - """ - Find providers used in a directory. - - Args: - directory: Directory to scan - - Returns: - Dictionary of provider information - """ - result = { - "provider_blocks": [], - "provider_references": {}, - "missing_sources": [], - } - - # Find all terraform files - tf_files = glob.glob(os.path.join(directory, "*.tf")) - - for file_path in tf_files: - try: - with open(file_path, "r") as f: - content = f.read() - - # Find provider blocks - provider_matches = re.finditer( - r'provider\s+"([^"]+)"\s*{([^}]*)}', content, re.DOTALL - ) - for match in provider_matches: - provider_name = match.group(1) - provider_block = match.group(2) - - # Check if block has version but no source - has_version = bool(re.search(r"version\s*=\s*", provider_block)) - has_source = bool(re.search(r"source\s*=\s*", provider_block)) - - result["provider_blocks"].append( - { - "name": provider_name, - "file": file_path, - "has_version": has_version, - "has_source": has_source, - "needs_source": has_version and not has_source, - } - ) - - # If needs source, add to missing list - if has_version and not has_source: - result["missing_sources"].append(provider_name) - - # Find provider references in resources and data sources - provider_refs = re.finditer( - r'(?:resource|data)\s+"[^"]+"\s+"[^"]+"\s*{[^}]*' - r"provider\s*=\s*([^}\n]+)", - content, - re.DOTALL, - ) - for ref in provider_refs: - provider_ref = ref.group(1).strip() - if provider_ref not in result["provider_references"]: - result["provider_references"][provider_ref] = [] - result["provider_references"][provider_ref].append(file_path) - - except Exception as e: - logger.warning(f"Error analyzing {file_path}: {str(e)}") - - # Remove duplicates from missing sources - result["missing_sources"] = list(set(result["missing_sources"])) - - return result - - def generate_required_providers_block(self, provider_info): - """ - Generate required_providers block for terraform configuration. - - Args: - provider_info: Provider information from find_providers - - Returns: - HCL string with required_providers block - """ - if not provider_info["missing_sources"]: - return None - - lines = [" required_providers {"] - - for provider in provider_info["missing_sources"]: - source = self.provider_sources.get(provider) - if source: - lines.append(f" {provider} = {{") - lines.append(f' source = "{source}"') - lines.append(" }") - - lines.append(" }") - - return "\n".join(lines) - - def update_terraform_block(self, content, required_providers_block): - """ - Update terraform block with required_providers. - - Args: - content: HCL content as string - required_providers_block: Generated required_providers block - - Returns: - Updated content - """ - if not required_providers_block: - return content - - # Check if terraform block exists - terraform_match = re.search(r"terraform\s*{([^}]*)}", content, re.DOTALL) - - if terraform_match: - # Terraform block exists, check if it has required_providers - block_content = terraform_match.group(1) - if "required_providers" in block_content: - # Already has required_providers, don't modify - return content - - # Add required_providers to existing block - updated_block = re.sub( - r"terraform\s*{", - f"terraform {{\n{required_providers_block}", - content, - count=1, - ) - return updated_block - else: - # No terraform block exists, create one - new_block = f"terraform {{\n{required_providers_block}\n}}\n\n" - return new_block + content - - def migrate_provider_format(self, directory): - """ - Update provider configurations to 0.13+ format. - - Args: - directory: Directory to update - - Returns: - Dictionary with migration results - """ - result = {"updated_files": 0, "providers_migrated": [], "errors": []} - - # Find providers needing migration - provider_info = self.find_providers(directory) - - if not provider_info["missing_sources"]: - logger.info(f"No provider migrations needed in {directory}") - return result - - # Generate required_providers block - required_block = self.generate_required_providers_block(provider_info) - if not required_block: - return result - - # Find a suitable file to add the terraform block - # Prefer existing versions.tf or main.tf - target_files = [ - os.path.join(directory, "versions.tf"), - os.path.join(directory, "main.tf"), - os.path.join(directory, "provider.tf"), - ] - - target_file = None - for file in target_files: - if os.path.exists(file): - target_file = file - break - - # If no suitable file found, create versions.tf - if not target_file: - target_file = os.path.join(directory, "versions.tf") - with open(target_file, "w") as f: - f.write(f"terraform {{\n{required_block}\n}}\n") - - result["updated_files"] += 1 - result["providers_migrated"] = provider_info["missing_sources"] - return result - - # Update existing file - try: - with open(target_file, "r") as f: - content = f.read() - - updated_content = self.update_terraform_block(content, required_block) - - if updated_content != content: - with open(target_file, "w") as f: - f.write(updated_content) - - result["updated_files"] += 1 - result["providers_migrated"] = provider_info["missing_sources"] - - except Exception as e: - error_msg = f"Error updating provider format in {target_file}: {str(e)}" - logger.error(error_msg) - result["errors"].append(error_msg) - - return result diff --git a/tf_upgrade/utils/terraform.py b/tf_upgrade/utils/terraform.py deleted file mode 100644 index 6c4ef054..00000000 --- a/tf_upgrade/utils/terraform.py +++ /dev/null @@ -1,579 +0,0 @@ -""" -Terraform utilities for the upgrade process. -""" - -import json -import logging -import os -import re -import shutil -import subprocess -import time -from datetime import datetime - -logger = logging.getLogger(__name__) - - -def get_aws_profile_for_directory(dir_path): - """Extract AWS profile from tfvars files similar to tf-run.sh.""" - profile = os.environ.get("AWS_PROFILE") - if not profile: - # Search for profile in tfvars files - for file in [f for f in os.listdir(dir_path) if f.endswith(".tfvars")]: - with open(os.path.join(dir_path, file), "r") as f: - content = f.read() - match = re.search( - r'^\bprofile\b.*=\s*["\']?([^"\']*)["\']?', content, re.MULTILINE - ) - if match: - profile = match.group(1).strip() - break - - return profile - - -def get_aws_region_for_directory(dir_path): - """Extract AWS region from tfvars files similar to tf-run.sh.""" - region = os.environ.get("AWS_REGION") - if not region: - # Search for region in tfvars files - for file in [f for f in os.listdir(dir_path) if f.endswith(".tfvars")]: - with open(os.path.join(dir_path, file), "r") as f: - content = f.read() - match = re.search( - r'^\bregion\b.*=\s*["\']?([^"\']*)["\']?', content, re.MULTILINE - ) - if match: - region = match.group(1).strip() - break - - return region - - -# Add these functions to avoid the circular import -def find_terraform_config_files(dir_path): - """ - Find Terraform configuration files including .tf-control. - - Args: - dir_path: Directory to search in - - Returns: - Dictionary with paths to found configuration files - """ - result = {"tf_files": [], "tfvars_files": [], "tf_control": None} - - # Check if directory exists - if not os.path.isdir(dir_path): - logger.error(f"Directory not found: {dir_path}") - return result - - # Look for tf_control in this directory - tf_control = os.path.join(dir_path, ".tf-control") - if os.path.isfile(tf_control): - result["tf_control"] = tf_control - - # Also check for .tf-control.tfrc - tf_control_tfrc = os.path.join(dir_path, ".tf-control.tfrc") - if os.path.isfile(tf_control_tfrc): - result["tf_control_tfrc"] = tf_control_tfrc - - # If tf_control not found in directory, check if it's in a git repository - if not result["tf_control"]: - try: - # Get git repository root - git_root = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], - cwd=dir_path, - universal_newlines=True, - ).strip() - - # Look for tf_control in git root - tf_control = os.path.join(git_root, ".tf-control") - if os.path.isfile(tf_control): - result["tf_control"] = tf_control - - # Check for .tf-control.tfrc in git root - tf_control_tfrc = os.path.join(git_root, ".tf-control.tfrc") - if os.path.isfile(tf_control_tfrc): - result["tf_control_tfrc"] = tf_control_tfrc - - except (subprocess.CalledProcessError, FileNotFoundError): - # Not a git repository or git not installed - pass - - # If tf_control still not found, check home directory - if not result["tf_control"]: - home_tf_control = os.path.expanduser("~/.tf-control") - if os.path.isfile(home_tf_control): - result["tf_control"] = home_tf_control - - # Find .tf and .tfvars files - if os.path.isdir(dir_path): - for item in os.listdir(dir_path): - if item.endswith(".tf"): - result["tf_files"].append(os.path.join(dir_path, item)) - elif item.endswith(".tfvars"): - result["tfvars_files"].append(os.path.join(dir_path, item)) - - return result - - -def parse_tf_control_file(tf_control_path): - """ - Parse a .tf-control file to extract configuration. - - Args: - tf_control_path: Path to .tf-control file - - Returns: - Dictionary with parsed configuration - """ - config = {} - - if not tf_control_path or not os.path.isfile(tf_control_path): - logger.warning(f"No .tf-control file found at {tf_control_path}") - return config - - try: - with open(tf_control_path, "r") as f: - content = f.read() - - # Extract TFCOMMAND - tfcommand_match = re.search(r'TFCOMMAND=["\']?([^"\']+)["\']?', content) - if tfcommand_match: - config["TFCOMMAND"] = tfcommand_match.group(1) - - # Extract other variables - for match in re.finditer(r'([A-Za-z0-9_]+)=["\']?([^"\']+)["\']?', content): - key = match.group(1) - value = match.group(2) - config[key] = value - - return config - - except Exception as e: - logger.error(f"Error parsing .tf-control file: {str(e)}") - return {} - - -def validate_aws_profile(directory_path): - """ - Validate AWS profiles for a given directory by checking: - 1. AWS_PROFILE environment variable - 2. Profile in tfvars files - 3. Default profile - - Then test the profile with STS get-caller-identity. - - Returns a tuple of (success, profile_name, account_info) - """ - # First check environment variable - profile = os.environ.get("AWS_PROFILE") - - # If no profile in environment, check tfvars files - if not profile: - tfvars_files = [f for f in os.listdir(directory_path) if f.endswith(".tfvars")] - for tfvars_file in tfvars_files: - with open(os.path.join(directory_path, tfvars_file), "r") as f: - content = f.read() - # Pattern matches both quoted and unquoted profiles - match = re.search( - r'^\s*profile\s*=\s*["\']?([^"\']+)["\']?', content, re.MULTILINE - ) - if match: - profile = match.group(1).strip() - logger.info(f"Found AWS profile '{profile}' in {tfvars_file}") - break - - if not profile: - # Try to get default profile from AWS CLI config - try: - result = subprocess.run( - ["aws", "configure", "list"], - capture_output=True, - text=True, - check=False, - ) - # Simple check for default profile - if "default" in result.stdout: - profile = "default" - logger.info("Using default AWS profile") - except Exception as e: - logger.warning(f"Error checking AWS config: {str(e)}") - - # Validate the profile by calling AWS STS - if profile: - try: - env = os.environ.copy() - env["AWS_PROFILE"] = profile - result = subprocess.run( - ["aws", "sts", "get-caller-identity"], - env=env, - capture_output=True, - text=True, - check=False, - ) - if result.returncode == 0: - account_info = json.loads(result.stdout) - logger.info(f"Successfully authenticated with profile '{profile}'") - logger.info(f"Account: {account_info.get('Account')}") - logger.info(f"User: {account_info.get('Arn')}") - - # Also get region information - region_result = subprocess.run( - ["aws", "configure", "get", "region"], - env=env, - capture_output=True, - text=True, - check=False, - ) - region = ( - region_result.stdout.strip() - if region_result.returncode == 0 - else None - ) - - if region: - logger.info(f"AWS Region: {region}") - account_info["Region"] = region - - return True, profile, account_info - else: - logger.error(f"Failed to authenticate profile '{profile}'") - logger.error(result.stderr) - return False, profile, None - except Exception as e: - logger.error(f"Error validating AWS profile '{profile}': {str(e)}") - return False, profile, None - else: - logger.warning("No AWS profile found") - return False, None, None - - -def get_terraform_binary(dir_path, version=None): - """ - Determine the appropriate Terraform binary to use based on: - 1. Specified version parameter - 2. .tf-control file - 3. Default terraform command - - Returns the command to execute. - """ - terraform_paths = ["/apps/terraform/bin"] - - # If version is explicitly specified, use versioned binary - if version: - # Map version to specific terraform version - version_map = { - "0.12": "terraform_0.12.31", - "0.13": "terraform_0.13.7", - "0.14": "terraform_0.14.11", - "0.15": "terraform_0.15.5", - "1.0": "terraform_1.0.11", - "1.10": "terraform_1.10.5", - "1.3": "terraform_current", # Add explicit handling for terraform_current - "current": "terraform_current", - } - - binary = version_map.get(version, f"terraform_{version}") - logger.info(f"Using specified terraform version: {binary}") - - # Check if binary exists in PATH - ensure we're using strings not bytes - binary_path = shutil.which(str(binary)) - if binary_path: - return str(binary_path) - - # Check in terraform_paths - for path in terraform_paths: - full_path = os.path.join(str(path), str(binary)) - if os.path.exists(full_path) and os.access(full_path, os.X_OK): - logger.info(f"Found terraform binary: {full_path}") - return full_path - - # If not found, try generic terraform binary - logger.warning( - f"Specified terraform binary {binary} not found, " f"checking alternatives" - ) - - # Check standard terraform binary - terraform_path = shutil.which("terraform") - if terraform_path is not None: - logger.warning("Falling back to standard terraform binary") - return str(terraform_path) - else: - logger.error("No terraform binary found") - raise FileNotFoundError(f"Terraform binary {binary} not found") - - # Look for .tf-control file - config_files = find_terraform_config_files(dir_path) - if config_files["tf_control"]: - tf_config = parse_tf_control_file(config_files["tf_control"]) - tfcommand = tf_config.get("TFCOMMAND", "terraform") - - # If the command is not an absolute path, check if it exists in PATH - if not os.path.isabs(tfcommand): - # First check in PATH - tf_path = shutil.which(str(tfcommand)) - if tf_path is not None: - return str(tf_path) - - # Then check in terraform_paths - for path in terraform_paths: - full_path = os.path.join(str(path), str(tfcommand)) - if os.path.exists(full_path) and os.access(full_path, os.X_OK): - logger.info(f"Found terraform command: {full_path}") - return full_path - - # If not found, try standard terraform - logger.warning( - f"Terraform command {tfcommand} specified in .tf-control " f"not found" - ) - terraform_path = shutil.which("terraform") - if terraform_path is not None: - logger.warning("Falling back to standard terraform binary") - return str(terraform_path) - else: - logger.error("No terraform binary found") - raise FileNotFoundError(f"Terraform binary {tfcommand} not found") - - logger.info(f"Using terraform command from .tf-control: {tfcommand}") - return tfcommand - - # Try explicitly using terraform_current for 1.x version - terraform_current = "/apps/terraform/bin/terraform_current" - if os.path.exists(terraform_current) and os.access(terraform_current, os.X_OK): - logger.info("Using terraform_current from /apps/terraform/bin") - return terraform_current - - # Default to standard terraform command - terraform_path = shutil.which("terraform") - if terraform_path is not None: - logger.info("Using default terraform binary") - return str(terraform_path) - - # Last resort: check in /apps/terraform/bin for any version - for path in terraform_paths: - # Look specifically for terraform_current first - current_path = os.path.join(str(path), "terraform_current") - if os.path.exists(current_path) and os.access(current_path, os.X_OK): - logger.info(f"Using terraform_current: {current_path}") - return current_path - - # List all terraform binaries and take the newest available one - if os.path.isdir(path): - terraform_binaries = [ - f - for f in os.listdir(path) - if f.startswith("terraform_") - and os.access(os.path.join(path, f), os.X_OK) - ] - - if terraform_binaries: - # Sort to get the latest version - terraform_binaries.sort(reverse=True) - binary = os.path.join(str(path), str(terraform_binaries[0])) - logger.info(f"Using available terraform binary: {binary}") - return binary - - logger.error("No terraform binary found") - raise FileNotFoundError("Terraform binary not found") - - -def run_terraform_command(command, dir_path, terraform_version=None, tf_env=None): - """ - Run terraform command with appropriate version and logging in a format - compatible with tf-control. - - Args: - command: The terraform command to run (e.g., 'init', 'plan') - dir_path: Directory in which to run the command - terraform_version: Specific terraform version to use (optional) - tf_env: Additional environment variables to set (optional) - - Returns: - Tuple of (success, output, log_file) - """ - # Determine terraform binary to use - tf_binary = get_terraform_binary(dir_path, terraform_version) - - # Create logs directory if it doesn't exist - logs_dir = os.path.join(dir_path, "logs") - os.makedirs(logs_dir, exist_ok=True) - - # Generate log filename similar to tf-control.sh - timestamp = int(time.time()) - date_stamp = time.strftime("%Y%m%d") - log_file = os.path.join(logs_dir, f"{command}.{date_stamp}.{timestamp}.log") - - # Set up environment - env = os.environ.copy() - if tf_env: - env.update(tf_env) - - # Get some additional context for logging - try: - # Get current git info if available - git_repo = "unknown" - git_branch = "unknown" - try: - git_repo = subprocess.check_output( - ["git", "remote", "get-url", "origin"], - cwd=dir_path, - universal_newlines=True, - stderr=subprocess.DEVNULL, - ).strip() - git_branch = subprocess.check_output( - ["git", "branch", "--show-current"], - cwd=dir_path, - universal_newlines=True, - stderr=subprocess.DEVNULL, - ).strip() - except Exception: - pass - - # Get terraform version info - tf_version = "unknown" - try: - tf_version_output = ( - subprocess.check_output( - [tf_binary, "-v"], - universal_newlines=True, - stderr=subprocess.DEVNULL, - ) - .strip() - .split("\n")[0] - ) - tf_version = tf_version_output - except Exception: - pass - - except Exception as exc: - logger.warning(f"Error getting context info: {str(exc)}") - git_repo = "error" - git_branch = "error" - tf_version = "error" - - # Run command with logging - logger.info( - "Running: {0} {1} in {2} (log: {3})".format( - tf_binary, command, dir_path, log_file - ) - ) - - # Write log header in tf-control style - with open(log_file, "w") as log: - header_line = ( - f"# starting terraform-upgrade-tool {command} file {log_file} " - f"stamp {date_stamp}.{timestamp} time {timestamp}\n" - ) - log.write(header_line) - log.write(f"# current_directory={os.path.abspath(dir_path)}\n") - log.write(f"# git_repository={git_repo}\n") - log.write(f"# git_current_branch={git_branch}\n") - log.write(f"# terraform_version={tf_version}\n") - log.write(f"# command={tf_binary} {command}\n") - log.write(f"# log_timestamp={datetime.now().isoformat()}\n\n") - - try: - # Run the terraform command and capture output - start_time = time.time() - process = subprocess.Popen( - [tf_binary] + command.split(), - cwd=dir_path, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=env, - universal_newlines=True, - bufsize=1, - ) - - # Stream output to both log file and collect it - output_lines = [] - for line in iter(process.stdout.readline, ""): - log.write(line) - output_lines.append(line) - log.flush() - - process.stdout.close() - return_code = process.wait() - end_time = time.time() - elapsed_time = end_time - start_time - - # Write log footer in tf-control style - footer = ( - f"\n# ending terraform-upgrade-tool {command} file {log_file} " - f"stamp {date_stamp}.{timestamp} start {int(start_time)} " - f"end {int(end_time)} elapsed {int(elapsed_time)}\n" - ) - log.write(footer) - - success = return_code == 0 - output = "".join(output_lines) - - if success: - logger.info( - f"Command completed successfully in" f" {int(elapsed_time)}s" - ) - else: - logger.error(f"Command failed with exit code {return_code}") - - return success, output, log_file - except Exception as e: - error_msg = f"\n# ERROR: {str(e)}\n" - log.write(error_msg) - logger.error(f"Failed to run terraform command: {str(e)}") - return False, str(e), log_file - - -def get_account_id_from_tfvars(directory): - """ - Attempt to extract the AWS account ID from tfvars files in the directory. - - Args: - directory: Directory to search for tfvars files - - Returns: - Account ID string or None if not found - """ - try: - config_files = find_terraform_config_files(directory) - tfvars_files = config_files.get("tfvars_files", []) - - for tfvars_file in tfvars_files: - with open(tfvars_file, "r") as f: - content = f.read() - - # Look for account_id variable - account_id_match = re.search(r'account_id\s*=\s*"(\d{12})"', content) - if account_id_match: - return account_id_match.group(1) - - # Look for account_number variable - account_number_match = re.search( - r'account_number\s*=\s*"(\d{12})"', content - ) - if account_number_match: - return account_number_match.group(1) - - return None - except Exception as ex: - logger.warning(f"Error extracting account ID from tfvars: {str(ex)}") - return None - - -def get_available_terraform_versions(): - """ - Gets a list of available Terraform versions installed on the system. - - Returns: - list: List of available Terraform versions as strings - """ - try: - # This is a placeholder implementation - # In a real implementation, we would scan for installed terraform versions - # or query the HashiCorp releases API - return ["0.12.31", "0.13.7", "0.14.11", "0.15.5", "1.3.10", "1.10.5"] - except Exception as e: - logger.error(f"Error getting available Terraform versions: {e}") - return [] diff --git a/tf_upgrade/utils/terraform_runner.py b/tf_upgrade/utils/terraform_runner.py deleted file mode 100644 index 0ea37419..00000000 --- a/tf_upgrade/utils/terraform_runner.py +++ /dev/null @@ -1,257 +0,0 @@ -""" -Utilities for running Terraform commands during upgrades. -""" - -import logging -import os -import subprocess - -logger = logging.getLogger(__name__) - - -class TerraformCommandError(Exception): - """Exception raised when a Terraform command fails.""" - - def __init__(self, message, command=None, log_file=None): - super().__init__(message) - self.command = command - self.log_file = log_file - - -class TerraformRunner: - """ - Executes terraform commands with proper error handling and logging. - """ - - def __init__(self, directory, terraform_version=None, env_vars=None): - """ - Initialize terraform runner. - - Args: - directory: Directory to run commands in - terraform_version: Specific terraform version to use - env_vars: Additional environment variables for commands - """ - self.directory = os.path.abspath(directory) - self.terraform_version = terraform_version - self.env_vars = env_vars or {} - self.log_dir = os.path.join(self.directory, "logs") - - # Create logs directory if needed - os.makedirs(self.log_dir, exist_ok=True) - - # Determine terraform binary to use - self.binary = self._get_terraform_binary() - - def _get_terraform_binary(self): - """Get the appropriate terraform binary based on version.""" - from tf_upgrade.utils.terraform import get_terraform_binary - - try: - # First attempt to get binary using the imported utility function - return get_terraform_binary(self.directory, self.terraform_version) - except Exception as e: - logger.warning(f"Error getting terraform binary via utils: {str(e)}") - - # Fallback to direct subprocess call - this ensures subprocess is used - try: - result = subprocess.run( - ["which", "terraform"], capture_output=True, text=True - ) - if result.returncode == 0 and result.stdout.strip(): - terraform_path = result.stdout.strip() - logger.info( - f"Found terraform via direct subprocess call: {terraform_path}" - ) - return terraform_path - except Exception as sub_err: - logger.error(f"Subprocess fallback also failed: {str(sub_err)}") - - raise FileNotFoundError("Terraform binary not found") - - def run_command(self, command, args=None, capture_output=True, raise_on_error=True): - """ - Run a terraform command. - - Args: - command: Terraform command (e.g., "init", "plan") - args: Additional arguments as list - capture_output: Whether to capture and return output - raise_on_error: Whether to raise exception on error - - Returns: - Tuple of (success, output, log_file) - """ - from tf_upgrade.utils.terraform import run_terraform_command - - full_command = command - if args: - full_command = f"{command} {' '.join(args)}" - - logger.info( - f"Running terraform command: " f"{full_command} in {self.directory}" - ) - - # Set up environment variables - env = os.environ.copy() - env.update(self.env_vars) - - try: - success, output, log_file = run_terraform_command( - command=full_command, - dir_path=self.directory, - terraform_version=self.terraform_version, - tf_env=env, - ) - - if not success and raise_on_error: - error_msg = ( - f"Terraform command '{full_command}'" - f" failed. See {log_file} for details." - ) - logger.error(error_msg) - raise TerraformCommandError( - error_msg, command=full_command, log_file=log_file - ) - - return success, output, log_file - - except Exception as e: - logger.error(f"Error running terraform command: {str(e)}") - if raise_on_error: - raise - return False, str(e), None - - def init(self, backend=True, upgrade=False, reconfigure=False): - """ - Run terraform init. - - Args: - backend: Whether to initialize backends - upgrade: Whether to upgrade modules and plugins - reconfigure: Whether to reconfigure backend - - Returns: - Tuple of (success, output, log_file) - """ - args = [] - - if not backend: - args.append("-backend=false") - - if upgrade: - args.append("-upgrade") - - if reconfigure: - args.append("-reconfigure") - - return self.run_command("init", args) - - def validate(self): - """Run terraform validate.""" - return self.run_command("validate") - - def plan(self, out_file=None, detailed_exitcode=True): - """ - Run terraform plan. - - Args: - out_file: Optional path to save plan file - detailed_exitcode: Whether to use detailed exit code - - Returns: - Tuple of (has_changes, output, log_file) - """ - args = [] - - if out_file: - out_path = os.path.join(self.directory, out_file) - args.extend(["-out", out_path]) - - if detailed_exitcode: - args.append("-detailed-exitcode") - - try: - success, output, log_file = self.run_command( - "plan", args, raise_on_error=False - ) - - # With detailed_exitcode: 0=no changes, 1=error, 2=changes needed - if not success and "exitcode: 2" in output: - # This is actually success but with changes needed - return True, output, log_file # has_changes = True - - return success, output, log_file - except Exception as e: - logger.error(f"Error running terraform plan: {str(e)}") - return False, str(e), None - - def apply(self, auto_approve=True, plan_file=None): - """ - Run terraform apply. - - Args: - auto_approve: Whether to auto-approve changes - plan_file: Optional plan file to apply - - Returns: - Tuple of (success, output, log_file) - """ - args = [] - - if auto_approve: - args.append("-auto-approve") - - if plan_file: - plan_path = os.path.join(self.directory, plan_file) - args.append(plan_path) - - return self.run_command("apply", args) - - def fmt(self, check=False, recursive=True, diff=False): - """ - Run terraform fmt. - - Args: - check: Check if files are formatted correctly but don't modify - recursive: Format recursively - diff: Show diff of formatting changes - - Returns: - Tuple of (success, output, log_file) - """ - args = [] - - if check: - args.append("-check") - - if recursive: - args.append("-recursive") - - if diff: - args.append("-diff") - - return self.run_command("fmt", args, raise_on_error=False) - - def get_state_items(self): - """ - Get items from terraform state. - - Returns: - Dictionary of resources and data sources in state - """ - success, output, _ = self.run_command("state list") - - if not success: - return {"resources": [], "data_sources": []} - - items = output.strip().split("\n") - - resources = [item for item in items if not item.startswith("data.")] - data_sources = [item for item in items if item.startswith("data.")] - - return { - "resources": resources, - "data_sources": data_sources, - "total": len(items), - } diff --git a/tf_upgrade/utils/validator.py b/tf_upgrade/utils/validator.py deleted file mode 100644 index c4873858..00000000 --- a/tf_upgrade/utils/validator.py +++ /dev/null @@ -1,260 +0,0 @@ -""" -Validation utilities for Terraform configurations. -""" - -import logging -import os -import re - -from tf_upgrade.utils.terraform_runner import TerraformRunner - -logger = logging.getLogger(__name__) - - -class TerraformValidator: - """ - Validates Terraform configurations for errors and warnings. - """ - - def __init__(self, directory): - """ - Initialize validator. - - Args: - directory: Directory containing terraform configurations - """ - self.directory = os.path.abspath(directory) - - def validate(self, terraform_version=None): - """ - Run terraform validate and parse results. - - Args: - terraform_version: Optional specific version to use - - Returns: - Dictionary with validation results - """ - # Create terraform runner - runner = TerraformRunner(self.directory, terraform_version) - - result = { - "valid": False, - "errors": [], - "warnings": [], - "init_succeeded": False, - "version_used": terraform_version, - "log_files": {}, - } - - # First run init (required before validate) - init_success, init_output, init_log = runner.init() - result["init_succeeded"] = init_success - result["log_files"]["init"] = init_log - - if not init_success: - result["errors"].append( - { - "type": "init_failed", - "message": "Terraform init failed. " - "Cannot proceed with validation.", - "log": init_log, - } - ) - return result - - # Now run validate - validate_success, validate_output, validate_log = runner.validate() - result["log_files"]["validate"] = validate_log - - if validate_success: - result["valid"] = True - else: - # Parse errors from output - # Match lines starting with "Error:" - error_pattern = r"Error: ([^\n]+)" - warning_pattern = r"Warning: ([^\n]+)" - - errors = re.findall(error_pattern, validate_output) - warnings = re.findall(warning_pattern, validate_output) - - for error in errors: - result["errors"].append( - {"type": "validation_error", "message": error.strip()} - ) - - for warning in warnings: - result["warnings"].append( - {"type": "validation_warning", "message": warning.strip()} - ) - - return result - - def check_plan_drift(self, terraform_version=None): - """ - Check if terraform plan would make changes. - - Args: - terraform_version: Optional specific version to use - - Returns: - Dictionary with plan results - """ - # Create terraform runner - runner = TerraformRunner(self.directory, terraform_version) - - result = { - "has_changes": False, - "resource_changes": [], - "errors": [], - "init_succeeded": False, - "plan_succeeded": False, - "log_files": {}, - } - - # First run init - init_success, _, init_log = runner.init() - result["init_succeeded"] = init_success - result["log_files"]["init"] = init_log - - if not init_success: - result["errors"].append( - { - "type": "init_failed", - "message": "Terraform init failed. Cannot proceed with plan.", - } - ) - return result - - # Now run plan - plan_success, plan_output, plan_log = runner.plan(detailed_exitcode=True) - # Even if changes needed (exitcode 2), it's still success - result["plan_succeeded"] = True - result["log_files"]["plan"] = plan_log - - # Parse plan for changes - # With detailed_exitcode=True, true means changes - result["has_changes"] = plan_success - - # Parse resource changes from output - change_pattern = r"([-+~]) ([^\s]+)" - changes = re.findall(change_pattern, plan_output) - - for change in changes: - change_type = change[0] - resource = change[1] - - change_desc = {"+": "create", "-": "destroy", "~": "update"}.get( - change_type, "unknown" - ) - - result["resource_changes"].append( - {"resource": resource, "action": change_desc} - ) - - return result - - def pre_upgrade_validation(self, terraform_version=None): - """ - Run comprehensive validation before an upgrade. - - Args: - terraform_version: Optional specific version to use - - Returns: - Dictionary with validation results - """ - result = { - "validation_result": None, - "plan_result": None, - "passed": False, - "warnings": [], - "errors": [], - } - - # Run validation - validation = self.validate(terraform_version) - result["validation_result"] = validation - - # If validation failed, report error - if not validation["valid"]: - result["errors"].append("Configuration is invalid before upgrade.") - - # Check if plan shows any changes (this may fail if validate failed) - if validation["valid"]: - plan_result = self.check_plan_drift(terraform_version) - result["plan_result"] = plan_result - - if plan_result["has_changes"]: - result["warnings"].append( - f"Terraform plan shows " - f"{len(plan_result['resource_changes'])} " - f"resource changes before upgrade. " - f"This might affect testing results." - ) - - # Passed if validation succeeded - result["passed"] = validation["valid"] - - return result - - def post_upgrade_validation(self, pre_upgrade_result, terraform_version=None): - """ - Run comprehensive validation after an upgrade. - - Args: - pre_upgrade_result: Results from pre_upgrade_validation - terraform_version: Optional specific version to use - - Returns: - Dictionary with validation results - """ - result = { - "validation_result": None, - "plan_result": None, - "passed": False, - "warnings": [], - "errors": [], - "unexpected_changes": [], - } - - # Run validation - validation = self.validate(terraform_version) - result["validation_result"] = validation - - # If validation failed, report error - if not validation["valid"]: - result["errors"].append("Configuration is invalid after upgrade.") - return result - - # Run plan to check for drift - plan_result = self.check_plan_drift(terraform_version) - result["plan_result"] = plan_result - - # Compare plans before and after to identify unexpected changes - pre_plan = pre_upgrade_result.get("plan_result", {"resource_changes": []}) - - # Find new changes that weren't in the pre-upgrade plan - pre_changes = { - f"{c['action']}:{c['resource']}" - for c in pre_plan.get("resource_changes", []) - } - - post_changes = { - f"{c['action']}:{c['resource']}" - for c in plan_result.get("resource_changes", []) - } - - # Identify new changes - new_changes = post_changes - pre_changes - if new_changes: - result["warnings"].append( - f"Found {len(new_changes)} unexpected " - f"resource changes after upgrade." - ) - result["unexpected_changes"] = list(new_changes) - - # Passed if validation succeeded - result["passed"] = validation["valid"] - - return result diff --git a/tf_upgrade/utils/version_validator.py b/tf_upgrade/utils/version_validator.py deleted file mode 100644 index 31347d0b..00000000 --- a/tf_upgrade/utils/version_validator.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Version validation utilities for Terraform upgrade tool. -""" - -import functools -import logging -import re - -from packaging import version - -logger = logging.getLogger(__name__) - - -@functools.lru_cache() -def is_valid_terraform_version(version_str): - """ - Check if the given string is a valid Terraform version. - - Args: - version_str: Version string to validate - - Returns: - bool: True if valid, False otherwise - """ - # Accept versions like 0.12, 0.13.7, 1.0, 1.1.5, 1.10, 1.10.5 - pattern = r"^(?:0\.\d+(?:\.\d+)?|[1-9]\d*\.\d+(?:\.\d+)?)$" - - if not re.match(pattern, version_str): - logger.debug(f"Version string '{version_str}' does not match pattern") - return False - - # Further validate with packaging.version - try: - version.parse(version_str) - return True - except ValueError: - logger.debug(f"Failed to parse '{version_str}' as a version") - return False - - -@functools.lru_cache() -def normalize_terraform_version(version_str): - """ - Normalize a Terraform version string. - - For example: - - "0.12" -> "0.12.0" - - "1.0" -> "1.0.0" - - "1.10" -> "1.10.0" - - Args: - version_str: Version string to normalize - - Returns: - str: Normalized version string - """ - if not is_valid_terraform_version(version_str): - raise ValueError(f"Invalid Terraform version: {version_str}") - - # Add .0 if needed to make it a 3-part version - parts = version_str.split(".") - if len(parts) < 3: - version_str = f"{version_str}.0" - - return version_str - - -@functools.lru_cache() -def compare_terraform_versions(version1, version2): - """ - Compare two Terraform versions. - - Args: - version1: First version string - version2: Second version string - - Returns: - int: -1 if version1 < version2, 0 if equal, 1 if version1 > version2 - """ - v1 = normalize_terraform_version(version1) - v2 = normalize_terraform_version(version2) - - parsed_v1 = version.parse(v1) - parsed_v2 = version.parse(v2) - - if parsed_v1 < parsed_v2: - return -1 - elif parsed_v1 > parsed_v2: - return 1 - else: - return 0 diff --git a/tf_upgrade/version_detector.py b/tf_upgrade/version_detector.py deleted file mode 100644 index d7bd4f28..00000000 --- a/tf_upgrade/version_detector.py +++ /dev/null @@ -1,261 +0,0 @@ -""" -Module for detecting Terraform version constraints and usage patterns. -""" - -import logging -import os -import re - -logger = logging.getLogger(__name__) - - -class VersionDetector: - """Detects Terraform version constraints and usage patterns.""" - - def __init__(self): - """Initialize the version detector.""" - self.version_patterns = { - # Common version constraint patterns - "exact": r"=\s*(\d+\.\d+\.\d+)", - "pessimistic": r"~>\s*(\d+\.\d+)", - "greater_than": r">=\s*(\d+\.\d+\.\d+)", - "range": r">=\s*(\d+\.\d+\.\d+)\s*,\s*<\s*(\d+\.\d+\.\d+)", - # Detect features specific to different versions - "0.12_features": [ - r"for_each\s*=", - r'dynamic\s+["\w]+\s*{', - r"depends_on\s*=\s*\[", - ], - "0.13_features": [ - r"required_providers\s*{", - r'source\s*=\s*"', - ], - "0.14_features": [ - r"sensitive\s*=\s*true", - r"module\.[\w\d_]+\.providers\s*\[", - r"(.+?)\s*=>\s*(.+?)", # For expressions - ], - "0.15_features": [ - r"moved\s*{", - r"precondition\s*{", - r"postcondition\s*{", - ], - } - - def detect_version_from_constraint(self, version_constraint): - """ - Determine compatible Terraform version from a constraint string. - - Args: - version_constraint: Version constraint string (e.g., "~> 0.12.0") - - Returns: - A normalized version string (e.g., "0.12") - """ - if not version_constraint: - return None - - # Try exact match - match = re.search(r"=\s*(\d+\.\d+)", version_constraint) - if match: - return match.group(1) - - # Try pessimistic match - match = re.search(r"~>\s*(\d+\.\d+)", version_constraint) - if match: - return match.group(1) - - # Try range match - match = re.search(r">=\s*(\d+\.\d+)", version_constraint) - if match: - return match.group(1) - - # Try raw version number - match = re.search(r"(\d+\.\d+)\.", version_constraint) - if match: - return match.group(1) - - return None - - def detect_version_from_features(self, terraform_files): - """ - Detect Terraform version based on syntax features used. - - Args: - terraform_files: List of paths to Terraform files - - Returns: - Dictionary with detected feature counts and likely version - """ - feature_counts = { - "0.12": 0, - "0.13": 0, - "0.14": 0, - "0.15": 0, - } - - for file_path in terraform_files: - try: - with open(file_path, "r") as f: - content = f.read() - - # Check for each feature - for pattern in self.version_patterns["0.12_features"]: - if re.search(pattern, content): - feature_counts["0.12"] += 1 - - for pattern in self.version_patterns["0.13_features"]: - if re.search(pattern, content): - feature_counts["0.13"] += 1 - - for pattern in self.version_patterns["0.14_features"]: - if re.search(pattern, content): - feature_counts["0.14"] += 1 - - for pattern in self.version_patterns["0.15_features"]: - if re.search(pattern, content): - feature_counts["0.15"] += 1 - - except Exception as e: - logger.warning(f"Error analyzing {file_path}: {str(e)}") - - # Determine likely version - highest_version = "0.12" # Default to oldest - for version in ["0.15", "0.14", "0.13", "0.12"]: - if feature_counts[version] > 0: - highest_version = version - break - - return {"feature_counts": feature_counts, "likely_version": highest_version} - - def analyze_directory(self, directory): - """ - Analyze a directory to determine Terraform version in use. - - Args: - directory: Path to directory containing Terraform files - - Returns: - Dictionary with version analysis - """ - tf_files = [ - os.path.join(directory, f) - for f in os.listdir(directory) - if f.endswith(".tf") - ] - - # Look for explicit version constraint - version_constraint = None - for file_path in tf_files: - try: - with open(file_path, "r") as f: - content = f.read() - - # Extract terraform version constraints - version_match = re.search( - r'terraform\s*{[^}]*required_version\s*=\s*["\']' r'([^"\']+)["\']', - content, - re.DOTALL, - ) - if version_match: - version_constraint = version_match.group(1) - break - - except Exception as e: - logger.warning(f"Error reading {file_path}: {str(e)}") - - # Determine version from constraint - constrained_version = self.detect_version_from_constraint(version_constraint) - - # Also check for features - feature_detection = self.detect_version_from_features(tf_files) - - # Determine final version estimate - if constrained_version: - final_version = constrained_version - else: - final_version = feature_detection["likely_version"] - - return { - "directory": directory, - "version_constraint": version_constraint, - "constrained_version": constrained_version, - "feature_detection": feature_detection, - "final_version": final_version, - "requires_upgrade": final_version != "1.0", - "upgrade_path": self._determine_upgrade_path(final_version), - } - - def _determine_upgrade_path(self, current_version): - """ - Determine the necessary upgrade path to reach Terraform 1.0. - - Args: - current_version: Current Terraform version - - Returns: - List of versions to upgrade through - """ - versions = ["0.12", "0.13", "0.14", "0.15", "1.0"] - - try: - current_index = versions.index(current_version) - # Skip current version, return next ones in upgrade path - return versions[current_index + 1 :] - except ValueError: - # If version not found, assume full upgrade path - return versions[1:] # Skip 0.12 - except IndexError: - # If already at latest version - return [] - - def find_deprecated_syntax(self, terraform_files): - """ - Find deprecated syntax patterns in Terraform files. - - Args: - terraform_files: List of paths to Terraform files - - Returns: - List of deprecated syntax instances found - """ - deprecated_patterns = { - # "${var.name}" style - "interpolation_only": r'"\${([^{}]+)}"', - # "prefix${var.name}suffix" style - "quoted_interpolation": r'"([^"]*)\${([^{}]+)}([^"]*)"', - # Old template directives - "template_directives": r"%{if\s+.+?}|%{for\s+.+?}|%{endfor}|%{endif}", - # resource.*.id style - "splat_without_brackets": r"\*\.\w+", - # Hashbang comments - "single_line_comment_hashbang": r"(^|\n)#!/", - } - - results = [] - - for file_path in terraform_files: - try: - with open(file_path, "r") as f: - content = f.read() - line_num = 1 - - for line in content.split("\n"): - for name, pattern in deprecated_patterns.items(): - matches = re.finditer(pattern, line) - for match in matches: - results.append( - { - "file": file_path, - "line": line_num, - "type": name, - "text": match.group(0), - "context": line.strip(), - } - ) - line_num += 1 - - except Exception as e: - logger.warning(f"Error analyzing {file_path}: {str(e)}") - - return results diff --git a/tf_upgrade/version_upgraders/__init__.py b/tf_upgrade/version_upgraders/__init__.py deleted file mode 100644 index 6b2a1934..00000000 --- a/tf_upgrade/version_upgraders/__init__.py +++ /dev/null @@ -1,275 +0,0 @@ -""" -Base classes and interfaces for Terraform version upgraders. -""" - -import logging -import os -from abc import ABC, abstractmethod - -from tf_upgrade.config import ConfigManager -from tf_upgrade.utils.file_manager import FileManager -from tf_upgrade.utils.terraform import run_terraform_command -from tf_upgrade.utils.terraform_runner import TerraformRunner -from tf_upgrade.utils.validator import TerraformValidator - -logger = logging.getLogger(__name__) - - -class TerraformUpgrader(ABC): - """Base class for version-specific Terraform upgraders.""" - - def __init__(self, directory, config_manager=None, backup=True): - """ - Initialize the upgrader. - - Args: - directory: Directory containing Terraform files to upgrade - config_manager: Optional configuration manager - backup: Whether to create backups before modifications - """ - self.directory = os.path.abspath(directory) - self.config = config_manager or ConfigManager() - self.create_backup = backup - self.file_manager = None - self.terraform_runner = None - self.validator = None - - # Initialize common utilities - self._initialize_utilities() - - def _initialize_utilities(self): - """Initialize common utilities for the upgrader.""" - # Set up file manager - self.file_manager = FileManager(self.directory) - - # Set up terraform runner with the appropriate version - self.terraform_runner = TerraformRunner(self.directory, self.source_version) - - # Set up validator - self.validator = TerraformValidator(self.directory) - - @property - @abstractmethod - def source_version(self): - """Version this upgrader upgrades from.""" - pass - - @property - @abstractmethod - def target_version(self): - """Version this upgrader upgrades to.""" - pass - - @abstractmethod - def pre_upgrade_check(self): - """ - Check if the directory is ready for upgrade. - - Returns: - Dictionary with check results - """ - pass - - @abstractmethod - def apply_syntax_transformations(self): - """ - Apply syntax transformations for this version upgrade. - - Returns: - Dictionary with transformation results - """ - pass - - @abstractmethod - def run_built_in_upgrade_commands(self): - """ - Run any built-in terraform upgrade commands. - - Returns: - Dictionary with command results - """ - pass - - @abstractmethod - def post_upgrade_actions(self): - """ - Perform post-upgrade actions like formatting. - - Returns: - Dictionary with action results - """ - pass - - @abstractmethod - def validate_upgrade(self): - """ - Validate the upgraded configuration. - - Returns: - Dictionary with validation results - """ - pass - - def upgrade(self): - """ - Perform the full upgrade process. - - Returns: - Dictionary with upgrade results - """ - results = { - "directory": self.directory, - "source_version": self.source_version, - "target_version": self.target_version, - "success": False, - "steps": [], - "errors": [], - "warnings": [], - } - - # Step 1: Create backup if requested - if self.create_backup: - backup_info = self.file_manager.create_backup() - results["backup_info"] = backup_info - results["steps"].append( - { - "name": "create_backup", - "success": True, - "details": f"Created backup at {backup_info['backup_dir']}", - } - ) - - # Step 2: Pre-upgrade check - try: - check_results = self.pre_upgrade_check() - results["pre_upgrade_check"] = check_results - if not check_results.get("ready", False): - results["errors"].append("Pre-upgrade check failed") - results["steps"].append( - { - "name": "pre_upgrade_check", - "success": False, - "details": check_results.get( - "message", "Configuration not ready for upgrade" - ), - } - ) - return results - - results["steps"].append( - { - "name": "pre_upgrade_check", - "success": True, - "details": check_results.get( - "message", "Configuration ready for upgrade" - ), - } - ) - except Exception as e: - results["errors"].append(f"Pre-upgrade check error: {str(e)}") - results["steps"].append( - {"name": "pre_upgrade_check", "success": False, "details": str(e)} - ) - return results - - # Step 3: Apply syntax transformations - try: - transform_results = self.apply_syntax_transformations() - results["syntax_transformations"] = transform_results - results["steps"].append( - { - "name": "syntax_transformations", - "success": True, - "details": ( - f"Applied {transform_results.get('total_changes', 0)} " - f"syntax transformations" - ), - } - ) - except Exception as e: - results["errors"].append(f"Syntax transformation error: {str(e)}") - results["steps"].append( - {"name": "syntax_transformations", "success": False, "details": str(e)} - ) - return results - - # Step 4: Run built-in upgrade commands - try: - command_results = self.run_built_in_upgrade_commands() - results["upgrade_commands"] = command_results - results["steps"].append( - { - "name": "upgrade_commands", - "success": command_results.get("success", False), - "details": command_results.get("details", ""), - } - ) - - if not command_results.get("success", False): - results["errors"].append("Built-in upgrade commands failed") - return results - except Exception as e: - results["errors"].append(f"Upgrade command error: {str(e)}") - results["steps"].append( - {"name": "upgrade_commands", "success": False, "details": str(e)} - ) - return results - - # Step 5: Post-upgrade actions - try: - post_results = self.post_upgrade_actions() - results["post_upgrade"] = post_results - results["steps"].append( - { - "name": "post_upgrade_actions", - "success": post_results.get("success", False), - "details": post_results.get("details", ""), - } - ) - except Exception as e: - results["errors"].append(f"Post-upgrade action error: {str(e)}") - results["steps"].append( - {"name": "post_upgrade_actions", "success": False, "details": str(e)} - ) - # Continue despite errors here - - # Step 6: Validate upgraded configuration - try: - validation_results = self.validate_upgrade() - results["validation"] = validation_results - results["steps"].append( - { - "name": "validation", - "success": validation_results.get("valid", False), - "details": validation_results.get("details", ""), - } - ) - - if not validation_results.get("valid", False): - results["warnings"].append("Validation after upgrade failed") - # Dont return yet, we completed the upgrade even if validation fails - except Exception as e: - results["errors"].append(f"Validation error: {str(e)}") - results["steps"].append( - {"name": "validation", "success": False, "details": str(e)} - ) - # Continue despite errors here - - # Update final success status - results["success"] = all( - step.get("success", False) for step in results["steps"] - ) - - return results - - self.progress.start_step("Running Terraform init") - success, output, log_file = run_terraform_command( - "init", self.directory, terraform_version=self.version - ) - if not success: - self.progress.complete_step( - success=False, - details=(f"Terraform init failed. See log for details: " f"{log_file}"), - ) - return False - self.progress.complete_step(success=True) diff --git a/tf_upgrade/version_upgraders/v0_13.py b/tf_upgrade/version_upgraders/v0_13.py deleted file mode 100644 index b5f212bf..00000000 --- a/tf_upgrade/version_upgraders/v0_13.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -Upgrader for Terraform 0.13 from 0.12. -""" - -import logging -import os - -from tf_upgrade.utils.terraform import run_terraform_command -from tf_upgrade.utils.terraform_runner import TerraformRunner -from tf_upgrade.utils.validator import TerraformValidator -from tf_upgrade.version_upgraders import TerraformUpgrader - -logger = logging.getLogger(__name__) - - -class Terraform013Upgrader(TerraformUpgrader): - """Handles upgrades from Terraform 0.12 to 0.13.""" - - def __init__(self, directory, config_manager=None, backup=True): - """ - Initialize the 0.13 upgrader. - - Args: - directory: Directory containing Terraform files to upgrade - config_manager: Optional configuration manager - backup: Whether to create backups before modifications - """ - super().__init__(directory, config_manager, backup) - self.progress = None - - @property - def source_version(self): - return "0.12" - - @property - def target_version(self): - return "0.13" - - @property - def version(self): - """Get the target Terraform version for this upgrader.""" - return self.target_version - - def pre_upgrade_check(self): - """Verify configuration is compatible with 0.13 upgrade.""" - # Run validations specific to 0.12 to 0.13 upgrade - results = { - "ready": True, - "requires_provider_updates": False, - "message": "Configuration ready for upgrade to 0.13", - } - - # Check for providers without required_providers block - from tf_upgrade.utils.provider_migration import ProviderMigration - - provider_helper = ProviderMigration() - provider_info = provider_helper.find_providers(self.directory) - - if provider_info["missing_sources"]: - results["requires_provider_updates"] = True - results["providers_to_migrate"] = provider_info["missing_sources"] - - # Run validate to check for basic issues - validation = self.validator.validate("0.12") - if not validation.get("valid", False): - results["ready"] = False - results["message"] = "Configuration not valid with 0.12" - - return results - - def apply_syntax_transformations(self): - """Apply syntax changes needed for 0.13.""" - from tf_upgrade.utils.hcl_transformer import HCLTransformer - - # Create transformer with 0.13 upgrade rules - transformer = HCLTransformer() - - # Add rules for upgrading to 0.13 - transformer.add_rule( - r'(resource|data)\s+"([^"]+)"\s+"([^"]+)"\s*{', - r'\1 "\2" "\3" {', - "Normalize resource/data block spacing", - ) - - # Handle provider configurations - from tf_upgrade.utils.provider_migration import ProviderMigration - - provider_helper = ProviderMigration() - provider_result = provider_helper.migrate_provider_format(self.directory) - - # Run transformer on all .tf files - transform_result = transformer.transform_directory(self.directory) - - # Combine results - result = { - "total_changes": ( - transform_result["total_changes"] + provider_result["updated_files"] - ), - "modified_files": ( - transform_result["modified_files"] + provider_result["updated_files"] - ), - "provider_updates": provider_result["providers_migrated"], - } - - return result - - def update_required_providers(self): - """ - Update the required_providers block for Terraform 0.13 compatibility. - This creates or updates the versions.tf - file with proper provider source attributes. - """ - try: - # Check if versions.tf exists - versions_file = os.path.join(self.directory, "versions.tf") - versions_content = """terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - } - required_version = ">= 0.13" -} -""" - # Write the versions.tf file if it doesn't exist - with open(versions_file, "w") as f: - f.write(versions_content) - - logger.info(f"Created/updated versions.tf file in {self.directory}") - return True - except Exception as e: - logger.error(f"Failed to update required providers: {str(e)}") - raise - - def run_built_in_upgrade_commands(self): - """Run the terraform 0.13upgrade command.""" - # Set up upgraded terraform runner - runner = TerraformRunner(self.directory, "0.13") - logs = {} - - # First run init - init_success, init_output, init_log = runner.init() - logs["init"] = init_log - if not init_success: - return { - "success": False, - "details": f"terraform init failed. See {init_log}", - "logs": logs, - } - - # Update required providers - try: - self.update_required_providers() - except Exception as e: - return { - "success": False, - "details": f"Failed to update required providers: {str(e)}", - "logs": logs, - } - - # Run 0.13upgrade command - success, output, log_file = run_terraform_command( - "0.13upgrade -yes", self.directory, terraform_version="0.13" - ) - logs["0.13upgrade"] = log_file - - if not success: - return { - "success": False, - "details": f"0.13upgrade failed. See log for details: {log_file}", - "logs": logs, - } - - # Validate the upgraded configuration - success, details = self.validate_configuration() - if not success: - return { - "success": False, - "details": details, - "logs": logs, - } - - return { - "success": True, - "details": "Successfully upgraded to Terraform 0.13", - "logs": logs, - } - - def post_upgrade_actions(self): - """Perform post-upgrade actions like formatting.""" - # Run terraform fmt - runner = TerraformRunner(self.directory, "0.13") - success, output, log_file = runner.fmt() - - return { - "success": success, - "details": "Post-upgrade formatting completed", - "logs": {"fmt": log_file}, - } - - def validate_upgrade(self): - """Validate the upgraded configuration.""" - # Validate with the new version - validator = TerraformValidator(self.directory) - validation = validator.validate("0.13") - - # Extract meaningful information - valid = validation.get("valid", False) - errors = validation.get("errors", []) - - return { - "valid": valid, - "details": ( - "Configuration validates successfully with " "Terraform 0.13" - if valid - else "Validation errors found" - ), - "errors": errors, - "validation_result": validation, - } - - def validate_configuration(self): - """Validates the Terraform configuration.""" - try: - return True, "Validation skipped" - except Exception as e: - return False, str(e) diff --git a/tf_upgrade/version_upgraders/v0_14.py b/tf_upgrade/version_upgraders/v0_14.py deleted file mode 100644 index b4511108..00000000 --- a/tf_upgrade/version_upgraders/v0_14.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Upgrader for Terraform 0.14 from 0.13. -""" - -import logging -import os - -from tf_upgrade.utils.terraform import run_terraform_command -from tf_upgrade.utils.terraform_runner import TerraformRunner -from tf_upgrade.utils.validator import TerraformValidator -from tf_upgrade.version_upgraders import TerraformUpgrader - -logger = logging.getLogger(__name__) - - -class Terraform014Upgrader(TerraformUpgrader): - """Handles upgrades from Terraform 0.13 to 0.14.""" - - @property - def source_version(self): - return "0.13" - - @property - def target_version(self): - return "0.14" - - def pre_upgrade_check(self): - """Verify configuration is compatible with 0.14 upgrade.""" - results = { - "ready": True, - "message": ("Configuration ready for upgrade to 0.14"), - } - - # Run validate to check for basic issues - validation = self.validator.validate("0.13") - if not validation.get("valid", False): - results["ready"] = False - results["message"] = "Configuration not valid with 0.13" - - return results - - def apply_syntax_transformations(self): - """Apply syntax changes needed for 0.14.""" - from tf_upgrade.utils.hcl_transformer import HCLTransformer - - # Create transformer with 0.14 upgrade rules - transformer = HCLTransformer() - - # In 0.14, there are minimal syntax changes required, - # most changes are related to dependency lock files - - # Identify sensitive outputs - transformer.add_rule( - r'(output\s+"[^"]+"[^{]*{\s*[^\}]*value\s*=[^\}]*)' - r"(sensitive\s*=\s*(?:true|false))?([^\}]*\})", - lambda m: self._handle_sensitive_outputs(m), - "Check for outputs that should be marked sensitive", - ) - - # Run transformer on all .tf files - transform_result = transformer.transform_directory(self.directory) - - return { - "total_changes": transform_result["total_changes"], - "modified_files": transform_result["modified_files"], - } - - def _handle_sensitive_outputs(self, match): - """Helper function to identify outputs - that should be marked sensitive.""" - # This is a placeholder - in a real implementation, we'd use heuristics - # to detect outputs that contain sensitive values (passwords, etc) - output_block = match.group(1) - has_sensitive = match.group(2) is not None - remainder = match.group(3) - - # If already has sensitive attribute, leave it as is - if has_sensitive: - return output_block + match.group(2) + remainder - - # Check if output probably contains sensitive info based on name/value - if "password" in output_block.lower() or "secret" in output_block.lower(): - return output_block + "sensitive = true " + remainder - - return output_block + remainder - - def run_built_in_upgrade_commands(self): - """Generate dependency lock files for 0.14.""" - # Set up upgraded terraform runner - runner = TerraformRunner(self.directory, "0.14") - - # Run init to generate dependency lock file - init_success, init_output, init_log = runner.init() - - # Check for the generated lock file - lock_file = os.path.join(self.directory, ".terraform.lock.hcl") - lock_file_created = os.path.exists(lock_file) - - return { - "success": init_success and lock_file_created, - "details": ( - "Dependency lock file created" - if lock_file_created - else "Failed to create dependency lock file" - ), - "logs": {"init": init_log}, - "lock_file_created": lock_file_created, - } - - def post_upgrade_actions(self): - """Perform post-upgrade actions like formatting.""" - # Run terraform fmt - runner = TerraformRunner(self.directory, "0.14") - success, output, log_file = runner.fmt() - - return { - "success": success, - "details": "Post-upgrade formatting completed", - "logs": {"fmt": log_file}, - } - - def validate_upgrade(self): - """Validate the upgraded configuration.""" - # Validate with the new version - validator = TerraformValidator(self.directory) - validation = validator.validate("0.14") - - # Also do a plan to check for drift - drift = validator.check_plan_drift("0.14") - - # Extract meaningful information - valid = validation.get("valid", False) - has_changes = drift.get("has_changes", False) - - return { - "valid": valid, - "has_changes": has_changes, - "details": ( - "Configuration validates successfully with" " Terraform 0.14" - if valid - else "Validation errors found" - ), - "validation_result": validation, - "drift_result": drift, - } - - def run_upgrade(self): - """Run the full upgrade process.""" - self.progress.start_step("Running Terraform init -upgrade") - success, output, log_file = run_terraform_command( - "init -upgrade", self.directory, terraform_version=self.version - ) - if not success: - self.progress.complete_step( - success=False, details=f"Init failed: {log_file}" - ) - return False - - self.progress.complete_step(success=True) - - # Apply required syntax transformations - self.progress.start_step("Applying syntax transformations") - transform_results = self.apply_syntax_transformations() - if transform_results.get("total_changes", 0) == 0: - self.progress.complete_step(success=True, details="No changes needed") - else: - details = ( - "Total changes: " - f"{transform_results.get('total_changes')}, " - "Files modified: " - f"{transform_results.get('modified_files')}" - ) - self.progress.complete_step(success=True, details=details) - - return True - - -def interpolation_only(match): - """Convert deprecated interpolation-only syntax.""" - original = match.group(0) - inner_value = match.group(1) - new_value = inner_value - logger.debug(f"Converting interpolation-only: {original} -> {new_value}") - return new_value - - -logger.warning("Ignoring format errors during post-upgrade actions") diff --git a/tf_upgrade/version_upgraders/v0_15.py b/tf_upgrade/version_upgraders/v0_15.py deleted file mode 100644 index 6c28f9ba..00000000 --- a/tf_upgrade/version_upgraders/v0_15.py +++ /dev/null @@ -1,225 +0,0 @@ -""" -Upgrader for Terraform 0.15 from 0.14. -""" - -import logging - -from tf_upgrade.utils.terraform import run_terraform_command -from tf_upgrade.utils.terraform_runner import TerraformRunner -from tf_upgrade.utils.validator import TerraformValidator -from tf_upgrade.version_upgraders import TerraformUpgrader - -logger = logging.getLogger(__name__) - - -class Terraform015Upgrader(TerraformUpgrader): - """Handles upgrades from Terraform 0.14 to 0.15.""" - - @property - def source_version(self): - return "0.14" - - @property - def target_version(self): - return "0.15" - - def pre_upgrade_check(self): - """Verify configuration is compatible with 0.15 upgrade.""" - results = {"ready": True, "message": "Configuration ready for upgrade to 0.15"} - - # Run validate to check for basic issues - validation = self.validator.validate("0.14") - if not validation.get("valid", False): - results["ready"] = False - results["message"] = "Configuration not valid with 0.14" - - return results - - def apply_syntax_transformations(self): - """Apply syntax changes needed for 0.15.""" - from tf_upgrade.utils.hcl_transformer import HCLTransformer - - # Create transformer with 0.15 upgrade rules - transformer = HCLTransformer() - - # 0.15 removes a number of deprecated functions and behaviors - - # Replace deprecated interpolation-only expressions - transformer.add_rule( - r'"\${([^{}]+)}"', - r"\1", - "Convert deprecated interpolation-only expressions", - ) - - # Update legacy splat expressions [*] - transformer.add_rule( - r"(\w+)\.(\w+)\.\*\.(\w+)", - r"\1.\2[*].\3", - "Update legacy splat expressions to current syntax", - ) - - # Update list/map type changes - compact() without quotes - transformer.add_rule( - r'(list|map)\(\s*([^"\')\s][^)]*)\)', - lambda m: self._handle_list_map_conversion(m), - "Convert list() and map() to square bracket syntax", - ) - - # Run transformer on all .tf files - transform_result = transformer.transform_directory(self.directory) - - return { - "total_changes": transform_result["total_changes"], - "modified_files": transform_result["modified_files"], - "rule_counts": transform_result.get("rule_counts", {}), - } - - def _handle_list_map_conversion(self, match): - """Helper to convert list() and map() calls to bracket syntax.""" - type_name = match.group(1) - content = match.group(2).strip() - - if type_name == "list": - return f"[{content}]" - elif type_name == "map": - # This is a simplistic approach - a real implementation would - # need to be more robust - entries = content.split(",") - result = "{" - for i in range(0, len(entries), 2): - if i + 1 < len(entries): - result += f"{entries[i].strip()} = " f"{entries[i+1].strip()}, " - result = result.rstrip(", ") + "}" - return result - - # Fallback to original if we can't handle it - return match.group(0) - - def run_built_in_upgrade_commands(self): - """Run init with upgrade for 0.15.""" - # Set up upgraded terraform runner - runner = TerraformRunner(self.directory, "0.15") - - # Run init with -upgrade flag - init_success, init_output, init_log = runner.init(upgrade=True) - - self.progress.start_step("Running Terraform init -upgrade") - success, output, log_file = run_terraform_command( - "init -upgrade", self.directory, terraform_version=self.version - ) - if not success: - self.progress.complete_step( - success=False, - details=( - f"Terraform init -upgrade failed. " - f"See log for details: {log_file}" - ), - ) - return False - - self.progress.complete_step(success=True) - - # Apply required syntax transformations - self.progress.start_step("Applying syntax transformations") - transform_results = self.apply_syntax_transformations() - if transform_results.get("total_changes", 0) == 0: - self.progress.complete_step(success=True, details="No changes needed") - else: - details = ( - "Total changes: " - f"{transform_results.get('total_changes')}, " - "Files modified: " - f"{transform_results.get('modified_files')}" - ) - self.progress.complete_step(success=True, details=details) - - return True - - def run_upgrade(self): - """Run the full upgrade process.""" - # Run init with -upgrade flag - runner = TerraformRunner(self.directory, "0.15") - init_success, init_output, init_log = runner.init(upgrade=True) - - self.progress.start_step("Running Terraform init -upgrade") - success, output, log_file = run_terraform_command( - "init -upgrade", self.directory, terraform_version=self.version - ) - if not success: - self.progress.complete_step( - success=False, - details=( - "Terraform init -upgrade failed. " - f"See log for details: {log_file}" - ), - ) - return False - - self.progress.complete_step(success=True) - - # Apply required syntax transformations - self.progress.start_step("Applying syntax transformations") - transform_results = self.apply_syntax_transformations() - if transform_results.get("total_changes", 0) == 0: - self.progress.complete_step(success=True, details="No changes needed") - else: - details = ( - "Total changes: " - f"{transform_results.get('total_changes')}, " - "Files modified: " - f"{transform_results.get('modified_files')}" - ) - self.progress.complete_step(success=True, details=details) - - return True - - def post_upgrade_actions(self): - """Perform post-upgrade actions like formatting.""" - # Run terraform fmt - runner = TerraformRunner(self.directory, "0.15") - success, output, log_file = runner.fmt() - - return { - "success": success, - "details": "Post-upgrade formatting completed", - "logs": {"fmt": log_file}, - } - - def validate_upgrade(self): - """Validate the upgraded configuration.""" - # Validate with the new version - validator = TerraformValidator(self.directory) - validation = validator.validate("0.15") - - # Extract meaningful information - valid = validation.get("valid", False) - errors = validation.get("errors", []) - - return { - "valid": valid, - "details": ( - "Configuration validates successfully with" " Terraform 0.15" - if valid - else "Validation errors found" - ), - "errors": errors, - "validation_result": validation, - } - - -def convert_required_providers(match): - """Convert required providers syntax.""" - original = match.group(0) - new_value = match.group(1) - logger.debug(f"Converting required providers: {original} -> {new_value}") - return new_value - - -logger.info("Multiple state file formats detected - conversion needed") - - -try: - # This is a placeholder for code that might raise an exception - pass -except Exception as e: - logger.warning(f"Unable to determine state file formats: {str(e)}") diff --git a/tf_upgrade/version_upgraders/v1_0.py b/tf_upgrade/version_upgraders/v1_0.py deleted file mode 100644 index 5d5d1bc3..00000000 --- a/tf_upgrade/version_upgraders/v1_0.py +++ /dev/null @@ -1,196 +0,0 @@ -""" -Upgrader for Terraform 1.0 from 0.15. -""" - -import logging -import os -import re - -from tf_upgrade.utils.hcl_transformer import HCLTransformer -from tf_upgrade.utils.terraform_runner import TerraformRunner -from tf_upgrade.utils.validator import TerraformValidator -from tf_upgrade.version_upgraders import TerraformUpgrader - -logger = logging.getLogger(__name__) - - -class Terraform10Upgrader(TerraformUpgrader): - """Handles upgrades from Terraform 0.15 to 1.0.""" - - @property - def source_version(self): - return "0.15" - - @property - def target_version(self): - return "1.0" - - def pre_upgrade_check(self): - """Verify configuration is compatible with 1.0 upgrade.""" - results = {"ready": True, "message": "Configuration ready for upgrade to 1.0"} - - # Run validate to check for basic issues - validation = self.validator.validate("0.15") - if not validation.get("valid", False): - results["ready"] = False # Remove semicolon - results["message"] = "Configuration not valid with 0.15" - - return results - - def apply_syntax_transformations(self): - """Apply syntax changes needed for 1.0.""" - - # Create transformer with 1.0 upgrade rules - transformer = HCLTransformer() - - # In 1.0, the only major syntax change is related to - # count/for_each with modules - - # Identify module blocks using count/for_each - transformer.add_rule( - r'(module\s+"[^"]+"\s*{[^{]*?)' r"((count|for_each)\s*=\s*.*?)" r"([^{]*})", - lambda m: self._handle_module_meta_attributes(m), - "Check for modules using count/for_each", - ) - - # Check for any deprecated features that were removed in 1.0 - # Many were already deprecated in 0.15, review needed - - # Run transformer on all .tf files - transform_result = transformer.transform_directory(self.directory) - - return { - "total_changes": transform_result["total_changes"], - "modified_files": transform_result["modified_files"], - } - - def _handle_module_meta_attributes(self, match): - """Helper function to update module meta-argument syntax.""" - # In Terraform 1.0, module `count` and `for_each` meta-arguments - # must be moved inside a `module` block - - module_block = match.group(1) - meta_attribute = match.group(2) - remainder = match.group(4) - - # Move the meta-attribute inside the module block - updated_block = module_block + " " + meta_attribute + "\n" + remainder - - return updated_block - - def run_built_in_upgrade_commands(self): - """Run init and apply for 1.0.""" - # Set up upgraded terraform runner - # runner = TerraformRunner(self.directory, "1.0") - - # Run init - self.progress.start_step("Running Terraform init -upgrade") - success, output, log_file = self.terraform_runner.init(upgrade=True) - if not success: - self.progress.complete_step( - success=False, - details=f"Terraform init failed. " f"See log for details: {log_file}", - ) - return False - - self.progress.complete_step(success=True) - - # Apply required syntax transformations - self.progress.start_step("Applying syntax transformations") - transform_results = self.apply_syntax_transformations() - if transform_results.get("total_changes", 0) == 0: - self.progress.complete_step(success=True, details="No changes needed") - else: - details = ( - f"Total changes: " - f"{transform_results.get('total_changes')}, " - f"Files modified: " - f"{transform_results.get('modified_files')}" - ) - self.progress.complete_step(success=True, details=details) - - return True - - def post_upgrade_actions(self): - """Perform post-upgrade actions like formatting.""" - # Run terraform fmt - runner = TerraformRunner(self.directory, "1.0") - success, output, log_file = runner.fmt() - - # Update .tf-control file if it exists - tf_control_path = os.path.join(self.directory, ".tf-control") - if os.path.exists(tf_control_path): - try: - with open(tf_control_path, "r") as f: - content = f.read() - - # Update TFCOMMAND if it exists - updated_content = re.sub( - r'TFCOMMAND=["\']?([^"\'\n]+)["\']?', - r'TFCOMMAND="terraform-1.0"', - content, - ) - - # If TFCOMMAND didn't exist, add it - if "TFCOMMAND=" not in content: - updated_content = f'TFCOMMAND="terraform-1.0"\n{content}' - - with open(tf_control_path, "w") as f: - f.write(updated_content) - - except Exception as e: - logger.warning(f"Failed to update .tf-control file: {str(e)}") - - return { - "success": success, - "details": "Post-upgrade formatting and configuration completed", - "logs": {"fmt": log_file}, - } - - def validate_upgrade(self): - """Validate the upgraded configuration.""" - # Validate with the new version - validator = TerraformValidator(self.directory) - validation = validator.validate("1.0") - - # Also do a plan to check for drift - drift = validator.check_plan_drift("1.0") - - # Extract meaningful information - valid = validation.get("valid", False) - has_changes = drift.get("has_changes", False) - - return { - "valid": valid, - "has_changes": has_changes, - "details": ( - "Configuration validates successfully with" " Terraform 1.0" - if valid - else "Validation errors found" - ), - "validation_result": validation, - "drift_result": drift, - } - - def convert_attribute_sensitive(match): - """Convert attribute sensitive to sensitive = true.""" - original = match.group(0) - # attribute_name = match.group(1) - new_value = "sensitive = true" - logger.debug(f"Converting attribute sensitive: {original} -> {new_value}") - return new_value - - def convert_terraform_remote_state(match): - """Convert terraform_remote_state data source to terraform block.""" - original = match.group(0) - backend_type = match.group(1) - config_string = match.group(2) - new_value = ( - f"terraform {{\n" - f' backend "{backend_type}" {{\n' - f" {config_string}\n" - f" }}\n" - f"}}" - ) - logger.debug(f"Converting terraform_remote_state: {original} -> {new_value}") - return new_value diff --git a/tf_upgrade/workspace_manager.py b/tf_upgrade/workspace_manager.py deleted file mode 100644 index 0629deb4..00000000 --- a/tf_upgrade/workspace_manager.py +++ /dev/null @@ -1,231 +0,0 @@ -""" -Workspace manager for Terraform upgrade tool. -Provides functions for cloning repositories, creating workspaces, -and managing the upgrade workflow in a dedicated workspace. -""" - -import logging -import os -import shutil -import subprocess -import tempfile - -# Configure logger -logger = logging.getLogger(__name__) - - -class WorkspaceManager: - """ - Manages workspaces for Terraform upgrades. - - A workspace is a temporary directory where we clone the target repository, - perform the upgrade process, and then create commits and PRs. - """ - - def __init__(self, base_dir=None): - """ - Initialize the workspace manager. - - Args: - base_dir: Base directory for workspaces. If None, will use system temp dir. - """ - self.base_dir = base_dir or tempfile.gettempdir() - self.workspace_dir = None - self.original_dir = None - - def create_workspace(self, name="terraform-upgrade"): - """ - Create a new workspace directory. - - Args: - name: Name prefix for the workspace directory - - Returns: - Path to the created workspace directory - """ - # Create a unique workspace directory - workspace_path = os.path.join(self.base_dir, f"{name}-{os.urandom(4).hex()}") - os.makedirs(workspace_path, exist_ok=True) - self.workspace_dir = workspace_path - logger.info(f"Created workspace at: {workspace_path}") - return workspace_path - - def clone_repository(self, repo_url, target_dir=None, branch="main"): - """ - Clone a git repository into the workspace. - - Args: - repo_url: URL of the git repository to clone - target_dir: Target directory name (defaults to last part of repo URL) - branch: Branch to checkout - - Returns: - Path to the cloned repository - """ - if not self.workspace_dir: - self.create_workspace() - - # Determine target directory name if not provided - if not target_dir: - target_dir = repo_url.rstrip("/").split("/")[-1] - if target_dir.endswith(".git"): - target_dir = target_dir[:-4] - - repo_path = os.path.join(self.workspace_dir, target_dir) - - try: - # Clone the repository - cmd = ["git", "clone", repo_url, repo_path] - logger.info(f"Cloning repository: {repo_url} to {repo_path}") - subprocess.run(cmd, check=True) - - # Checkout the specified branch - if branch: - subprocess.run(["git", "checkout", branch], cwd=repo_path, check=True) - - return repo_path - except subprocess.CalledProcessError as e: - logger.error(f"Error cloning repository: {e}") - return None - - def enter_workspace(self, directory=None): - """ - Change to the workspace directory or a subdirectory of the workspace. - - Args: - directory: Subdirectory within the workspace to enter - - Returns: - True if successful, False otherwise - """ - if not self.workspace_dir: - logger.error("No workspace directory created yet") - return False - - target_dir = self.workspace_dir - if directory: - target_dir = os.path.join(self.workspace_dir, directory) - if not os.path.isdir(target_dir): - logger.error(f"Directory does not exist: {target_dir}") - return False - - # Remember the original directory - self.original_dir = os.getcwd() - os.chdir(target_dir) - logger.info(f"Changed directory to: {target_dir}") - return True - - def exit_workspace(self): - """ - Return to the original directory before entering the workspace. - - Returns: - True if successful, False otherwise - """ - if not self.original_dir: - logger.warning("No original directory saved, cannot exit workspace") - return False - - os.chdir(self.original_dir) - logger.info(f"Returned to directory: {self.original_dir}") - return True - - def commit_changes(self, repo_path, message="Terraform upgrade changes"): - """ - Commit changes in the workspace repository. - - Args: - repo_path: Path to the repository in the workspace - message: Commit message - - Returns: - True if successful, False otherwise - """ - try: - # Check if there are changes to commit - result = subprocess.run( - ["git", "status", "--porcelain"], - cwd=repo_path, - check=True, - capture_output=True, - text=True, - ) - - if not result.stdout.strip(): - logger.info("No changes to commit") - return True - - # Stage all changes - subprocess.run(["git", "add", "."], cwd=repo_path, check=True) - - # Commit changes - subprocess.run(["git", "commit", "-m", message], cwd=repo_path, check=True) - logger.info(f"Changes committed with message: {message}") - return True - except subprocess.CalledProcessError as e: - logger.error(f"Error committing changes: {e}") - return False - - def create_branch(self, repo_path, branch_name): - """ - Create and checkout a new branch in the repository. - - Args: - repo_path: Path to the repository in the workspace - branch_name: Name of the new branch - - Returns: - True if successful, False otherwise - """ - try: - subprocess.run( - ["git", "checkout", "-b", branch_name], cwd=repo_path, check=True - ) - logger.info(f"Created and checked out branch: {branch_name}") - return True - except subprocess.CalledProcessError as e: - logger.error(f"Error creating branch: {e}") - return False - - def push_changes(self, repo_path, branch_name, remote="origin"): - """ - Push changes to the remote repository. - - Args: - repo_path: Path to the repository in the workspace - branch_name: Name of the branch to push - remote: Remote name (default: origin) - - Returns: - True if successful, False otherwise - """ - try: - subprocess.run( - ["git", "push", remote, branch_name], cwd=repo_path, check=True - ) - logger.info(f"Pushed changes to {remote}/{branch_name}") - return True - except subprocess.CalledProcessError as e: - logger.error(f"Error pushing changes: {e}") - return False - - def cleanup_workspace(self): - """ - Clean up the workspace directory. - - Returns: - True if successful, False otherwise - """ - if not self.workspace_dir or not os.path.exists(self.workspace_dir): - logger.warning("No workspace to clean up") - return False - - # Return to original directory if needed - if self.original_dir: - self.exit_workspace() - - # Remove the workspace directory - shutil.rmtree(self.workspace_dir) - logger.info(f"Workspace cleaned up: {self.workspace_dir}") - self.workspace_dir = None - return True diff --git a/tf_upgrade_diagnostics.py b/tf_upgrade_diagnostics.py deleted file mode 100644 index 61698b00..00000000 --- a/tf_upgrade_diagnostics.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 -""" -Diagnostic tools for troubleshooting Terraform Upgrade Tool issues. -""" - -import argparse -import logging -import os -import sys -import time - -# Configure basic logging -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", -) - -logger = logging.getLogger("tf-upgrade-diagnostics") - - -def scan_directory_tree(root_dir, max_depth=20, ignored_dirs=None): - """ - Scan a directory tree and report all Terraform files found. - - This is a simplified version of the scanner that only focuses on - finding Terraform files to help diagnose scanning issues. - """ - if ignored_dirs is None: - ignored_dirs = [".git", ".terraform", "node_modules", "logs", ".github"] - - tf_dirs = [] - scanned_dirs = 0 - start_time = time.time() - - def _scan_dir(current_dir, depth=0): - nonlocal scanned_dirs - scanned_dirs += 1 - - if depth > max_depth: - logger.warning(f"Max depth {max_depth} reached at {current_dir}") - return - - if os.path.basename(current_dir) in ignored_dirs: - logger.debug(f"Skipping ignored directory: {current_dir}") - return - - logger.debug(f"Scanning at depth {depth}: {current_dir}") - - try: - dir_items = os.listdir(current_dir) - - # Find Terraform files - tf_files = [f for f in dir_items if f.endswith(".tf")] - if tf_files: - logger.info(f"Found {len(tf_files)} Terraform files in: {current_dir}") - tf_dirs.append({"path": current_dir, "files": tf_files, "depth": depth}) - - # Check subdirectories - for item in dir_items: - item_path = os.path.join(current_dir, item) - if os.path.isdir(item_path): - _scan_dir(item_path, depth + 1) - - except (PermissionError, FileNotFoundError) as e: - logger.error(f"Error accessing {current_dir}: {str(e)}") - - # Start the recursive scan - logger.info(f"Starting diagnostic scan of {root_dir}") - _scan_dir(root_dir) - - elapsed_time = time.time() - start_time - - # Report results - logger.info(f"Scan completed in {elapsed_time:.2f} seconds") - logger.info(f"Total directories scanned: {scanned_dirs}") - logger.info(f"Found {len(tf_dirs)} directories with Terraform files") - - # Show directory structure with indentation to visualize hierarchy - print("\nDirectory structure with Terraform files:") - for i, tf_dir in enumerate(sorted(tf_dirs, key=lambda d: d["path"]), 1): - indent = " " * tf_dir["depth"] - rel_path = os.path.relpath(tf_dir["path"], root_dir) - print(f"{indent}├── {rel_path} ({len(tf_dir['files'])} .tf files)") - - return { - "terraform_dirs": tf_dirs, - "total_dirs_scanned": scanned_dirs, - "elapsed_time": elapsed_time, - } - - -def check_cli_command(): - """Check if CLI command is working correctly.""" - try: - import tf_upgrade.cli - - print("\nCLI Module Information:") - print(f"Module path: {tf_upgrade.cli.__file__}") - - # Check scan command options - scan_command = next( - (cmd for cmd in tf_upgrade.cli.cli.commands.values() if cmd.name == "scan"), - None, - ) - if scan_command: - print("\nScan Command Options:") - for param in scan_command.params: - if param.name == "recursive": - print(f" --recursive option default: {param.default}") - print(f" --recursive option is_flag: {param.is_flag}") - elif param.name == "max_depth": - print(f" --max-depth option default: {param.default}") - else: - print("Scan command not found in CLI") - except ImportError: - print("Could not import tf_upgrade.cli module") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Terraform Upgrade Tool Diagnostics") - parser.add_argument("directory", help="Directory to scan") - parser.add_argument( - "--max-depth", type=int, default=20, help="Maximum directory depth" - ) - parser.add_argument( - "--check-cli", action="store_true", help="Check CLI command settings" - ) - parser.add_argument( - "--verbose", "-v", action="store_true", help="Enable verbose output" - ) - - args = parser.parse_args() - - if args.verbose: - logger.setLevel(logging.DEBUG) - - if not os.path.isdir(args.directory): - print(f"Error: {args.directory} is not a directory") - sys.exit(1) - - if args.check_cli: - check_cli_command() - - results = scan_directory_tree(args.directory, args.max_depth) - - print("\nScan Summary:") - print(f" Total directories scanned: {results['total_dirs_scanned']}") - print(f" Terraform directories found: {len(results['terraform_dirs'])}") - print(f" Scan completed in: {results['elapsed_time']:.2f} seconds")