diff --git a/Makefile b/Makefile index 3bf8830d..5848d16b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install develop clean test lint format scan dry-run upgrade verify-tools all verify-format install-dev +.PHONY: help install develop clean test lint format scan dry-run upgrade verify-tools all verify-format # Variables DIR ?= . @@ -68,12 +68,9 @@ test-validation: ## Run validation tests only export PYTHONPATH := $(PYTHONPATH):$(shell pwd) lint: ## Run linting checks - flake8 tf_upgrade - isort --check-only tf_upgrade - -format: ## Format code with black and isort - black tf_upgrade - isort tf_upgrade + black . + isort . + flake8 . verify-tools: ## Verify required tools are installed @echo "Verifying required tools..." @@ -176,13 +173,6 @@ verify-error-handling: ## Verify error handling @python -m tf_upgrade.cli upgrade $(TEST_FIXTURES_DIR)/0.12/error-test --target-version 0.13 || echo "✅ Expected error occurred" | tee -a $(VERIFICATION_LOG) @rm -rf $(TEST_FIXTURES_DIR)/0.12/error-test -verify-interactive: ## Verify interactive mode (requires manual input) - @echo "Verifying interactive mode (requires manual input)..." | tee -a $(VERIFICATION_LOG) - @cp -r $(TEST_FIXTURES_DIR)/0.12/simple $(TEST_FIXTURES_DIR)/0.12/interactive-test - @echo "Run the interactive upgrade with 'y' responses when prompted" - @python -m tf_upgrade.cli upgrade $(TEST_FIXTURES_DIR)/0.12/interactive-test --interactive | tee -a $(VERIFICATION_LOG) - @rm -rf $(TEST_FIXTURES_DIR)/0.12/interactive-test - verify-all: setup-fixtures verify-env verify-scan verify-dry-run verify-backup verify-upgrade verify-error-handling ## Run all verification tests (except interactive) @echo "======================================================" @echo "🎉 All verification tests completed successfully! 🎉" @@ -204,3 +194,8 @@ install-dev: ## Install development dependencies setup-env: @echo "Setting up test environment..." @./setup_test_env.sh + +report: ## Generate comprehensive project report + @echo "Generating project report..." + @python -m tf_upgrade.reporting.generate_report + @echo "✅ Report generated in reports/" diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md new file mode 100644 index 00000000..96e466aa --- /dev/null +++ b/PROJECT_PLAN.md @@ -0,0 +1,122 @@ +# Terraform Upgrade Tool - Project Plan + +## 1. Core Functionality Enhancements + +### 1.1 Version Support +- [x] Support for upgrading from Terraform 0.12 to 0.13 +- [x] Support for upgrading from 0.13 to 0.14 +- [x] Support for upgrading from 0.14 to 0.15 +- [x] Support for upgrading from 0.15 to 1.10 +- [ ] Support for upgrading to future versions (1.11+) + +### 1.2 Upgrade Process Improvements +- [x] Safe backup mechanism +- [ ] Improved error handling during upgrade steps +- [ ] Rollback mechanism for failed upgrades +- [ ] Support for ignoring specific files or directories +- [ ] Custom upgrade paths configuration + +### 1.3 Testing Infrastructure +- [x] Unit tests for core components +- [x] Integration tests for workflows +- [ ] Add more comprehensive test fixtures +- [ ] Test with real-world complex Terraform projects +- [ ] Test coverage improvements (aim for >85%) + +## 2. User Experience + +### 2.1 Documentation +- [ ] Comprehensive README with examples +- [ ] User guide with detailed usage instructions +- [ ] Best practices for different upgrade scenarios +- [ ] Troubleshooting guide +- [ ] API documentation for developers + +### 2.2 CLI Improvements +- [ ] Interactive mode with guidance +- [ ] Configuration profiles for different upgrade strategies +- [ ] Progress indication for long-running operations +- [ ] Better error messages and suggestions +- [ ] Color-coded output + +### 2.3 Reporting +- [ ] HTML report generation +- [ ] Summary statistics for changes made +- [ ] Risk assessment reports +- [ ] Export capabilities (JSON, CSV) + +## 3. DevOps Integration + +### 3.1 CI/CD +- [ ] GitHub Actions workflow +- [ ] Automated testing on multiple Python versions +- [ ] Release automation +- [ ] Version tagging + +### 3.2 Development Infrastructure +- [x] Makefile targets for common operations +- [x] Environment setup script +- [ ] Pre-commit hooks +- [ ] Code quality gates +- [ ] Dependency management improvements + +### 3.3 Deployment +- [ ] Package for PyPI +- [ ] Docker container for isolated execution +- [ ] Installation script for enterprise environments + +## 4. Advanced Features + +### 4.1 Code Analysis +- [ ] Complexity analysis for Terraform code +- [ ] Dependency graph visualization +- [ ] Unused resource detection +- [ ] Security scanning integration + +### 4.2 Integrations +- [ ] Support for Terraform Cloud +- [ ] Integration with git workflows +- [ ] Support for module registries +- [ ] AWS/Azure/GCP specific optimizations + +### 4.3 Extensibility +- [ ] Plugin system for custom transformations +- [ ] Hook points for organization-specific rules +- [ ] Custom linting rules + +## 5. Project Management + +### 5.1 Release Planning +- [ ] v0.1.0: Basic upgrade path from 0.12 to 1.10 +- [ ] v0.2.0: Improved reporting and user experience +- [ ] v0.3.0: Advanced analysis features +- [ ] v1.0.0: Production-ready with all core features + +### 5.2 Feedback Loop +- [ ] User testing protocol +- [ ] Feedback collection mechanism +- [ ] Prioritization framework for feature requests + +## Timeline + +| Phase | Duration | Focus Areas | Major Deliverables | +|-------|----------|-------------|-------------------| +| 1 | 2 weeks | Core Functionality | Working upgrade path, tests | +| 2 | 2 weeks | User Experience | Documentation, improved CLI | +| 3 | 2 weeks | DevOps | CI/CD, deployment | +| 4 | 4 weeks | Advanced Features | Analysis, integrations | +| 5 | Ongoing | Maintenance | Bug fixes, minor enhancements | + +## Resource Requirements + +- Development: 1-2 Python developers +- Testing: Access to various Terraform projects at different version levels +- Infrastructure: Test environment with multiple Terraform versions installed +- Documentation: Technical writer (part-time) + +## Success Metrics + +- 100% successful upgrades on test fixtures +- >85% test coverage +- Positive user feedback +- Adoption rate within organization diff --git a/checklist.md b/checklist.md index 8b36b309..b9dfdd5f 100644 --- a/checklist.md +++ b/checklist.md @@ -59,22 +59,96 @@ - [x] Create comprehensive logging for all operations - [x] Add rollback/restore capabilities -## Phase 7: Documentation and Delivery ✅ - -- [x] Complete CLI documentation -- [x] Write detailed usage guides -- [x] Create example workflows -- [x] Add troubleshooting section -- [x] Complete API documentation for developers -- [x] Create user-friendly error messages -- [x] Implement final packaging for distribution - -## Phase 8: Pre-Release Verification +## Phase 7: Documentation and Testing ⏳ + +- [ ] Complete CLI documentation +- [ ] Write detailed usage guides +- [ ] Create example workflows +- [ ] Add troubleshooting section +- [ ] Complete API documentation for developers +- [x] Implement basic unit tests for core components +- [x] Implement basic integration tests for workflows +- [x] Create simple test fixtures +- [ ] Create complex test fixtures +- [ ] Test with real-world Census Bureau configurations +- [ ] Achieve >85% test coverage + +## Phase 8: GitHub Project Integration ⬜ + +- [ ] Implement GitHub API client +- [ ] Add authentication support for GitHub API +- [ ] Create ticket status management functions +- [ ] Implement commands for updating ticket status +- [ ] Add functionality to create/update tickets +- [ ] Implement comment and report attachment +- [ ] Create GitHub-compatible report generators +- [ ] Add PR creation and management features +- [ ] Implement CLI commands for GitHub interaction +- [ ] Create Makefile targets for GitHub operations +- [ ] Document GitHub integration features +- [ ] Test GitHub workflow automation + +## Phase 9: Test Case Collection and Management ⬜ + +- [ ] Create test case extraction utility +- [ ] Implement pattern detection for test cases +- [ ] Develop test fixture minimization tool +- [ ] Create directory structure for real-world test fixtures +- [ ] Implement metadata tracking for test fixtures +- [ ] Add automated test case validation +- [ ] Create test coverage tracking for patterns +- [ ] Implement regression test framework +- [ ] Document test case contribution workflow +- [ ] Create tools to clean and normalize test fixtures +- [ ] Build pattern library from collected test cases +- [ ] Add Census-specific pattern documentation + +## Phase 10: Pre-Release Verification ⬜ - [ ] Verify all CLI commands function properly -- [ ] Check backup and restore functionality +- [ ] Check backup and restore functionality on varied configurations - [ ] Test on representative configuration samples - [ ] Validate dry-run output matches actual changes - [ ] Confirm interactive mode functions correctly - [ ] Verify log files contain appropriate detail - [ ] Test on multiple target environments +- [ ] Validate AWS partition handling (commercial vs GovCloud) +- [ ] Test cross-account resource handling +- [ ] Conduct performance testing on large repositories +- [ ] Verify GitHub integration features +- [ ] Test end-to-end workflow with GitHub project + +## Phase 11: Advanced Features (Future Work) ⬜ + +- [ ] Improve error handling for complex edge cases +- [ ] Enhance rollback mechanism for failed upgrades +- [ ] Add support for ignoring specific files or directories +- [ ] Implement custom upgrade paths configuration +- [ ] Add support for upgrading to future versions (1.11+) +- [ ] Create HTML report generation +- [ ] Implement summary statistics for changes made +- [ ] Add export capabilities (JSON, CSV) +- [ ] Create pull request generation for automated review +- [ ] Add dashboard for upgrade progress monitoring +- [ ] Implement automated pattern recommendations +- [ ] Create upgrade analytics for project-wide insights + +## Current Status Summary + +- **Core functionality**: ✅ COMPLETE +- **Validation and testing**: ⏳ IN PROGRESS (basic tests complete, more needed) +- **Documentation**: ⏳ IN PROGRESS (needs significant work) +- **GitHub integration**: ⬜ NOT STARTED +- **Test case management**: ⬜ NOT STARTED +- **Pre-release verification**: ⬜ NOT STARTED +- **Advanced features**: ⬜ FUTURE WORK + +## Next Priorities + +1. Complete documentation (especially CLI usage guide and examples) +2. Begin GitHub project integration implementation +3. Design test case collection framework +4. Improve test coverage with more complex fixtures +5. Begin pre-release verification process +6. Test with real Census Bureau configurations +7. Fix any issues identified during testing diff --git a/debug_scanner.py b/debug_scanner.py new file mode 100644 index 00000000..a5f1154f --- /dev/null +++ b/debug_scanner.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +Diagnostic script to debug directory scanning issues. +""" + +import logging +import os +import sys + +# Configure basic logging +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) + +logger = logging.getLogger("debug-scanner") + + +def list_terraform_files(root_dir, max_depth=20): + """ + Recursively list all Terraform files under a directory. + """ + tf_dirs = [] + total_dirs_scanned = 0 + + def _walk(current_dir, depth=0): + nonlocal total_dirs_scanned + total_dirs_scanned += 1 + + if depth > max_depth: + logger.warning(f"Max depth reached at: {current_dir}") + return + + logger.debug(f"Scanning directory at depth {depth}: {current_dir}") + + try: + dir_items = os.listdir(current_dir) + + # Check for .tf files + tf_files = [f for f in dir_items if f.endswith(".tf")] + if tf_files: + logger.info(f"Found {len(tf_files)} Terraform files in: {current_dir}") + tf_dirs.append({"path": current_dir, "files": tf_files}) + + # Recursively scan subdirectories + for item in dir_items: + item_path = os.path.join(current_dir, item) + if os.path.isdir(item_path) and item not in [ + ".git", + ".terraform", + "node_modules", + ]: + _walk(item_path, depth + 1) + + except (PermissionError, FileNotFoundError) as e: + logger.error(f"Error accessing {current_dir}: {str(e)}") + + # Start the recursive scan + _walk(root_dir) + + return {"terraform_dirs": tf_dirs, "total_dirs_scanned": total_dirs_scanned} + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python debug_scanner.py ") + sys.exit(1) + + dir_path = sys.argv[1] + if not os.path.isdir(dir_path): + print(f"Error: {dir_path} is not a directory") + sys.exit(1) + + print(f"Starting diagnostic scan of {dir_path}") + results = list_terraform_files(dir_path) + + print("\n----- SCAN RESULTS -----") + print(f"Total directories scanned: {results['total_dirs_scanned']}") + print(f"Found {len(results['terraform_dirs'])} directories with Terraform files") + + for i, tf_dir in enumerate(results["terraform_dirs"], 1): + print(f"\n{i}. {tf_dir['path']}") + print(f" Files: {', '.join(tf_dir['files'])}") + + print("\nDiagnostic scan complete") diff --git a/plan.md b/plan.md index b5c037e7..fcd64839 100644 --- a/plan.md +++ b/plan.md @@ -7,18 +7,18 @@ This document outlines a comprehensive plan for upgrading Terraform configuratio ## 2. Prerequisites - ~~Backup of all existing Terraform configurations~~ (Git provides version control, so separate backups are not needed) -- Verification of required tools via Makefile (see "Tool Verification" section) -- Required AWS profiles configured in `~/.aws/config` or retrievable via `aws configure list-profiles` +- ✅ Verification of required tools via Makefile (see "Tool Verification" section) +- ✅ Required AWS profiles configured in `~/.aws/config` or retrievable via `aws configure list-profiles` - Completion of all pending infrastructure changes before beginning the upgrade - Access to testing environments to validate upgraded configurations ### Tool Verification -The Makefile provides a "verify-tools" target to check for required tool installation. This has been implemented in the repository's Makefile. +✅ The Makefile provides a "verify-tools" target to check for required tool installation. This has been implemented in the repository's Makefile. ### AWS Profile Configuration -Before beginning the upgrade process: +✅ The AWS profile configuration detection and validation has been implemented. The tool can now: 1. Verify AWS profile setup: ```bash @@ -1049,3 +1049,162 @@ echo "Verification complete. Review output for any issues." ``` This script should be run before each release to ensure basic functionality is working correctly. + +## 26. GitHub Project Integration + +To efficiently manage the upgrade process across multiple Terraform directories, we'll integrate with the GitHub Enterprise project board that tracks upgrade tickets. This will allow for automated status updates and a consistent workflow. + +### 26.1 GitHub API Integration + +The upgrade tool will be extended to interact with GitHub projects through the GitHub API: + +1. **Authentication Methods** + - ✅ Personal access token (PAT) support + - ✅ GitHub App authentication support + - ⬜ Optional environment variable configuration (GITHUB_TOKEN) + - ⬜ Support for .netrc file credentials + +2. **Project Board Interaction** + - ⬜ Fetch tickets from specified GitHub project board + - ⬜ Filter tickets by labels, status, and other metadata + - ⬜ Update ticket status based on upgrade progress + - ⬜ Add comments with upgrade results and logs + - ⬜ Attach reports and summaries to tickets + +3. **Implementation Components** + - ⬜ GitHub client module in `tf_upgrade/integrations/github.py` + - ⬜ Configuration for GitHub settings in config files + - ⬜ CLI commands for GitHub project integration + - ⬜ Progress reporting extensions for GitHub updates + +### 26.2 Ticket Workflow Integration + +The tool will support the standard upgrade workflow as represented in the project board: + +1. **Ticket States** + - **Todo**: Initial state for directories needing upgrade + - **In Progress**: Directories currently being upgraded + - **In Review**: Upgrade completed, awaiting validation + - **Done**: Successfully upgraded and verified + +2. **Automated State Transitions** + - `tf-upgrade scan` - Identifies directories and can create tickets if they don't exist + - `tf-upgrade start-upgrade ` - Moves ticket to "In Progress" + - `tf-upgrade complete-upgrade ` - Moves ticket to "In Review" and adds results + - `tf-upgrade verify-upgrade ` - Moves ticket to "Done" after validation + +3. **Integration Commands** + - ⬜ `make github-list` - List all upgrade tickets and their status + - ⬜ `make github-claim DIR=` - Claim a directory for upgrading (moves to "In Progress") + - ⬜ `make github-update DIR= STATUS=` - Update ticket status + - ⬜ `make github-comment DIR= MESSAGE="comment"` - Add comment to ticket + - ⬜ `make github-attach-report DIR=` - Attach upgrade report to ticket + +### 26.3 Reporting and Feedback + +The tool will enhance its reporting capabilities to work with GitHub: + +1. **GitHub-Compatible Reports** + - ⬜ Generate Markdown reports compatible with GitHub comments + - ⬜ Create summary reports for ticket descriptions + - ⬜ Support for before/after comparisons in GitHub-friendly format + - ⬜ Generate upgrade statistics for project-level reporting + +2. **Pull Request Integration** + - ⬜ Create pull requests for upgrade changes + - ⬜ Link PRs to project tickets + - ⬜ Include upgrade summary in PR description + - ⬜ Add validation results to PR comments + +## 27. Test Case Collection From Real Upgrades + +As upgrades are performed on real-world configurations, we'll systematically build a library of test cases to improve testing coverage and tool reliability. + +### 27.1 Test Case Generation + +1. **Automated Test Case Extraction** + - ⬜ Implement `tf-upgrade extract-test-case DIR=` command + - ⬜ Create minimal reproducible examples from complex configurations + - ⬜ Generate before/after snapshots for verification + - ⬜ Document patterns and edge cases encountered + +2. **Test Case Organization** + - ⬜ Create a structured test case library in `test/fixtures/real-world/` + - ⬜ Categorize by complexity, patterns, and upgrade challenges + - ⬜ Include metadata about source configurations + - ⬜ Document testing strategy for each case + +### 27.2 Test Coverage Enhancement + +1. **Pattern-Based Test Matrix** + - ⬜ Identify common patterns from real configurations + - ⬜ Create a coverage matrix of pattern vs. upgrade version + - ⬜ Track testing coverage for identified patterns + - ⬜ Prioritize testing for frequently encountered patterns + +2. **Regression Testing** + - ⬜ Add regression tests for each bug or edge case discovered + - ⬜ Implement automated regression test suite + - ⬜ Link regression tests to GitHub issues/tickets + - ⬜ Create verification matrix for common edge cases + +### 27.3 Implementation Approach + +1. **Test Fixture Manager** + - ⬜ Create `tf_upgrade/test_fixtures/manager.py` for test case management + - ⬜ Implement fixture extraction and minimization + - ⬜ Add pattern detection and classification + - ⬜ Provide fixture metadata management + +2. **Integration with Upgrade Process** + - ⬜ Add hooks in upgrade process to capture interesting patterns + - ⬜ Implement opt-in test case generation during upgrades + - ⬜ Create consistent directory structure for generated test cases + - ⬜ Add documentation for test case contribution workflow + +3. **Test Case Validation** + - ⬜ Implement verification of test case validity + - ⬜ Create tools for test case cleanup and normalization + - ⬜ Add validation of before/after state equivalence + - ⬜ Support for parameterized test case variants + +### 27.4 Census Bureau Pattern Library + +1. **Census-Specific Patterns** + - ⬜ Document Census-specific Terraform patterns (EDL, etc.) + - ⬜ Create targeted test fixtures for Census patterns + - ⬜ Validate upgrade paths for Census-specific modules + - ⬜ Verify handling of Census AWS account structure + +2. **Pattern Documentation** + - ⬜ Create pattern guide with examples and upgrade notes + - ⬜ Document recommended practices for new Terraform 1.x syntax + - ⬜ Provide migration guides for Census-specific patterns + - ⬜ Link patterns to relevant test fixtures + +## 28. Comprehensive Workflow with GitHub and Testing Integration + +With GitHub integration and test case collection in place, the comprehensive upgrade workflow becomes: + +1. **Discovery and Planning** + - Run `tf-upgrade github-scan REPO_ROOT` to identify directories and update GitHub project + - Review and prioritize tickets in GitHub project + - Assign upgrade tasks to team members + +2. **Upgrade Execution** + - Claim ticket with `make github-claim DIR=` + - Perform upgrade with `make upgrade DIR=` + - Collect test cases with `make extract-test-case DIR=` + - Submit PR and update ticket with `make github-update-pr DIR=` + +3. **Validation and Completion** + - Review PRs and validation reports + - Move tickets to "Done" after verification + - Extract lessons learned for process improvement + +4. **Continuous Improvement** + - Analyze test cases for common patterns + - Update tool based on real-world feedback + - Improve documentation with real examples + +This integrated workflow ensures a systematic approach to managing the upgrade process across many configurations while continuously improving the tool and building knowledge for future efforts. diff --git a/setup_test_env.sh b/setup_test_env.sh index 35085af9..6bb63705 100755 --- a/setup_test_env.sh +++ b/setup_test_env.sh @@ -1,5 +1,13 @@ #!/bin/bash +# Source environment variables from .env file +if [ -f .env ]; then + echo "Loading environment variables from .env file" + export $(grep -v '^#' .env | xargs) +else + echo "Warning: .env file not found, using default values" +fi + # Initialize Git repository if not already initialized if [ ! -d .git ]; then git init @@ -9,10 +17,10 @@ if [ ! -d .git ]; then git commit -m "Initial commit for testing" fi -# Export AWS environment variables if not set +# Export AWS environment variables if not set from .env file if [ -z "$AWS_PROFILE" ]; then - AWS_PROFILE=252960665057-ma6-gov.inf-admin-t2 - AWS_REGION=us-gov-east-1 + export AWS_PROFILE=252960665057-ma6-gov.inf-admin-t2 + export AWS_REGION=us-gov-east-1 # Uncomment if needed: # export AWS_ACCESS_KEY_ID=dummy-key # export AWS_SECRET_ACCESS_KEY=dummy-secret @@ -20,4 +28,5 @@ fi echo "Test environment setup complete!" echo "AWS Profile: $AWS_PROFILE" +echo "AWS Region: $AWS_REGION" echo "Git repository initialized." diff --git a/test-results.log b/test-results.log deleted file mode 100644 index d32f4e3b..00000000 --- a/test-results.log +++ /dev/null @@ -1,73 +0,0 @@ -====== TEST EXECUTION SUMMARY ====== -Starting unit tests at Thu Mar 27 12:33:54 EDT 2025 -test_create_backup (test_file_manager.TestFileManager.test_create_backup) ... ok -test_restore_backup (test_file_manager.TestFileManager.test_restore_backup) ... ok -test_transform_file (test_file_manager.TestFileManager.test_transform_file) ... ok -test_callable_transform (test_hcl_transformer.TestHCLTransformer.test_callable_transform) ... ok -test_multiple_transformations (test_hcl_transformer.TestHCLTransformer.test_multiple_transformations) ... ok -test_regex_transform (test_hcl_transformer.TestHCLTransformer.test_regex_transform) ... ok -test_init (test_terraform_runner.TestTerraformRunner.test_init) ... ok -test_plan (test_terraform_runner.TestTerraformRunner.test_plan) ... ok -test_run_command (test_terraform_runner.TestTerraformRunner.test_run_command) ... ok -test_specific_version (test_terraform_runner.TestTerraformRunner.test_specific_version) ... ok -test_validate (test_terraform_runner.TestTerraformRunner.test_validate) ... ok - ----------------------------------------------------------------------- -Ran 11 tests in 0.020s - -OK -Starting integration tests at Thu Mar 27 12:33:54 EDT 2025 -test_complete_upgrade_workflow (test_upgrade_workflow.TestUpgradeWorkflow.test_complete_upgrade_workflow) -Test complete upgrade workflow with mocked git commands. ... fatal: not a git repository (or any parent up to mount point /) -Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set). -Invalid version specified: 1.10 -🔄 [ 0%] Starting: Upgrading to Terraform 0.13... -✅ [ 25%] Completed: Upgrading to Terraform 0.13 - Est. remaining: 0s - Successfully upgraded to Terraform 0.13 -🔄 [ 25%] Starting: Upgrading to Terraform 0.14... -✅ [ 50%] Completed: Upgrading to Terraform 0.14 - Est. remaining: 0s - Successfully upgraded to Terraform 0.14 -🔄 [ 50%] Starting: Upgrading to Terraform 0.15... -✅ [ 75%] Completed: Upgrading to Terraform 0.15 - Est. remaining: 0s - Successfully upgraded to Terraform 0.15 -🔄 [ 75%] Starting: Upgrading to Terraform 1.10... -✅ [100%] Completed: Upgrading to Terraform 1.10 - Est. remaining: 0s - Successfully upgraded to Terraform 1.10 -✅ Operation Terraform upgrade to 1.10 completed successfully in 0s -ok -test_step_by_step_upgrade (test_upgrade_workflow.TestUpgradeWorkflow.test_step_by_step_upgrade) -Test step-by-step upgrade with mocked git commands. ... fatal: not a git repository (or any parent up to mount point /) -Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set). -🔄 [ 0%] Starting: Upgrading to Terraform 0.13... -✅ [100%] Completed: Upgrading to Terraform 0.13 - Est. remaining: 0s - Successfully upgraded to Terraform 0.13 -✅ Operation Terraform upgrade to 0.13 completed successfully in 0s -🔄 [ 0%] Starting: Upgrading to Terraform 0.14... -✅ [100%] Completed: Upgrading to Terraform 0.14 - Est. remaining: 0s - Successfully upgraded to Terraform 0.14 -✅ Operation Terraform upgrade to 0.14 completed successfully in 0s -🔄 [ 0%] Starting: Upgrading to Terraform 0.15... -✅ [100%] Completed: Upgrading to Terraform 0.15 - Est. remaining: 0s - Successfully upgraded to Terraform 0.15 -✅ Operation Terraform upgrade to 0.15 completed successfully in 0s -Invalid version specified: 1.10 -🔄 [ 0%] Starting: Upgrading to Terraform 1.10... -✅ [100%] Completed: Upgrading to Terraform 1.10 - Est. remaining: 0s - Successfully upgraded to Terraform 1.10 -✅ Operation Terraform upgrade to 1.10 completed successfully in 0s -ok - ----------------------------------------------------------------------- -Ran 2 tests in 0.028s - -OK -Starting validation tests at Thu Mar 27 12:33:54 EDT 2025 -test_complex_configuration_upgrade (test_upgrade_validation.TestUpgradeValidation.test_complex_configuration_upgrade) ... ok -test_medium_configuration_upgrade (test_upgrade_validation.TestUpgradeValidation.test_medium_configuration_upgrade) ... ok -test_simple_configuration_upgrade (test_upgrade_validation.TestUpgradeValidation.test_simple_configuration_upgrade) ... ok - ----------------------------------------------------------------------- -Ran 3 tests in 0.008s - -OK -====== TEST EXECUTION COMPLETED ====== diff --git a/testplan.md b/testplan.md index ac206018..42c7524f 100644 --- a/testplan.md +++ b/testplan.md @@ -347,3 +347,417 @@ The Terraform Upgrade Tool is ready for release when: 3. AWS account and profile management functions work reliably 4. Census Bureau-specific patterns are correctly handled 5. Performance is acceptable on large repositories (100+ files) + +## 11. GitHub Project Integration Testing + +### TC-19: GitHub Authentication and Connection + +**Objective:** Verify the tool can authenticate and connect to GitHub Enterprise instance + +**Steps:** +1. Configure various authentication methods (PAT, GitHub App, environment variables) +2. Test connection to GitHub API with each auth method +3. Verify proper error handling for authentication failures +4. Test access to repositories and project boards + +**Expected Outcome:** +- Tool successfully authenticates with all supported methods +- Connection to GitHub API is established +- Appropriate error messages are shown for invalid credentials +- Access to required repositories and project boards is confirmed + +### TC-20: Project Board Ticket Management + +**Objective:** Verify the tool can interact with project board tickets + +**Steps:** +1. Create test project board with sample tickets +2. Run tool with commands to fetch and update tickets +3. Test ticket status transitions +4. Verify comment and attachment capabilities + +**Expected Outcome:** +- Tool correctly retrieves tickets from project board +- Ticket status is updated appropriately +- Comments are added with proper formatting +- Reports and logs are attached to tickets + +### TC-21: Directory-to-Ticket Mapping + +**Objective:** Verify the tool correctly maps Terraform directories to GitHub tickets + +**Steps:** +1. Run `tf-upgrade github-scan` on a directory with multiple Terraform configurations +2. Verify tickets are created for each directory needing upgrade +3. Test directory path normalization and matching +4. Verify existing tickets are updated rather than duplicated + +**Expected Outcome:** +- Each directory maps to exactly one ticket +- Directory paths are normalized consistently +- Existing tickets are found and updated +- New tickets are created only when necessary + +### TC-22: Workflow Automation + +**Objective:** Test end-to-end workflow with GitHub integration + +**Steps:** +1. Start with a new directory requiring upgrade +2. Create ticket with `make github-create DIR=` +3. Claim ticket with `make github-claim DIR=` +4. Perform upgrade with `make upgrade DIR=` +5. Update ticket with results using `make github-update DIR=` +6. Generate PR with `make github-pr DIR=` + +**Expected Outcome:** +- Complete workflow succeeds end-to-end +- Ticket transitions through expected states +- PR is created with appropriate content +- Ticket and PR are properly linked + +## 12. Test Case Collection and Management + +### TC-23: Test Case Extraction + +**Objective:** Verify automatic extraction of test cases from real configurations + +**Steps:** +1. Run `make extract-test-case DIR=` on a complex configuration +2. Check generated test fixture structure and content +3. Verify key patterns are preserved in the minimal example +4. Test the extracted fixture with the upgrade process + +**Expected Outcome:** +- Test case is extracted with minimal required elements +- Important upgrade patterns are preserved +- Test fixture can be run independently +- Fixture includes before/after states for validation + +### TC-24: Pattern Detection + +**Objective:** Test pattern detection capability for test case categorization + +**Steps:** +1. Run pattern analysis on diverse test configurations +2. Check automatic categorization of configurations +3. Test pattern matching against known patterns +4. Verify coverage tracking for patterns + +**Expected Outcome:** +- Common patterns are correctly identified +- Configurations are assigned to appropriate categories +- Pattern matching works consistently +- Coverage reporting shows accurate statistics + +### TC-25: Test Fixture Management + +**Objective:** Verify organization and management of collected test cases + +**Steps:** +1. Populate test fixture library with multiple fixtures +2. Test search and filtering of fixtures by pattern +3. Verify metadata storage and retrieval +4. Check normalization and cleaning of fixtures + +**Expected Outcome:** +- Fixtures are organized in logical structure +- Search returns appropriate fixtures by criteria +- Metadata accurately describes fixture contents +- Normalized fixtures maintain essential patterns + +### TC-26: Regression Test Generation + +**Objective:** Test generation of regression tests from identified issues + +**Steps:** +1. Document a specific upgrade issue or bug +2. Create regression test using test case framework +3. Verify the test fails when issue is present +4. Fix the issue and verify test now passes + +**Expected Outcome:** +- Regression test accurately represents the issue +- Test fails when issue exists in the code +- Test passes when issue is fixed +- Test can be integrated into automated test suite + +## 13. Census Bureau Specific Testing + +### TC-27: Census Bureau Pattern Library + +**Objective:** Test handling of Census Bureau-specific Terraform patterns + +**Steps:** +1. Create test fixtures for Census-specific patterns +2. Run pattern detection on Census configurations +3. Verify documentation of Census patterns +4. Test upgrade paths for Census-specific modules + +**Expected Outcome:** +- Census-specific patterns are correctly identified +- Census module references are properly handled +- Documentation covers Census-specific recommendations +- Test fixtures represent true Census patterns + +### TC-28: Cross-Team Collaboration + +**Objective:** Test workflow for multi-team collaboration on upgrades + +**Steps:** +1. Set up test environment with multiple user roles +2. Configure GitHub permissions to match team structure +3. Test assignment and handoff of tickets between teams +4. Verify notification and status update flows + +**Expected Outcome:** +- Multi-team workflow functions correctly +- Assignment and handoff work as expected +- Status updates are visible to all stakeholders +- Notifications are sent to appropriate teams + +## 14. End-to-End Testing with GitHub and Test Case Collection + +### TC-29: Integrated Workflow Testing + +**Objective:** Verify complete end-to-end workflow with all features enabled + +**Steps:** +1. Start with repository containing multiple terraform configurations +2. Run full workflow from scanning to completion with GitHub integration +3. Extract test cases during the upgrade process +4. Complete upgrade and verify all components worked together + +**Expected Outcome:** +- Complete workflow succeeds end-to-end +- GitHub project board reflects accurate state +- Test cases are collected and categorized +- Upgraded configurations pass validation + +### TC-30: Performance Testing with Large Repositories + +**Objective:** Test tool performance with large repositories and many tickets + +**Steps:** +1. Set up test environment with 50+ terraform directories +2. Configure GitHub project board with matching tickets +3. Run full workflow on complete repository +4. Measure performance metrics + +**Expected Outcome:** +- Tool handles large repository without excessive memory usage +- GitHub API rate limits are respected +- Performance remains acceptable with many tickets +- Parallel processing options improve performance + +## 15. Census Bureau Infrastructure Pattern Testing + +### TC-31: VPC Upgrade Script Compatibility + +**Objective:** Test upgrade tool compatibility with VPC upgrade scripts + +**Steps:** +1. Locate and analyze `upgrade.vpc-code.sh` scripts +2. Test tool's handling of curl-based file downloads in scripts +3. Verify module reference checking (git::https to git@ format) +4. Test analysis of ref=tf-upgrade references + +**Expected Outcome:** +- Tool correctly identifies scripts that modify Terraform files +- Upgrade recommendations don't conflict with script-based upgrades +- Tool can properly analyze files before and after script execution +- Proper handling of alternative upgrade mechanisms + +### TC-32: EC2 Keypair Generation Patterns + +**Objective:** Verify handling of EC2 keypair generation patterns during upgrade + +**Steps:** +1. Identify configurations with null_resource keypair generation +2. Test upgrade of different keypair generation patterns +3. Verify DSA to RSA key migration recommendations +4. Test handling of different key sizes (1024, 2048) + +**Expected Outcome:** +- Correct transformation of provisioner blocks in null_resource +- Proper handling of depends_on references after upgrade +- Key size recommendations align with current security best practices +- SSH-keygen command formats are preserved through upgrade + +### TC-33: Module Version Reference Patterns + +**Objective:** Test upgrade of various module reference patterns + +**Steps:** +1. Create test fixtures from repository patterns of module references +2. Test different git reference formats (git::https://, git@github) +3. Verify handling of ref= parameters in module sources +4. Test branch and tag reference recommendations + +**Expected Outcome:** +- Module references are correctly upgraded with proper syntax +- Git URL formats are consistently handled +- Version constraints are preserved or upgraded appropriately +- Census-specific module branches (tf-upgrade) are correctly recognized + +### TC-34: DNS Integration Handling + +**Objective:** Test Terraform DNS provider configuration upgrades + +**Steps:** +1. Create test fixtures with dns_a_record_set and dns_ptr_record resources +2. Test upgrade of TSIG key configurations +3. Verify preservation of DNS update server configurations +4. Test record type transformations + +**Expected Outcome:** +- DNS provider configurations are correctly upgraded +- TSIG key authentications are preserved +- Record set configurations maintain their structure +- DNS validation logic is preserved through upgrade + +### TC-35: DynamoDB Table Attribute Patterns + +**Objective:** Test upgrade of DynamoDB table definitions + +**Steps:** +1. Create test fixtures with various DynamoDB table configurations +2. Test attribute definitions with different types (S, N, B) +3. Verify handling of global secondary indexes +4. Test multiline item declarations with heredoc syntax + +**Expected Outcome:** +- Table attribute definitions are correctly preserved +- Global secondary index configurations are properly upgraded +- Heredoc syntax for item declarations is correctly transformed +- Proper handling of JSON document structure in heredocs + +### TC-36: Container Service Definitions + +**Objective:** Test upgrade of ECS/Fargate task definitions + +**Steps:** +1. Create test fixtures with container definitions and port mappings +2. Test jsonencode function usage in container definitions +3. Verify upgrade of log configuration blocks +4. Test complex nested block structures in task definitions + +**Expected Outcome:** +- Task definition structures remain valid after upgrade +- jsonencode functions are properly upgraded +- Nested container configuration blocks are correctly transformed +- Array syntax in container definitions is preserved + +### TC-37: Infrastructure Setup Files + +**Objective:** Test handling of Census-specific infrastructure setup files + +**Steps:** +1. Analyze tf-run.data.bkp files and similar bootstrapping configurations +2. Test upgrade tool's handling of LINKTOP commands +3. Verify COMMAND, COMMENT pattern recognition +4. Test integration with tf-directory-setup.py references + +**Expected Outcome:** +- Tool correctly recognizes and preserves infrastructure setup patterns +- LINKTOP references are properly maintained or updated +- COMMAND directives are preserved during upgrade +- Integration with existing Census Bureau tooling is maintained + +### TC-38: Tag Structure Transformation + +**Objective:** Test census-specific tag structure upgrades + +**Steps:** +1. Create test fixtures with various Census-specific tag patterns +2. Test upgrades of boc: prefixed tags +3. Verify CostAllocation tag format preservation +4. Test special tag handling (boc:dns:name, boc:dns:zone) + +**Expected Outcome:** +- Census-specific tag structures are preserved +- Tag format standardization is applied where appropriate +- Special tag prefixes are recognized and handled correctly +- Tag value format (string interpolation) is correctly upgraded + +### TC-39: Pull Request Template Integration + +**Objective:** Test integration with Census Bureau PR templates + +**Steps:** +1. Analyze existing PR template format (.github/PULL_REQUEST_TEMPLATE.md) +2. Test generated PR content against template requirements +3. Verify tool can populate template sections +4. Test Plan integration into PR description + +**Expected Outcome:** +- Generated PRs follow the Census Bureau template format +- Required sections of PR template are populated +- Plan output is correctly formatted for template +- Tool helps users meet PR requirements + +### TC-40: Remote State Reference Handling + +**Objective:** Test upgrade of remote state references + +**Steps:** +1. Create test fixtures with terraform_remote_state data sources +2. Test cross-account state references +3. Verify workspace-specific state path formats +4. Test handling of output references from remote state + +**Expected Outcome:** +- Remote state references are preserved or correctly upgraded +- Output references using remote state are correctly transformed +- Workspace-specific state paths are correctly handled +- Cross-account references are identified and properly managed + +## 16. Census-Specific AWS Architecture Patterns + +### TC-41: VPC Organization Structure + +**Objective:** Test upgrade tool's handling of Census VPC directory structure + +**Steps:** +1. Analyze repository structure of account/vpc/region/vpc# patterns +2. Test scanning and file relationship detection across structure +3. Verify output variable references between parent/child directories +4. Test handling of linked files and modules across directory structure + +**Expected Outcome:** +- Tool correctly identifies related configurations across directory structure +- Upgrade maintains relationships between parent/child directories +- Output references between dirs are correctly transformed +- Tool respects organizational boundaries for upgrade units + +### TC-42: Multi-Environment Module References + +**Objective:** Test upgrade of modules used across different environments + +**Steps:** +1. Identify modules used in multiple env contexts (dev, prod, etc) +2. Test upgrade consistency across different environment references +3. Verify version constraint recommendations align across environments +4. Test handling of environment-specific module configuration + +**Expected Outcome:** +- Consistent upgrade recommendations across environments +- Environment-specific configurations are preserved +- Version constraints are handled appropriately for each environment +- Proper handling of environment-conditional blocks + +### TC-43: Terraform Workflow Script Integration + +**Objective:** Test integration with Census-specific Terraform workflow scripts + +**Steps:** +1. Analyze tf-control, tf-run, and related scripts +2. Test upgrade tool's compatibility with these workflow patterns +3. Verify proper handling of .tf-control.tfrc files +4. Test log format compatibility with existing scripts + +**Expected Outcome:** +- Tool integrates properly with existing workflow scripts +- Command execution follows expected patterns +- Log formats are compatible with existing processing +- Tool respects configuration from .tf-control files diff --git a/tests/fixtures/0.12/backup-test/.terraform-upgrade-backup-20250327123415/main.tf b/tests/fixtures/0.12/backup-test/.terraform-upgrade-backup-20250327123415/main.tf deleted file mode 100644 index 4d50d6a8..00000000 --- a/tests/fixtures/0.12/backup-test/.terraform-upgrade-backup-20250327123415/main.tf +++ /dev/null @@ -1,13 +0,0 @@ -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -resource "aws_s3_bucket" "simple" { - bucket = "my-simple-bucket" - acl = "private" -} - -output "bucket_id" { - value = "${aws_s3_bucket.simple.id}" -} diff --git a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093246.log b/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093246.log deleted file mode 100644 index e26cdaa4..00000000 --- a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093246.log +++ /dev/null @@ -1,26 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093246.log stamp 20250327.1743093246 time 1743093246 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.12.31 -# command=/apps/terraform/bin/terraform_0.12.31 init -# log_timestamp=2025-03-27T12:34:07.001310 - - -Initializing the backend... - -Initializing provider plugins... -- Checking for available provider plugins... -- Downloading plugin for provider "aws" (hashicorp/aws) 2.70.4... - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093246.log stamp 20250327.1743093246 start 1743093247 end 1743093247 elapsed 0 diff --git a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093248.log b/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093248.log deleted file mode 100644 index 7d0542d2..00000000 --- a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093248.log +++ /dev/null @@ -1,42 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093248.log stamp 20250327.1743093248 time 1743093248 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.13.7 -# command=/apps/terraform/bin/terraform_0.13.7 init -# log_timestamp=2025-03-27T12:34:08.906639 - - -Initializing the backend... - -Initializing provider plugins... -- Finding hashicorp/aws versions matching "~> 2.0"... -- Using hashicorp/aws v2.70.4 from the shared cache directory - - -Warning: Interpolation-only expressions are deprecated - - on main.tf line 12, in output "bucket_id": - 12: value = "${aws_s3_bucket.simple.id}" - -Terraform 0.11 and earlier required all non-constant expressions to be -provided via interpolation syntax, but this pattern is now deprecated. To -silence this warning, remove the "${ sequence from the start and the }" -sequence from the end of this expression, leaving just the inner expression. - -Template interpolation syntax is still used to construct strings from -expressions when the template includes multiple interpolation sequences or a -mixture of literal strings and interpolations. This deprecation applies only -to templates that consist entirely of a single interpolation sequence. - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093248.log stamp 20250327.1743093248 start 1743093248 end 1743093249 elapsed 0 diff --git a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093250.log b/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093250.log deleted file mode 100644 index 503551a8..00000000 --- a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093250.log +++ /dev/null @@ -1,58 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093250.log stamp 20250327.1743093250 time 1743093250 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.14.11 -# command=/apps/terraform/bin/terraform_0.14.11 init -# log_timestamp=2025-03-27T12:34:10.833010 - - -Initializing the backend... - -Initializing provider plugins... -- Finding hashicorp/aws versions matching "~> 2.0"... -- Using hashicorp/aws v2.70.4 from the shared cache directory - -Terraform has created a lock file .terraform.lock.hcl to record the provider -selections it made above. Include this file in your version control repository -so that Terraform can guarantee to make the same selections by default when -you run "terraform init" in the future. - - -Warning: Version constraints inside provider configuration blocks are deprecated - - on main.tf line 2, in provider "aws": - 2: version = "~> 2.0" - -Terraform 0.13 and earlier allowed provider version constraints inside the -provider configuration block, but that is now deprecated and will be removed -in a future version of Terraform. To silence this warning, move the provider -version constraint into the required_providers block. - - -Warning: Interpolation-only expressions are deprecated - - on main.tf line 12, in output "bucket_id": - 12: value = "${aws_s3_bucket.simple.id}" - -Terraform 0.11 and earlier required all non-constant expressions to be -provided via interpolation syntax, but this pattern is now deprecated. To -silence this warning, remove the "${ sequence from the start and the }" -sequence from the end of this expression, leaving just the inner expression. - -Template interpolation syntax is still used to construct strings from -expressions when the template includes multiple interpolation sequences or a -mixture of literal strings and interpolations. This deprecation applies only -to templates that consist entirely of a single interpolation sequence. - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093250.log stamp 20250327.1743093250 start 1743093250 end 1743093251 elapsed 0 diff --git a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093252.log b/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093252.log deleted file mode 100644 index 64bebcc0..00000000 --- a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093252.log +++ /dev/null @@ -1,38 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093252.log stamp 20250327.1743093252 time 1743093252 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.15.5 -# command=/apps/terraform/bin/terraform_0.15.5 init -# log_timestamp=2025-03-27T12:34:12.749403 - - -Initializing the backend... - -Initializing provider plugins... -- Reusing previous version of hashicorp/aws from the dependency lock file -- Using previously-installed hashicorp/aws v2.70.4 - -╷ -│ Warning: Version constraints inside provider configuration blocks are deprecated -│  -│  on main.tf line 2, in provider "aws": -│  2: version = "~> 2.0" -│  -│ Terraform 0.13 and earlier allowed provider version constraints inside the -│ provider configuration block, but that is now deprecated and will be -│ removed in a future version of Terraform. To silence this warning, move the -│ provider version constraint into the required_providers block. -╵ - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093252.log stamp 20250327.1743093252 start 1743093252 end 1743093253 elapsed 0 diff --git a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093255.log b/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093255.log deleted file mode 100644 index 19c8e128..00000000 --- a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093255.log +++ /dev/null @@ -1,24 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093255.log stamp 20250327.1743093255 time 1743093255 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/backup-test -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.12.31 -# command=/apps/terraform/bin/terraform_0.12.31 init -# log_timestamp=2025-03-27T12:34:15.798235 - - -Initializing the backend... - -Initializing provider plugins... - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093255.log stamp 20250327.1743093255 start 1743093255 end 1743093256 elapsed 0 diff --git a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093257.log b/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093257.log deleted file mode 100644 index 2dc36933..00000000 --- a/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093257.log +++ /dev/null @@ -1,41 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093257.log stamp 20250327.1743093257 time 1743093257 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/backup-test -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.13.7 -# command=/apps/terraform/bin/terraform_0.13.7 init -# log_timestamp=2025-03-27T12:34:17.675602 - - -Initializing the backend... - -Initializing provider plugins... -- Using previously-installed hashicorp/aws v2.70.4 - - -Warning: Interpolation-only expressions are deprecated - - on main.tf line 20, in output "bucket_id": - 20: value = "${aws_s3_bucket.simple.id}" - -Terraform 0.11 and earlier required all non-constant expressions to be -provided via interpolation syntax, but this pattern is now deprecated. To -silence this warning, remove the "${ sequence from the start and the }" -sequence from the end of this expression, leaving just the inner expression. - -Template interpolation syntax is still used to construct strings from -expressions when the template includes multiple interpolation sequences or a -mixture of literal strings and interpolations. This deprecation applies only -to templates that consist entirely of a single interpolation sequence. - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/backup-test/logs/init.20250327.1743093257.log stamp 20250327.1743093257 start 1743093257 end 1743093258 elapsed 0 diff --git a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093247.log b/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093247.log deleted file mode 100644 index ffe347c3..00000000 --- a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093247.log +++ /dev/null @@ -1,12 +0,0 @@ -# starting terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093247.log stamp 20250327.1743093247 time 1743093247 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.12.31 -# command=/apps/terraform/bin/terraform_0.12.31 validate -# log_timestamp=2025-03-27T12:34:07.628772 - -Success! The configuration is valid. - - -# ending terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093247.log stamp 20250327.1743093247 start 1743093247 end 1743093248 elapsed 1 diff --git a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093249.log b/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093249.log deleted file mode 100644 index b1bd6fec..00000000 --- a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093249.log +++ /dev/null @@ -1,28 +0,0 @@ -# starting terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093249.log stamp 20250327.1743093249 time 1743093249 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.13.7 -# command=/apps/terraform/bin/terraform_0.13.7 validate -# log_timestamp=2025-03-27T12:34:09.535292 - - -Warning: Interpolation-only expressions are deprecated - - on main.tf line 12, in output "bucket_id": - 12: value = "${aws_s3_bucket.simple.id}" - -Terraform 0.11 and earlier required all non-constant expressions to be -provided via interpolation syntax, but this pattern is now deprecated. To -silence this warning, remove the "${ sequence from the start and the }" -sequence from the end of this expression, leaving just the inner expression. - -Template interpolation syntax is still used to construct strings from -expressions when the template includes multiple interpolation sequences or a -mixture of literal strings and interpolations. This deprecation applies only -to templates that consist entirely of a single interpolation sequence. - -Success! The configuration is valid, but there were some validation warnings as shown above. - - -# ending terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093249.log stamp 20250327.1743093249 start 1743093249 end 1743093250 elapsed 1 diff --git a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093251.log b/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093251.log deleted file mode 100644 index f3b76690..00000000 --- a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093251.log +++ /dev/null @@ -1,39 +0,0 @@ -# starting terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093251.log stamp 20250327.1743093251 time 1743093251 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.14.11 -# command=/apps/terraform/bin/terraform_0.14.11 validate -# log_timestamp=2025-03-27T12:34:11.573362 - - -Warning: Version constraints inside provider configuration blocks are deprecated - - on main.tf line 2, in provider "aws": - 2: version = "~> 2.0" - -Terraform 0.13 and earlier allowed provider version constraints inside the -provider configuration block, but that is now deprecated and will be removed -in a future version of Terraform. To silence this warning, move the provider -version constraint into the required_providers block. - - -Warning: Interpolation-only expressions are deprecated - - on main.tf line 12, in output "bucket_id": - 12: value = "${aws_s3_bucket.simple.id}" - -Terraform 0.11 and earlier required all non-constant expressions to be -provided via interpolation syntax, but this pattern is now deprecated. To -silence this warning, remove the "${ sequence from the start and the }" -sequence from the end of this expression, leaving just the inner expression. - -Template interpolation syntax is still used to construct strings from -expressions when the template includes multiple interpolation sequences or a -mixture of literal strings and interpolations. This deprecation applies only -to templates that consist entirely of a single interpolation sequence. - -Success! The configuration is valid, but there were some validation warnings as shown above. - - -# ending terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093251.log stamp 20250327.1743093251 start 1743093251 end 1743093252 elapsed 0 diff --git a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093253.log b/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093253.log deleted file mode 100644 index 77a7ada4..00000000 --- a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093253.log +++ /dev/null @@ -1,24 +0,0 @@ -# starting terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093253.log stamp 20250327.1743093253 time 1743093253 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.15.5 -# command=/apps/terraform/bin/terraform_0.15.5 validate -# log_timestamp=2025-03-27T12:34:13.806263 - -╷ -│ Warning: Version constraints inside provider configuration blocks are deprecated -│  -│  on main.tf line 2, in provider "aws": -│  2: version = "~> 2.0" -│  -│ Terraform 0.13 and earlier allowed provider version constraints inside the -│ provider configuration block, but that is now deprecated and will be -│ removed in a future version of Terraform. To silence this warning, move the -│ provider version constraint into the required_providers block. -╵ -Success! The configuration is valid, but there were some -validation warnings as shown above. - - -# ending terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093253.log stamp 20250327.1743093253 start 1743093253 end 1743093254 elapsed 1 diff --git a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093256.log b/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093256.log deleted file mode 100644 index d67e158a..00000000 --- a/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093256.log +++ /dev/null @@ -1,12 +0,0 @@ -# starting terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093256.log stamp 20250327.1743093256 time 1743093256 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/backup-test -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.12.31 -# command=/apps/terraform/bin/terraform_0.12.31 validate -# log_timestamp=2025-03-27T12:34:16.538214 - -Success! The configuration is valid. - - -# ending terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/backup-test/logs/validate.20250327.1743093256.log stamp 20250327.1743093256 start 1743093256 end 1743093257 elapsed 0 diff --git a/tests/fixtures/0.12/backup-test/main.tf b/tests/fixtures/0.12/backup-test/main.tf deleted file mode 100644 index 3760d41a..00000000 --- a/tests/fixtures/0.12/backup-test/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - } -} - -provider "aws" { - version = "~> 2.0" - region = "us-west-2" -} - -resource "aws_s3_bucket" "simple" { - bucket = "my-simple-bucket" - acl = "private" -} - -output "bucket_id" { - value = "${aws_s3_bucket.simple.id}" -} diff --git a/tests/fixtures/0.12/backup-test/terraform-upgrade-dryrun-report.md b/tests/fixtures/0.12/backup-test/terraform-upgrade-dryrun-report.md deleted file mode 100644 index ae9bf5f1..00000000 --- a/tests/fixtures/0.12/backup-test/terraform-upgrade-dryrun-report.md +++ /dev/null @@ -1,12 +0,0 @@ - -## Summary - -- Current Version: 0.12 -- Upgrade Path: 0.13 → 0.14 → 0.15 → 1.0 -- Complexity Level: Low -- Complexity Score: 8.5 - -### Deprecated Syntax - -- interpolation_only: `"${aws_s3_bucket.simple.id}"` in main.tf -- quoted_interpolation: `"${aws_s3_bucket.simple.id}"` in main.tf diff --git a/tests/fixtures/0.12/backup-test/terraform-upgrade-reports/upgrade-report-20250327-123415.md b/tests/fixtures/0.12/backup-test/terraform-upgrade-reports/upgrade-report-20250327-123415.md deleted file mode 100644 index 46e1aa48..00000000 --- a/tests/fixtures/0.12/backup-test/terraform-upgrade-reports/upgrade-report-20250327-123415.md +++ /dev/null @@ -1,29 +0,0 @@ -# Terraform Upgrade Report - -Report generated on 2025-03-27 12:34:15 - -## Progress - -| Step | Status | Duration | Details | -|------|--------|----------|--------| -| 1: Upgrading to Terraform 0.13 | ❌ Failed | 2s | **Error:** Failed to upgrade to Terraform 0.13 | - -## Summary - -- **Status:** ❌ Failed -- **Total Duration:** 2s -- **Steps Completed:** 1/1 -- **Successful Steps:** 0 -- **Failed Steps:** 1 - -## Details - -- **Started:** 2025-03-27 12:34:15 -- **Completed:** 2025-03-27 12:34:18 - -### Failed Steps - -#### Step 1: Upgrading to Terraform 0.13 - -- **Details:** Failed to upgrade to Terraform 0.13 - diff --git a/tests/fixtures/0.12/backup-test/upgrade-logs/upgrade-progress.json b/tests/fixtures/0.12/backup-test/upgrade-logs/upgrade-progress.json deleted file mode 100644 index fe698f65..00000000 --- a/tests/fixtures/0.12/backup-test/upgrade-logs/upgrade-progress.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "started_at": "2025-03-27T12:34:06.725924", - "steps": [], - "completed": 0, - "total": 0, - "status": "in_progress" -} \ No newline at end of file diff --git a/tests/fixtures/0.12/simple/logs/init.20250327.1743093246.log b/tests/fixtures/0.12/simple/logs/init.20250327.1743093246.log deleted file mode 100644 index e26cdaa4..00000000 --- a/tests/fixtures/0.12/simple/logs/init.20250327.1743093246.log +++ /dev/null @@ -1,26 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093246.log stamp 20250327.1743093246 time 1743093246 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.12.31 -# command=/apps/terraform/bin/terraform_0.12.31 init -# log_timestamp=2025-03-27T12:34:07.001310 - - -Initializing the backend... - -Initializing provider plugins... -- Checking for available provider plugins... -- Downloading plugin for provider "aws" (hashicorp/aws) 2.70.4... - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093246.log stamp 20250327.1743093246 start 1743093247 end 1743093247 elapsed 0 diff --git a/tests/fixtures/0.12/simple/logs/init.20250327.1743093248.log b/tests/fixtures/0.12/simple/logs/init.20250327.1743093248.log deleted file mode 100644 index 7d0542d2..00000000 --- a/tests/fixtures/0.12/simple/logs/init.20250327.1743093248.log +++ /dev/null @@ -1,42 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093248.log stamp 20250327.1743093248 time 1743093248 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.13.7 -# command=/apps/terraform/bin/terraform_0.13.7 init -# log_timestamp=2025-03-27T12:34:08.906639 - - -Initializing the backend... - -Initializing provider plugins... -- Finding hashicorp/aws versions matching "~> 2.0"... -- Using hashicorp/aws v2.70.4 from the shared cache directory - - -Warning: Interpolation-only expressions are deprecated - - on main.tf line 12, in output "bucket_id": - 12: value = "${aws_s3_bucket.simple.id}" - -Terraform 0.11 and earlier required all non-constant expressions to be -provided via interpolation syntax, but this pattern is now deprecated. To -silence this warning, remove the "${ sequence from the start and the }" -sequence from the end of this expression, leaving just the inner expression. - -Template interpolation syntax is still used to construct strings from -expressions when the template includes multiple interpolation sequences or a -mixture of literal strings and interpolations. This deprecation applies only -to templates that consist entirely of a single interpolation sequence. - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093248.log stamp 20250327.1743093248 start 1743093248 end 1743093249 elapsed 0 diff --git a/tests/fixtures/0.12/simple/logs/init.20250327.1743093250.log b/tests/fixtures/0.12/simple/logs/init.20250327.1743093250.log deleted file mode 100644 index 503551a8..00000000 --- a/tests/fixtures/0.12/simple/logs/init.20250327.1743093250.log +++ /dev/null @@ -1,58 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093250.log stamp 20250327.1743093250 time 1743093250 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.14.11 -# command=/apps/terraform/bin/terraform_0.14.11 init -# log_timestamp=2025-03-27T12:34:10.833010 - - -Initializing the backend... - -Initializing provider plugins... -- Finding hashicorp/aws versions matching "~> 2.0"... -- Using hashicorp/aws v2.70.4 from the shared cache directory - -Terraform has created a lock file .terraform.lock.hcl to record the provider -selections it made above. Include this file in your version control repository -so that Terraform can guarantee to make the same selections by default when -you run "terraform init" in the future. - - -Warning: Version constraints inside provider configuration blocks are deprecated - - on main.tf line 2, in provider "aws": - 2: version = "~> 2.0" - -Terraform 0.13 and earlier allowed provider version constraints inside the -provider configuration block, but that is now deprecated and will be removed -in a future version of Terraform. To silence this warning, move the provider -version constraint into the required_providers block. - - -Warning: Interpolation-only expressions are deprecated - - on main.tf line 12, in output "bucket_id": - 12: value = "${aws_s3_bucket.simple.id}" - -Terraform 0.11 and earlier required all non-constant expressions to be -provided via interpolation syntax, but this pattern is now deprecated. To -silence this warning, remove the "${ sequence from the start and the }" -sequence from the end of this expression, leaving just the inner expression. - -Template interpolation syntax is still used to construct strings from -expressions when the template includes multiple interpolation sequences or a -mixture of literal strings and interpolations. This deprecation applies only -to templates that consist entirely of a single interpolation sequence. - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093250.log stamp 20250327.1743093250 start 1743093250 end 1743093251 elapsed 0 diff --git a/tests/fixtures/0.12/simple/logs/init.20250327.1743093252.log b/tests/fixtures/0.12/simple/logs/init.20250327.1743093252.log deleted file mode 100644 index 64bebcc0..00000000 --- a/tests/fixtures/0.12/simple/logs/init.20250327.1743093252.log +++ /dev/null @@ -1,38 +0,0 @@ -# starting terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093252.log stamp 20250327.1743093252 time 1743093252 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.15.5 -# command=/apps/terraform/bin/terraform_0.15.5 init -# log_timestamp=2025-03-27T12:34:12.749403 - - -Initializing the backend... - -Initializing provider plugins... -- Reusing previous version of hashicorp/aws from the dependency lock file -- Using previously-installed hashicorp/aws v2.70.4 - -╷ -│ Warning: Version constraints inside provider configuration blocks are deprecated -│  -│  on main.tf line 2, in provider "aws": -│  2: version = "~> 2.0" -│  -│ Terraform 0.13 and earlier allowed provider version constraints inside the -│ provider configuration block, but that is now deprecated and will be -│ removed in a future version of Terraform. To silence this warning, move the -│ provider version constraint into the required_providers block. -╵ - -Terraform has been successfully initialized! - -You may now begin working with Terraform. Try running "terraform plan" to see -any changes that are required for your infrastructure. All Terraform commands -should now work. - -If you ever set or change modules or backend configuration for Terraform, -rerun this command to reinitialize your working directory. If you forget, other -commands will detect it and remind you to do so if necessary. - -# ending terraform-upgrade-tool init file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/init.20250327.1743093252.log stamp 20250327.1743093252 start 1743093252 end 1743093253 elapsed 0 diff --git a/tests/fixtures/0.12/simple/logs/validate.20250327.1743093247.log b/tests/fixtures/0.12/simple/logs/validate.20250327.1743093247.log deleted file mode 100644 index ffe347c3..00000000 --- a/tests/fixtures/0.12/simple/logs/validate.20250327.1743093247.log +++ /dev/null @@ -1,12 +0,0 @@ -# starting terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093247.log stamp 20250327.1743093247 time 1743093247 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.12.31 -# command=/apps/terraform/bin/terraform_0.12.31 validate -# log_timestamp=2025-03-27T12:34:07.628772 - -Success! The configuration is valid. - - -# ending terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093247.log stamp 20250327.1743093247 start 1743093247 end 1743093248 elapsed 1 diff --git a/tests/fixtures/0.12/simple/logs/validate.20250327.1743093249.log b/tests/fixtures/0.12/simple/logs/validate.20250327.1743093249.log deleted file mode 100644 index b1bd6fec..00000000 --- a/tests/fixtures/0.12/simple/logs/validate.20250327.1743093249.log +++ /dev/null @@ -1,28 +0,0 @@ -# starting terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093249.log stamp 20250327.1743093249 time 1743093249 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.13.7 -# command=/apps/terraform/bin/terraform_0.13.7 validate -# log_timestamp=2025-03-27T12:34:09.535292 - - -Warning: Interpolation-only expressions are deprecated - - on main.tf line 12, in output "bucket_id": - 12: value = "${aws_s3_bucket.simple.id}" - -Terraform 0.11 and earlier required all non-constant expressions to be -provided via interpolation syntax, but this pattern is now deprecated. To -silence this warning, remove the "${ sequence from the start and the }" -sequence from the end of this expression, leaving just the inner expression. - -Template interpolation syntax is still used to construct strings from -expressions when the template includes multiple interpolation sequences or a -mixture of literal strings and interpolations. This deprecation applies only -to templates that consist entirely of a single interpolation sequence. - -Success! The configuration is valid, but there were some validation warnings as shown above. - - -# ending terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093249.log stamp 20250327.1743093249 start 1743093249 end 1743093250 elapsed 1 diff --git a/tests/fixtures/0.12/simple/logs/validate.20250327.1743093251.log b/tests/fixtures/0.12/simple/logs/validate.20250327.1743093251.log deleted file mode 100644 index f3b76690..00000000 --- a/tests/fixtures/0.12/simple/logs/validate.20250327.1743093251.log +++ /dev/null @@ -1,39 +0,0 @@ -# starting terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093251.log stamp 20250327.1743093251 time 1743093251 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.14.11 -# command=/apps/terraform/bin/terraform_0.14.11 validate -# log_timestamp=2025-03-27T12:34:11.573362 - - -Warning: Version constraints inside provider configuration blocks are deprecated - - on main.tf line 2, in provider "aws": - 2: version = "~> 2.0" - -Terraform 0.13 and earlier allowed provider version constraints inside the -provider configuration block, but that is now deprecated and will be removed -in a future version of Terraform. To silence this warning, move the provider -version constraint into the required_providers block. - - -Warning: Interpolation-only expressions are deprecated - - on main.tf line 12, in output "bucket_id": - 12: value = "${aws_s3_bucket.simple.id}" - -Terraform 0.11 and earlier required all non-constant expressions to be -provided via interpolation syntax, but this pattern is now deprecated. To -silence this warning, remove the "${ sequence from the start and the }" -sequence from the end of this expression, leaving just the inner expression. - -Template interpolation syntax is still used to construct strings from -expressions when the template includes multiple interpolation sequences or a -mixture of literal strings and interpolations. This deprecation applies only -to templates that consist entirely of a single interpolation sequence. - -Success! The configuration is valid, but there were some validation warnings as shown above. - - -# ending terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093251.log stamp 20250327.1743093251 start 1743093251 end 1743093252 elapsed 0 diff --git a/tests/fixtures/0.12/simple/logs/validate.20250327.1743093253.log b/tests/fixtures/0.12/simple/logs/validate.20250327.1743093253.log deleted file mode 100644 index 77a7ada4..00000000 --- a/tests/fixtures/0.12/simple/logs/validate.20250327.1743093253.log +++ /dev/null @@ -1,24 +0,0 @@ -# starting terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093253.log stamp 20250327.1743093253 time 1743093253 -# current_directory=/data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple -# git_repository=git@github.e.it.census.gov:terraform/support -# git_current_branch=gliffy -# terraform_version=Terraform v0.15.5 -# command=/apps/terraform/bin/terraform_0.15.5 validate -# log_timestamp=2025-03-27T12:34:13.806263 - -╷ -│ Warning: Version constraints inside provider configuration blocks are deprecated -│  -│  on main.tf line 2, in provider "aws": -│  2: version = "~> 2.0" -│  -│ Terraform 0.13 and earlier allowed provider version constraints inside the -│ provider configuration block, but that is now deprecated and will be -│ removed in a future version of Terraform. To silence this warning, move the -│ provider version constraint into the required_providers block. -╵ -Success! The configuration is valid, but there were some -validation warnings as shown above. - - -# ending terraform-upgrade-tool validate file /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures/0.12/simple/logs/validate.20250327.1743093253.log stamp 20250327.1743093253 start 1743093253 end 1743093254 elapsed 1 diff --git a/tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md b/tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md deleted file mode 100644 index ae9bf5f1..00000000 --- a/tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md +++ /dev/null @@ -1,12 +0,0 @@ - -## Summary - -- Current Version: 0.12 -- Upgrade Path: 0.13 → 0.14 → 0.15 → 1.0 -- Complexity Level: Low -- Complexity Score: 8.5 - -### Deprecated Syntax - -- interpolation_only: `"${aws_s3_bucket.simple.id}"` in main.tf -- quoted_interpolation: `"${aws_s3_bucket.simple.id}"` in main.tf diff --git a/tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json b/tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json deleted file mode 100644 index fe698f65..00000000 --- a/tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "started_at": "2025-03-27T12:34:06.725924", - "steps": [], - "completed": 0, - "total": 0, - "status": "in_progress" -} \ No newline at end of file diff --git a/tests/fixtures/logs/terraform-init-20250327-001102.log b/tests/fixtures/logs/terraform-init-20250327-001102.log deleted file mode 100644 index 833f65e3..00000000 --- a/tests/fixtures/logs/terraform-init-20250327-001102.log +++ /dev/null @@ -1,7 +0,0 @@ -=== Terraform init command === -Date: 2025-03-27 00:11:02 -Directory: /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool/tests/fixtures -Terraform binary: terraform-0.15 - - -Error: [Errno 2] No such file or directory: 'terraform-0.15' diff --git a/tests/integration/test_upgrade_workflow.py b/tests/integration/test_upgrade_workflow.py index a14b986d..84b458f2 100644 --- a/tests/integration/test_upgrade_workflow.py +++ b/tests/integration/test_upgrade_workflow.py @@ -26,6 +26,7 @@ } """ + # The key to fixing the git errors is to apply the patches at the module level # before any of the test functions run @patch("subprocess.run") @@ -35,8 +36,15 @@ @patch("tf_upgrade.utils.git.os.path.exists") @patch("shutil.which") class TestUpgradeWorkflow(unittest.TestCase): - def setUp(self, mock_run=None, mock_check_output=None, mock_tf_check_output=None, - mock_git_check_output=None, mock_git_exists=None, mock_which=None): + def setUp( + self, + mock_run=None, + mock_check_output=None, + mock_tf_check_output=None, + mock_git_check_output=None, + mock_git_exists=None, + mock_which=None, + ): """Set up test environment with pre-configured mocks.""" # Create a temporary directory for the test self.test_dir = tempfile.mkdtemp() @@ -56,105 +64,146 @@ def tearDown(self): shutil.rmtree(self.test_dir) @patch("tf_upgrade.version_detector.VersionDetector.analyze_directory") - def test_complete_upgrade_workflow(self, mock_analyze, mock_which, mock_git_exists, - mock_git_check_output, mock_tf_check_output, - mock_check_output, mock_run): + def test_complete_upgrade_workflow( + self, + mock_analyze, + mock_which, + mock_git_exists, + mock_git_check_output, + mock_tf_check_output, + mock_check_output, + mock_run, + ): """Test complete upgrade workflow with mocked git commands.""" # Configure mocks for this test - self._configure_mocks(mock_which, mock_git_exists, mock_git_check_output, - mock_tf_check_output, mock_check_output, mock_run) - + self._configure_mocks( + mock_which, + mock_git_exists, + mock_git_check_output, + mock_tf_check_output, + mock_check_output, + mock_run, + ) + # Mock version detection mock_analyze.return_value = { - "final_version": "0.12", + "final_version": "0.12", "upgrade_path": ["0.13", "0.14", "0.15", "1.10"], - "requires_upgrade": True + "requires_upgrade": True, } - + # Replace the upgrade method with a mock that always succeeds - with patch.object(self.controller, 'upgrade_to_version', - return_value={"success": True, "message": "Successful upgrade"}): + with patch.object( + self.controller, + "upgrade_to_version", + return_value={"success": True, "message": "Successful upgrade"}, + ): # Run the upgrade process result = self.controller.upgrade(target_version="1.10", backup=True) - + # Verify the result self.assertTrue(result["success"]) @patch("tf_upgrade.version_detector.VersionDetector.analyze_directory") - def test_step_by_step_upgrade(self, mock_analyze, mock_which, mock_git_exists, - mock_git_check_output, mock_tf_check_output, - mock_check_output, mock_run): + def test_step_by_step_upgrade( + self, + mock_analyze, + mock_which, + mock_git_exists, + mock_git_check_output, + mock_tf_check_output, + mock_check_output, + mock_run, + ): """Test step-by-step upgrade with mocked git commands.""" # Configure mocks for this test - self._configure_mocks(mock_which, mock_git_exists, mock_git_check_output, - mock_tf_check_output, mock_check_output, mock_run) - + self._configure_mocks( + mock_which, + mock_git_exists, + mock_git_check_output, + mock_tf_check_output, + mock_check_output, + mock_run, + ) + # Mock version detection for each version mock_analyze.side_effect = [ { "final_version": "0.12", "upgrade_path": ["0.13"], - "requires_upgrade": True + "requires_upgrade": True, }, { "final_version": "0.13", "upgrade_path": ["0.14"], - "requires_upgrade": True + "requires_upgrade": True, }, { "final_version": "0.14", "upgrade_path": ["0.15"], - "requires_upgrade": True + "requires_upgrade": True, }, { "final_version": "0.15", "upgrade_path": ["1.10"], - "requires_upgrade": True - } + "requires_upgrade": True, + }, ] - + # Replace the upgrade_to_version method to always succeed - with patch.object(self.controller, 'upgrade_to_version', - return_value={"success": True, "message": "Successful upgrade"}): + with patch.object( + self.controller, + "upgrade_to_version", + return_value={"success": True, "message": "Successful upgrade"}, + ): # Run individual upgrades result_0_13 = self.controller.upgrade(target_version="0.13") self.assertTrue(result_0_13["success"], "Upgrade to 0.13 failed") - + result_0_14 = self.controller.upgrade(target_version="0.14") self.assertTrue(result_0_14["success"], "Upgrade to 0.14 failed") - + result_0_15 = self.controller.upgrade(target_version="0.15") self.assertTrue(result_0_15["success"], "Upgrade to 0.15 failed") - + result_1_10 = self.controller.upgrade(target_version="1.10") self.assertTrue(result_1_10["success"], "Upgrade to 1.10 failed") - def _configure_mocks(self, mock_which, mock_git_exists, mock_git_check_output, - mock_tf_check_output, mock_check_output, mock_run): + def _configure_mocks( + self, + mock_which, + mock_git_exists, + mock_git_check_output, + mock_tf_check_output, + mock_check_output, + mock_run, + ): """Configure the mocks with common behavior.""" # Configure all mock outputs for git commands mock_which.return_value = "/usr/bin/git" # Make git appear to be installed - mock_git_exists.return_value = False # Make it appear as if .git directory doesn't exist + mock_git_exists.return_value = ( + False # Make it appear as if .git directory doesn't exist + ) mock_git_check_output.return_value = b"dummy/git/path" mock_tf_check_output.return_value = b"dummy/git/path" mock_check_output.return_value = b"dummy/git/path" - + # Configure subprocess.run to handle git commands specially def mock_subprocess_run(*args, **kwargs): - cmd = args[0] if args else kwargs.get('args', []) - if isinstance(cmd, list) and cmd and cmd[0] == 'git': + cmd = args[0] if args else kwargs.get("args", []) + if isinstance(cmd, list) and cmd and cmd[0] == "git": # Return failed for git commands with appropriate error result = MagicMock() result.returncode = 128 result.stdout = b"" return result - + # For non-git commands, return success result = MagicMock() result.returncode = 0 result.stdout = b"Success" return result - + mock_run.side_effect = mock_subprocess_run diff --git a/tests/unit/test_file_manager.py b/tests/unit/test_file_manager.py index eaa07aeb..144f9764 100644 --- a/tests/unit/test_file_manager.py +++ b/tests/unit/test_file_manager.py @@ -40,7 +40,10 @@ def test_create_backup(self): with open(os.path.join(backup_dir, "test1.tf"), "r") as f: content = f.read() - self.assertEqual(content, 'resource "aws_instance" "example" {\n ami = "ami-123456"\n instance_type = "t2.micro"\n}') + self.assertEqual( + content, + 'resource "aws_instance" "example" {\n ami = "ami-123456"\n instance_type = "t2.micro"\n}', + ) def test_restore_backup(self): # Test the backup restore functionality @@ -52,7 +55,10 @@ def test_restore_backup(self): self.file_manager.restore_backup(backup_dir) with open(self.test_file1, "r") as f: content = f.read() - self.assertEqual(content, 'resource "aws_instance" "example" {\n ami = "ami-123456"\n instance_type = "t2.micro"\n}') + self.assertEqual( + content, + 'resource "aws_instance" "example" {\n ami = "ami-123456"\n instance_type = "t2.micro"\n}', + ) def test_transform_file(self): # Test file transformation diff --git a/tests/unit/test_hcl_transformer.py b/tests/unit/test_hcl_transformer.py index 11f07c50..79954a79 100644 --- a/tests/unit/test_hcl_transformer.py +++ b/tests/unit/test_hcl_transformer.py @@ -34,7 +34,9 @@ def tearDown(self): def test_regex_transform(self): # Test regex transformation transformer = HCLTransformer() - transformer.add_regex_transformation(r"version = \"~> 2.0\"", "version = \"~> 3.0\"") + transformer.add_regex_transformation( + r"version = \"~> 2.0\"", 'version = "~> 3.0"' + ) transformer.transform_file(self.test_file) with open(self.test_file, "r") as f: @@ -44,7 +46,7 @@ def test_regex_transform(self): def test_callable_transform(self): # Test callable transformation def transform_func(content): - return content.replace("region = \"us-west-2\"", "region = \"us-east-1\"") + return content.replace('region = "us-west-2"', 'region = "us-east-1"') transformer = HCLTransformer() transformer.add_callable_transformation(transform_func) @@ -57,10 +59,12 @@ def transform_func(content): def test_multiple_transformations(self): # Test multiple transformations transformer = HCLTransformer() - transformer.add_regex_transformation(r"version = \"~> 2.0\"", "version = \"~> 3.0\"") + transformer.add_regex_transformation( + r"version = \"~> 2.0\"", 'version = "~> 3.0"' + ) def transform_func(content): - return content.replace("region = \"us-west-2\"", "region = \"us-east-1\"") + return content.replace('region = "us-west-2"', 'region = "us-east-1"') transformer.add_callable_transformation(transform_func) transformer.transform_file(self.test_file) diff --git a/tests/unit/test_terraform_runner.py b/tests/unit/test_terraform_runner.py index 58b2e96d..8f16e5e5 100644 --- a/tests/unit/test_terraform_runner.py +++ b/tests/unit/test_terraform_runner.py @@ -11,7 +11,7 @@ class TestTerraformRunner(unittest.TestCase): def setUp(self): self.test_dir = tempfile.mkdtemp() # Mock git-related operations first - with patch('subprocess.run') as mock_subprocess: + with patch("subprocess.run") as mock_subprocess: # Make subprocess.run return success for git commands mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") self.runner = TerraformRunner(self.test_dir) @@ -20,95 +20,99 @@ def tearDown(self): shutil.rmtree(self.test_dir) @patch.object(TerraformRunner, "run_command") - @patch('subprocess.run') # Add this to mock any Git subprocess calls + @patch("subprocess.run") # Add this to mock any Git subprocess calls def test_run_command(self, mock_subprocess, mock_run_command): # Mock subprocess to avoid Git errors mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - + # Mock the run_command method directly mock_run_command.return_value = (True, "Success output", "log_file.log") - + # Call the method (which is now mocked) success, output, log_file = self.runner.run_command("version") - + # Assert the mock was called with the right arguments mock_run_command.assert_called_once_with("version") - + # Assert success self.assertTrue(success) self.assertEqual(output, "Success output") self.assertEqual(log_file, "log_file.log") @patch.object(TerraformRunner, "run_command") - @patch('subprocess.run') # Add this to mock any Git subprocess calls + @patch("subprocess.run") # Add this to mock any Git subprocess calls def test_init(self, mock_subprocess, mock_run_command): # Mock subprocess to avoid Git errors mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - + # Mock for init command mock_run_command.return_value = (True, "Terraform initialized", "init_log.log") - + # Run the method to test success, output, log_file = self.runner.init() - + # Assert the mock was called with the expected arguments mock_run_command.assert_called_once() self.assertEqual(mock_run_command.call_args[0][0], "init") - + # Assert success self.assertTrue(success) self.assertEqual(output, "Terraform initialized") self.assertEqual(log_file, "init_log.log") @patch.object(TerraformRunner, "run_command") - @patch('subprocess.run') # Add this to mock any Git subprocess calls + @patch("subprocess.run") # Add this to mock any Git subprocess calls def test_validate(self, mock_subprocess, mock_run_command): # Mock subprocess to avoid Git errors mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - + # Mock for validate command - mock_run_command.return_value = (True, "Success! Configuration is valid", "validate_log.log") - + mock_run_command.return_value = ( + True, + "Success! Configuration is valid", + "validate_log.log", + ) + # Run the method to test success, output, log_file = self.runner.validate() - + # Assert success self.assertTrue(success) self.assertEqual(output, "Success! Configuration is valid") self.assertEqual(log_file, "validate_log.log") @patch.object(TerraformRunner, "run_command") - @patch('subprocess.run') # Add this to mock any Git subprocess calls + @patch("subprocess.run") # Add this to mock any Git subprocess calls def test_plan(self, mock_subprocess, mock_run_command): # Mock subprocess to avoid Git errors mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - + # Mock for plan command mock_run_command.return_value = (True, "No changes needed", "plan_log.log") - + # Run the method to test success, output, log_file = self.runner.plan() - + # Assert success self.assertTrue(success) self.assertEqual(output, "No changes needed") self.assertEqual(log_file, "plan_log.log") @patch.object(TerraformRunner, "run_command") - @patch('subprocess.run') # Add this to mock any Git subprocess calls + @patch("subprocess.run") # Add this to mock any Git subprocess calls def test_specific_version(self, mock_subprocess, mock_run_command): # Mock subprocess to avoid Git errors mock_subprocess.return_value = MagicMock(returncode=0, stdout="test_output") - + # Mock for specific version mock_run_command.return_value = (True, "Terraform v0.13.7", "version_log.log") - + # Create a TerraformRunner instance with a specific version runner = TerraformRunner(self.test_dir, terraform_version="0.13") - + # Run the method (which uses the mocked run_command) success, output, log_file = runner.run_command("version") - + # Verify the result self.assertTrue(success) self.assertEqual(output, "Terraform v0.13.7") diff --git a/tests/validation/test_upgrade_validation.py b/tests/validation/test_upgrade_validation.py index 8bc4e7ef..c78700ef 100644 --- a/tests/validation/test_upgrade_validation.py +++ b/tests/validation/test_upgrade_validation.py @@ -193,64 +193,70 @@ def tearDown(self): @patch("tf_upgrade.utils.terraform.find_terraform_config_files") @patch("subprocess.run") - @patch.object(TerraformRunner, "run_command") - def test_simple_configuration_upgrade(self, mock_run_command, mock_subprocess_run, mock_find_config): + @patch.object(TerraformRunner, "run_command") + def test_simple_configuration_upgrade( + self, mock_run_command, mock_subprocess_run, mock_find_config + ): # Mock find_terraform_config_files to avoid git repository detection mock_find_config.return_value = { "tf_files": [os.path.join(self.simple_dir, "main.tf")], "tfvars_files": [], - "tf_control": None + "tf_control": None, } - + # Mock subprocess calls mock_subprocess_run.return_value = MagicMock(returncode=0, stdout=b"Success") mock_run_command.return_value = (True, "Success", "log_file.log") - + # Initialize controller with mocked upgrade method controller = UpgradeController(self.simple_dir) - with patch.object(controller, 'upgrade', return_value={"success": True}): + with patch.object(controller, "upgrade", return_value={"success": True}): result = controller.upgrade(target_version="1.0.0") self.assertTrue(result["success"]) @patch("tf_upgrade.utils.terraform.find_terraform_config_files") @patch("subprocess.run") @patch.object(TerraformRunner, "run_command") - def test_medium_configuration_upgrade(self, mock_run_command, mock_subprocess_run, mock_find_config): + def test_medium_configuration_upgrade( + self, mock_run_command, mock_subprocess_run, mock_find_config + ): # Mock find_terraform_config_files to avoid git repository detection mock_find_config.return_value = { "tf_files": [os.path.join(self.medium_dir, "main.tf")], "tfvars_files": [], - "tf_control": None + "tf_control": None, } - + # Mock subprocess calls mock_subprocess_run.return_value = MagicMock(returncode=0, stdout=b"Success") mock_run_command.return_value = (True, "Success", "log_file.log") - + # Initialize controller with mocked upgrade method controller = UpgradeController(self.medium_dir) - with patch.object(controller, 'upgrade', return_value={"success": True}): + with patch.object(controller, "upgrade", return_value={"success": True}): result = controller.upgrade(target_version="1.0.0") self.assertTrue(result["success"]) @patch("tf_upgrade.utils.terraform.find_terraform_config_files") @patch("subprocess.run") @patch.object(TerraformRunner, "run_command") - def test_complex_configuration_upgrade(self, mock_run_command, mock_subprocess_run, mock_find_config): + def test_complex_configuration_upgrade( + self, mock_run_command, mock_subprocess_run, mock_find_config + ): # Mock find_terraform_config_files to avoid git repository detection mock_find_config.return_value = { "tf_files": [os.path.join(self.complex_dir, "main.tf")], "tfvars_files": [], - "tf_control": None + "tf_control": None, } - + # Mock subprocess calls mock_subprocess_run.return_value = MagicMock(returncode=0, stdout=b"Success") mock_run_command.return_value = (True, "Success", "log_file.log") - + # Initialize controller with mocked upgrade method controller = UpgradeController(self.complex_dir) - with patch.object(controller, 'upgrade', return_value={"success": True}): + with patch.object(controller, "upgrade", return_value={"success": True}): result = controller.upgrade(target_version="1.0.0") self.assertTrue(result["success"]) diff --git a/tf_upgrade/cli.py b/tf_upgrade/cli.py index 20d8bb59..7b3b0c51 100644 --- a/tf_upgrade/cli.py +++ b/tf_upgrade/cli.py @@ -98,12 +98,17 @@ def cli(verbose, debug, aws_profile): @cli.command() @click.argument("directory", type=click.Path(exists=True), required=False) -@click.option("--recursive", "-r", is_flag=True, help="Scan directories recursively") +@click.option( + "--recursive/--no-recursive", + "-r", + default=True, + help="Scan directories recursively (enabled by default)", +) @click.option( "--max-depth", "-d", type=int, - default=10, + default=20, help="Maximum directory depth for recursive scans", ) @click.option( @@ -123,7 +128,13 @@ def scan(directory, recursive, max_depth, parallel, output, format): """Scan for Terraform configurations requiring upgrades""" directory = directory or os.getcwd() - click.echo(f"Scanning directory: {directory} {'recursively' if recursive else ''}") + # Make the recursive behavior more explicit in the output + if recursive: + click.echo( + f"Scanning directory recursively: {directory} (max depth: {max_depth})" + ) + else: + click.echo(f"Scanning directory (non-recursive): {directory}") try: results = scan_command( @@ -134,6 +145,7 @@ def scan(directory, recursive, max_depth, parallel, output, format): module_count = len(results.get("module_dirs", [])) error_count = len(results.get("errors", [])) + # Add more details in the output click.echo( f"\nFound {terraform_count} Terraform configurations" f" ({module_count} modules)" diff --git a/tf_upgrade/scanner.py b/tf_upgrade/scanner.py index 255bcfea..d4944794 100644 --- a/tf_upgrade/scanner.py +++ b/tf_upgrade/scanner.py @@ -6,6 +6,7 @@ import logging import os import re +from pprint import pformat from tf_upgrade.config import ConfigManager @@ -15,7 +16,7 @@ class TerraformScanner: """Scanner for identifying and analyzing Terraform configurations.""" - def __init__(self, config_manager=None, max_depth=10): + def __init__(self, config_manager=None, max_depth=20): """ Initialize the scanner with configuration. @@ -25,6 +26,17 @@ def __init__(self, config_manager=None, max_depth=10): """ self.config = config_manager or ConfigManager() self.max_depth = max_depth + # Add common ignored directories + self.ignored_dirs = [ + ".git", + ".terraform", + "node_modules", + ".ci", + ".github", + "logs", + ] + if self.config.get("scanner.ignore_dirs"): + self.ignored_dirs.extend(self.config.get("scanner.ignore_dirs")) def scan_directory(self, root_dir): """ @@ -43,36 +55,61 @@ def scan_directory(self, root_dir): "backend_configs": {}, "errors": [], "warnings": [], + "scanned_dirs": 0, # Add counter for total directories scanned } + logger.info(f"Starting scan of {root_dir} with max depth {self.max_depth}") self._scan_recursive(root_dir, results, 0) + logger.info( + f"Scan complete. Found {len(results['terraform_dirs'])} Terraform configurations in {results['scanned_dirs']} directories" + ) + + # Add a detailed breakdown for better visibility + if len(results["terraform_dirs"]) > 0: + logger.debug( + f"Terraform directories found:\n{pformat([d['path'] for d in results['terraform_dirs']])}" + ) + return results def _scan_recursive(self, current_dir, results, depth): """Recursively scan directories.""" + results["scanned_dirs"] += 1 # Increment the counter for each directory visited + if depth > self.max_depth: results["warnings"].append( f"Max depth {self.max_depth} reached at {current_dir}" ) return - # Skip .git, .terraform directories - if os.path.basename(current_dir) in [".git", ".terraform", "node_modules"]: + # Skip ignored directories + if os.path.basename(current_dir) in self.ignored_dirs: + logger.debug(f"Skipping ignored directory: {current_dir}") return + # Add debug message to show the recursion progression + logger.debug(f"Scanning at depth {depth}: {current_dir}") + # Check if current directory contains Terraform files try: - tf_files = [f for f in os.listdir(current_dir) if f.endswith(".tf")] + dir_items = os.listdir(current_dir) + tf_files = [f for f in dir_items if f.endswith(".tf")] + + # Log what we're examining to help with troubleshooting + if tf_files: + logger.debug(f"Found {len(tf_files)} .tf files in: {current_dir}") if tf_files: # This is a terraform directory try: dir_result = self._analyze_directory(current_dir, tf_files) results["terraform_dirs"].append(dir_result) + logger.debug(f"Added as terraform directory: {current_dir}") # Check if it's a module if dir_result.get("is_module", False): results["module_dirs"].append(dir_result) + logger.debug(f"Identified as module: {current_dir}") # Collect provider configurations for provider, version in dir_result.get("providers", {}).items(): @@ -92,19 +129,35 @@ def _scan_recursive(self, current_dir, results, depth): ) except Exception as e: + error_msg = f"Error analyzing {current_dir}: {str(e)}" + logger.error(error_msg) results["errors"].append( - {"directory": current_dir, "error": str(e)} + {"directory": current_dir, "error": error_msg} ) - # Scan subdirectories - for item in os.listdir(current_dir): - item_path = os.path.join(current_dir, item) - if os.path.isdir(item_path): - self._scan_recursive(item_path, results, depth + 1) + # Scan subdirectories - make sure we're recursing + subdirs = [ + os.path.join(current_dir, item) + for item in dir_items + if os.path.isdir(os.path.join(current_dir, item)) + ] + + if subdirs: + logger.debug( + f"Found {len(subdirs)} subdirectories to scan in {current_dir}" + ) + + for subdir in subdirs: + # Skip ignored directories one more time (in case basename check isn't sufficient) + if os.path.basename(subdir) in self.ignored_dirs: + continue + + self._scan_recursive(subdir, results, depth + 1) + except (PermissionError, FileNotFoundError) as e: - results["errors"].append( - {"directory": current_dir, "error": f"Access error: {str(e)}"} - ) + error_msg = f"Access error for {current_dir}: {str(e)}" + logger.error(error_msg) + results["errors"].append({"directory": current_dir, "error": error_msg}) def _analyze_directory(self, directory, tf_files): """ @@ -131,28 +184,38 @@ def _analyze_directory(self, directory, tf_files): "complexity": 0, } - # Check if it's a module by looking for variables.tf, outputs.tf - result["has_variables"] = "variables.tf" in tf_files - result["has_outputs"] = "outputs.tf" in tf_files - result["is_module"] = result["has_variables"] or result["has_outputs"] + # Check if it's a module by looking for variables.tf, outputs.tf or other indicators + result["has_variables"] = any( + f in tf_files for f in ["variables.tf", "vars.tf"] + ) + result["has_outputs"] = any(f in tf_files for f in ["outputs.tf", "output.tf"]) + + # Also check for main.tf as an additional indicator + has_main = "main.tf" in tf_files + + # Consider it a module if it has variables, outputs, or main.tf + result["is_module"] = ( + result["has_variables"] or result["has_outputs"] or has_main + ) # Look for version constraints, providers, and resources for tf_file in tf_files: file_path = os.path.join(directory, tf_file) try: - with open(file_path, "r") as f: + with open(file_path, "r", encoding="utf-8", errors="replace") as f: content = f.read() # Extract terraform version constraints version_match = re.search( - r'terraform\s*{[^}]*required_version\s*=\s*["\']' r'([^"\']+)["\']', + r'terraform\s*{[^}]*required_version\s*=\s*["\']([^"\']+)["\']', content, re.DOTALL, ) if version_match and not result["version"]: result["version"] = version_match.group(1) - # Extract providers + # Extract providers from both older style and newer style declarations + # Older style: provider "aws" { ... } provider_matches = re.finditer( r'provider\s+"([^"]+)"\s*{([^}]*)}', content, re.DOTALL ) @@ -167,6 +230,24 @@ def _analyze_directory(self, directory, tf_files): else: result["providers"][provider_name] = None + # Newer style: required_providers block + required_block_match = re.search( + r"required_providers\s*{([^}]*)}", content, re.DOTALL + ) + if required_block_match: + required_block = required_block_match.group(1) + provider_entries = re.finditer( + r"([a-z0-9_]+)\s*=\s*{([^}]*)}", required_block, re.DOTALL + ) + for entry in provider_entries: + provider_name = entry.group(1) + provider_config = entry.group(2) + version_match = re.search( + r'version\s*=\s*["\']([^"\']+)["\']', provider_config + ) + if version_match: + result["providers"][provider_name] = version_match.group(1) + # Count resources result["resource_count"] += len( re.findall(r'resource\s+"[^"]+"\s+"[^"]+"\s*{', content) @@ -272,7 +353,7 @@ def export_to_csv(self, results, output_file): def scan_command( - dir_path, recursive=True, max_depth=10, parallel=True, output_format="json" + dir_path, recursive=True, max_depth=20, parallel=True, output_format="json" ): """ Command function for scanning directories. @@ -293,88 +374,43 @@ def scan_command( if not os.path.isdir(dir_path): raise ValueError(f"Not a directory: {dir_path}") + # Print basic information before scanning + logger.info( + f"Scanning directory: {dir_path} (recursive={recursive}, max_depth={max_depth})" + ) + if not recursive: # Just scan the single directory + tf_files = [f for f in os.listdir(dir_path) if f.endswith(".tf")] + logger.info(f"Found {len(tf_files)} .tf files in {dir_path}") + results = { - "terraform_dirs": [ - scanner._analyze_directory( - dir_path, [f for f in os.listdir(dir_path) if f.endswith(".tf")] - ) - ] + "terraform_dirs": ( + [scanner._analyze_directory(dir_path, tf_files)] if tf_files else [] + ), + "module_dirs": [], + "scanned_dirs": 1, # Just scanned one directory } return results else: - if parallel and config.get("parallel.scan.enabled", True): - max_workers = config.get("parallel.scan.max_workers", 10) - - # Get all subdirectories first - all_dirs = [] - for root, dirs, files in os.walk(dir_path): - depth = root[len(dir_path) :].count(os.path.sep) - if depth <= max_depth: - # Skip .git and .terraform directories - dirs[:] = [ - d - for d in dirs - if d not in [".git", ".terraform", "node_modules"] - ] - if any(f.endswith(".tf") for f in files): - all_dirs.append(root) - - # Use the parallel implementation - from tf_upgrade.utils.parallel import parallel_scan_directories - - # Create a function that scans a single directory - def scan_single_dir(d): - try: - tf_files = [f for f in os.listdir(d) if f.endswith(".tf")] - return scanner._analyze_directory(d, tf_files) - except Exception as e: - return {"error": str(e), "path": d} + # Use either parallel or sequential scanning + results = scanner.scan_directory(dir_path) + + # Print a more informative summary + logger.info(f"\nSummary:") + logger.info(f" Total directories scanned: {results.get('scanned_dirs', 0)}") + logger.info(f" Total Terraform directories: {len(results['terraform_dirs'])}") + logger.info(f" Module directories: {len(results['module_dirs'])}") + + # Add more details for debugging + if results["errors"]: + logger.warning(f" Errors encountered: {len(results['errors'])}") + for error in results["errors"][:5]: # Show first 5 errors + logger.warning(f" - {error['directory']}: {error['error']}") + if len(results["errors"]) > 5: + logger.warning(f" ... and {len(results['errors'])-5} more errors") + + if results["warnings"]: + logger.info(f" Warnings: {len(results['warnings'])}") - # Scan all directories in parallel - dir_results = parallel_scan_directories( - all_dirs, scan_single_dir, max_workers - ) - - # Combine results - combined_results = { - "terraform_dirs": [], - "module_dirs": [], - "provider_configs": {}, - "backend_configs": {}, - "errors": [], - "warnings": [], - } - - for dir_path, result in dir_results.items(): - if "error" in result: - combined_results["errors"].append( - {"directory": dir_path, "error": result["error"]} - ) - else: - combined_results["terraform_dirs"].append(result) - if result.get("is_module", False): - combined_results["module_dirs"].append(result) - - # Collect provider configurations - for provider, version in result.get("providers", {}).items(): - if provider not in combined_results["provider_configs"]: - combined_results["provider_configs"][provider] = [] - combined_results["provider_configs"][provider].append( - {"version": version, "directory": dir_path} - ) - - # Collect backend configurations - if "backend" in result: - backend_type = result["backend"].get("type") - if backend_type not in combined_results["backend_configs"]: - combined_results["backend_configs"][backend_type] = [] - combined_results["backend_configs"][backend_type].append( - {"directory": dir_path, "config": result["backend"]} - ) - - return combined_results - else: - # Use the recursive scanner - return scanner.scan_directory(dir_path) + return results diff --git a/verification-results.log b/verification-results.log deleted file mode 100644 index 39120479..00000000 --- a/verification-results.log +++ /dev/null @@ -1,138 +0,0 @@ -Verifying environment... -Verifying environment for directory: /data/terraform/workspaces/morga471/terraform/support/local-app/python-tools/terraform-upgrade-tool - -=== ENVIRONMENT VERIFICATION RESULTS === - -Terraform Versions: - 0.12: True - 0.13: True - 0.14: True - 0.15: True - 1.0: False - 1.10: True - -Terraform Config Files: - -AWS Profile: - Profile validation failed for 'None' - -Git Repository Access: - Not a Git repository - -Overall Status: - ❌ Environment verification FAILED - -Warnings: - - No Terraform configuration files found in directory - - AWS profile validation failed for 'None' - - Not in a Git repository - - No .terraform directory found, terraform init may be required - - AWS profile validation failed for 'None' - - AWS profile validation failed for profile 'None' - -Errors: - - Missing Terraform versions: 1.0 - - No Terraform configuration files found -Verifying scanning functionality... -Scanning directory: tests/fixtures/0.12/simple - -Found 1 Terraform configurations (0 modules) - -Summary: - Total Terraform directories: 1 - Module directories: 0 - -Version detection results for tests/fixtures/0.12/simple: - Version constraint: None - Detected version: 0.12 - Requires upgrade: Yes - Upgrade path: 0.13 → 0.14 → 0.15 → 1.0 - -Feature detection: - 0.12 features: 0 - 0.13 features: 0 - 0.14 features: 0 - 0.15 features: 0 - -Found 2 deprecated syntax: - - interpolation_only: "${aws_s3_bucket.simple.id}" (in main.tf:12) - - quoted_interpolation: "${aws_s3_bucket.simple.id}" (in main.tf:12) - -Complexity analysis for tests/fixtures/0.12/simple: - Complexity score: 8.5 - Risk level: Low - -Metrics: - Resources: 1 - Data sources: 0 - Modules: 0 - Variables: 0 - Outputs: 1 - -Advanced metrics: - Dynamic blocks: 0 - Count usage: 0 - For_each usage: 0 - Conditional expressions: 0 - -Deprecated syntax (2 instances): - Interpolation Only: 1 - Quoted Interpolation: 1 - -Examples: - - interpolation_only: "${aws_s3_bucket.simple.id}" in main.tf - - quoted_interpolation: "${aws_s3_bucket.simple.id}" in main.tf -Verifying dry run functionality... -Performing dry run for directory: tests/fixtures/0.12/simple - -Current Terraform version: 0.12 -Upgrade path: 0.13 → 0.14 → 0.15 → 1.0 -Complexity level: Low - -Analyzing upgrade steps: - -=== Upgrade to 0.13 === - ✓ Configuration ready for upgrade - - requires_provider_updates: True - - providers_to_migrate: ['aws'] - • Syntax changes that would be made: - - Total changes: 2 - - Files modified: 2 - • Built-in commands that would be executed: - - terraform init - - terraform 0.13upgrade -yes - -=== Upgrade to 0.14 === - ✓ Configuration ready for upgrade - • Syntax changes that would be made: - - Total changes: 1 - - Files modified: 1 - • Built-in commands that would be executed: - - terraform init -upgrade - -=== Upgrade to 0.15 === - ✓ Configuration ready for upgrade - • Syntax changes that would be made: - - Total changes: 1 - - Files modified: 1 - - Convert deprecated interpolation-only expressions: 1 changes - • Built-in commands that would be executed: - - terraform init -upgrade - -=== Upgrade to 1.0 === - ✓ Configuration ready for upgrade - • Syntax changes that would be made: - - No syntax changes needed - • Built-in commands that would be executed: - - terraform init -upgrade - -Dry run completed. Report saved to tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md -Verifying backup functionality... -Upgrading directory to 0.13: tests/fixtures/0.12/backup-test -🔄 [ 0%] Starting: Upgrading to Terraform 0.13... -❌ [100%] Completed: Upgrading to Terraform 0.13 - Est. remaining: 0s - Failed to upgrade to Terraform 0.13 -❌ Operation Terraform upgrade to 0.13 failed in 2s: 1/1 steps completed, 1 failed - -❌ Upgrade failed. See report for details. -Detailed report saved to: tests/fixtures/0.12/backup-test/terraform-upgrade-reports/upgrade-report-20250327-123415.md