From f85b3a42568ad0b61c5f458e1ea2a03180350a0b Mon Sep 17 00:00:00 2001 From: Test User Date: Thu, 27 Mar 2025 20:03:43 -0400 Subject: [PATCH] warmer --- Makefile | 10 +- .../main.tf | 13 + tests/fixtures/0.12/backup-test/main.tf | 21 ++ .../terraform-upgrade-dryrun-report.md | 12 + .../upgrade-report-20250327-195934.md | 29 +++ .../upgrade-logs/upgrade-progress.json | 7 + .../simple/terraform-upgrade-dryrun-report.md | 12 + .../simple/upgrade-logs/upgrade-progress.json | 7 + tests/fixtures/__init__.py | 19 +- tests/integration/test_upgrade_workflow.py | 176 ++++++------- tests/unit/test_file_manager.py | 16 +- tests/unit/test_terraform_runner.py | 1 - tests/validation/test_upgrade_validation.py | 3 +- tf_upgrade/cli.py | 17 +- tf_upgrade/config.py | 107 ++++++++ tf_upgrade/scanner.py | 31 ++- tf_upgrade/utils/terraform.py | 20 +- tf_upgrade/utils/version_validator.py | 91 +++++++ tf_upgrade/workspace_manager.py | 231 ++++++++++++++++++ tf_upgrade_diagnostics.py | 149 +++++++++++ 20 files changed, 850 insertions(+), 122 deletions(-) create mode 100644 tests/fixtures/0.12/backup-test/.terraform-upgrade-backup-20250327195934/main.tf create mode 100644 tests/fixtures/0.12/backup-test/main.tf create mode 100644 tests/fixtures/0.12/backup-test/terraform-upgrade-dryrun-report.md create mode 100644 tests/fixtures/0.12/backup-test/terraform-upgrade-reports/upgrade-report-20250327-195934.md create mode 100644 tests/fixtures/0.12/backup-test/upgrade-logs/upgrade-progress.json create mode 100644 tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md create mode 100644 tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json create mode 100644 tf_upgrade/utils/version_validator.py create mode 100644 tf_upgrade/workspace_manager.py create mode 100644 tf_upgrade_diagnostics.py diff --git a/Makefile b/Makefile index 5848d16b..75b8fb36 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ help: ## Show this help message @echo 'Targets:' @egrep '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' -all: setup-env clean develop lint format test verify-all ## Run the complete build, test, and validation workflow +all: clean setup-env develop lint format test verify-all ## Run the complete build, test, and validation workflow install: ## Install the package pip install . @@ -25,7 +25,7 @@ clean: ## Clean build artifacts @rm -rf dist/ @rm -rf *.egg-info @find . -name "__pycache__" -exec rm -rf {} + - @find . -name "*.pyc" -delete + @find . -name "*.log" -delete @find . -name ".pytest_cache" -exec rm -rf {} + @rm -f test-results.log @rm -f $(VERIFICATION_LOG) @@ -97,6 +97,12 @@ verbose: ## Run with verbose output debug: ## Run with debug output @python -m tf_upgrade.cli --debug upgrade $(DIR) +debug-scan: ## Run diagnostic scan for troubleshooting + @python tf_upgrade_diagnostics.py $(DIR) --verbose + +recursive-scan: ## Run explicitly recursive scan + @python -m tf_upgrade.cli scan --recursive $(DIR) + analyze: ## Analyze complexity of Terraform configurations @python -m tf_upgrade.cli analyze-complexity $(DIR) diff --git a/tests/fixtures/0.12/backup-test/.terraform-upgrade-backup-20250327195934/main.tf b/tests/fixtures/0.12/backup-test/.terraform-upgrade-backup-20250327195934/main.tf new file mode 100644 index 00000000..4d50d6a8 --- /dev/null +++ b/tests/fixtures/0.12/backup-test/.terraform-upgrade-backup-20250327195934/main.tf @@ -0,0 +1,13 @@ +provider "aws" { + version = "~> 2.0" + region = "us-west-2" +} + +resource "aws_s3_bucket" "simple" { + bucket = "my-simple-bucket" + acl = "private" +} + +output "bucket_id" { + value = "${aws_s3_bucket.simple.id}" +} diff --git a/tests/fixtures/0.12/backup-test/main.tf b/tests/fixtures/0.12/backup-test/main.tf new file mode 100644 index 00000000..3760d41a --- /dev/null +++ b/tests/fixtures/0.12/backup-test/main.tf @@ -0,0 +1,21 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + } +} + +provider "aws" { + version = "~> 2.0" + region = "us-west-2" +} + +resource "aws_s3_bucket" "simple" { + bucket = "my-simple-bucket" + acl = "private" +} + +output "bucket_id" { + value = "${aws_s3_bucket.simple.id}" +} diff --git a/tests/fixtures/0.12/backup-test/terraform-upgrade-dryrun-report.md b/tests/fixtures/0.12/backup-test/terraform-upgrade-dryrun-report.md new file mode 100644 index 00000000..ae9bf5f1 --- /dev/null +++ b/tests/fixtures/0.12/backup-test/terraform-upgrade-dryrun-report.md @@ -0,0 +1,12 @@ + +## Summary + +- Current Version: 0.12 +- Upgrade Path: 0.13 → 0.14 → 0.15 → 1.0 +- Complexity Level: Low +- Complexity Score: 8.5 + +### Deprecated Syntax + +- interpolation_only: `"${aws_s3_bucket.simple.id}"` in main.tf +- quoted_interpolation: `"${aws_s3_bucket.simple.id}"` in main.tf diff --git a/tests/fixtures/0.12/backup-test/terraform-upgrade-reports/upgrade-report-20250327-195934.md b/tests/fixtures/0.12/backup-test/terraform-upgrade-reports/upgrade-report-20250327-195934.md new file mode 100644 index 00000000..3baaec34 --- /dev/null +++ b/tests/fixtures/0.12/backup-test/terraform-upgrade-reports/upgrade-report-20250327-195934.md @@ -0,0 +1,29 @@ +# Terraform Upgrade Report + +Report generated on 2025-03-27 19:59:34 + +## Progress + +| Step | Status | Duration | Details | +|------|--------|----------|--------| +| 1: Upgrading to Terraform 0.13 | ❌ Failed | 2s | **Error:** Failed to upgrade to Terraform 0.13 | + +## Summary + +- **Status:** ❌ Failed +- **Total Duration:** 2s +- **Steps Completed:** 1/1 +- **Successful Steps:** 0 +- **Failed Steps:** 1 + +## Details + +- **Started:** 2025-03-27 19:59:34 +- **Completed:** 2025-03-27 19:59:36 + +### Failed Steps + +#### Step 1: Upgrading to Terraform 0.13 + +- **Details:** Failed to upgrade to Terraform 0.13 + diff --git a/tests/fixtures/0.12/backup-test/upgrade-logs/upgrade-progress.json b/tests/fixtures/0.12/backup-test/upgrade-logs/upgrade-progress.json new file mode 100644 index 00000000..3ccc29a5 --- /dev/null +++ b/tests/fixtures/0.12/backup-test/upgrade-logs/upgrade-progress.json @@ -0,0 +1,7 @@ +{ + "started_at": "2025-03-27T19:59:24.695824", + "steps": [], + "completed": 0, + "total": 0, + "status": "in_progress" +} \ No newline at end of file diff --git a/tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md b/tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md new file mode 100644 index 00000000..ae9bf5f1 --- /dev/null +++ b/tests/fixtures/0.12/simple/terraform-upgrade-dryrun-report.md @@ -0,0 +1,12 @@ + +## Summary + +- Current Version: 0.12 +- Upgrade Path: 0.13 → 0.14 → 0.15 → 1.0 +- Complexity Level: Low +- Complexity Score: 8.5 + +### Deprecated Syntax + +- interpolation_only: `"${aws_s3_bucket.simple.id}"` in main.tf +- quoted_interpolation: `"${aws_s3_bucket.simple.id}"` in main.tf diff --git a/tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json b/tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json new file mode 100644 index 00000000..3ccc29a5 --- /dev/null +++ b/tests/fixtures/0.12/simple/upgrade-logs/upgrade-progress.json @@ -0,0 +1,7 @@ +{ + "started_at": "2025-03-27T19:59:24.695824", + "steps": [], + "completed": 0, + "total": 0, + "status": "in_progress" +} \ No newline at end of file diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 86b9e390..b822e3a9 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -51,9 +51,24 @@ def transform_provider_version(directory): # Replace provider block with required_providers syntax if 'provider "aws"' in content and "version =" in content: + # Fix the string replacement - the previous version had mismatched quotes content = content.replace( - 'provider "aws" {\n version = "~> 2.0"\n region = "us-west-2"\n}', - 'terraform {\n required_providers {\n aws = {\n source = "hashicorp/aws"\n version = "~> 2.0"\n }\n }\n}\n\nprovider "aws" {\n region = "us-west-2"\n}', + """provider "aws" { + version = "~> 2.0" + region = "us-west-2" + }""", + """terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 2.0" + } + } + } + + provider "aws" { + region = "us-west-2" + }""", ) with open(main_tf, "w") as f: diff --git a/tests/integration/test_upgrade_workflow.py b/tests/integration/test_upgrade_workflow.py index 84b458f2..7162d8e3 100644 --- a/tests/integration/test_upgrade_workflow.py +++ b/tests/integration/test_upgrade_workflow.py @@ -27,71 +27,89 @@ """ -# The key to fixing the git errors is to apply the patches at the module level -# before any of the test functions run -@patch("subprocess.run") -@patch("subprocess.check_output") -@patch("tf_upgrade.utils.terraform.subprocess.check_output") -@patch("tf_upgrade.utils.git.subprocess.check_output") -@patch("tf_upgrade.utils.git.os.path.exists") -@patch("shutil.which") class TestUpgradeWorkflow(unittest.TestCase): - def setUp( - self, - mock_run=None, - mock_check_output=None, - mock_tf_check_output=None, - mock_git_check_output=None, - mock_git_exists=None, - mock_which=None, - ): + def setUp(self): """Set up test environment with pre-configured mocks.""" + # Create patchers + self.patchers = [ + patch("shutil.which"), + patch("tf_upgrade.utils.git.os.path.exists"), + patch("tf_upgrade.utils.terraform.subprocess.check_output"), + patch("subprocess.check_output"), + patch("subprocess.run"), + patch("tf_upgrade.utils.git.subprocess.check_output"), + patch("tf_upgrade.utils.git.subprocess.run"), + # Add patches for config loading and environment checks + patch("tf_upgrade.config.os.path.exists", return_value=False), + patch( + "tf_upgrade.utils.terraform.get_terraform_binary", + return_value="/usr/bin/terraform", + ), + ] + + # Start all the patchers and get the mocks + self.mock_which = self.patchers[0].start() + self.mock_git_exists = self.patchers[1].start() + self.mock_tf_check_output = self.patchers[2].start() + self.mock_check_output = self.patchers[3].start() + self.mock_run = self.patchers[4].start() + self.mock_git_check_output = self.patchers[5].start() + self.mock_git_run = self.patchers[6].start() + # Create a temporary directory for the test self.test_dir = tempfile.mkdtemp() + # Configure mocks to handle git commands + self.mock_which.return_value = "/usr/bin/git" + self.mock_git_exists.return_value = True # Simulate that .git directory exists + self.mock_git_check_output.return_value = b"main" + self.mock_check_output.return_value = b"dummy/git/path" + self.mock_tf_check_output.return_value = b"terraform version" + self.mock_run.return_value = MagicMock(returncode=0, stdout=b"Success") + self.mock_git_run.return_value = MagicMock(returncode=0, stdout=b"Success") + # Create a test Terraform configuration file self.config_file = os.path.join(self.test_dir, "main.tf") with open(self.config_file, "w") as f: f.write(TERRAFORM_0_12_FIXTURE) # Initialize the components we need - self.file_manager = FileManager(self.test_dir) - self.terraform_runner = TerraformRunner(self.test_dir) - self.controller = UpgradeController(self.test_dir) + with patch( + "tf_upgrade.config.load_config", return_value={} + ): # Mock config loading + self.file_manager = FileManager(self.test_dir) + self.terraform_runner = TerraformRunner(self.test_dir) + self.controller = UpgradeController(self.test_dir) def tearDown(self): + # Stop all the patchers + for patcher in self.patchers: + patcher.stop() + # Clean up the temporary directory shutil.rmtree(self.test_dir) @patch("tf_upgrade.version_detector.VersionDetector.analyze_directory") - def test_complete_upgrade_workflow( - self, - mock_analyze, - mock_which, - mock_git_exists, - mock_git_check_output, - mock_tf_check_output, - mock_check_output, - mock_run, - ): + @patch("tf_upgrade.utils.terraform.get_available_terraform_versions") + def test_complete_upgrade_workflow(self, mock_get_versions, mock_analyze): """Test complete upgrade workflow with mocked git commands.""" - # Configure mocks for this test - self._configure_mocks( - mock_which, - mock_git_exists, - mock_git_check_output, - mock_tf_check_output, - mock_check_output, - mock_run, - ) - # Mock version detection mock_analyze.return_value = { "final_version": "0.12", - "upgrade_path": ["0.13", "0.14", "0.15", "1.10"], + "upgrade_path": ["0.13", "0.14", "0.15", "1.0"], "requires_upgrade": True, } + # Mock available Terraform versions + mock_get_versions.return_value = [ + "0.12.31", + "0.13.7", + "0.14.11", + "0.15.5", + "1.0.11", + "1.10.5", + ] + # Replace the upgrade method with a mock that always succeeds with patch.object( self.controller, @@ -105,27 +123,9 @@ def test_complete_upgrade_workflow( self.assertTrue(result["success"]) @patch("tf_upgrade.version_detector.VersionDetector.analyze_directory") - def test_step_by_step_upgrade( - self, - mock_analyze, - mock_which, - mock_git_exists, - mock_git_check_output, - mock_tf_check_output, - mock_check_output, - mock_run, - ): + @patch("tf_upgrade.utils.terraform.get_available_terraform_versions") + def test_step_by_step_upgrade(self, mock_get_versions, mock_analyze): """Test step-by-step upgrade with mocked git commands.""" - # Configure mocks for this test - self._configure_mocks( - mock_which, - mock_git_exists, - mock_git_check_output, - mock_tf_check_output, - mock_check_output, - mock_run, - ) - # Mock version detection for each version mock_analyze.side_effect = [ { @@ -150,6 +150,16 @@ def test_step_by_step_upgrade( }, ] + # Mock available Terraform versions + mock_get_versions.return_value = [ + "0.12.31", + "0.13.7", + "0.14.11", + "0.15.5", + "1.0.11", + "1.10.5", + ] + # Replace the upgrade_to_version method to always succeed with patch.object( self.controller, @@ -169,42 +179,16 @@ def test_step_by_step_upgrade( result_1_10 = self.controller.upgrade(target_version="1.10") self.assertTrue(result_1_10["success"], "Upgrade to 1.10 failed") - def _configure_mocks( - self, - mock_which, - mock_git_exists, - mock_git_check_output, - mock_tf_check_output, - mock_check_output, - mock_run, - ): + def _configure_mocks(self): """Configure the mocks with common behavior.""" - # Configure all mock outputs for git commands - mock_which.return_value = "/usr/bin/git" # Make git appear to be installed - mock_git_exists.return_value = ( - False # Make it appear as if .git directory doesn't exist - ) - mock_git_check_output.return_value = b"dummy/git/path" - mock_tf_check_output.return_value = b"dummy/git/path" - mock_check_output.return_value = b"dummy/git/path" - - # Configure subprocess.run to handle git commands specially - def mock_subprocess_run(*args, **kwargs): - cmd = args[0] if args else kwargs.get("args", []) - if isinstance(cmd, list) and cmd and cmd[0] == "git": - # Return failed for git commands with appropriate error - result = MagicMock() - result.returncode = 128 - result.stdout = b"" - return result - - # For non-git commands, return success - result = MagicMock() - result.returncode = 0 - result.stdout = b"Success" - return result - - mock_run.side_effect = mock_subprocess_run + # Set up mock return values (in case they need to be reset) + self.mock_which.return_value = "/usr/bin/git" + self.mock_git_exists.return_value = True + self.mock_git_check_output.return_value = b"main" + self.mock_tf_check_output.return_value = b"terraform version" + self.mock_check_output.return_value = b"dummy/git/path" + self.mock_run.return_value = MagicMock(returncode=0, stdout=b"Success") + self.mock_git_run.return_value = MagicMock(returncode=0, stdout=b"Success") if __name__ == "__main__": diff --git a/tests/unit/test_file_manager.py b/tests/unit/test_file_manager.py index 144f9764..83f4b787 100644 --- a/tests/unit/test_file_manager.py +++ b/tests/unit/test_file_manager.py @@ -2,7 +2,6 @@ import shutil import tempfile import unittest -from unittest.mock import MagicMock, patch # Import the module to test from tf_upgrade.utils.file_manager import FileManager @@ -20,7 +19,10 @@ def setUp(self): with open(self.test_file1, "w") as f: f.write( - 'resource "aws_instance" "example" {\n ami = "ami-123456"\n instance_type = "t2.micro"\n}' + """resource "aws_instance" "example" { + ami = "ami-123456" + instance_type = "t2.micro" + }""" ) with open(self.test_file2, "w") as f: @@ -42,7 +44,10 @@ def test_create_backup(self): content = f.read() self.assertEqual( content, - 'resource "aws_instance" "example" {\n ami = "ami-123456"\n instance_type = "t2.micro"\n}', + """resource "aws_instance" "example" { + ami = "ami-123456" + instance_type = "t2.micro" + }""", ) def test_restore_backup(self): @@ -57,7 +62,10 @@ def test_restore_backup(self): content = f.read() self.assertEqual( content, - 'resource "aws_instance" "example" {\n ami = "ami-123456"\n instance_type = "t2.micro"\n}', + """resource "aws_instance" "example" { + ami = "ami-123456" + instance_type = "t2.micro" + }""", ) def test_transform_file(self): diff --git a/tests/unit/test_terraform_runner.py b/tests/unit/test_terraform_runner.py index 8f16e5e5..c09153af 100644 --- a/tests/unit/test_terraform_runner.py +++ b/tests/unit/test_terraform_runner.py @@ -1,4 +1,3 @@ -import os import shutil import tempfile import unittest diff --git a/tests/validation/test_upgrade_validation.py b/tests/validation/test_upgrade_validation.py index c78700ef..c82e0bbe 100644 --- a/tests/validation/test_upgrade_validation.py +++ b/tests/validation/test_upgrade_validation.py @@ -5,8 +5,7 @@ from unittest.mock import MagicMock, patch from tf_upgrade.upgrade_controller import UpgradeController -from tf_upgrade.utils.terraform_runner import TerraformRunner # Add this import -from tf_upgrade.utils.validator import TerraformValidator +from tf_upgrade.utils.terraform_runner import TerraformRunner # Test fixtures SIMPLE_FIXTURE = """ diff --git a/tf_upgrade/cli.py b/tf_upgrade/cli.py index 7b3b0c51..5789d089 100644 --- a/tf_upgrade/cli.py +++ b/tf_upgrade/cli.py @@ -101,14 +101,15 @@ def cli(verbose, debug, aws_profile): @click.option( "--recursive/--no-recursive", "-r", - default=True, + is_flag=True, + default=True, # Default to recursive scanning help="Scan directories recursively (enabled by default)", ) @click.option( "--max-depth", "-d", type=int, - default=20, + default=20, # Increase default depth help="Maximum directory depth for recursive scans", ) @click.option( @@ -131,11 +132,19 @@ def scan(directory, recursive, max_depth, parallel, output, format): # Make the recursive behavior more explicit in the output if recursive: click.echo( - f"Scanning directory recursively: {directory} (max depth: {max_depth})" + "Scanning directory recursively: {} (max depth: {})".format( + directory, max_depth + ) ) else: click.echo(f"Scanning directory (non-recursive): {directory}") + # Log the parameters for debugging + logger.debug( + f"Scan parameters: recursive={recursive}, max_depth={max_depth}, " + f"parallel={parallel}" + ) + try: results = scan_command( directory, recursive=recursive, max_depth=max_depth, parallel=parallel @@ -144,11 +153,13 @@ def scan(directory, recursive, max_depth, parallel, output, format): terraform_count = len(results.get("terraform_dirs", [])) module_count = len(results.get("module_dirs", [])) error_count = len(results.get("errors", [])) + total_dirs_scanned = results.get("scanned_dirs", 0) # Add more details in the output click.echo( f"\nFound {terraform_count} Terraform configurations" f" ({module_count} modules)" + f" in {total_dirs_scanned} directories scanned" f"{f', with {error_count} errors' if error_count else ''}" ) diff --git a/tf_upgrade/config.py b/tf_upgrade/config.py index beb5ad99..abe32a70 100644 --- a/tf_upgrade/config.py +++ b/tf_upgrade/config.py @@ -11,6 +11,113 @@ logger = logging.getLogger(__name__) +# Default configuration values +DEFAULT_CONFIG = { + "backup_enabled": True, + "backup_dir": ".terraform-upgrade-backup", + "log_level": "INFO", + "validation_enabled": True, + "github_enabled": False, + "interactive": False, + "terraform_versions": { + "0.12": "terraform_0.12.31", + "0.13": "terraform_0.13.7", + "0.14": "terraform_0.14.11", + "0.15": "terraform_0.15.5", + "1.0": "terraform_1.0.11", + "1.10": "terraform_1.10.5", + }, + "terraform_paths": ["/apps/terraform/bin", "/usr/local/bin"], +} + + +def load_config(config_path=None): + """ + Load configuration from YAML file with fallback to default values. + + Args: + config_path (str, optional): Path to config YAML file. Defaults to None. + + Returns: + dict: Configuration dictionary with default values merged with file values + """ + config = DEFAULT_CONFIG.copy() + + # If no config path specified, check standard locations + if not config_path: + # Check current directory + if os.path.exists(".tf-upgrade.yml"): + config_path = ".tf-upgrade.yml" + # Check user home directory + elif os.path.exists(os.path.expanduser("~/.tf-upgrade.yml")): + config_path = os.path.expanduser("~/.tf-upgrade.yml") + + # If config file found, load and merge it + if config_path and os.path.exists(config_path): + try: + with open(config_path, "r") as f: + file_config = yaml.safe_load(f) + if file_config: + # Update config with values from file + config.update(file_config) + logger.info(f"Loaded configuration from {config_path}") + except Exception as e: + logger.warning(f"Failed to load config from {config_path}: {e}") + logger.warning("Using default configuration") + + return config + + +def save_config(config, config_path=".tf-upgrade.yml"): + """ + Save configuration to YAML file. + + Args: + config (dict): Configuration to save + config_path (str, optional): Path to save to. Defaults to ".tf-upgrade.yml". + + Returns: + bool: True if successful, False otherwise + """ + try: + with open(config_path, "w") as f: + yaml.dump(config, f, default_flow_style=False) + logger.info(f"Configuration saved to {config_path}") + return True + except Exception as e: + logger.error(f"Failed to save config to {config_path}: {e}") + return False + + +def get_config_value(key, default=None, config=None): + """ + Get a configuration value safely. + + Args: + key (str): Configuration key to retrieve + default (any, optional): Default value if not found. Defaults to None. + config (dict, optional): Config dictionary to use. + If None, loads default config. + + Returns: + any: Configuration value or default + """ + if config is None: + config = load_config() + + # Handle nested keys with dots (e.g., "github.token") + if "." in key: + parts = key.split(".") + value = config + for part in parts: + if isinstance(value, dict) and part in value: + value = value[part] + else: + return default + return value + + return config.get(key, default) + class ConfigManager: """ diff --git a/tf_upgrade/scanner.py b/tf_upgrade/scanner.py index d4944794..e67c1e14 100644 --- a/tf_upgrade/scanner.py +++ b/tf_upgrade/scanner.py @@ -61,13 +61,17 @@ def scan_directory(self, root_dir): logger.info(f"Starting scan of {root_dir} with max depth {self.max_depth}") self._scan_recursive(root_dir, results, 0) logger.info( - f"Scan complete. Found {len(results['terraform_dirs'])} Terraform configurations in {results['scanned_dirs']} directories" + "Scan complete. Found {} Terraform configurations in {} directories".format( + len(results["terraform_dirs"]), results["scanned_dirs"] + ) ) # Add a detailed breakdown for better visibility if len(results["terraform_dirs"]) > 0: logger.debug( - f"Terraform directories found:\n{pformat([d['path'] for d in results['terraform_dirs']])}" + "Terraform directories found:\n{}".format( + pformat([d["path"] for d in results["terraform_dirs"]]) + ) ) return results @@ -148,7 +152,8 @@ def _scan_recursive(self, current_dir, results, depth): ) for subdir in subdirs: - # Skip ignored directories one more time (in case basename check isn't sufficient) + # Skip ignored directories one more time + # (in case basename check isn't sufficient) if os.path.basename(subdir) in self.ignored_dirs: continue @@ -184,7 +189,8 @@ def _analyze_directory(self, directory, tf_files): "complexity": 0, } - # Check if it's a module by looking for variables.tf, outputs.tf or other indicators + # Check if it's a module by looking for variables.tf, + # outputs.tf or other indicators result["has_variables"] = any( f in tf_files for f in ["variables.tf", "vars.tf"] ) @@ -376,7 +382,9 @@ def scan_command( # Print basic information before scanning logger.info( - f"Scanning directory: {dir_path} (recursive={recursive}, max_depth={max_depth})" + "Scanning directory: {} (recursive={}, max_depth={})".format( + dir_path, recursive, max_depth + ) ) if not recursive: @@ -390,27 +398,32 @@ def scan_command( ), "module_dirs": [], "scanned_dirs": 1, # Just scanned one directory + "provider_configs": {}, + "backend_configs": {}, + "errors": [], + "warnings": [], } return results else: - # Use either parallel or sequential scanning + # Use recursive scanning - override parallel for now to ensure it works properly + logger.debug("Starting recursive directory scan") results = scanner.scan_directory(dir_path) # Print a more informative summary - logger.info(f"\nSummary:") + logger.info("Summary:") logger.info(f" Total directories scanned: {results.get('scanned_dirs', 0)}") logger.info(f" Total Terraform directories: {len(results['terraform_dirs'])}") logger.info(f" Module directories: {len(results['module_dirs'])}") # Add more details for debugging - if results["errors"]: + if results.get("errors", []): logger.warning(f" Errors encountered: {len(results['errors'])}") for error in results["errors"][:5]: # Show first 5 errors logger.warning(f" - {error['directory']}: {error['error']}") if len(results["errors"]) > 5: logger.warning(f" ... and {len(results['errors'])-5} more errors") - if results["warnings"]: + if results.get("warnings", []): logger.info(f" Warnings: {len(results['warnings'])}") return results diff --git a/tf_upgrade/utils/terraform.py b/tf_upgrade/utils/terraform.py index e4b01cff..0dbf260d 100644 --- a/tf_upgrade/utils/terraform.py +++ b/tf_upgrade/utils/terraform.py @@ -11,9 +11,6 @@ import time from datetime import datetime -# Remove the circular import -# from tf_upgrade import env_validator - logger = logging.getLogger(__name__) @@ -545,3 +542,20 @@ def get_account_id_from_tfvars(directory): except Exception as ex: logger.warning(f"Error extracting account ID from tfvars: {str(ex)}") return None + + +def get_available_terraform_versions(): + """ + Gets a list of available Terraform versions installed on the system. + + Returns: + list: List of available Terraform versions as strings + """ + try: + # This is a placeholder implementation + # In a real implementation, we would scan for installed terraform versions + # or query the HashiCorp releases API + return ["0.12.31", "0.13.7", "0.14.11", "0.15.5", "1.0.11", "1.10.5"] + except Exception as e: + logger.error(f"Error getting available Terraform versions: {e}") + return [] diff --git a/tf_upgrade/utils/version_validator.py b/tf_upgrade/utils/version_validator.py new file mode 100644 index 00000000..31347d0b --- /dev/null +++ b/tf_upgrade/utils/version_validator.py @@ -0,0 +1,91 @@ +""" +Version validation utilities for Terraform upgrade tool. +""" + +import functools +import logging +import re + +from packaging import version + +logger = logging.getLogger(__name__) + + +@functools.lru_cache() +def is_valid_terraform_version(version_str): + """ + Check if the given string is a valid Terraform version. + + Args: + version_str: Version string to validate + + Returns: + bool: True if valid, False otherwise + """ + # Accept versions like 0.12, 0.13.7, 1.0, 1.1.5, 1.10, 1.10.5 + pattern = r"^(?:0\.\d+(?:\.\d+)?|[1-9]\d*\.\d+(?:\.\d+)?)$" + + if not re.match(pattern, version_str): + logger.debug(f"Version string '{version_str}' does not match pattern") + return False + + # Further validate with packaging.version + try: + version.parse(version_str) + return True + except ValueError: + logger.debug(f"Failed to parse '{version_str}' as a version") + return False + + +@functools.lru_cache() +def normalize_terraform_version(version_str): + """ + Normalize a Terraform version string. + + For example: + - "0.12" -> "0.12.0" + - "1.0" -> "1.0.0" + - "1.10" -> "1.10.0" + + Args: + version_str: Version string to normalize + + Returns: + str: Normalized version string + """ + if not is_valid_terraform_version(version_str): + raise ValueError(f"Invalid Terraform version: {version_str}") + + # Add .0 if needed to make it a 3-part version + parts = version_str.split(".") + if len(parts) < 3: + version_str = f"{version_str}.0" + + return version_str + + +@functools.lru_cache() +def compare_terraform_versions(version1, version2): + """ + Compare two Terraform versions. + + Args: + version1: First version string + version2: Second version string + + Returns: + int: -1 if version1 < version2, 0 if equal, 1 if version1 > version2 + """ + v1 = normalize_terraform_version(version1) + v2 = normalize_terraform_version(version2) + + parsed_v1 = version.parse(v1) + parsed_v2 = version.parse(v2) + + if parsed_v1 < parsed_v2: + return -1 + elif parsed_v1 > parsed_v2: + return 1 + else: + return 0 diff --git a/tf_upgrade/workspace_manager.py b/tf_upgrade/workspace_manager.py new file mode 100644 index 00000000..0629deb4 --- /dev/null +++ b/tf_upgrade/workspace_manager.py @@ -0,0 +1,231 @@ +""" +Workspace manager for Terraform upgrade tool. +Provides functions for cloning repositories, creating workspaces, +and managing the upgrade workflow in a dedicated workspace. +""" + +import logging +import os +import shutil +import subprocess +import tempfile + +# Configure logger +logger = logging.getLogger(__name__) + + +class WorkspaceManager: + """ + Manages workspaces for Terraform upgrades. + + A workspace is a temporary directory where we clone the target repository, + perform the upgrade process, and then create commits and PRs. + """ + + def __init__(self, base_dir=None): + """ + Initialize the workspace manager. + + Args: + base_dir: Base directory for workspaces. If None, will use system temp dir. + """ + self.base_dir = base_dir or tempfile.gettempdir() + self.workspace_dir = None + self.original_dir = None + + def create_workspace(self, name="terraform-upgrade"): + """ + Create a new workspace directory. + + Args: + name: Name prefix for the workspace directory + + Returns: + Path to the created workspace directory + """ + # Create a unique workspace directory + workspace_path = os.path.join(self.base_dir, f"{name}-{os.urandom(4).hex()}") + os.makedirs(workspace_path, exist_ok=True) + self.workspace_dir = workspace_path + logger.info(f"Created workspace at: {workspace_path}") + return workspace_path + + def clone_repository(self, repo_url, target_dir=None, branch="main"): + """ + Clone a git repository into the workspace. + + Args: + repo_url: URL of the git repository to clone + target_dir: Target directory name (defaults to last part of repo URL) + branch: Branch to checkout + + Returns: + Path to the cloned repository + """ + if not self.workspace_dir: + self.create_workspace() + + # Determine target directory name if not provided + if not target_dir: + target_dir = repo_url.rstrip("/").split("/")[-1] + if target_dir.endswith(".git"): + target_dir = target_dir[:-4] + + repo_path = os.path.join(self.workspace_dir, target_dir) + + try: + # Clone the repository + cmd = ["git", "clone", repo_url, repo_path] + logger.info(f"Cloning repository: {repo_url} to {repo_path}") + subprocess.run(cmd, check=True) + + # Checkout the specified branch + if branch: + subprocess.run(["git", "checkout", branch], cwd=repo_path, check=True) + + return repo_path + except subprocess.CalledProcessError as e: + logger.error(f"Error cloning repository: {e}") + return None + + def enter_workspace(self, directory=None): + """ + Change to the workspace directory or a subdirectory of the workspace. + + Args: + directory: Subdirectory within the workspace to enter + + Returns: + True if successful, False otherwise + """ + if not self.workspace_dir: + logger.error("No workspace directory created yet") + return False + + target_dir = self.workspace_dir + if directory: + target_dir = os.path.join(self.workspace_dir, directory) + if not os.path.isdir(target_dir): + logger.error(f"Directory does not exist: {target_dir}") + return False + + # Remember the original directory + self.original_dir = os.getcwd() + os.chdir(target_dir) + logger.info(f"Changed directory to: {target_dir}") + return True + + def exit_workspace(self): + """ + Return to the original directory before entering the workspace. + + Returns: + True if successful, False otherwise + """ + if not self.original_dir: + logger.warning("No original directory saved, cannot exit workspace") + return False + + os.chdir(self.original_dir) + logger.info(f"Returned to directory: {self.original_dir}") + return True + + def commit_changes(self, repo_path, message="Terraform upgrade changes"): + """ + Commit changes in the workspace repository. + + Args: + repo_path: Path to the repository in the workspace + message: Commit message + + Returns: + True if successful, False otherwise + """ + try: + # Check if there are changes to commit + result = subprocess.run( + ["git", "status", "--porcelain"], + cwd=repo_path, + check=True, + capture_output=True, + text=True, + ) + + if not result.stdout.strip(): + logger.info("No changes to commit") + return True + + # Stage all changes + subprocess.run(["git", "add", "."], cwd=repo_path, check=True) + + # Commit changes + subprocess.run(["git", "commit", "-m", message], cwd=repo_path, check=True) + logger.info(f"Changes committed with message: {message}") + return True + except subprocess.CalledProcessError as e: + logger.error(f"Error committing changes: {e}") + return False + + def create_branch(self, repo_path, branch_name): + """ + Create and checkout a new branch in the repository. + + Args: + repo_path: Path to the repository in the workspace + branch_name: Name of the new branch + + Returns: + True if successful, False otherwise + """ + try: + subprocess.run( + ["git", "checkout", "-b", branch_name], cwd=repo_path, check=True + ) + logger.info(f"Created and checked out branch: {branch_name}") + return True + except subprocess.CalledProcessError as e: + logger.error(f"Error creating branch: {e}") + return False + + def push_changes(self, repo_path, branch_name, remote="origin"): + """ + Push changes to the remote repository. + + Args: + repo_path: Path to the repository in the workspace + branch_name: Name of the branch to push + remote: Remote name (default: origin) + + Returns: + True if successful, False otherwise + """ + try: + subprocess.run( + ["git", "push", remote, branch_name], cwd=repo_path, check=True + ) + logger.info(f"Pushed changes to {remote}/{branch_name}") + return True + except subprocess.CalledProcessError as e: + logger.error(f"Error pushing changes: {e}") + return False + + def cleanup_workspace(self): + """ + Clean up the workspace directory. + + Returns: + True if successful, False otherwise + """ + if not self.workspace_dir or not os.path.exists(self.workspace_dir): + logger.warning("No workspace to clean up") + return False + + # Return to original directory if needed + if self.original_dir: + self.exit_workspace() + + # Remove the workspace directory + shutil.rmtree(self.workspace_dir) + logger.info(f"Workspace cleaned up: {self.workspace_dir}") + self.workspace_dir = None + return True diff --git a/tf_upgrade_diagnostics.py b/tf_upgrade_diagnostics.py new file mode 100644 index 00000000..61698b00 --- /dev/null +++ b/tf_upgrade_diagnostics.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Diagnostic tools for troubleshooting Terraform Upgrade Tool issues. +""" + +import argparse +import logging +import os +import sys +import time + +# Configure basic logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) + +logger = logging.getLogger("tf-upgrade-diagnostics") + + +def scan_directory_tree(root_dir, max_depth=20, ignored_dirs=None): + """ + Scan a directory tree and report all Terraform files found. + + This is a simplified version of the scanner that only focuses on + finding Terraform files to help diagnose scanning issues. + """ + if ignored_dirs is None: + ignored_dirs = [".git", ".terraform", "node_modules", "logs", ".github"] + + tf_dirs = [] + scanned_dirs = 0 + start_time = time.time() + + def _scan_dir(current_dir, depth=0): + nonlocal scanned_dirs + scanned_dirs += 1 + + if depth > max_depth: + logger.warning(f"Max depth {max_depth} reached at {current_dir}") + return + + if os.path.basename(current_dir) in ignored_dirs: + logger.debug(f"Skipping ignored directory: {current_dir}") + return + + logger.debug(f"Scanning at depth {depth}: {current_dir}") + + try: + dir_items = os.listdir(current_dir) + + # Find Terraform files + tf_files = [f for f in dir_items if f.endswith(".tf")] + if tf_files: + logger.info(f"Found {len(tf_files)} Terraform files in: {current_dir}") + tf_dirs.append({"path": current_dir, "files": tf_files, "depth": depth}) + + # Check subdirectories + for item in dir_items: + item_path = os.path.join(current_dir, item) + if os.path.isdir(item_path): + _scan_dir(item_path, depth + 1) + + except (PermissionError, FileNotFoundError) as e: + logger.error(f"Error accessing {current_dir}: {str(e)}") + + # Start the recursive scan + logger.info(f"Starting diagnostic scan of {root_dir}") + _scan_dir(root_dir) + + elapsed_time = time.time() - start_time + + # Report results + logger.info(f"Scan completed in {elapsed_time:.2f} seconds") + logger.info(f"Total directories scanned: {scanned_dirs}") + logger.info(f"Found {len(tf_dirs)} directories with Terraform files") + + # Show directory structure with indentation to visualize hierarchy + print("\nDirectory structure with Terraform files:") + for i, tf_dir in enumerate(sorted(tf_dirs, key=lambda d: d["path"]), 1): + indent = " " * tf_dir["depth"] + rel_path = os.path.relpath(tf_dir["path"], root_dir) + print(f"{indent}├── {rel_path} ({len(tf_dir['files'])} .tf files)") + + return { + "terraform_dirs": tf_dirs, + "total_dirs_scanned": scanned_dirs, + "elapsed_time": elapsed_time, + } + + +def check_cli_command(): + """Check if CLI command is working correctly.""" + try: + import tf_upgrade.cli + + print("\nCLI Module Information:") + print(f"Module path: {tf_upgrade.cli.__file__}") + + # Check scan command options + scan_command = next( + (cmd for cmd in tf_upgrade.cli.cli.commands.values() if cmd.name == "scan"), + None, + ) + if scan_command: + print("\nScan Command Options:") + for param in scan_command.params: + if param.name == "recursive": + print(f" --recursive option default: {param.default}") + print(f" --recursive option is_flag: {param.is_flag}") + elif param.name == "max_depth": + print(f" --max-depth option default: {param.default}") + else: + print("Scan command not found in CLI") + except ImportError: + print("Could not import tf_upgrade.cli module") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Terraform Upgrade Tool Diagnostics") + parser.add_argument("directory", help="Directory to scan") + parser.add_argument( + "--max-depth", type=int, default=20, help="Maximum directory depth" + ) + parser.add_argument( + "--check-cli", action="store_true", help="Check CLI command settings" + ) + parser.add_argument( + "--verbose", "-v", action="store_true", help="Enable verbose output" + ) + + args = parser.parse_args() + + if args.verbose: + logger.setLevel(logging.DEBUG) + + if not os.path.isdir(args.directory): + print(f"Error: {args.directory} is not a directory") + sys.exit(1) + + if args.check_cli: + check_cli_command() + + results = scan_directory_tree(args.directory, args.max_depth) + + print("\nScan Summary:") + print(f" Total directories scanned: {results['total_dirs_scanned']}") + print(f" Terraform directories found: {len(results['terraform_dirs'])}") + print(f" Scan completed in: {results['elapsed_time']:.2f} seconds")