diff --git a/scripts/tf-run.py b/scripts/tf-run.py new file mode 100644 index 0000000..fd385d5 --- /dev/null +++ b/scripts/tf-run.py @@ -0,0 +1,665 @@ +#!/usr/bin/env python3 +""" +tf-run Python port - A Terraform workflow automation tool +Port of the original bash tf-run script to Python for better maintainability and integration +""" + +import os +import sys +import re +import subprocess +import time +import json +import argparse +from pathlib import Path +from typing import List, Dict, Optional, Tuple +import tempfile +from datetime import datetime +import shutil + + +class TfRun: + VERSION = "2.0.0" + + def __init__(self): + self.log_dir = "logs" + self.targets = [] + self.targets_status = [] + self.tags = {} + self.profile = None + self.region = None + self.short_region = None + self.git_root = None + self.log_file = None + self.error_get_profile = 0 + self.error_get_region = 0 + + def get_git_root(self) -> Optional[str]: + """Get the git repository root directory""" + try: + result = subprocess.run(['git', 'rev-parse', '--show-toplevel'], + capture_output=True, text=True, check=True) + self.git_root = result.stdout.strip() + return self.git_root + except subprocess.CalledProcessError: + return None + + def get_relative_to_git_root(self) -> str: + """Get relative path from git root to current directory""" + if not self.git_root: + self.get_git_root() + + if not self.git_root: + return "" + + git_base = os.path.basename(self.git_root) + cwd = os.getcwd() + try: + relative_path = cwd.split(git_base, 1)[1] + return relative_path.replace('/', '../') if relative_path else "" + except IndexError: + return "" + + def get_relative_directory(self, filename: str) -> str: + """Find a file in current or parent directories and return relative path""" + cwd = Path(os.getcwd()) + relative_path = "" + + for parent in [cwd] + list(cwd.parents): + file_path = parent / filename + if file_path.exists() and not file_path.is_symlink(): + return relative_path if relative_path else "./" + relative_path += "../" + + # Stop at git root or filesystem root + if (parent / ".git").exists() or parent == parent.parent: + break + + return "" + + def get_profile(self) -> bool: + """Extract AWS profile from *.tfvars files""" + if os.environ.get('AWS_PROFILE'): + self.profile = os.environ['AWS_PROFILE'] + return True + + tfvars_files = [f for f in os.listdir('.') if f.endswith('.tfvars')] + if not tfvars_files: + if self.error_get_profile > 0: + print("* [WARNING] cannot determine profile from *.tfvars") + self.error_get_profile += 1 + return False + + for file in tfvars_files: + try: + with open(file, 'r') as f: + for line in f: + match = re.match(r'^\s*profile\s*=\s*["\']?([^"\']+)["\']?', line) + if match: + self.profile = match.group(1).strip() + self.error_get_profile = 0 + return True + except IOError: + continue + + return False + + def get_region(self) -> bool: + """Extract AWS region from *.tfvars files""" + if os.environ.get('AWS_REGION'): + self.region = os.environ['AWS_REGION'] + else: + tfvars_files = [f for f in os.listdir('.') if f.endswith('.tfvars')] + if not tfvars_files: + if self.error_get_region > 0: + print("* [WARNING] cannot determine region from *.tfvars") + self.error_get_region += 1 + return False + + for file in tfvars_files: + try: + with open(file, 'r') as f: + for line in f: + match = re.match(r'^\s*region\s*=\s*["\']?([^"\']+)["\']?', line) + if match: + self.region = match.group(1).strip() + break + if self.region: + break + except IOError: + continue + + if self.region: + if 'gov' in self.region: + self.short_region = re.sub(r'^us-gov-', '', self.region) + self.short_region = re.sub(r'-[0-9]$', '', self.short_region) + else: + self.short_region = re.sub(r'^us-', '', self.region) + self.error_get_region = 0 + return True + + return False + + def replace_placeholders(self, text: str, next_step: int = 0, previous_step: int = 0) -> str: + """Replace placeholder variables in text""" + if next_step: + text = text.replace('%%NEXT%%', str(next_step)) + if previous_step: + text = text.replace('%%PREVIOUS%%', str(previous_step)) + if self.profile: + text = text.replace('%%PROFILE%%', self.profile) + if self.region: + text = text.replace('%%REGION%%', self.region) + if self.short_region: + text = text.replace('%%SHORT_REGION%%', self.short_region) + return text + + def get_file_from_git(self, filename: str, url: str) -> bool: + """Download a file from git repository if it doesn't exist""" + if os.path.exists(filename): + print(f"* file {filename} exists, not overwriting") + return True + + print(f"* getting init file {filename}") + try: + result = subprocess.run(['curl', '-q', '-s', '-k', '-o', filename, url], + check=True) + return result.returncode == 0 + except subprocess.CalledProcessError: + return False + + def do_clean(self, action: str = "clean"): + """Remove remote_state files and symlinks""" + print(f"* executing {action}, removing remote_state.*") + print("> ", end="") + + # Remove remote_state files + for file in Path('.').glob('remote_state.*'): + if file.is_file(): + file.unlink() + print(f" {file}", end="") + print("") + + print("* executing {action}, removing links") + print("> ", end="") + + # Remove symlinks + for file in Path('.').iterdir(): + if file.is_symlink(): + file.unlink() + print(f" {file}", end="") + print("") + + def do_superclean(self): + """Perform clean plus remove logs, .terraform files, and tfstate files""" + self.do_clean("superclean") + print("* executing superclean, removing logs, .terraform files, terraform.tfstate files") + print("> ", end="") + + to_remove = ['logs', '.terraform', '.terraform.lock.hcl'] + to_remove.extend([str(p) for p in Path('.').glob('terraform.tfstate*')]) + + for item in to_remove: + path = Path(item) + if path.exists(): + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + print(f" {item}", end="") + print("") + + def ask_continue(self, prefix: str = "", duration: int = 10, default: str = "y") -> str: + """Ask user if they want to continue with timeout""" + try: + import select + import sys + import termios + + print(f"{prefix}continue [y|n: default={default}]? ", end="", flush=True) + + # Simple implementation - in a real scenario might want more sophisticated input handling + old_settings = termios.tcgetattr(sys.stdin) + try: + termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings) + ready, _, _ = select.select([sys.stdin], [], [], duration) + if ready: + response = sys.stdin.read(1).lower() + else: + response = default + finally: + termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings) + + print() + return response if response in ['y', 'n'] else 'n' + + except ImportError: + # Fallback for systems without select/termios + response = input(f"{prefix}continue [y|n: default={default}]? ").lower() + return response if response in ['y', 'n'] else default + + def read_run_file(self, filename: str) -> bool: + """Read and parse the tf-run data file""" + self.targets = [] + self.targets_status = [] + self.tags = {} + + if not os.path.exists(filename): + if filename == "ALL": + self.targets.append("ALL") + self.targets_status.append(0) + return True + else: + print(f"* unable to open {filename}, exiting") + return False + + print(f"* reading from {filename}") + pos = 1 + + try: + with open(filename, 'r') as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + # Skip comments + if line.startswith('#') or not line: + continue + + self.targets.append(line) + self.targets_status.append(0) + + words = line.split() + if words[0] == "VERSION": + self.tfrunfile_version = words[1] if len(words) > 1 else "" + pos -= 1 + elif words[0] == "TAG": + self.tags[words[1]] = pos + pos -= 1 + + pos += 1 + + print(f"* read {len(self.targets)} entries from {filename}") + return True + + except IOError as e: + print(f"* error reading {filename}: {e}") + return False + + def execute_action(self, action: str, start: Optional[int] = None, end: Optional[int] = None, + list_only: bool = False, tf_options: str = "") -> int: + """Execute the specified action on the targets""" + status = 0 + c = 0 + total_targets = len([t for t in self.targets if not t.split()[0] in ['VERSION', 'TAG']]) + + print(f">> START: action={action} start={start} end={end}") + print(f"- profile={self.profile} region={self.region} short_region={self.short_region}") + + for target in self.targets: + c += 1 + words = target.split() + if not words: + continue + + command = words[0] + rest = ' '.join(words[1:]) if len(words) > 1 else "" + next_step = c + 1 + + rest = self.replace_placeholders(rest, next_step, c - 1) + + # Skip VERSION and TAG entries for target counting + if command in ['VERSION', 'TAG']: + total_targets -= 1 + + # Check start condition + if start is not None and start > c: + if command in ['VERSION', 'TAG']: + c -= 1 + continue + + # Check end condition + if end is not None and end != 0 and c > end: + break + + # Handle special commands + if command == "REMOTE-STATE": + print(f"> [{c}] {command}> generate-remote-state") + if not list_only: + status = self._handle_remote_state() + self.targets_status[c-1] = status + continue + + elif command == "BACKUP-STATE": + print(f"> [{c}] {command}> backup-state") + if not list_only: + status = self._handle_backup_state() + self.targets_status[c-1] = status + c -= 1 + continue + + elif command == "COMMAND": + print(f"> [{c}] {command}> {rest}") + if not list_only: + status = subprocess.call(rest, shell=True) + print(f"= Complete: {c} {command}> {rest} | status={status}") + self.targets_status[c-1] = status + self.get_profile() + self.get_region() + continue + + elif command in ["LINK", "LINKTOP"]: + self._handle_link(command, rest, c, list_only) + continue + + elif command == "VERSION": + c -= 1 + continue + + elif command == "TAG": + print(f"\n# [{command}] {rest}") + c -= 1 + continue + + elif command in ["COMMENT", "CHECK", "PAUSE", "STOP"]: + status = self._handle_special_command(command, rest, c, list_only) + if command == "STOP": + break + continue + + elif command == "POLICY": + status = self._handle_policy(words[1:] if len(words) > 1 else ["*.tf"], c, list_only) + if not target: + continue + + # Handle regular terraform targets + if status != 0: + print(f"* error encountered, status={status}; exiting") + return status + + tf_args = "" + if target != "ALL": + for tt in target.split(): + tf_args += f"-target={tt} " + + if list_only: + print(f"> [{c}] tf-{action} {tf_options} {tf_args}") + continue + + print(f"> [{c}] tf-{action} {tf_options} {tf_args}") + + # Execute terraform command + cmd = f"tf-{action} {tf_options} {tf_args}".strip() + status = subprocess.call(cmd, shell=True) + self.targets_status[c-1] = status + + if status != 0: + print(f"> [{c}] exiting status={status}") + break + + print(f"= Complete: {c} {command}> {rest} | status={status}") + + # Ask to continue + response = self.ask_continue(f"}} Next: {next_step}, ") + if response == 'n': + break + + return status + + def _handle_remote_state(self) -> int: + """Handle REMOTE-STATE command""" + parent_remote_state = Path("../remote_state.yml") + if parent_remote_state.exists(): + with open(parent_remote_state, 'r') as f: + content = f.read() + + # Replace directory path + current_dir = os.path.basename(os.getcwd()) + content = re.sub(r'^(directory.*)"', rf'\1/{current_dir}"', content, flags=re.MULTILINE) + + with open("remote_state.yml", 'w') as f: + f.write(content) + + print(f"* generated line: {current_dir}") + return 0 + else: + print("* missing parent remote_state.yml, exiting") + return 1 + + def _handle_backup_state(self) -> int: + """Handle BACKUP-STATE command""" + timestamp = datetime.now().strftime("%Y%m%d.%s") + backup_file = f"{self.log_dir}/backup.{timestamp}.tfstate" + + try: + result = subprocess.run(['tf-state', 'pull'], capture_output=True, text=True, check=True) + with open(backup_file, 'w') as f: + f.write(result.stdout) + return 0 + except subprocess.CalledProcessError as e: + return e.returncode + + def _handle_link(self, command: str, rest: str, step: int, list_only: bool): + """Handle LINK and LINKTOP commands""" + link_arg = self.replace_placeholders(rest.split()[0] if rest.split() else "") + + if command == "LINKTOP": + relative_path = self.get_relative_to_git_root() + else: + relative_path = self.get_relative_directory(link_arg) + + print(f"> [{step}] {command}> ln -sf {relative_path}{link_arg} ./") + + if not list_only: + source_path = f"{relative_path}{link_arg}" + if os.path.exists(source_path) and relative_path != "./": + try: + if os.path.islink(link_arg): + os.unlink(link_arg) + os.symlink(source_path, link_arg) + status = 0 + except OSError: + status = 1 + else: + print(f"* linked-to file {source_path} does not exist, skipping") + status = 0 + + print(f"= Complete: {step} {command}> {rest} | status={status}") + self.targets_status[step-1] = status + + def _handle_special_command(self, command: str, rest: str, step: int, list_only: bool) -> int: + """Handle special commands like COMMENT, CHECK, PAUSE, STOP""" + print(f"> [{step}] {command}> {rest}") + + if command == "PAUSE" and not list_only: + sleep_time = int(rest) if rest.isdigit() else 15 + time.sleep(sleep_time) + elif command == "CHECK" and not list_only: + response = self.ask_continue(f"}} Next: {step + 1}, ") + if response == 'n': + return 1 + elif command == "STOP": + print(f"= Complete: {step} {command}> {rest} | status=0") + print(f"- Continue {step + 1}: tf-run {step + 1}") + return 1 + + return 0 + + def _handle_policy(self, files: List[str], step: int, list_only: bool) -> int: + """Handle POLICY command to find IAM policies""" + file_pattern = files[0] if files else "*.tf" + + try: + result = subprocess.run(['grep', '-iE', r'^resource\b.*aws_iam_policy\b'] + files, + capture_output=True, text=True) + + policies = [] + for line in result.stdout.split('\n'): + if line.strip(): + parts = line.split() + if len(parts) >= 3: + policies.append(f"{parts[1]}.{parts[2]}".replace('"', '')) + + print(f"> [{step}] POLICY> ({file_pattern}) {' '.join(policies)}") + + if not policies: + if not list_only: + print(f"= No policy targets found, skipping this step | status=0") + return 0 + + return 0 + + except subprocess.CalledProcessError: + return 1 + + +def main(): + parser = argparse.ArgumentParser(description='tf-run: Terraform workflow automation tool') + parser.add_argument('action', choices=['plan', 'apply', 'destroy', 'list', 'init', + 'init-upgrade', 'check', 'clean', 'superclean', 'tags', 'help'], + help='Action to perform') + parser.add_argument('start', nargs='?', help='Start step number or tag:name') + parser.add_argument('end', nargs='?', help='End step number, tag:name, only, or +N') + parser.add_argument('--dry-run', action='store_true', help='Show what would be done') + parser.add_argument('--tf-options', default='', help='Additional terraform options') + + args = parser.parse_args() + + tf_run = TfRun() + + # Setup logging directory + os.makedirs(tf_run.log_dir, exist_ok=True) + + if args.action == 'help': + parser.print_help() + print("\nAdditional help:") + print(" init: get base files and setup tf-control files") + print(" clean: removes remote_state.*, links") + print(" superclean: removes remote_state.*, links, logs/, .terraform files") + print(" tags: list available tags and step numbers") + return 0 + + # Handle init actions + if args.action == 'init': + tf_run.get_git_root() + base_url = "https://github.e.it.census.gov/raw/terraform/support/master/local-app/tf-run/applications/base" + + files_to_get = [ + ("tf-run.data", f"{base_url}/tf-run.data"), + ("region.tf", f"{base_url}/region.tf"), + ("locals.tf", f"{base_url}/locals.tf"), + ("versions.tf", f"{base_url}/versions.tf") + ] + + for filename, url in files_to_get: + tf_run.get_file_from_git(filename, url) + + # Get tf-control files + control_url = "https://github.e.it.census.gov/raw/terraform/support/master/local-app/aws-account-setup/ansible/roles/setup-git-repo/files" + if tf_run.git_root: + tf_control_path = os.path.join(tf_run.git_root, ".tf-control") + tf_control_tfrc_path = os.path.join(tf_run.git_root, ".tf-control.tfrc") + + if not os.path.exists(tf_control_path): + tf_run.get_file_from_git(tf_control_path, f"{control_url}/.tf-control") + if not os.path.exists(tf_control_tfrc_path): + tf_run.get_file_from_git(tf_control_tfrc_path, f"{control_url}/.tf-control.tfrc") + + return 0 + + elif args.action == 'init-upgrade': + tf_run.get_git_root() + control_url = "https://github.e.it.census.gov/raw/terraform/support/master/local-app/aws-account-setup/ansible/roles/setup-git-repo/files" + if tf_run.git_root: + tf_control_path = os.path.join(tf_run.git_root, ".tf-control") + tf_control_tfrc_path = os.path.join(tf_run.git_root, ".tf-control.tfrc") + + if not os.path.exists(tf_control_path): + tf_run.get_file_from_git(tf_control_path, f"{control_url}/.tf-control") + if not os.path.exists(tf_control_tfrc_path): + tf_run.get_file_from_git(tf_control_tfrc_path, f"{control_url}/.tf-control.tfrc") + return 0 + + elif args.action in ['clean', 'superclean']: + print(f"\nAbout to execute {args.action}. This is destructive and will remove files.") + response = tf_run.ask_continue("Continue (y|n)? ", default="n") + if response == 'y': + if args.action == 'clean': + tf_run.do_clean() + else: + tf_run.do_superclean() + else: + print(f"* action {args.action} declined") + return 0 + + # Get AWS profile and region + tf_run.get_profile() + tf_run.get_region() + + # Determine run file + if args.action == 'destroy': + run_file = "tf-run.destroy.data" if os.path.exists("tf-run.destroy.data") else "ALL" + else: + run_file = "tf-run.data" + + # Read the run file + if not tf_run.read_run_file(run_file): + return 1 + + # Handle tags action + if args.action == 'tags': + print("* available TAGS and step numbers") + for tag, step in tf_run.tags.items(): + print(f"TAG {tag} = {step}") + return 0 + + # Parse start/end parameters + start = None + end = None + list_only = False + + if args.start: + if args.start == 'list': + list_only = True + elif args.start.startswith('tag:'): + start_tag = args.start[4:] + start = tf_run.tags.get(start_tag) + if start is None: + print(f"* start tag:{start_tag} not found") + return 1 + else: + try: + start = int(args.start) + except ValueError: + print(f"* invalid start value: {args.start}") + return 1 + + if args.end: + if args.end == 'only': + end = start + elif args.end.startswith('+'): + try: + offset = int(args.end[1:]) + end = start + offset if start else offset + except ValueError: + print(f"* invalid end offset: {args.end}") + return 1 + elif args.end.startswith('tag:'): + end_tag = args.end[4:] + end = tf_run.tags.get(end_tag) + if end is not None: + end -= 1 # Stop before the tag + else: + print(f"* end tag:{end_tag} not found") + return 1 + else: + try: + end = int(args.end) + except ValueError: + print(f"* invalid end value: {args.end}") + return 1 + + # Execute the action + return tf_run.execute_action(args.action, start, end, list_only, args.tf_options) + + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file