From b24cbae924257f0fed833244b14bc659ceede408 Mon Sep 17 00:00:00 2001 From: badra001 Date: Fri, 2 Jan 2026 08:29:54 -0500 Subject: [PATCH] update output --- .../cross-organization/check_config.py | 60 +++------ .../cross-organization/org_runner.py | 121 ++++++++++-------- 2 files changed, 90 insertions(+), 91 deletions(-) diff --git a/local-app/python-tools/cross-organization/check_config.py b/local-app/python-tools/cross-organization/check_config.py index 14b28f55..c48d3ed0 100644 --- a/local-app/python-tools/cross-organization/check_config.py +++ b/local-app/python-tools/cross-organization/check_config.py @@ -2,70 +2,52 @@ import time from datetime import datetime, timedelta -__version__ = "1.0.6" +__version__ = "1.0.7" def get_s3_metrics(session, bucket_name, region): cw = session.client('cloudwatch', region_name=region) metrics = {"bucket_size_bytes": 0, "object_count": 0} end = datetime.utcnow() start = end - timedelta(days=2) - try: - for metric_name in ['BucketSizeBytes', 'NumberOfObjects']: - response = cw.get_metric_statistics( - Namespace='AWS/S3', - MetricName=metric_name, - Dimensions=[ - {'Name': 'BucketName', 'Value': bucket_name}, - {'Name': 'StorageType', 'Value': 'StandardStorage'} - ], - StartTime=start, EndTime=end, Period=86400, Statistics=['Average'] - ) - if response['Datapoints']: - key = "bucket_size_bytes" if metric_name == 'BucketSizeBytes' else "object_count" - metrics[key] = int(response['Datapoints'][-1]['Average']) + response = cw.get_metric_statistics( + Namespace='AWS/S3', MetricName='BucketSizeBytes', + Dimensions=[{'Name': 'BucketName', 'Value': bucket_name}, {'Name': 'StorageType', 'Value': 'StandardStorage'}], + StartTime=start, EndTime=end, Period=86400, Statistics=['Average'] + ) + if response['Datapoints']: metrics["bucket_size_bytes"] = int(response['Datapoints'][-1]['Average']) + + response = cw.get_metric_statistics( + Namespace='AWS/S3', MetricName='NumberOfObjects', + Dimensions=[{'Name': 'BucketName', 'Value': bucket_name}, {'Name': 'StorageType', 'Value': 'AllStorageTypes'}], + StartTime=start, EndTime=end, Period=86400, Statistics=['Average'] + ) + if response['Datapoints']: metrics["object_count"] = int(response['Datapoints'][-1]['Average']) except: pass return metrics def account_task(account_session, account_id, account_name, region): - results = {"alias": "None", "data": {}} + results = {"alias": "N/A", "data": {}} try: - iam = account_session.client('iam') - aliases = iam.list_account_aliases().get('AccountAliases', []) - results["alias"] = aliases[0] if aliases else "None" - - ec2 = account_session.client('ec2', region_name=region) - regions = [r['RegionName'] for r in ec2.describe_regions()['Regions']] + results["alias"] = account_session.client('iam').list_account_aliases().get('AccountAliases', ["N/A"])[0] + regions = [r['RegionName'] for r in account_session.client('ec2', region_name=region).describe_regions()['Regions']] for reg in regions: - # START REGIONAL TIMER reg_start = time.perf_counter() - config = account_session.client('config', region_name=reg) channels = config.describe_delivery_channels().get('DeliveryChannels', []) recorders = config.describe_configuration_recorders().get('ConfigurationRecorders', []) - retention = config.describe_retention_configurations().get('RetentionConfigurations', []) - reg_data = {"s3_bucket": "N/A", "recorder_status": "OFF", "retention_days": "Default"} - if recorders: reg_data["recorder_status"] = "ON" - if retention: reg_data["retention_days"] = str(retention[0]['RetentionPeriodInDays']) - + reg_data = {"recorder_status": "ON" if recorders else "OFF", "s3_bucket": "N/A"} if channels: bucket = channels[0].get('s3BucketName', 'N/A') reg_data.update({ "s3_bucket": bucket, - "sns_topic": channels[0].get('snsTopicARN', 'N/A').split(':')[-1], "delivery_freq": channels[0].get('configSnapshotDeliveryProperties', {}).get('deliveryFrequency', 'N/A') }) - if bucket != "N/A": - reg_data.update(get_s3_metrics(account_session, bucket, reg)) - - # END REGIONAL TIMER - reg_elapsed = time.perf_counter() - reg_start - reg_data["check_elapsed_sec"] = round(reg_elapsed, 3) + if bucket != "N/A": reg_data.update(get_s3_metrics(account_session, bucket, reg)) + reg_data["check_elapsed_sec"] = round(time.perf_counter() - reg_start, 3) results["data"][reg] = reg_data - - except Exception as e: - results["error"] = str(e) + except Exception as e: results["error"] = str(e) return results diff --git a/local-app/python-tools/cross-organization/org_runner.py b/local-app/python-tools/cross-organization/org_runner.py index 3f95b757..52500777 100755 --- a/local-app/python-tools/cross-organization/org_runner.py +++ b/local-app/python-tools/cross-organization/org_runner.py @@ -12,40 +12,68 @@ from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed -# Try to import tqdm, fallback to a dummy if not installed +# tqdm integration try: from tqdm import tqdm except ImportError: - print("Warning: 'tqdm' not found. Install it with 'pip install tqdm' for a progress bar.") - # Simple fallback to avoid crashing + print("Warning: 'tqdm' not found. Install with 'pip install tqdm'.") def tqdm(iterable, **kwargs): return iterable # --- VERSIONING --- -__version__ = "1.5.1" +__version__ = "1.5.2" class OrgTaskRunner: def __init__(self, args): self.args = args self.output_log = [] self.full_results = [] + self.hierarchy_cache = {} self.start_time = 0 def log_print(self, message=""): - # We only log to the internal buffer for the final file write - # Standard printing is handled by tqdm to avoid screen flickering + # Terminal output is managed by tqdm; this is for internal log buffering self.output_log.append(message) - def process_account(self, acc_id, acc_name, partition, tasks): - """Worker function for each thread.""" - # Create a fresh session for thread safety + def get_ou_path(self, org_client, entity_id): + """Recursive OU path resolver with caching.""" + if entity_id in self.hierarchy_cache: return self.hierarchy_cache[entity_id] + if entity_id.startswith('r-'): + self.hierarchy_cache[entity_id] = (None, entity_id) + return None, entity_id + try: + parents = org_client.list_parents(ChildId=entity_id)['Parents'] + p_id = parents[0]['Id'] if parents else None + ou_name = org_client.describe_organizational_unit(OrganizationalUnitId=entity_id)['OrganizationalUnit']['Name'] + p_path, _ = self.get_ou_path(org_client, p_id) if p_id else (None, None) + path = f"{p_path}:{ou_name}" if p_path else ou_name + self.hierarchy_cache[entity_id] = (path, entity_id) + return path, entity_id + except: return "Unknown", entity_id + + def process_account(self, acc, partition, tasks): + """Worker thread for each account.""" thread_session = boto3.Session(profile_name=self.args.profile, region_name=self.args.region) - sts = thread_session.client('sts') + sts, org = thread_session.client('sts'), thread_session.client('organizations') + acc_id, acc_name = acc['Id'], acc['Name'] role_arn = f"arn:{partition}:iam::{acc_id}:role/{self.args.role_name}" - account_data = {"account_id": acc_id, "account_name": acc_name, "checks": {}, "alias": "N/A"} + + # Resolve OU Data (inherited from v1.2) + parents = org.list_parents(ChildId=acc_id).get('Parents', []) + raw_path, ou_id = self.get_ou_path(org, parents[0]['Id']) if parents else ("Orphaned", "N/A") + ou_path = raw_path if raw_path else "Root" + + account_data = { + "account_id": acc_id, + "account_name": acc_name, + "alias": "N/A", + "ou_path": ou_path, + "ou_id": ou_id, + "checks": {} + } try: - assumed = sts.assume_role(RoleArn=role_arn, RoleSessionName="OrgRunnerTask") + assumed = sts.assume_role(RoleArn=role_arn, RoleSessionName="OrgRunner") m_sess = boto3.Session( aws_access_key_id=assumed['Credentials']['AccessKeyId'], aws_secret_access_key=assumed['Credentials']['SecretAccessKey'], @@ -55,75 +83,66 @@ def process_account(self, acc_id, acc_name, partition, tasks): for t_func in tasks: res = t_func(m_sess, acc_id, acc_name, self.args.region) - account_data["alias"] = res.get("alias") + account_data["alias"] = res.get("alias", "N/A") account_data["checks"].update(res.get("data", {})) - return account_data, None # Success + return account_data, None except Exception as e: return None, f"FAILED {acc_name} ({acc_id}): {str(e)}" def run(self): self.start_time = time.perf_counter() - - main_session = boto3.Session(profile_name=self.args.profile, region_name=self.args.region) - org = main_session.client('organizations') - sts = main_session.client('sts') + session = boto3.Session(profile_name=self.args.profile, region_name=self.args.region) + sts = session.client('sts') partition = sts.get_caller_identity()['Arn'].split(':')[1] - tasks = [] + # Load tasks & build dynamic filename suffix + tasks, check_names = [], [] if self.args.enable_checks: sys.path.append(os.getcwd()) for m in self.args.enable_checks: - module = importlib.import_module(m.replace('.py', '')) + module_name = m.replace('.py', '') + module = importlib.import_module(module_name) tasks.append(getattr(module, 'account_task')) + check_names.append(module_name) - print("-" * 100) - print(f"AWS ORG TASK RUNNER - v{__version__} | Workers: {self.args.max_workers}") - print("-" * 100) - - all_accounts = [acc for page in org.get_paginator('list_accounts').paginate() + check_suffix = ".".join(check_names) if check_names else "connectivity" + + all_accounts = [acc for page in session.client('organizations').get_paginator('list_accounts').paginate() for acc in page['Accounts'] if acc['Status'] == 'ACTIVE'] all_accounts.sort(key=lambda x: x['Name' if self.args.sort == 'name' else 'Id'].lower()) - # PARALLEL EXECUTION WITH TQDM + print("-" * 100) + print(f"AWS ORG TASK RUNNER - v{__version__} | Check: {check_suffix}") + print("-" * 100) + with ThreadPoolExecutor(max_workers=self.args.max_workers) as executor: - futures = { - executor.submit(self.process_account, acc['Id'], acc['Name'], partition, tasks): acc - for acc in all_accounts - } - - # tqdm context manager wraps the as_completed generator - with tqdm(total=len(all_accounts), desc="Processing Accounts", unit="acc", colour="green") as pbar: + futures = {executor.submit(self.process_account, acc, partition, tasks): acc for acc in all_accounts} + with tqdm(total=len(all_accounts), desc="Processing", unit="acc", colour="green") as pbar: for future in as_completed(futures): - data, error = future.result() - if data: - self.full_results.append(data) - self.log_print(f"SUCCESS: {data['account_name']}") - else: - self.log_print(error) + data, err = future.result() + if data: self.full_results.append(data) pbar.update(1) # FINAL EXPORTS if self.args.output: ds = datetime.now().strftime("%Y-%m-%dT%H%M%S") - base = self.args.output if self.args.output != 'DEFAULT' else "audit_results" + base = self.args.output if self.args.output != 'DEFAULT' else f"audit_results.{check_suffix}" - # Save CSV - csv_file = f"{base}.{ds}.csv" - with open(csv_file, 'w', newline='') as f: + # Save CSV (Long Format) + with open(f"{base}.{ds}.csv", 'w', newline='') as f: writer = csv.writer(f) - writer.writerow(["account_id", "account_alias", "region", "field_name", "field_value"]) + writer.writerow(["account_id", "account_name", "account_alias", "ou_path", "ou_id", "region", "field_name", "field_value"]) for acc in self.full_results: for reg, fields in acc["checks"].items(): for k, v in fields.items(): - writer.writerow([acc["account_id"], acc.get("alias"), reg, k, v]) + writer.writerow([acc["account_id"], acc["account_name"], acc["alias"], acc["ou_path"], acc["ou_id"], reg, k, v]) - # Save JSON - json_file = f"{base}.{ds}.json" - with open(json_file, 'w') as f: + # Save JSON (Hierarchical) + with open(f"{base}.{ds}.json", 'w') as f: json.dump(self.full_results, f, indent=2) - print(f"\nTotal Time: {round(time.perf_counter() - self.start_time, 2)}s") + print(f"\nCompleted in {round(time.perf_counter() - self.start_time, 2)}s.") if __name__ == "__main__": p = argparse.ArgumentParser() @@ -133,7 +152,5 @@ def run(self): p.add_argument("--sort", choices=['id', 'name'], default='name') p.add_argument("--output", nargs='?', const='DEFAULT') p.add_argument("--enable-checks", nargs='+') - p.add_argument("--max-workers", type=int, default=8) + p.add_argument("--max-workers", type=int, default=4) OrgTaskRunner(p.parse_args()).run() - -