From 3dfb338a7d7bc2c1b7d9f24835a538b7db3d2176 Mon Sep 17 00:00:00 2001 From: badra001 Date: Fri, 2 Jan 2026 10:27:43 -0500 Subject: [PATCH] update to add org and ou ids --- .../assess_check_cloudtrail.py | 43 +++++-------- .../cross-organization/assess_check_config.py | 17 ++--- .../cross-organization/check_cloudtrail.py | 9 ++- .../cross-organization/check_config.py | 5 +- .../cross-organization/org_runner.py | 62 +++++++++++++------ 5 files changed, 72 insertions(+), 64 deletions(-) diff --git a/local-app/python-tools/cross-organization/assess_check_cloudtrail.py b/local-app/python-tools/cross-organization/assess_check_cloudtrail.py index f3f6137b..74b277ed 100644 --- a/local-app/python-tools/cross-organization/assess_check_cloudtrail.py +++ b/local-app/python-tools/cross-organization/assess_check_cloudtrail.py @@ -1,16 +1,16 @@ #!/usr/bin/env python - import json import argparse import sys +import re from collections import Counter -# --- VERSIONING --- -__version__ = "1.0.7" +__version__ = "1.0.8" def main(): parser = argparse.ArgumentParser(description="AWS CloudTrail Audit Assessor") parser.add_argument("--input", required=True, help="JSON file from check_cloudtrail") + parser.add_argument("--central-bucket-regex", default=".*", help="Regex for central bucket") args = parser.parse_args() try: @@ -19,24 +19,27 @@ def main(): except: print("Error reading input file"); sys.exit(1) + org_id = data[0].get("org_id", "Unknown") if data else "Unknown" + + print("-" * 160) + print(f"CLOUDTRAIL ASSESSMENT REPORT | Organization: {org_id}") print("-" * 160) - print(f"{'Account ID':<15} | {'Global Summary':<25} | {'Active/Stopped':<15} | {'Security/Cost Issues'}") + print(f"{'Account ID':<15} | {'OU Path':<25} | {'Global Summary':<25} | {'Active/Stopped':<15} | {'Security Issues'}") print("-" * 160) - # Organizational Stats stats = { "s3_bytes": 0, "s3_objects": 0, "s3_buckets": set(), - "cw_bytes": 0, "cw_group_arns": set(), # Now using ARNs for accurate unique count + "cw_bytes": 0, "cw_group_arns": set(), "sns_topics": set(), "kms_cmk_count": 0, "sse_s3_count": 0, "local_trails_total": 0, "local_trails_by_reg": {}, "logging_active": 0, "logging_stopped": 0 } - # Counter for retention days distribution retention_distribution = Counter() for account in data: acc_id = account.get("account_id") + ou_path = account.get("ou_path", "Root") checks = account.get("data", {}) summary = checks.get("account_summary", {}).get("_summary", "UNKNOWN") @@ -47,70 +50,54 @@ def main(): for key, val in checks.items(): if not key.startswith("trail:"): continue - # Status tracking if val.get("is_logging") == "True": acc_active += 1; stats["logging_active"] += 1 else: acc_stopped += 1; stats["logging_stopped"] += 1 - # Local trails regional breakdown if val.get("is_org_trail") == "False": stats["local_trails_total"] += 1 reg = val.get("home_region", "unknown") stats["local_trails_by_reg"][reg] = stats["local_trails_by_reg"].get(reg, 0) + 1 - # S3 Metrics bucket = val.get("s3_bucket") if bucket and bucket != "N/A": stats["s3_buckets"].add(bucket) stats["s3_bytes"] += val.get("bucket_size_bytes", 0) stats["s3_objects"] += val.get("object_count", 0) + if not re.match(args.central_bucket_regex, bucket): + issues.append(f"Non-Central:{bucket}") - # CloudWatch Logs Metrics - Using ARN to ensure accuracy across accounts/regions - # Note: We use a placeholder if ARN isn't in JSON, but check_cloudtrail v1.1.1+ provides metrics if "cw_logs_size_bytes" in val: - # To be perfectly accurate, we use the account + region + trail name as a unique key - # if the specific log group ARN wasn't captured. unique_lg_key = f"{acc_id}:{val.get('home_region')}:{val.get('trail_name')}" stats["cw_group_arns"].add(unique_lg_key) stats["cw_bytes"] += val["cw_logs_size_bytes"] - - # Retention tracking retention = val.get("cw_logs_retention_days", "Never Expire") retention_distribution[retention] += 1 - # Notifications & Encryption sns = val.get("sns_topic") if sns and sns != "N/A": stats["sns_topics"].add(sns) if val.get("kms_key_id") == "SSE-S3": stats["sse_s3_count"] += 1 else: stats["kms_cmk_count"] += 1 - print(f"{acc_id:<15} | {summary:<25} | {f'{acc_active} ON / {acc_stopped} OFF':<15} | {', '.join(issues) if issues else 'COMPLIANT'}") + print(f"{acc_id:<15} | {ou_path[:25]:<25} | {summary:<25} | {f'{acc_active} ON / {acc_stopped} OFF':<15} | {', '.join(issues) if issues else 'COMPLIANT'}") - # Summary Calculations s3_gb = stats["s3_bytes"] / (1024**3) cw_gb = stats["cw_bytes"] / (1024**3) print("-" * 160) - print(f"ORGANIZATION CLOUDTRAIL FOOTPRINT SUMMARY:") + print(f"ORGANIZATION CLOUDTRAIL FOOTPRINT SUMMARY | Org ID: {org_id}") print(f" Logging Status: {stats['logging_active']} Active Trails | {stats['logging_stopped']} Stopped Trails") print(f" S3 Storage: {s3_gb:.2f} GB | {stats['s3_objects']:,} objects | {len(stats['s3_buckets'])} unique buckets") print(f" CloudWatch Logs: {cw_gb:.2f} GB | {len(stats['cw_group_arns'])} unique log groups") - print(f" Log Group Retention Distribution:") - # Sort: Numerical values first, then "Never Expire" string - sorted_retention = sorted(retention_distribution.keys(), - key=lambda x: (0, x) if isinstance(x, int) else (1, str(x))) + sorted_retention = sorted(retention_distribution.keys(), key=lambda x: (0, x) if isinstance(x, int) else (1, str(x))) for period in sorted_retention: label = f"{period} days" if isinstance(period, int) else str(period) print(f" - {label:<15}: {retention_distribution[period]} group(s)") - print(f" Notifications: {len(stats['sns_topics'])} unique SNS Topics") print(f" Encryption: {stats['kms_cmk_count']} KMS CMK | {stats['sse_s3_count']} SSE-S3") - print(f" Local Trails: {stats['local_trails_total']} total") - for r, c in sorted(stats["local_trails_by_reg"].items()): - print(f" - {r:<15}: {c}") print("-" * 160) if __name__ == "__main__": diff --git a/local-app/python-tools/cross-organization/assess_check_config.py b/local-app/python-tools/cross-organization/assess_check_config.py index 80601071..0b5d769d 100755 --- a/local-app/python-tools/cross-organization/assess_check_config.py +++ b/local-app/python-tools/cross-organization/assess_check_config.py @@ -1,12 +1,10 @@ #!/usr/bin/env python - import json import argparse import re import sys -# --- VERSIONING --- -__version__ = "1.0.3" +__version__ = "1.0.4" def main(): parser = argparse.ArgumentParser(description=f"AWS Config Audit Assessor v{__version__}") @@ -20,8 +18,12 @@ def main(): except Exception as e: print(f"Error: {e}"); sys.exit(1) + org_id = data[0].get("org_id", "Unknown") if data else "Unknown" + + print("-" * 125) + print(f"AWS CONFIG ASSESSMENT | Org ID: {org_id}") print("-" * 125) - print(f"{'Account ID':<15} | {'Global Status':<12} | {'S3 Compliance'}") + print(f"{'Account ID':<15} | {'OU Path':<25} | {'Global Status':<12} | {'S3 Compliance'}") print("-" * 125) total_stats = {"objects": 0, "size_bytes": 0, "non_central_buckets": 0, "accounts": len(data)} @@ -29,13 +31,13 @@ def main(): for account in data: acc_id = account.get("account_id") + ou_path = account.get("ou_path", "Root") checks = account.get("data", {}) summary = checks.get("account_summary", {}).get("_summary", "UNKNOWN") s3_issues = [] for reg, reg_data in checks.items(): if reg == "account_summary": continue - bucket = reg_data.get("s3_bucket", "N/A") total_stats["objects"] += reg_data.get("object_count", 0) total_stats["size_bytes"] += reg_data.get("bucket_size_bytes", 0) @@ -46,12 +48,11 @@ def main(): unique_non_central.add(bucket) s3_status = "NON_COMPLIANT" if s3_issues else "COMPLIANT" - print(f"{acc_id:<15} | {summary:<12} | {s3_status}") + print(f"{acc_id:<15} | {ou_path[:25]:<25} | {summary:<12} | {s3_status}") - # Summary Section size_gb = total_stats["size_bytes"] / (1024**3) print("-" * 125) - print(f"ORGANIZATION STORAGE SUMMARY (CONFIG):") + print(f"ORGANIZATION STORAGE SUMMARY (CONFIG) | Org ID: {org_id}") print(f" Total S3 Objects: {total_stats['objects']:,}") print(f" Total S3 Storage: {size_gb:.2f} GB") print(f" Non-Central Buckets: {len(unique_non_central)}") diff --git a/local-app/python-tools/cross-organization/check_cloudtrail.py b/local-app/python-tools/cross-organization/check_cloudtrail.py index 95626159..90541ab0 100644 --- a/local-app/python-tools/cross-organization/check_cloudtrail.py +++ b/local-app/python-tools/cross-organization/check_cloudtrail.py @@ -2,7 +2,8 @@ import time from datetime import datetime, timedelta -__version__ = "1.1.1" +# --- VERSIONING --- +__version__ = "1.1.2" def get_s3_metrics(session, bucket_name, region): cw = session.client('cloudwatch', region_name=region) @@ -49,8 +50,8 @@ def account_task(account_session, account_id, account_name, region): found_trail_arns = set() for reg in enabled_regions: + reg_start = time.perf_counter() ct = account_session.client('cloudtrail', region_name=reg) - # IncludeShadowTrails catches regional replications of Org/Multi-region trails trails = ct.describe_trails(includeShadowTrails=True).get('trailList', []) for trail in trails: @@ -64,7 +65,6 @@ def account_task(account_session, account_id, account_name, region): else: local_trail_count += 1 t_name = trail['Name'] - # Use .get() for optional fields like SnsTopicARN and KmsKeyId t_data = { "trail_name": t_name, "home_region": trail.get('HomeRegion', reg), @@ -75,7 +75,7 @@ def account_task(account_session, account_id, account_name, region): "log_file_validation": str(trail.get('LogFileValidationEnabled', False)), "sns_topic": trail.get('SnsTopicARN', 'N/A'), "kms_key_id": trail.get('KmsKeyId', 'SSE-S3'), - "check_elapsed_sec": 0 + "check_elapsed_sec": round(time.perf_counter() - reg_start, 3) } if 'CloudWatchLogsLogGroupArn' in trail: @@ -86,7 +86,6 @@ def account_task(account_session, account_id, account_name, region): results["data"][f"trail:{t_name}:{t_data['home_region']}"] = t_data - # Logic summary summary = "OK" if org_trail_count > 0 and local_trail_count > 0: summary = f"DUPLICATIVE/{local_trail_count}_LOCAL_WITH_ORG" diff --git a/local-app/python-tools/cross-organization/check_config.py b/local-app/python-tools/cross-organization/check_config.py index e20bb10c..872003b3 100644 --- a/local-app/python-tools/cross-organization/check_config.py +++ b/local-app/python-tools/cross-organization/check_config.py @@ -2,7 +2,8 @@ import time from datetime import datetime, timedelta -__version__ = "1.0.8" +# --- VERSIONING --- +__version__ = "1.0.9" def get_s3_metrics(session, bucket_name, region): cw = session.client('cloudwatch', region_name=region) @@ -30,7 +31,6 @@ def account_task(account_session, account_id, account_name, region): recorders = config.describe_configuration_recorders().get('ConfigurationRecorders', []) channels = config.describe_delivery_channels().get('DeliveryChannels', []) - # Re-added Global Resource Check is_global = any(r.get('recordingGroup', {}).get('includeGlobalResourceTypes') for r in recorders) if is_global: global_count += 1 @@ -43,7 +43,6 @@ def account_task(account_session, account_id, account_name, region): reg_data["check_elapsed_sec"] = round(time.perf_counter() - reg_start, 3) results["data"][reg] = reg_data - # Per-Account Summary record summary_val = f"OK/1" if global_count == 1 else f"MULTIPLE/{global_count}" if global_count > 1 else "NONE/0" results["data"]["account_summary"] = {"_summary": summary_val} diff --git a/local-app/python-tools/cross-organization/org_runner.py b/local-app/python-tools/cross-organization/org_runner.py index ef98fc50..621ed580 100755 --- a/local-app/python-tools/cross-organization/org_runner.py +++ b/local-app/python-tools/cross-organization/org_runner.py @@ -18,7 +18,7 @@ def tqdm(iterable, **kwargs): return iterable # --- VERSIONING --- -__version__ = "1.6.2" +__version__ = "1.6.3" class OrgTaskRunner: def __init__(self, args): @@ -27,8 +27,10 @@ def __init__(self, args): self.hierarchy_cache = {} self.created_files = [] self.start_time = 0 + self.org_id = "unknown" def get_ou_path(self, org_client, entity_id): + """Recursively resolves OU path and caches results.""" if entity_id in self.hierarchy_cache: return self.hierarchy_cache[entity_id] if entity_id.startswith('r-'): self.hierarchy_cache[entity_id] = (None, entity_id) @@ -45,6 +47,7 @@ def get_ou_path(self, org_client, entity_id): except: return "Unknown", entity_id def process_account(self, acc, partition, tasks): + """Worker thread logic for cross-account execution.""" thread_session = boto3.Session(profile_name=self.args.profile, region_name=self.args.region) sts, org = thread_session.client('sts'), thread_session.client('organizations') acc_id, acc_name = acc['Id'], acc['Name'] @@ -54,9 +57,14 @@ def process_account(self, acc, partition, tasks): ou_path, ou_id = self.get_ou_path(org, parents[0]['Id']) if parents else ("Orphaned", "N/A") ou_path = ou_path if ou_path else "Root" + # Metadata including Org and OU details account_metadata = { - "account_id": acc_id, "account_name": acc_name, "alias": "N/A", - "ou_path": ou_path, "ou_id": ou_id + "org_id": self.org_id, + "account_id": acc_id, + "account_name": acc_name, + "alias": "N/A", + "ou_path": ou_path, + "ou_id": ou_id } account_results = {"metadata": account_metadata, "checks": {}} @@ -83,6 +91,11 @@ def run(self): org_client = session.client('organizations') partition = session.client('sts').get_caller_identity()['Arn'].split(':')[1] + # Resolve Organization ID + try: + self.org_id = org_client.describe_organization()['Organization']['Id'] + except Exception: pass + # Load tasks & build dynamic metadata tasks, check_info = [], [] if self.args.enable_checks: @@ -94,20 +107,19 @@ def run(self): v = getattr(module, '__version__', '?.?.?') check_info.append(f"{mod_name} (v{v})") - check_suffix = ".".join([m.replace('.py', '') for m in (self.args.enable_checks or [])]) or "connectivity" - - # Gather accounts and count them before processing + # Gather accounts all_accounts = [acc for page in org_client.get_paginator('list_accounts').paginate() for acc in page['Accounts'] if acc['Status'] == 'ACTIVE'] all_accounts.sort(key=lambda x: x['Name' if self.args.sort == 'name' else 'Id'].lower()) - # RESTORED HEADER + # HEADER print("-" * 100) print(f"AWS ORG TASK RUNNER - v{__version__}") - print(f"Target Role: {self.args.role_name}") - print(f"Runners: {self.args.max_workers}") - print(f"Enabled Checks: {', '.join(check_info) if check_info else 'None'}") - print(f"Accounts Found: {len(all_accounts)}") + print(f"Organization ID: {self.org_id}") + print(f"Target Role: {self.args.role_name}") + print(f"Runners: {self.args.max_workers}") + print(f"Enabled Checks: {', '.join(check_info) if check_info else 'None'}") + print(f"Accounts Found: {len(all_accounts)}") print("-" * 100) with ThreadPoolExecutor(max_workers=self.args.max_workers) as executor: @@ -121,11 +133,12 @@ def run(self): if self.args.output: ds = datetime.now().strftime("%Y%m%dT%H%M%S") - # 1. ACCOUNT BASELINE + # 1. ACCOUNT BASELINE (JSON/CSV) acc_base = f"audit_results.account.{ds}" - with open(f"{acc_base}.json", 'w') as f: json.dump([r['metadata'] for r in self.full_results], f, indent=2) + with open(f"{acc_base}.json", 'w') as f: + json.dump([r['metadata'] for r in self.full_results], f, indent=2) with open(f"{acc_base}.csv", 'w', newline='') as f: - w = csv.DictWriter(f, fieldnames=["account_id", "account_name", "alias", "ou_path", "ou_id"]) + w = csv.DictWriter(f, fieldnames=["org_id", "account_id", "account_name", "alias", "ou_path", "ou_id"]) w.writeheader() w.writerows([r['metadata'] for r in self.full_results]) self.created_files.extend([f"{acc_base}.json", f"{acc_base}.csv"]) @@ -133,17 +146,24 @@ def run(self): # 2. CHECK SPECIFIC FILES for cn in [m.replace('.py', '') for m in (self.args.enable_checks or [])]: chk_base = f"audit_results.{cn}.{ds}" - # Save CSV + # Save CSV (Long Format) with open(f"{chk_base}.csv", 'w', newline='') as f: w = csv.writer(f) - w.writerow(["account_id", "account_alias", "region", "field_name", "field_value"]) + w.writerow(["org_id", "account_id", "account_alias", "region", "field_name", "field_value"]) for res in self.full_results: for reg, fields in res["checks"].items(): for k, v in fields.items(): - w.writerow([res["metadata"]["account_id"], res["metadata"]["alias"], reg, k, v]) - # Save JSON + w.writerow([self.org_id, res["metadata"]["account_id"], res["metadata"]["alias"], reg, k, v]) + # Save JSON (Nested Format) with open(f"{chk_base}.json", 'w') as f: - json.dump([{"account_id": r["metadata"]["account_id"], "alias": r["metadata"]["alias"], "data": r["checks"]} for r in self.full_results], f, indent=2) + json.dump([{ + "org_id": self.org_id, + "account_id": r["metadata"]["account_id"], + "alias": r["metadata"]["alias"], + "ou_path": r["metadata"]["ou_path"], + "ou_id": r["metadata"]["ou_id"], + "data": r["checks"] + } for r in self.full_results], f, indent=2) self.created_files.extend([f"{chk_base}.csv", f"{chk_base}.json"]) @@ -156,5 +176,7 @@ def run(self): p.add_argument("--output", nargs='?', const='DEFAULT') p.add_argument("--enable-checks", nargs='+') p.add_argument("--max-workers", type=int, default=8) - p.add_argument("--profile"); p.add_argument("--region", default="us-east-1"); p.add_argument("--sort", default="name") + p.add_argument("--profile") + p.add_argument("--region", default="us-east-1") + p.add_argument("--sort", default="name") OrgTaskRunner(p.parse_args()).run()