diff --git a/local-app/python-tools/cross-organization/assess_check_s3_buckets.py b/local-app/python-tools/cross-organization/assess_check_s3_buckets.py index 2b71ab69..84615642 100755 --- a/local-app/python-tools/cross-organization/assess_check_s3_buckets.py +++ b/local-app/python-tools/cross-organization/assess_check_s3_buckets.py @@ -9,7 +9,7 @@ from collections import defaultdict # --- VERSIONING --- -__version__ = "1.3.1" +__version__ = "1.4.0" UNIT_CONVERSION_MAP = { "B": {"divisor": 1, "label": "Bytes"}, @@ -24,7 +24,7 @@ def find_latest_file(pattern): return max(files, key=os.path.getctime) if files else None def main(): - parser = argparse.ArgumentParser(description="AWS S3 Fleet Storage Filtered Assessor Suite v1.3.1") + parser = argparse.ArgumentParser(description="AWS S3 Fleet Storage Segmented Filter Assessor Suite v1.4.0") parser.add_argument("--input", help="JSON file (default: latest audit_results.check_s3_buckets.*.json)") parser.add_argument("--size-units", default="GB", choices=["B", "MB", "GB", "TB", "PB"], help="Target size scale unit representation to output in reports (Default: GB)") @@ -58,9 +58,15 @@ def main(): divisor = scale_spec["divisor"] unit_label = scale_spec["label"] - total_buckets = 0 - total_global_bytes = 0 - total_global_objects = 0 + # Global Footprint Accumulators + global_total_buckets = 0 + global_total_bytes = 0 + global_total_objects = 0 + + # Filtered Footprint Accumulators + filtered_total_buckets = 0 + filtered_total_bytes = 0 + filtered_total_objects = 0 account_summary = defaultdict(lambda: {"count": 0, "bytes": 0, "objects": 0}) storage_class_summary = defaultdict(int) @@ -78,15 +84,23 @@ def main(): max_alias_w = max(max_alias_w, len(str(alias))) for b in buckets: + b_bytes = b.get("size_bytes", 0) + b_objects = b.get("object_count", 0) + + # 1. Unconditionally tally raw global metric layers + global_total_buckets += 1 + global_total_bytes += b_bytes + global_total_objects += b_objects + + # 2. Evaluate Regex filter threshold criteria if bucket_regex and not bucket_regex.search(b["name"]): continue - total_buckets += 1 - b_bytes = b.get("size_bytes", 0) - b_objects = b.get("object_count", 0) + # 3. Populate downstream metrics for matching resources only + filtered_total_buckets += 1 + filtered_total_bytes += b_bytes + filtered_total_objects += b_objects - total_global_bytes += b_bytes - total_global_objects += b_objects max_bucket_w = max(max_bucket_w, len(b["name"])) account_summary[f"{alias} ({acc_id})"]["count"] += 1 @@ -101,6 +115,7 @@ def main(): b["versioning_status"], b["kms_key_id"], b["has_lifecycle"] ]) + # Write Filtered Inventory CSV File with open(csv_filename, "w", newline="", encoding="utf-8") as cf: writer = csv.writer(cf) writer.writerow(["Account ID", "Account Alias", "Bucket Name", "Region", "ARN", f"Size ({target_unit})", "Object Count", "Versioning Status", "KMS Key ID", "Has Lifecycle"]) @@ -113,16 +128,25 @@ def main(): tf.write(f"AWS ORGANIZATIONAL S3 METRICS STORAGE OPTIMIZATION REPORT | Source: {input_base_name}\n") tf.write("=" * 155 + "\n\n") - # Section 1 + # --- SECTION 1: EXPANDED DUAL SEGMENT FOOTPRINT MATRIX --- tf.write("[SECTION 1: FLEET METRICS PROFILE OVERVIEW]\n") - if bucket_regex: - tf.write(f" [FILTER ACTIVE] Target Bucket Name Regex Match Pattern: '{args.bucket_name}'\n") - tf.write(f" ▶ Total Corporate S3 Buckets Managed : {total_buckets}\n") - scaled_global_size = total_global_bytes if target_unit == "B" else (total_global_bytes / divisor) + # Format specifiers logic map block fmt_spec = ",d" if target_unit == "B" else ",.2f" - tf.write(f" ▶ Total Global Footprint Aggregation : {scaled_global_size:{fmt_spec}} {unit_label}\n") - tf.write(f" ▶ Total Allocated Object Count : {total_global_objects:,} Objects\n\n") + scaled_global_size = global_total_bytes if target_unit == "B" else (global_total_bytes / divisor) + + tf.write(f" ▶ Global Unfiltered Footprint:\n") + tf.write(f" • Total S3 Buckets Managed : {global_total_buckets} Buckets\n") + tf.write(f" • Cumulative Raw Storage Volume : {scaled_global_size:{fmt_spec}} {unit_label}\n") + tf.write(f" • Aggregate Object Count : {global_total_objects:,} Objects\n") + + if bucket_regex: + scaled_filtered_size = filtered_total_bytes if target_unit == "B" else (filtered_total_bytes / divisor) + tf.write(f"\n ▶ Filtered Subset Footprint (Match Pattern: '{args.bucket_name}'):\n") + tf.write(f" • Total Matching S3 Buckets : {filtered_total_buckets} Buckets\n") + tf.write(f" • Cumulative Subset Storage Vol : {scaled_filtered_size:{fmt_spec}} {unit_label}\n") + tf.write(f" • Subset Target Object Count : {filtered_total_objects:,} Objects\n") + tf.write("\n") # Section 2 tf.write("[SECTION 2: ACCOUNT BILLING SECTOR SUMMARY BREAKDOWN]\n") @@ -144,12 +168,12 @@ def main(): fmt_tier = f" • Storage Tier: {{:<30}} | Footprint ({target_unit}): {{:>17,.3f}} | Share %: {{:>8.2f}}\n" for s_tier, t_bytes in sorted(storage_class_summary.items(), key=lambda x: x[1], reverse=True): - share = (t_bytes / total_global_bytes * 100) if total_global_bytes > 0 else 0.0 + share = (t_bytes / filtered_total_bytes * 100) if filtered_total_bytes > 0 else 0.0 scaled_tier_size = t_bytes if target_unit == "B" else (t_bytes / divisor) tf.write(fmt_tier.format(s_tier, scaled_tier_size, share)) tf.write("\n") - # --- RE-ORDERED SECTION 4: TOP 20 LARGEST BUCKETS --- + # Section 4: Top 20 Largest Buckets tf.write("[SECTION 4: TOP 20 LARGEST BUCKETS BY INSTALLED VOLUME FOOTPRINT]\n") size_data_spec = ",d" if target_unit == "B" else ",.2f" fmt_top_size_header = f" {{:<4}} | {{:<12}} | {{:<{max_alias_w}}} | {{:<{max_bucket_w}}} | {{:<12}} | {{:>18}} | {{:>12}}\n" @@ -165,7 +189,7 @@ def main(): tf.write(fmt_top_size_data.format(idx, r[0], r[1], r[2], r[3], scaled_bucket_size, r[6])) tf.write("\n") - # --- RE-ORDERED SECTION 5: TOP 20 BUCKETS BY TOTAL OBJECT COUNT --- + # Section 5: Top 20 Most Populated Buckets tf.write("[SECTION 5: TOP 20 MOST POPULATED BUCKETS BY ALLOCATED OBJECT DENSITY]\n") fmt_top_obj_header = f" {{:<4}} | {{:<12}} | {{:<{max_alias_w}}} | {{:<{max_bucket_w}}} | {{:<12}} | {{:>18}} | {{:>12}}\n" fmt_top_obj_data = f" #{{:<3}} | {{:<12}} | {{:<{max_alias_w}}} | {{:<{max_bucket_w}}} | {{:<12}} | {{:>18{size_data_spec}}} | {{:>12,}}\n" @@ -179,7 +203,7 @@ def main(): tf.write(fmt_top_obj_data.format(idx, r[0], r[1], r[2], r[3], scaled_bucket_size, r[6])) tf.write("\n") - # --- RE-ORDERED SECTION 6: MASTER MATRIX TABLE --- + # Section 6: Master Matrix Table tf.write("[SECTION 6: GRANULAR BUCKET METADATA INVENTORY MATRIX]\n") fmt_row_header = f" {{:<12}} | {{:<{max_alias_w}}} | {{:<{max_bucket_w}}} | {{:<12}} | {{:>18}} | {{:>12}} | {{:<10}} | {{:<13}}\n" fmt_row_data = f" {{:<12}} | {{:<{max_alias_w}}} | {{:<{max_bucket_w}}} | {{:<12}} | {{:>18{size_data_spec}}} | {{:>12,}} | {{:<10}} | {{:<13}}\n"