discover size header better

local-app/python-tools/cross-organization/purge_check_s3_buckets.py

@@ -18,7 +18,7 @@
 from rich.progress import Progress, BarColumn, TextColumn, TimeElapsedColumn, SpinnerColumn
 
 # --- VERSIONING ---
-__version__ = "2.3.0"
+__version__ = "2.4.0"
 
 console = Console()
 
@@ -176,19 +176,28 @@ def main():
     with open(args.csv, "r", encoding="utf-8") as f:
         reader = csv.DictReader(f)
         for row in reader:
-            b_name = row["Bucket Name"]
-            acct_id = row["Account ID"]
-            acct_alias = row["Account Alias"]
-            b_region = row.get("Region", args.region)
+            # --- FIXED: Explicitly strip whitespace properties out of column values and headers ---
+            clean_row = {str(k).strip(): str(v).strip() for k, v in row.items() if k is not None}
 
+            b_name = clean_row.get("Bucket Name")
+            acct_id = clean_row.get("Account ID")
+            acct_alias = clean_row.get("Account Alias")
+            b_region = clean_row.get("Region", args.region)
+
+            if not b_name: continue
+
+            # --- FIXED: Robust fuzzy column mapping ensuring comma stripping does not zero weights ---
             raw_size = 0
             raw_objects = 0
-            for k, v in row.items():
-                if "size" in k.lower():
-                    try: raw_size = int(float(v))
+            for key_str, val_str in clean_row.items():
+                normalized_key = key_str.lower()
+                clean_val = val_str.replace(",", "")
+
+                if "size" in normalized_key:
+                    try: raw_size = int(float(clean_val))
                     except Exception: pass
-                if "object" in k.lower():
-                    try: raw_objects = int(v.replace(",", ""))
+                if "object" in normalized_key:
+                    try: raw_objects = int(float(clean_val))
                     except Exception: pass
 
             if not bucket_pattern.search(b_name):
@@ -230,7 +239,6 @@ def main():
     )
     macro_task_id = overall_progress.add_task("Purge Fleet", total=len(targets))
 
-    # --- FIXED: Isolate pure full bucket namespace variable mapping outputs ---
     lane_progress = Progress(
         TextColumn("[bold blue]Lane #{task.fields[lane_idx]}[/bold blue]"),
         TextColumn("([bold white]{task.fields[bucket_name]}[/bold white])"),
@@ -275,10 +283,7 @@ def main():
                         pct = int((current_val / total_w) * 100) if total_w > 0 else 0
                         if pct > 100: pct = 100
 
-                        # --- FIXED: Pass full string directly to layout engine with zero index clipping boundaries ---
-                        b_display = signal["bucket_name"]
-
-                        lane_progress.update(t_id, bucket_name=b_display, completed=pct, current_test=signal["current_test"], total_tests=signal["total_tests"])
+                        lane_progress.update(t_id, bucket_name=signal["bucket_name"], completed=pct, current_test=signal["current_test"], total_tests=signal["total_tests"])
                     except queue.Empty:
                         break
 

local-app/python-tools/cross-organization/run.check_s3_buckets.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+VERSION="1.0.0"
+export PYTHONUNBUFFERED=1
+
+if [ -z "$1" ]
+then
+  ORG="ent-gov"
+else
+  ORG="$1"
+fi
+
+WORKERS=8
+if [ "$ORG" == "ent-gov" ]
+then
+  REGION="us-gov-east-1"
+  REGIONS="us-gov-east-1 us-gov-west-1"
+  WORKERS=16
+fi
+if [ "$ORG" == "lab-gov" ]
+then
+  REGION="us-gov-east-1"
+  REGIONS="us-gov-east-1 us-gov-west-1"
+fi
+if [ "$ORG" == "ent-ew" ]
+then
+  REGION="us-east-1"
+  REGIONS="us-east-1 us-east-1 us-west-1 us-west-2"
+  WORKERS=16
+fi
+
+if [ ! -z "$2" ]
+then
+  REGION="$2"
+fi
+
+CHECK="check_s3_buckets"
+
+if [ $CHECK == "NAME" ]
+then
+  echo "* configure for the appropriate check"
+  exit 1
+fi
+
+# --dry-run
+# --no-logfile
+echo "* executing for ORG $ORG region $REGION check $CHECK workers $WORKERS"
+echo "./org_runner3.py --profile $ORG.org --region $REGION \
+  --progress-account \
+  --role-name r-inf-org-controller \
+  --output \
+  --module $CHECK \
+  --regions $REGIONS \
+  --max-workers $WORKERS "