From 9f06bad3349535477b2ff9bb5aa7380a7b645732 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 14 Mar 2025 18:50:33 -0400 Subject: [PATCH] add support for combined tags in eks --- .../gfl-resource-actions/discovery.py | 91 +++- .../gfl-resource-actions/logging_utils.py | 65 ++- .../gfl-resource-actions/managers/base.py | 25 +- .../gfl-resource-actions/managers/eks.py | 446 +++++++++++------- 4 files changed, 402 insertions(+), 225 deletions(-) diff --git a/local-app/python-tools/gfl-resource-actions/discovery.py b/local-app/python-tools/gfl-resource-actions/discovery.py index 0fde7627..de336b82 100644 --- a/local-app/python-tools/gfl-resource-actions/discovery.py +++ b/local-app/python-tools/gfl-resource-actions/discovery.py @@ -263,6 +263,8 @@ def get_eks_clusters(credentials, region, account_id=None, account_name=None): response = eks_client.list_clusters() cluster_names = response.get('clusters', []) + logger.info(f"Found {len(cluster_names)} EKS clusters in region {region}") + # Get detailed info for each cluster for cluster_name in cluster_names: try: @@ -271,6 +273,43 @@ def get_eks_clusters(credentials, region, account_id=None, account_name=None): # Get tags tags = cluster_info.get('tags', {}) + # Look for both individual scaling tags and combined format + min_nodes = tags.get('eks-min-nodes', '') + max_nodes = tags.get('eks-max-nodes', '') + desired_nodes = tags.get('eks-desired-nodes', '') + + # Parse the combined cluster:size tag if it exists + if 'cluster:size' in tags: + size_tag = tags['cluster:size'] + logger.debug(f"Found cluster:size tag: {size_tag} for cluster {cluster_name}") + try: + # Parse min:X-max:Y-desired:Z format + if 'min:' in size_tag and 'max:' in size_tag and 'desired:' in size_tag: + # Extract values using regex or string parsing + min_match = size_tag.split('min:')[1].split('-')[0] if 'min:' in size_tag else None + max_match = size_tag.split('max:')[1].split('-')[0] if 'max:' in size_tag else None + desired_match = size_tag.split('desired:')[1].split('-')[0] if 'desired:' in size_tag else None + + min_nodes = min_match if min_match and not min_nodes else min_nodes + max_nodes = max_match if max_match and not max_nodes else max_nodes + desired_nodes = desired_match if desired_match and not desired_nodes else desired_nodes + + logger.debug(f"Parsed cluster:size: min={min_nodes}, max={max_nodes}, desired={desired_nodes}") + except Exception as parse_error: + logger.warning(f"Error parsing cluster:size tag for cluster {cluster_name}: {parse_error}") + + # Check for backup tags that store original values + original_min = tags.get('eks-original-min-nodes', '') + original_max = tags.get('eks-original-max-nodes', '') + original_desired = tags.get('eks-original-desired-nodes', '') + + # Check for schedule tag + schedule = None + for tag_key, tag_value in tags.items(): + if 'schedule' in tag_key.lower(): + schedule = tag_value + break + cluster_data = { "name": cluster_name, "id": cluster_name, # Using name as ID for consistency @@ -278,12 +317,22 @@ def get_eks_clusters(credentials, region, account_id=None, account_name=None): "region": region, "endpoint": cluster_info.get('endpoint', ''), "version": cluster_info.get('version', ''), + "min_nodes": min_nodes, + "max_nodes": max_nodes, + "desired_nodes": desired_nodes, + "original_min": original_min, + "original_max": original_max, + "original_desired": original_desired, + "schedule": schedule, "tags": tags } clusters.append(cluster_data) - # Log the discovery + # Log the discovery with scaling info and schedule + scale_details = f"Min: {min_nodes or 'N/A'}, Max: {max_nodes or 'N/A'}, Desired: {desired_nodes or 'N/A'}" + schedule_info = f", Schedule: {schedule}" if schedule else "" + if account_id: log_action_to_csv( account_id=account_id, @@ -294,7 +343,8 @@ def get_eks_clusters(credentials, region, account_id=None, account_name=None): action="discover", region=region, status="success", - details=f"Status: {cluster_data['status']}, Version: {cluster_data['version']}" + details=f"Status: {cluster_data['status']}, Version: {cluster_data['version']}, Scaling: {scale_details}{schedule_info}", + existing_schedule=schedule ) except Exception as e: @@ -312,6 +362,16 @@ def get_eks_clusters(credentials, region, account_id=None, account_name=None): # Get tags tags = cluster_info.get('tags', {}) + # Extract scaling parameters from tags if they exist + min_nodes = tags.get('eks-min-nodes', '') + max_nodes = tags.get('eks-max-nodes', '') + desired_nodes = tags.get('eks-desired-nodes', '') + + # Check for backup tags that store original values + original_min = tags.get('eks-original-min-nodes', '') + original_max = tags.get('eks-original-max-nodes', '') + original_desired = tags.get('eks-original-desired-nodes', '') + cluster_data = { "name": cluster_name, "id": cluster_name, # Using name as ID for consistency @@ -319,12 +379,20 @@ def get_eks_clusters(credentials, region, account_id=None, account_name=None): "region": region, "endpoint": cluster_info.get('endpoint', ''), "version": cluster_info.get('version', ''), + "min_nodes": min_nodes, + "max_nodes": max_nodes, + "desired_nodes": desired_nodes, + "original_min": original_min, + "original_max": original_max, + "original_desired": original_desired, "tags": tags } clusters.append(cluster_data) - # Log the discovery + # Log the discovery with scaling info + scale_details = f"Min: {min_nodes or 'N/A'}, Max: {max_nodes or 'N/A'}, Desired: {desired_nodes or 'N/A'}" + if account_id: log_action_to_csv( account_id=account_id, @@ -335,13 +403,24 @@ def get_eks_clusters(credentials, region, account_id=None, account_name=None): action="discover", region=region, status="success", - details=f"Status: {cluster_data['status']}, Version: {cluster_data['version']}" + details=f"Status: {cluster_data['status']}, Version: {cluster_data['version']}, Scaling: {scale_details}" ) except Exception as e: logger.warning(f"Error getting details for EKS cluster {cluster_name} in region {region}: {e}") - logger.info(f"Found {len(clusters)} EKS clusters in region {region}") + # Log the summary + scaling_stats = {} + for cluster in clusters: + has_scaling = any([cluster.get('min_nodes'), cluster.get('max_nodes'), cluster.get('desired_nodes')]) + scaling_stats["with_scaling_tags"] = scaling_stats.get("with_scaling_tags", 0) + (1 if has_scaling else 0) + scaling_stats["without_scaling_tags"] = scaling_stats.get("without_scaling_tags", 0) + (0 if has_scaling else 1) + + if scaling_stats: + logger.info(f"EKS clusters in {region}: {len(clusters)} total, " + f"{scaling_stats.get('with_scaling_tags', 0)} with scaling tags, " + f"{scaling_stats.get('without_scaling_tags', 0)} without scaling tags") + return clusters except Exception as e: @@ -370,7 +449,7 @@ def get_emr_clusters(credentials, region, account_id=None, account_name=None): emr_client = create_boto3_client('emr', credentials, region) clusters = [] - # List active clusters (RUNNING, WAITING, STARTING, BOOTSTRAPPING, TERMINATING) + # List active clusters (RUNNING, WAITING, STARTING, BOOTSTRAPPING, TERMINATING, TERMINATED_WITH_ERRORS) response = emr_client.list_clusters( ClusterStates=['RUNNING', 'WAITING', 'STARTING', 'BOOTSTRAPPING', 'TERMINATING', 'TERMINATED_WITH_ERRORS'] ) diff --git a/local-app/python-tools/gfl-resource-actions/logging_utils.py b/local-app/python-tools/gfl-resource-actions/logging_utils.py index 6f84b6a9..c9c51b94 100644 --- a/local-app/python-tools/gfl-resource-actions/logging_utils.py +++ b/local-app/python-tools/gfl-resource-actions/logging_utils.py @@ -69,43 +69,40 @@ def initialize_csv_log(csv_file=None): return csv_path -def log_action_to_csv(account_id, account_name, resource_type, resource_id, - resource_name, action, region, status, details='', csv_file=None): - """ - Log an action to the CSV file. +def log_action_to_csv(account_id, account_name, resource_type, resource_id, resource_name, action, region, status, details="", dry_run=False, existing_schedule=None): + """Log resource action to CSV file for tracking.""" + timestamp = datetime.datetime.now().isoformat() - Args: - account_id (str): AWS account ID - account_name (str): AWS account name - resource_type (str): Type of resource (e.g., 'ec2', 'rds') - resource_id (str): Resource identifier - resource_name (str): Resource name - action (str): Action performed (e.g., 'discover', 'stop', 'start') - region (str): AWS region - status (str): Status of the action (e.g., 'success', 'failed') - details (str, optional): Additional details - csv_file (str, optional): CSV file name (default: resource_actions.csv) - """ - if csv_file is None: - csv_file = ACTION_CSV_FILE + # Create directory if it doesn't exist + os.makedirs(os.path.dirname(CSV_LOG_DIR), exist_ok=True) - csv_path = initialize_csv_log(csv_file) - timestamp = datetime.datetime.now().isoformat() + # Check if file exists to determine if we need to write headers + file_exists = os.path.isfile(CSV_LOG_DIR) - with open(csv_path, 'a', newline='') as f: - writer = csv.writer(f) - writer.writerow([ - timestamp, - account_id, - account_name, - resource_type, - resource_id, - resource_name, - action, - region, - status, - details - ]) + with open(CSV_LOG_DIR, 'a', newline='') as csvfile: + fieldnames = ['timestamp', 'account_id', 'account_name', 'resource_type', 'resource_id', + 'resource_name', 'action', 'region', 'status', 'details', 'dry_run', 'schedule'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + # Write header if file is new + if not file_exists: + writer.writeheader() + + # Write the log entry + writer.writerow({ + 'timestamp': timestamp, + 'account_id': account_id, + 'account_name': account_name, + 'resource_type': resource_type, + 'resource_id': resource_id, + 'resource_name': resource_name, + 'action': action, + 'region': region, + 'status': status, + 'details': details, + 'dry_run': 'Yes' if dry_run else 'No', + 'schedule': existing_schedule or '' + }) # Create a logger instance for direct import logger = setup_logging() diff --git a/local-app/python-tools/gfl-resource-actions/managers/base.py b/local-app/python-tools/gfl-resource-actions/managers/base.py index de2273c6..3c3a38a9 100644 --- a/local-app/python-tools/gfl-resource-actions/managers/base.py +++ b/local-app/python-tools/gfl-resource-actions/managers/base.py @@ -30,16 +30,21 @@ def get_timestamp(self): """Get ISO 8601 timestamp.""" return datetime.datetime.now().isoformat() - def log_action(self, resource_id, region, action, resource_name=None, dry_run=False): - """Log resource action.""" - action_prefix = "[DRY RUN] " if dry_run else "" - resource_info = f"name={resource_name}, " if resource_name else "" - timestamp = self.get_timestamp() - - logger.info(f"{action_prefix}ACTION: resource={resource_id}, {resource_info}region={region}, " - f"account_id={self.account_id}, action={action}, timestamp={timestamp}") - - return timestamp + def log_action(self, resource_id, region, action, details="", dry_run=False, existing_schedule=None): + """Log a resource action to CSV for audit trail.""" + log_action_to_csv( + account_id=self.account_id, + account_name=self.account_name, + resource_type=self.resource_type, + resource_id=resource_id, + resource_name=resource_id, # Using ID as name for simplicity + action=action, + region=region, + status="simulated" if dry_run else "success", + details=details, + dry_run=dry_run, + existing_schedule=existing_schedule + ) def should_exclude(self, resource): """Check if resource should be excluded based on tags.""" diff --git a/local-app/python-tools/gfl-resource-actions/managers/eks.py b/local-app/python-tools/gfl-resource-actions/managers/eks.py index ac0f12ed..f0a2c643 100644 --- a/local-app/python-tools/gfl-resource-actions/managers/eks.py +++ b/local-app/python-tools/gfl-resource-actions/managers/eks.py @@ -7,207 +7,303 @@ logger = setup_logging() +# Tag names for EKS scaling +EKS_MIN_NODES_TAG = 'eks-min-nodes' +EKS_MAX_NODES_TAG = 'eks-max-nodes' +EKS_DESIRED_NODES_TAG = 'eks-desired-nodes' +EKS_ORIGINAL_MIN_NODES_TAG = 'eks-original-min-nodes' +EKS_ORIGINAL_MAX_NODES_TAG = 'eks-original-max-nodes' +EKS_ORIGINAL_DESIRED_NODES_TAG = 'eks-original-desired-nodes' +CLUSTER_SIZE_TAG = 'cluster:size' +EKS_ORIGINAL_CLUSTER_SIZE_TAG = 'eks-original-cluster-size' + class EKSManager(base.ResourceManager): """Manager for EKS clusters.""" - def stop(self, clusters, dry_run=False): - """ - Stop EKS clusters by scaling down node groups to zero. - This doesn't actually delete clusters, just scales down node groups. - """ + def scale_down(self, clusters, dry_run=False): + """Scale down EKS clusters by setting node counts to 0.""" + logger.info(f"Evaluating {len(clusters)} EKS clusters for scale down action") + scaled_count = 0 + skipped_count = 0 + for cluster in clusters: # Skip clusters with the exclusion tag if self.should_exclude(cluster): logger.info(f"Skipping EKS cluster {cluster['name']} with exclusion tag {config.EXCLUSION_TAG}") + skipped_count += 1 continue - if cluster["status"] == "ACTIVE": - try: - region = cluster["region"] - eks_client = self.create_client('eks', region) - autoscaling_client = self.create_client('autoscaling', region) - - # Get nodegroups - nodegroups = eks_client.list_nodegroups(clusterName=cluster['name']).get('nodegroups', []) - + try: + region = cluster["region"] + eks_client = self.create_client('eks', region) + + # Get current scaling parameters from tags or nodegroups + min_nodes = cluster.get('min_nodes', '') + max_nodes = cluster.get('max_nodes', '') + desired_nodes = cluster.get('desired_nodes', '') + schedule = cluster.get('schedule', '') + + # Check if we have a combined cluster:size tag + has_combined_size_tag = CLUSTER_SIZE_TAG in cluster.get('tags', {}) + + # Only proceed if we have some scaling values to work with + if min_nodes or max_nodes or desired_nodes or has_combined_size_tag: logger.info(f"{'[DRY RUN] Would scale down' if dry_run else 'Scaling down'} EKS cluster {cluster['name']} in region {region}") - timestamp = self.get_timestamp() - - # Tag the cluster before scaling down - logger.info(f"{'[DRY RUN] Would tag' if dry_run else 'Tagging'} EKS cluster {cluster['name']} with {config.STOP_TAG}={timestamp}") if not dry_run: - eks_client.tag_resource( - resourceArn=f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}", - tags={ - config.STOP_TAG: timestamp - } - ) - - # Scale down each nodegroup - for nodegroup in nodegroups: - ng_info = eks_client.describe_nodegroup( - clusterName=cluster['name'], - nodegroupName=nodegroup - ) + # First, backup the current values in special tags + tags_to_update = {} - if 'autoScalingGroups' in ng_info['nodegroup']: - for asg in ng_info['nodegroup']['autoScalingGroups']: - asg_name = asg['name'] - - # Get current ASG configuration - asg_info = autoscaling_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[asg_name] - ) - - if len(asg_info['AutoScalingGroups']) > 0: - current_asg = asg_info['AutoScalingGroups'][0] - - # Save the current min size and desired capacity as tags - current_min_size = current_asg['MinSize'] - current_desired_size = current_asg['DesiredCapacity'] - - # Only save if greater than 0 - if current_min_size > 0 or current_desired_size > 0: - logger.info(f"{'[DRY RUN] Would save' if dry_run else 'Saving'} original ASG {asg_name} sizes: min={current_min_size}, desired={current_desired_size}") - if not dry_run: - autoscaling_client.create_or_update_tags( - Tags=[ - { - 'ResourceId': asg_name, - 'ResourceType': 'auto-scaling-group', - 'Key': config.TAG_KEY_ORIGINAL_MIN_SIZE, - 'Value': str(current_min_size), - 'PropagateAtLaunch': False - }, - { - 'ResourceId': asg_name, - 'ResourceType': 'auto-scaling-group', - 'Key': config.TAG_KEY_ORIGINAL_DESIRED_SIZE, - 'Value': str(current_desired_size), - 'PropagateAtLaunch': False - } - ] - ) - - # Now scale down the ASG - logger.info(f"{'[DRY RUN] Would scale down' if dry_run else 'Scaling down'} ASG {asg_name} for nodegroup {nodegroup}") - if not dry_run: - autoscaling_client.update_auto_scaling_group( - AutoScalingGroupName=asg_name, - MinSize=0, - DesiredCapacity=0 - ) + # Handle combined size tag if it exists + if has_combined_size_tag: + original_size_tag = cluster['tags'][CLUSTER_SIZE_TAG] + tags_to_update[EKS_ORIGINAL_CLUSTER_SIZE_TAG] = original_size_tag + tags_to_update[CLUSTER_SIZE_TAG] = "min:0-max:0-desired:0" + else: + # Store original values in backup tags (individual tags) + if min_nodes: + tags_to_update[EKS_ORIGINAL_MIN_NODES_TAG] = min_nodes + tags_to_update[EKS_MIN_NODES_TAG] = '0' + if max_nodes: + tags_to_update[EKS_ORIGINAL_MAX_NODES_TAG] = max_nodes + tags_to_update[EKS_MAX_NODES_TAG] = '0' + if desired_nodes: + tags_to_update[EKS_ORIGINAL_DESIRED_NODES_TAG] = desired_nodes + tags_to_update[EKS_DESIRED_NODES_TAG] = '0' + + # Apply the tag updates + if tags_to_update: + logger.info(f"Updating scaling tags for EKS cluster {cluster['name']}: {tags_to_update}") + eks_client.tag_resource( + resourceArn=cluster.get('arn', f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}"), + tags=tags_to_update + ) + + # Now actually scale down the nodegroups + self._scale_nodegroups(eks_client, cluster['name'], 0, region, dry_run=False) + else: + # Dry run reporting + if has_combined_size_tag: + original_size = cluster['tags'][CLUSTER_SIZE_TAG] + logger.info(f"[DRY RUN] Would backup combined size tag: {original_size} and set to min:0-max:0-desired:0") + else: + logger.info(f"[DRY RUN] Would backup scaling values: Min={min_nodes}, Max={max_nodes}, Desired={desired_nodes}") + + logger.info(f"[DRY RUN] Would set scaling values to 0 for EKS cluster {cluster['name']}") + self._scale_nodegroups(eks_client, cluster['name'], 0, region, dry_run=True) - # Log with detailed information - self.log_action(cluster['name'], region, "scale_down", dry_run=dry_run) + # Log the action with schedule information + schedule_info = f", Schedule: {schedule}" if schedule else "" + self.log_action( + cluster['name'], region, "scale_down", + details=f"Min={min_nodes}->{0}, Max={max_nodes}->{0}, Desired={desired_nodes}->{0}{schedule_info}", + dry_run=dry_run, + existing_schedule=schedule + ) + scaled_count += 1 + else: + logger.info(f"Skipping EKS cluster {cluster['name']}, no scaling tags found") + skipped_count += 1 - except Exception as e: - logger.error(f"Failed to {'simulate scaling down' if dry_run else 'scale down'} EKS cluster {cluster['name']}: {e}") - - def start(self, clusters, dry_run=False): - """ - Start EKS clusters by scaling up node groups. - Uses saved tags to restore original capacity if available. - """ + except Exception as e: + logger.error(f"Failed to {'simulate scaling down' if dry_run else 'scale down'} EKS cluster {cluster['name']}: {e}") + + logger.info(f"EKS scale down summary: {scaled_count} clusters processed, {skipped_count} clusters skipped") + + def scale_up(self, clusters, dry_run=False): + """Scale up EKS clusters using original node counts from tags.""" + logger.info(f"Evaluating {len(clusters)} EKS clusters for scale up action") + scaled_count = 0 + skipped_count = 0 + for cluster in clusters: # Skip clusters with the exclusion tag if self.should_exclude(cluster): logger.info(f"Skipping EKS cluster {cluster['name']} with exclusion tag {config.EXCLUSION_TAG}") + skipped_count += 1 continue + + try: + region = cluster["region"] + eks_client = self.create_client('eks', region) - if cluster["status"] == "ACTIVE": - try: - region = cluster["region"] - eks_client = self.create_client('eks', region) - autoscaling_client = self.create_client('autoscaling', region) - - # Get nodegroups - nodegroups = eks_client.list_nodegroups(clusterName=cluster['name']).get('nodegroups', []) - + # Check for original values in backup tags + original_min = cluster.get('original_min', '') + original_max = cluster.get('original_max', '') + original_desired = cluster.get('original_desired', '') + schedule = cluster.get('schedule', '') + + # Check for original combined size tag + tags = cluster.get('tags', {}) + has_original_combined_tag = EKS_ORIGINAL_CLUSTER_SIZE_TAG in tags + + # If we have backup values or original combined tag, restore them + if original_min or original_max or original_desired or has_original_combined_tag: logger.info(f"{'[DRY RUN] Would scale up' if dry_run else 'Scaling up'} EKS cluster {cluster['name']} in region {region}") - timestamp = self.get_timestamp() - - # Scale up each nodegroup - for nodegroup in nodegroups: - ng_info = eks_client.describe_nodegroup( - clusterName=cluster['name'], - nodegroupName=nodegroup - ) + if not dry_run: + # Restore the original values from backup tags + tags_to_update = {} + tags_to_remove = [] + + # Handle original combined size tag if it exists + if has_original_combined_tag: + original_size_tag = tags[EKS_ORIGINAL_CLUSTER_SIZE_TAG] + tags_to_update[CLUSTER_SIZE_TAG] = original_size_tag + tags_to_remove.append(EKS_ORIGINAL_CLUSTER_SIZE_TAG) + + # Parse the original desired value from the size tag + desired = None + if 'desired:' in original_size_tag: + try: + desired_str = original_size_tag.split('desired:')[1].split('-')[0] + desired = int(desired_str) if desired_str.isdigit() else None + except: + logger.warning(f"Could not parse desired value from combined size tag: {original_size_tag}") + else: + # Restore original values for individual tags + if original_min: + tags_to_update[EKS_MIN_NODES_TAG] = original_min + tags_to_remove.append(EKS_ORIGINAL_MIN_NODES_TAG) + if original_max: + tags_to_update[EKS_MAX_NODES_TAG] = original_max + tags_to_remove.append(EKS_ORIGINAL_MAX_NODES_TAG) + if original_desired: + tags_to_update[EKS_DESIRED_NODES_TAG] = original_desired + tags_to_remove.append(EKS_ORIGINAL_DESIRED_NODES_TAG) + + # Parse desired value + desired = int(original_desired) if original_desired and original_desired.isdigit() else None - if 'autoScalingGroups' in ng_info['nodegroup']: - for asg in ng_info['nodegroup']['autoScalingGroups']: - asg_name = asg['name'] - - # Check current capacity - asg_info = autoscaling_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[asg_name] + # Apply the tag updates + if tags_to_update: + logger.info(f"Restoring original scaling tags for EKS cluster {cluster['name']}: {tags_to_update}") + eks_client.tag_resource( + resourceArn=cluster.get('arn', f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}"), + tags=tags_to_update + ) + + # Now actually scale up the nodegroups to desired capacity + self._scale_nodegroups(eks_client, cluster['name'], desired, region, dry_run=False) + + # Remove the backup tags + if tags_to_remove: + logger.info(f"Removing backup scaling tags: {tags_to_remove}") + eks_client.untag_resource( + resourceArn=cluster.get('arn', f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}"), + tagKeys=tags_to_remove ) - - if len(asg_info['AutoScalingGroups']) > 0: - current_asg = asg_info['AutoScalingGroups'][0] - - # Only scale up if currently at zero - if current_asg['MinSize'] == 0 and current_asg['DesiredCapacity'] == 0: - # Check if we have saved sizes in tags - min_size = config.DEFAULT_NG_MIN_SIZE - desired_size = config.DEFAULT_NG_DESIRED_SIZE - - # Look for original size tags - for tag in current_asg.get('Tags', []): - if tag['Key'] == config.TAG_KEY_ORIGINAL_MIN_SIZE: - try: - min_size = int(tag['Value']) - except (ValueError, TypeError): - logger.warning(f"Invalid tag value for {config.TAG_KEY_ORIGINAL_MIN_SIZE} on ASG {asg_name}: {tag['Value']}") - - if tag['Key'] == config.TAG_KEY_ORIGINAL_DESIRED_SIZE: - try: - desired_size = int(tag['Value']) - except (ValueError, TypeError): - logger.warning(f"Invalid tag value for {config.TAG_KEY_ORIGINAL_DESIRED_SIZE} on ASG {asg_name}: {tag['Value']}") - - logger.info(f"{'[DRY RUN] Would scale up' if dry_run else 'Scaling up'} ASG {asg_name} for nodegroup {nodegroup} to minimum={min_size}, desired={desired_size}") - if not dry_run: - autoscaling_client.update_auto_scaling_group( - AutoScalingGroupName=asg_name, - MinSize=min_size, - DesiredCapacity=desired_size - ) - - # Clean up the tags after restoring - if not dry_run: - try: - autoscaling_client.delete_tags( - Tags=[ - { - 'ResourceId': asg_name, - 'ResourceType': 'auto-scaling-group', - 'Key': config.TAG_KEY_ORIGINAL_MIN_SIZE - }, - { - 'ResourceId': asg_name, - 'ResourceType': 'auto-scaling-group', - 'Key': config.TAG_KEY_ORIGINAL_DESIRED_SIZE - } - ] - ) - except Exception as tag_e: - logger.warning(f"Failed to clean up size tags on ASG {asg_name}: {tag_e}") - else: - logger.info(f"ASG {asg_name} already has capacity. Skipping scale up.") + else: + # Dry run reporting + if has_original_combined_tag: + original_size = tags[EKS_ORIGINAL_CLUSTER_SIZE_TAG] + logger.info(f"[DRY RUN] Would restore combined size tag: {original_size}") + + # Try to parse desired value for nodegroup scaling + desired = None + if 'desired:' in original_size: + try: + desired_str = original_size.split('desired:')[1].split('-')[0] + desired = int(desired_str) if desired_str.isdigit() else None + except: + pass + else: + logger.info(f"[DRY RUN] Would restore scaling values: Min={original_min}, Max={original_max}, Desired={original_desired}") + desired = int(original_desired) if original_desired and original_desired.isdigit() else None + + self._scale_nodegroups(eks_client, cluster['name'], desired, region, dry_run=True) - # Remove the stop tag after successful scale up - if not dry_run: - logger.info(f"Removing {config.STOP_TAG} tag from EKS cluster {cluster['name']}") - eks_client.untag_resource( - resourceArn=f"arn:{self.get_partition_for_region(region)}:eks:{region}:{self.account_id}:cluster/{cluster['name']}", - tagKeys=[config.STOP_TAG] - ) + # Log the action with schedule information + scale_details = "" + if has_original_combined_tag: + original_size = tags.get(EKS_ORIGINAL_CLUSTER_SIZE_TAG, "Unknown") + scale_details = f"Size tag: 0->'{original_size}'" + else: + scale_details = f"Min=0->{original_min}, Max=0->{original_max}, Desired=0->{original_desired}" - # Log with detailed information - self.log_action(cluster['name'], region, "scale_up", dry_run=dry_run) + schedule_info = f", Schedule: {schedule}" if schedule else "" + self.log_action( + cluster['name'], region, "scale_up", + details=f"{scale_details}{schedule_info}", + dry_run=dry_run, + existing_schedule=schedule + ) + scaled_count += 1 + else: + logger.info(f"Skipping EKS cluster {cluster['name']}, no original scaling values found in tags") + skipped_count += 1 + except Exception as e: + logger.error(f"Failed to {'simulate scaling up' if dry_run else 'scale up'} EKS cluster {cluster['name']}: {e}") + + logger.info(f"EKS scale up summary: {scaled_count} clusters processed, {skipped_count} clusters skipped") + + def _scale_nodegroups(self, eks_client, cluster_name, desired_capacity, region, dry_run=False): + """Helper method to scale nodegroups to the specified capacity.""" + try: + # List all nodegroups for this cluster + response = eks_client.list_nodegroups(clusterName=cluster_name) + nodegroup_names = response.get('nodegroups', []) + + # Handle pagination + while 'nextToken' in response: + response = eks_client.list_nodegroups( + clusterName=cluster_name, + nextToken=response['nextToken'] + ) + nodegroup_names.extend(response.get('nodegroups', [])) + + if not nodegroup_names: + logger.info(f"No nodegroups found for EKS cluster {cluster_name}") + return + + for nodegroup_name in nodegroup_names: + try: + # Get nodegroup details + nodegroup = eks_client.describe_nodegroup( + clusterName=cluster_name, + nodegroupName=nodegroup_name + ).get('nodegroup', {}) + + # Get current scaling configuration + current_min = nodegroup.get('scalingConfig', {}).get('minSize') + current_max = nodegroup.get('scalingConfig', {}).get('maxSize') + current_desired = nodegroup.get('scalingConfig', {}).get('desiredSize') + + # Use current min and max when scaling up if desired capacity is provided + if desired_capacity is not None: + # When scaling up, we need a valid min and max + new_min = current_min if desired_capacity == 0 else min(current_min, desired_capacity) + new_max = max(current_max, desired_capacity) + else: + # When scaling down to zero + new_min = 0 + new_max = current_max + desired_capacity = 0 + + if dry_run: + logger.info(f"[DRY RUN] Would update nodegroup {nodegroup_name} scaling: " + + f"Min: {current_min}->{new_min}, " + + f"Max: {current_max}->{new_max}, " + + f"Desired: {current_desired}->{desired_capacity}") + else: + logger.info(f"Updating nodegroup {nodegroup_name} scaling: " + + f"Min: {current_min}->{new_min}, " + + f"Max: {current_max}->{new_max}, " + + f"Desired: {current_desired}->{desired_capacity}") + + # Update the nodegroup scaling configuration + eks_client.update_nodegroup_config( + clusterName=cluster_name, + nodegroupName=nodegroup_name, + scalingConfig={ + 'minSize': new_min, + 'maxSize': new_max, + 'desiredSize': desired_capacity + } + ) except Exception as e: - logger.error(f"Failed to {'simulate scaling up' if dry_run else 'scale up'} EKS cluster {cluster['name']}: {e}") + logger.error(f"Error updating nodegroup {nodegroup_name}: {e}") + + except Exception as e: + logger.error(f"Error listing nodegroups for cluster {cluster_name} in region {region}: {e}")