diff --git a/CHANGELOG.md b/CHANGELOG.md index 797f957..2e60ba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,3 +131,7 @@ - add detection of a runnign EMR cluster with aws:elasticmapreduce: job-flow-id (cluster) and instance-group-role and use this to set an alias defined in boc:dns:cname ({friendlyname}.master), but if it is a cluser node, only use the cname if it is master. If it is not a cluster, set the cname + +* 2.0.1 -- 2023-04-29 + - code 2.0.1 + - add pagination for list hosted zone by vpc (as some vpcs have hundreds of zones, specifically the one with dns endpoints) diff --git a/code/ddns-lambda.py b/code/ddns-lambda.py index fa18018..0394481 100755 --- a/code/ddns-lambda.py +++ b/code/ddns-lambda.py @@ -73,7 +73,7 @@ LOGGER = logging.getLogger() account_id = None region = None -VERSION = '2.0.0' +VERSION = '2.0.1a1' # Read Env variables DEBUG_LOG_LEVEL = os.environ.get('DebugLogLevel', 'INFO') @@ -540,13 +540,14 @@ def lambda_handler( # LOGGER.debug("hosted_zones for vpc_id %s: %s", vpc_id, str(hosted_zones) + lineno()) hosted_zones_by_vpc = new_list_hosted_zones_by_vpc( route53, instance_id, vpc_id, region) - LOGGER.debug("hosted_zones_by_vpc for vpc_id %s: %s", - vpc_id, str(hosted_zones_by_vpc) + lineno()) + LOGGER.debug( + "hosted_zones_by_vpc for vpc_id {vpc_id}: {pformat(hosted_zones_by_vpc)} {lineno()}") # private_hosted_zones = get_private_hosted_zones(hosted_zones) # LOGGER.debug("private_hosted_zones: %s", str(list(private_hosted_zones)) + lineno()) - private_hosted_zones_by_vpc = get_private_hosted_zones_by_vpc(hosted_zones_by_vpc) - LOGGER.debug("private_hosted_zones_by_vpc: vpc_id: %s, %s", - vpc_id, str(list(private_hosted_zones_by_vpc)) + lineno()) + private_hosted_zones_by_vpc = hosted_zones_by_vpc +# private_hosted_zones_by_vpc = get_private_hosted_zones_by_vpc(hosted_zones_by_vpc) +# LOGGER.debug("private_hosted_zones_by_vpc: vpc_id: %s, %s", +# vpc_id, str(list(private_hosted_zones_by_vpc)) + lineno()) # private_hosted_zone_collection = get_private_hosted_zone_collection( # private_hosted_zones) # LOGGER.debug("private_hosted_zone_collection: %s", @@ -1256,32 +1257,50 @@ def new_list_hosted_zones_by_vpc(client, instance_id, vpc_id, region): :return: """ +# if we need to retry ,we do not want to start the list over i = 0 - hosted_zones = {} + c = 0 + token = None + time_start = datetime.datetime.now() + hosted_zones = [] # retry to handle errors in the possible API call while i < MAX_API_RETRY: try: - hosted_zones = client.list_hosted_zones_by_vpc( - VPCId=vpc_id, VPCRegion=region) - LOGGER.debug( - "list_hosted_zones_by_vpc returned without error. %s", lineno()) + if not token: + this_hosted_zones = client.list_hosted_zones_by_vpc( + VPCId=vpc_id, VPCRegion=region, MaxItems='100') + hosted_zones.extend(this_hosted_zone['HostedZoneSummaries']) + token = this_hosted_zones.get('NextToken', None) + LOGGER.debug( + f"list_hosted_zones_by_vpc retry {i} iteration {c} returned without error (more available) {token!=None}. {lineno()}") + while token: + c += 1 + this_hosted_zones = client.list_hosted_zones_by_vpc( + VPCId=vpc_id, VPCRegion=region, MaxItems='100', NextToken=token) + hosted_zones.extend(this_hosted_zone['HostedZoneSummaries']) + token = this_hosted_zones.get('NextToken', None) + LOGGER.debug( + f"list_hosted_zones_by_vpc retry {i} iteration {c} returned without error (more available) {token!=None}. {lineno()}") break except ClientError as err: error_message = str(err) if "(Throttling)" in str(err): LOGGER.debug( - "list_hosted_zones_by_vpc throttled due to API limit, retrying: %s", str(err) + lineno()) + f"list_hosted_zones_by_vpc throttled due to API limit, retrying: {err} {lineno()}") else: - LOGGER.info("vpc: %s, instance: %s, unexpected error. %s\n", - vpc_id, instance_id, error_message + lineno()) + LOGGER.info( + f"vpc: {vpc_id}, instance: {instance_id}, unexpected error. {error_messge} {lineno()}") + i += 1 - LOGGER.info("instance: %s, list_hosted_zones_by_vpc %v returned error, waiting before retry. %s", - vpc_id, instance_id, str(i) + lineno()) + LOGGER.info( + f"instance: {instance_id} list_hosted_zones_by_vpc {vpc_id} returned error, waiting {i} sec before retry. {lineno()}") time.sleep(i) count['sleep.count'] += 1 count['sleep.time'] += i count['retry'] += 1 + count['zones.pagination'] = c + time_end = datetime.datetime.now() if hosted_zones == {}: LOGGER.error("vpc_id: %s, instance: %s, list_hosted_zones_by_vpc returned error. Timed out. %s", vpc_id, instance_id, str(i) + lineno()) @@ -1302,6 +1321,10 @@ def new_list_hosted_zones_by_vpc(client, instance_id, vpc_id, region): LOGGER.info("vpc_id: %s, instance: %s, error: %s", vpc_id, instance_id, str(sys.exc_info()[0]) + lineno()) + elapsed_time = time_end - time_start + count['zones'] = len(hosted_zones) + LOGGER.info( + f"list_hosted_zones_by_vpc stats: elapsed_time.ms {elapsed_time.total_seconds()*1000.0} zones {len(hosted_zones)} iterations {c}") return hosted_zones @@ -1449,45 +1472,45 @@ def get_private_hosted_zone_collection_by_vpc(private_hosted_zones): LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno()) -def get_private_hosted_zones(hosted_zones): - """ - Get private hosted zones - :param hosted_zones: - :return: - """ - try: - private_hosted_zones = [] - - for item in hosted_zones['HostedZones']: - LOGGER.debug("item: %s", str(item) + lineno()) - - if item['Config']['PrivateZone']: - private_hosted_zones.append(item) - - return private_hosted_zones - except: - LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno()) - +# def get_private_hosted_zones(hosted_zones): +# """ +# Get private hosted zones +# :param hosted_zones: +# :return: +# """ +# try: +# private_hosted_zones = [] +# +# for item in hosted_zones['HostedZones']: +# LOGGER.debug("item: %s", str(item) + lineno()) +# +# if item['Config']['PrivateZone']: +# private_hosted_zones.append(item) +# +# return private_hosted_zones +# except: +# LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno()) +# # 'HostedZoneSummaries': [{'HostedZoneId': 'Z03702572CGNJTW5OPG68', 'Name': '0.192.10.in-addr.arpa.', 'Owner': {'OwningAccount': '057405694017'}}, {'HostedZoneId': 'Z07556021WTTHLOJ6S3C', 'Name': -def get_private_hosted_zones_by_vpc(hosted_zones): - """ - Get private hosted zones - :param hosted_zones: - :return: - """ - try: - private_hosted_zones = [] - - for item in hosted_zones['HostedZoneSummaries']: - LOGGER.debug("item: %s", str(item) + lineno()) - - private_hosted_zones.append(item) - - return private_hosted_zones - except: - LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno()) +# def get_private_hosted_zones_by_vpc(hosted_zones): +# """ +# Get private hosted zones +# :param list(str) hosted_zones: List of hosted zone elements from HostedZoneSummaries +# :return: +# """ +# try: +# private_hosted_zones = [] +# +# for item in hosted_zones: +# LOGGER.debug("item: %s", str(item) + lineno()) +# +# private_hosted_zones.append(item) +# +# return private_hosted_zones +# except: +# LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno()) def get_dhcp_option_set_id_for_vpc(client, instance_id, vpc_id): @@ -3048,7 +3071,8 @@ def tags_to_dict(tags): tag_dict = {} if len(tags) > 0: - tag_dict = {tag.get('Key', '').lstrip().rstrip() : tag.get('Value', '') for tag in tags} + tag_dict = {tag.get('Key', '').lstrip().rstrip() + : tag.get('Value', '') for tag in tags} return tag_dict diff --git a/code/ddns-lambda.zip b/code/ddns-lambda.zip index 22e2872..493e094 100644 Binary files a/code/ddns-lambda.zip and b/code/ddns-lambda.zip differ diff --git a/version.tf b/version.tf index 6b49608..100daf2 100644 --- a/version.tf +++ b/version.tf @@ -1,3 +1,3 @@ locals { - _module_version = "2.0.0" + _module_version = "2.0.1" }