fix

terraform-modules · Apr 29, 2023 · 2fcd20b · 2fcd20b
1 parent 9f9cf09
commit 2fcd20b
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 54 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -131,3 +131,7 @@
     - add detection of a runnign EMR cluster with aws:elasticmapreduce: job-flow-id (cluster) and instance-group-role and use this to
       set an alias defined in boc:dns:cname ({friendlyname}.master), but if it is a cluser node, only use the cname if it is master.  If 
       it is not a cluster, set the cname
+
+* 2.0.1 -- 2023-04-29
+  - code 2.0.1
+    - add pagination for list hosted zone by vpc (as some vpcs have hundreds of zones, specifically the one with dns endpoints)
diff --git a/code/ddns-lambda.py b/code/ddns-lambda.py
@@ -73,7 +73,7 @@
 LOGGER = logging.getLogger()
 account_id = None
 region = None
-VERSION = '2.0.0'
+VERSION = '2.0.1a1'
 
 # Read Env variables
 DEBUG_LOG_LEVEL = os.environ.get('DebugLogLevel', 'INFO')
@@ -540,13 +540,14 @@ def lambda_handler(
 # LOGGER.debug("hosted_zones for vpc_id %s: %s", vpc_id, str(hosted_zones) + lineno())
     hosted_zones_by_vpc = new_list_hosted_zones_by_vpc(
         route53, instance_id, vpc_id, region)
-    LOGGER.debug("hosted_zones_by_vpc for vpc_id %s: %s",
-                 vpc_id, str(hosted_zones_by_vpc) + lineno())
+    LOGGER.debug(
+        "hosted_zones_by_vpc for vpc_id {vpc_id}: {pformat(hosted_zones_by_vpc)} {lineno()}")
 # private_hosted_zones = get_private_hosted_zones(hosted_zones)
 # LOGGER.debug("private_hosted_zones: %s", str(list(private_hosted_zones)) + lineno())
-    private_hosted_zones_by_vpc = get_private_hosted_zones_by_vpc(hosted_zones_by_vpc)
-    LOGGER.debug("private_hosted_zones_by_vpc: vpc_id: %s, %s",
-                 vpc_id, str(list(private_hosted_zones_by_vpc)) + lineno())
+    private_hosted_zones_by_vpc = hosted_zones_by_vpc
+#    private_hosted_zones_by_vpc = get_private_hosted_zones_by_vpc(hosted_zones_by_vpc)
+#    LOGGER.debug("private_hosted_zones_by_vpc: vpc_id: %s, %s",
+#                 vpc_id, str(list(private_hosted_zones_by_vpc)) + lineno())
 # private_hosted_zone_collection = get_private_hosted_zone_collection(
 # private_hosted_zones)
 # LOGGER.debug("private_hosted_zone_collection: %s",
@@ -1256,32 +1257,50 @@ def new_list_hosted_zones_by_vpc(client, instance_id, vpc_id, region):
     :return:
     """
 
+# if we need to retry ,we do not want to start the list over
     i = 0
-    hosted_zones = {}
+    c = 0
+    token = None
+    time_start = datetime.datetime.now()
+    hosted_zones = []
     # retry to handle errors in the possible API call
     while i < MAX_API_RETRY:
         try:
-            hosted_zones = client.list_hosted_zones_by_vpc(
-                VPCId=vpc_id, VPCRegion=region)
-            LOGGER.debug(
-                "list_hosted_zones_by_vpc returned without error. %s", lineno())
+            if not token:
+                this_hosted_zones = client.list_hosted_zones_by_vpc(
+                    VPCId=vpc_id, VPCRegion=region, MaxItems='100')
+                hosted_zones.extend(this_hosted_zone['HostedZoneSummaries'])
+                token = this_hosted_zones.get('NextToken', None)
+                LOGGER.debug(
+                    f"list_hosted_zones_by_vpc retry {i} iteration {c} returned without error (more available) {token!=None}. {lineno()}")
+            while token:
+                c += 1
+                this_hosted_zones = client.list_hosted_zones_by_vpc(
+                    VPCId=vpc_id, VPCRegion=region, MaxItems='100', NextToken=token)
+                hosted_zones.extend(this_hosted_zone['HostedZoneSummaries'])
+                token = this_hosted_zones.get('NextToken', None)
+                LOGGER.debug(
+                    f"list_hosted_zones_by_vpc retry {i} iteration {c} returned without error (more available) {token!=None}. {lineno()}")
             break
         except ClientError as err:
             error_message = str(err)
             if "(Throttling)" in str(err):
                 LOGGER.debug(
-                    "list_hosted_zones_by_vpc throttled due to API limit, retrying: %s", str(err) + lineno())
+                    f"list_hosted_zones_by_vpc throttled due to API limit, retrying: {err} {lineno()}")
             else:
-                LOGGER.info("vpc: %s, instance: %s, unexpected error. %s\n",
-                            vpc_id, instance_id, error_message + lineno())
+                LOGGER.info(
+                    f"vpc: {vpc_id}, instance: {instance_id}, unexpected error. {error_messge} {lineno()}")
+
         i += 1
-        LOGGER.info("instance: %s, list_hosted_zones_by_vpc %v returned error, waiting before retry. %s",
-                    vpc_id, instance_id, str(i) + lineno())
+        LOGGER.info(
+            f"instance: {instance_id} list_hosted_zones_by_vpc {vpc_id} returned error, waiting {i} sec before retry. {lineno()}")
         time.sleep(i)
         count['sleep.count'] += 1
         count['sleep.time'] += i
         count['retry'] += 1
 
+    count['zones.pagination'] = c
+    time_end = datetime.datetime.now()
     if hosted_zones == {}:
         LOGGER.error("vpc_id: %s, instance: %s, list_hosted_zones_by_vpc returned error. Timed out. %s",
                      vpc_id, instance_id, str(i) + lineno())
@@ -1302,6 +1321,10 @@ def new_list_hosted_zones_by_vpc(client, instance_id, vpc_id, region):
                 LOGGER.info("vpc_id: %s, instance: %s, error: %s", vpc_id, instance_id,
                             str(sys.exc_info()[0]) + lineno())
 
+    elapsed_time = time_end - time_start
+    count['zones'] = len(hosted_zones)
+    LOGGER.info(
+        f"list_hosted_zones_by_vpc stats: elapsed_time.ms {elapsed_time.total_seconds()*1000.0} zones {len(hosted_zones)} iterations {c}")
     return hosted_zones
 
 
@@ -1449,45 +1472,45 @@ def get_private_hosted_zone_collection_by_vpc(private_hosted_zones):
         LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
 
 
-def get_private_hosted_zones(hosted_zones):
-    """
-    Get private hosted zones
-    :param hosted_zones:
-    :return:
-    """
-    try:
-        private_hosted_zones = []
-
-        for item in hosted_zones['HostedZones']:
-            LOGGER.debug("item: %s", str(item) + lineno())
-
-            if item['Config']['PrivateZone']:
-                private_hosted_zones.append(item)
-
-        return private_hosted_zones
-    except:
-        LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
-
+# def get_private_hosted_zones(hosted_zones):
+#     """
+#     Get private hosted zones
+#     :param hosted_zones:
+#     :return:
+#     """
+#     try:
+#         private_hosted_zones = []
+#
+#         for item in hosted_zones['HostedZones']:
+#             LOGGER.debug("item: %s", str(item) + lineno())
+#
+#             if item['Config']['PrivateZone']:
+#                 private_hosted_zones.append(item)
+#
+#         return private_hosted_zones
+#     except:
+#         LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+#
 
 # 'HostedZoneSummaries': [{'HostedZoneId': 'Z03702572CGNJTW5OPG68', 'Name': '0.192.10.in-addr.arpa.', 'Owner': {'OwningAccount': '057405694017'}}, {'HostedZoneId': 'Z07556021WTTHLOJ6S3C', 'Name':
 
-def get_private_hosted_zones_by_vpc(hosted_zones):
-    """
-    Get private hosted zones
-    :param hosted_zones:
-    :return:
-    """
-    try:
-        private_hosted_zones = []
-
-        for item in hosted_zones['HostedZoneSummaries']:
-            LOGGER.debug("item: %s", str(item) + lineno())
-
-            private_hosted_zones.append(item)
-
-        return private_hosted_zones
-    except:
-        LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+# def get_private_hosted_zones_by_vpc(hosted_zones):
+#     """
+#     Get private hosted zones
+#     :param list(str) hosted_zones: List of hosted zone elements from HostedZoneSummaries
+#     :return:
+#     """
+#     try:
+#         private_hosted_zones = []
+#
+#         for item in hosted_zones:
+#             LOGGER.debug("item: %s", str(item) + lineno())
+#
+#             private_hosted_zones.append(item)
+#
+#         return private_hosted_zones
+#     except:
+#         LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
 
 
 def get_dhcp_option_set_id_for_vpc(client, instance_id, vpc_id):
@@ -3048,7 +3071,8 @@ def tags_to_dict(tags):
 
     tag_dict = {}
     if len(tags) > 0:
-        tag_dict = {tag.get('Key', '').lstrip().rstrip()                    : tag.get('Value', '') for tag in tags}
+        tag_dict = {tag.get('Key', '').lstrip().rstrip()
+                            : tag.get('Value', '') for tag in tags}
     return tag_dict
 
 

diff --git a/code/ddns-lambda.zip b/code/ddns-lambda.zip
diff --git a/version.tf b/version.tf
@@ -1,3 +1,3 @@
 locals {
-  _module_version = "2.0.0"
+  _module_version = "2.0.1"
 }