diff --git a/.tf-control b/.tf-control
new file mode 100644
index 0000000..280f449
--- /dev/null
+++ b/.tf-control
@@ -0,0 +1,20 @@
+# .tf-control
+# allows for setting a specific command to be used for tf-* commands under this git repo
+# see tf-control.sh help for more info
+
+TFCONTROL_VERSION="1.0.5"
+
+TFCOMMAND="terraform_latest"
+# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc
+# TFARGS=""
+# TFNOLOG=""
+# TFNOCOLOR=""
+
+# use the following to force a specific version. An upgrade of an existing 0.12.31 to 1.x
+# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed). Other
+# steps in between. See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details
+#
+#TFCOMMAND="terraform_0.12.31"
+#TFCOMMAND="terraform_0.13.7"
+#TFCOMMAND="terraform_0.14.11"
+#TFCOMMAND="terraform_0.15.5"
diff --git a/.tf-control.tfrc b/.tf-control.tfrc
new file mode 100644
index 0000000..7425488
--- /dev/null
+++ b/.tf-control.tfrc
@@ -0,0 +1,24 @@
+TFCONTROL_VERSION="1.0.5"
+
+# https://www.terraform.io/docs/cli/config/config-file.html
+plugin_cache_dir = "/data/terraform/terraform.d/plugin-cache"
+#disable_checkpoint = true
+
+provider_installation {
+# filesystem_mirror {
+# path = "/apps/terraform/terraform.d/providers"
+# include = [ "*/*/*" ]
+# }
+ filesystem_mirror {
+ path = "/data/terraform/terraform.d/providers"
+ include = [ "*/*/*" ]
+ }
+# filesystem_mirror {
+# path = "/apps/terraform/terraform.d/providers"
+# include = [ "external.terraform.census.gov/*/*" ]
+# }
+ direct {
+ include = [ "*/*/*" ]
+ }
+}
+
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ff2bdc4..797f957 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -116,3 +116,18 @@
* 1.0.1 -- 2022-10-28
- change map() to {}
+* 1.0.2 -- 2023-04-22
+ - add alllow assume role in (a) org and (b) to remote role r-inf-dynamic-reoute53-actions
+
+## Release 2.x
+
+* 2.0.0 -- 2023-04-28
+ - code 2.0.0
+ - use sessions
+ - make assume role call to remote account where PHZ is defined
+ - add dns entries to the DDB item, so that on stop/terminate we delete only what was added
+ - add flags: noforward, noptr, noheritage, nocname
+ - add data: boc:dns:ptrname
+ - add detection of a runnign EMR cluster with aws:elasticmapreduce: job-flow-id (cluster) and instance-group-role and use this to
+ set an alias defined in boc:dns:cname ({friendlyname}.master), but if it is a cluser node, only use the cname if it is master. If
+ it is not a cluster, set the cname
diff --git a/README.md b/README.md
index 2f19d90..f9bc045 100644
--- a/README.md
+++ b/README.md
@@ -134,6 +134,7 @@ No modules.
| [aws_iam_policy_document.queue_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_iam_policy_document.queue_sqs_deadletter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_iam_policy_document.topic](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_organizations_organization.org](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/organizations_organization) | data source |
| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
## Inputs
@@ -148,7 +149,7 @@ No modules.
| [enable\_sns](#input\_enable\_sns) | Enable use of SNS for reporting errors | `bool` | `false` | no |
| [enable\_sqs](#input\_enable\_sqs) | Enable use of SQS for SNS to send errors. Requires the use of enable\_sns as well | `bool` | `false` | no |
| [kms\_key\_name](#input\_kms\_key\_name) | Different KMS Key (for SNS and SQS) to override default of var.name | `string` | `null` | no |
-| [lambda\_environment\_variables](#input\_lambda\_environment\_variables) | Map of lambda environment variables and values | `map(string)` | {
"DNS_RR_TimeToLive": 60,
"DebugLogLevel": "INFO",
"DynamoDBName": null,
"HeritageIdentifier": "dynr53",
"HeritageTXTRecordPrefix": "_txt",
"MaxApiRetry": 10,
"SleepTime": 60,
"SnsEnable": false,
"SnsTopicArn": "",
"TagKeyCname": "boc:dns:cname",
"TagKeyHostName": "boc:dns:name",
"TagKeyZone": "boc:dns:zone"
} | no |
+| [lambda\_environment\_variables](#input\_lambda\_environment\_variables) | Map of lambda environment variables and values | `map(string)` | {
"DNS_RR_TimeToLive": 60,
"DebugLogLevel": "INFO",
"DynamoDBName": null,
"EMRTagPrefix": "aws",
"HeritageIdentifier": "dynr53",
"HeritageTXTRecordPrefix": "_txt",
"MaxApiRetry": 10,
"RemoteRoleArnFormat": "arn:%s:iam::%s:role/r-inf-dynamic-route53-actions",
"SleepTime": 60,
"SnsEnable": false,
"SnsTopicArn": "",
"TagKeyCname": "boc:dns:cname",
"TagKeyFlags": "boc:dns:flags",
"TagKeyHostName": "boc:dns:name",
"TagKeyPtrname": "boc:dns:ptrname",
"TagKeyZone": "boc:dns:zone"
} | no |
| [lambda\_environment\_variables\_override](#input\_lambda\_environment\_variables\_override) | Map of lambda environment variables and values to override from the defaults | `map(string)` | `{}` | no |
| [lambda\_name](#input\_lambda\_name) | Different Lambda name to override default of var.name | `string` | `null` | no |
| [name](#input\_name) | Name to use within all the created resources (default: inf-dynamic-route53) | `string` | `"inf-dynamic-route53"` | no |
diff --git a/code/.terraform.lock.hcl b/code/.terraform.lock.hcl
new file mode 100644
index 0000000..d4e3a63
--- /dev/null
+++ b/code/.terraform.lock.hcl
@@ -0,0 +1,26 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/archive" {
+ version = "2.3.0"
+ constraints = ">= 2.0.0"
+ hashes = [
+ "h1:OmE1tPjiST8iQp6fC0N3Xzur+q2RvgvD7Lz0TpKSRBw=",
+ ]
+}
+
+provider "registry.terraform.io/hashicorp/aws" {
+ version = "4.64.0"
+ constraints = ">= 4.55.0"
+ hashes = [
+ "h1:4xXf+eZtKPiRyjle7HUPaVzF3h/6S8seNEIIbWlDbuk=",
+ ]
+}
+
+provider "registry.terraform.io/hashicorp/null" {
+ version = "3.2.1"
+ constraints = ">= 1.0.0"
+ hashes = [
+ "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=",
+ ]
+}
diff --git a/code/ddns-lambda.py b/code/ddns-lambda.py
index 508d018..fa18018 100755
--- a/code/ddns-lambda.py
+++ b/code/ddns-lambda.py
@@ -64,15 +64,16 @@
import os
import ipaddress
from botocore.exceptions import ClientError
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict, namedtuple, Counter
from pprint import pformat
from dateutil.parser import parse as date_parse
# Setting Global Variables
+APPNAME = 'inf-dynamic-route53'
LOGGER = logging.getLogger()
-ACCOUNT = None
-REGION = None
-VERSION = '1.0.0'
+account_id = None
+region = None
+VERSION = '2.0.0'
# Read Env variables
DEBUG_LOG_LEVEL = os.environ.get('DebugLogLevel', 'INFO')
@@ -81,10 +82,16 @@
TAGKEY_CNAME = os.environ.get('TagKeyCname', 'boc:dns:cname')
TAGKEY_ZONE = os.environ.get('TagKeyZone', 'boc:dns:zone')
TAGKEY_HOSTNAME = os.environ.get('TagKeyHostName', 'boc:dns:name')
+TAGKEY_PTRNAME = os.environ.get('TagKeyPtrName', 'boc:dns:ptrname')
+TAGKEY_FLAGS = os.environ.get('TagKeyFlags', 'boc:dns:flags')
DNS_RR_TTL = int(os.environ.get('DNS_RR_TimeToLive', '60'))
DNS_RR_TTL = 60 if DNS_RR_TTL == 0 else DNS_RR_TTL
TF_MODULE_VERSION = os.environ.get('tf_module_version', '(unknown)')
MAX_API_RETRY = int(os.environ.get('MaxApiRetry', '10'))
+REMOTE_ROLE_ARN_FORMAT = os.environ.get(
+ 'RemoteRoleArnFormat', 'arn:%s:iam::%s:role/r-inf-dynamic-route53-actions')
+REMOTE_ROLE_NAME = REMOTE_ROLE_ARN_FORMAT.partition('/')[2]
+EMR_TAG_PREFIX = os.environ.get('EMRTagPrefix', 'aws')
# for SNS topic
SNS_TOPIC_ARN = os.environ.get('SnsTopicArn', '')
@@ -112,6 +119,20 @@
print('Loading function v{} tf_module_version {}: {}'.format(
VERSION, TF_MODULE_VERSION, datetime.datetime.now().time().isoformat()))
+# ---
+# session setup. sessions is used for assume roles into other accounts where PHZs are defined
+# ---
+session = None
+sessions = {}
+dynamodb_client = None
+compute = None
+route53 = None
+sns_client = None
+sts_client = None
+partition = None
+phz_collection_by_vpc = {}
+count = Counter()
+
def lineno(): # pragma: no cover
"""
@@ -127,7 +148,7 @@ def get_sns_client():
:return:
"""
try:
- return boto3.client('sns')
+ return session.client('sns')
except ClientError as err:
print("Unexpected error: %s" % err)
@@ -138,7 +159,7 @@ def get_route53_client():
:return:
"""
try:
- return boto3.client('route53')
+ return session.client('route53')
except ClientError as err:
print("Unexpected error: %s" % err)
@@ -149,7 +170,7 @@ def get_ec2_client():
:return:
"""
try:
- return boto3.client('ec2')
+ return session.client('ec2')
except ClientError as err:
print("Unexpected error: %s" % err)
@@ -160,7 +181,18 @@ def get_dynamodb_client():
:return:
"""
try:
- return boto3.client('dynamodb')
+ return session.client('dynamodb')
+ except ClientError as err:
+ print("Unexpected error: %s" % err)
+
+
+def get_sts_client():
+ """
+ Get STS client
+ :return:
+ """
+ try:
+ return session.client('sts')
except ClientError as err:
print("Unexpected error: %s" % err)
@@ -171,18 +203,75 @@ def get_caller_account_id():
:return str: AWS Account ID
"""
try:
- return boto3.client('sts').get_caller_identity()['Account']
+ return sts_client.get_caller_identity()['Account']
except ClientError as err:
print("Unexpected error: %s" % err)
+def get_caller_partition(region):
+ """
+ Get AWS Partition Name from session
+ :param region:
+ :return str: AWS Partition name (aws, aws-us-gov)
+ """
+ try:
+ return session.get_partition_for_region(region)
+ except ClientError as err:
+ print("Unexpected error: %s" % err)
+
+
+def initialize_clients():
+ """
+ Set up all of the API clients from the main session, done before invocation of the handlers
+ :return:
+ """
+ global session
+ global sessions
+ global dynamodb_client
+ global compute
+ global route53
+ global sns_client
+ global sts_client
+ global account_id
+
+ LOGGER.info("initializing boto3 session and clients: %s", lineno())
+ if session is None:
+ session = boto3.session.Session()
+ if sessions is None:
+ sessions = {}
+
+ LOGGER.debug(" boto3 client: %s", 'dynamodb' + lineno())
+ dynamodb_client = get_dynamodb_client()
+
+ LOGGER.debug(" boto3 client: %s", 'ec2' + lineno())
+ compute = get_ec2_client()
+
+ LOGGER.debug(" boto3 client: %s", 'route53' + lineno())
+ route53 = get_route53_client()
+
+ LOGGER.debug(" boto3 client: %s", 'sns' + lineno())
+ sns_client = get_sns_client()
+
+ LOGGER.debug(" boto3 client: %s", 'sts' + lineno())
+ sts_client = get_sts_client()
+
+ LOGGER.debug("getting account_id: %s", lineno())
+ account_id = get_caller_account_id()
+ sessions[account_id] = session
+
+ return
+
+
+initialize_clients()
+
+
def lambda_handler(
event,
context,
- dynamodb_client=get_dynamodb_client(),
- compute=get_ec2_client(),
- route53=get_route53_client(),
- sns_client=get_sns_client()
+ # dynamodb_client=get_dynamodb_client(),
+ # compute=get_ec2_client(),
+ # route53=get_route53_client(),
+ # sns_client=get_sns_client()
):
@@ -199,11 +288,41 @@ def lambda_handler(
:param sns_client:
:return:
"""
+ global session
+ global dynamodb_client
+ global compute
+ global route53
+ global sns_client
+ global sts_client
+ global account_id
+ global region
+ global partition
+ global phz_collection_by_vpc
+ global count
+
+ count['start'] = datetime.datetime.now()
LOGGER.info("event: %s", str(event) + lineno())
LOGGER.info("context: %s", str(context) + lineno())
LOGGER.info("Sns Topic Mode: %s, sending to %s", str(SNS_ENABLE), SNS_TOPIC_ARN)
+ dns_data_fields = ['zone_id', 'rr_name', 'zone_name', 'rr_type', 'rr_value']
+ dns_data_tuple = namedtuple('DnsData', dns_data_fields)
+
caller_response = []
+# if not an ec2 start or stop, skip processing, send SNS, and exit
+ action = evaluate_event_action(event)
+ if not action:
+ caller_response.append(f"Event structure not an aws.ec2 event, exiting")
+ if SNS_ENABLE:
+ sns_msg = {}
+ sns_msg['account_id'] = account_id
+ sns_msg['region'] = event['region']
+ sns_msg['event'] = str(event)
+ sns_msg['context'] = str(context)
+ sns_msg['message'] = caller_response[-1]
+ publish_to_sns(sns_client, json.dumps(sns_msg))
+ return caller_response
+
# Checking to make sure there is a dynamodb table named in the Env Variable
tables = list_tables(dynamodb_client)
@@ -217,7 +336,7 @@ def lambda_handler(
if SNS_ENABLE:
sns_msg = {}
sns_msg['instance_id'] = event['detail']['instance-id']
- sns_msg['account_id'] = get_caller_account_id()
+ sns_msg['account_id'] = account_id
sns_msg['region'] = event['region']
sns_msg['message'] = 'DynamoDB table does not exist: ' + DDBNAME
publish_to_sns(sns_client, json.dumps(sns_msg))
@@ -225,6 +344,7 @@ def lambda_handler(
# Set variables
# Get the state from the Event stream
+ event_source = event['source']
state = event['detail']['state']
LOGGER.debug("instance state: %s", str(state) + lineno())
@@ -233,8 +353,10 @@ def lambda_handler(
LOGGER.debug("instance id: %s", str(instance_id) + lineno())
region = event['region']
LOGGER.debug("region: %s", str(region) + lineno())
- account_id = get_caller_account_id()
+# account_id = get_caller_account_id()
LOGGER.debug("account_id: %s", str(account_id) + lineno())
+ partition = get_caller_partition(region)
+ LOGGER.debug("partition: %s", str(partition) + lineno())
# Only doing something if the state is running
LOGGER.debug("instance state is {} {}".format(state, lineno()))
@@ -246,6 +368,8 @@ def lambda_handler(
while i < SLEEPTIME:
LOGGER.debug("waiting count: %s", str(i) + lineno())
time.sleep(1)
+ count['sleep.count'] += 1
+ count['sleep.time'] += 1
i += 1
try:
@@ -266,6 +390,10 @@ def lambda_handler(
LOGGER.info("instance: %s, no instance data, repeat check: %s",
instance_id, lineno())
+# instance object to/from DDB contains the instance data from get_instances (Reservations).
+# We are adding a fake entry to it _DnsEntries to store what was added by the code, so we can also easily remove it later
+# to avoid orphaned records
+
# Remove response metadata from the response
if 'ResponseMetadata' in instance:
instance.pop('ResponseMetadata')
@@ -275,7 +403,7 @@ def lambda_handler(
instance = remove_empty_from_dict(instance)
instance_dump = json.dumps(instance, default=json_serial)
- # dont' change to dictionary. Keep it as string to be written to DDB
+ # don't change to dictionary. Keep it as string to be written to DDB
# instance_attributes = json.loads(instance_dump)
instance_attributes = instance_dump
LOGGER.info("instance_attributes: %s", str(instance_attributes) + lineno())
@@ -284,11 +412,27 @@ def lambda_handler(
put_item_in_dynamodb_table(dynamodb_client, DDBNAME,
instance_id, instance_attributes)
LOGGER.debug("done putting item in dynamo table %s", lineno())
+ dns_data = []
else:
+ # state is stopping or terminating
# Fetch item from DynamoDB
LOGGER.info("Fetching instance information from dynamodb %s", lineno())
instance = get_item_from_dynamodb_table(dynamodb_client, DDBNAME, instance_id)
LOGGER.info("instance attributes: %s", str(instance) + lineno())
+ dns_data = []
+ if len(instance) > 0:
+ try:
+ dns_data_raw = instance.get('_DnsEntries', [])
+ LOGGER.debug(
+ f"got _DnsEntries type {type(dns_data_raw)} value {dns_data_raw} {lineno()}")
+ dns_data = [dns_data_tuple(**item) for item in dns_data_raw]
+ LOGGER.debug(
+ f"converted _DnsEntries to namdtuples {dns_data} {lineno()}")
+ LOGGER.info(
+ f"Found entries from DDB for DNS records: {str(dns_data)} {lineno()}")
+ except Exception as err:
+ LOGGER.error(
+ f"Cannot deserialize instance data from DDB: {err} {lineno()}")
# Get the instance tags and reorder them because we want a zone created before CNAME
try:
@@ -339,31 +483,15 @@ def lambda_handler(
LOGGER.debug("subnet_id: %s", str(subnet_id) + lineno())
cidr_block = get_subnet_cidr_block(compute, instance_id, subnet_id)
LOGGER.debug("cidr_block: %s", str(cidr_block) + lineno())
- subnet_mask = int(cidr_block.split('/')[-1])
+# subnet_mask = int(cidr_block.split('/')[-1])
+ subnet_mask = ipaddress.ip_network(cidr_block).prefixlen
LOGGER.debug("subnet_mask: %s", str(subnet_mask) + lineno())
- reversed_ip_address = reverse_list(private_ip)
-
- if reversed_ip_address == None:
- LOGGER.error("Error in getting reverse IP address for: %s",
- str(private_ip) + lineno())
- caller_response.append(
- "Error in getting reverse IP address for: " + str(private_ip))
- return caller_response
-
- reversed_domain_prefix = get_reversed_domain_prefix(subnet_mask, private_ip)
- reversed_domain_prefix = reverse_list(reversed_domain_prefix)
-
- if reversed_domain_prefix == None:
- LOGGER.error("Error in getting reverse domain prefix for: %s",
- str(private_ip) + lineno())
- caller_response.append(
- "Error in getting reverse domain prefix for: " + str(private_ip))
- return caller_response
+# reversed_ip_address = reverse_list(private_ip)
+ reversed_ip_address = new_reverse_list(private_ip)
- LOGGER.debug("reversed_domain_prefix is: %s",
- str(reversed_domain_prefix) + lineno())
- # Set the reverse lookup zone
- reversed_lookup_zone = reversed_domain_prefix + 'in-addr.arpa.'
+ reversed_entry = reversed_ip_address.split('.')[0]
+ reversed_entry = new_reverse_entry(private_ip)
+ reversed_lookup_zone = new_reverse_domain(private_ip)
LOGGER.info("instance: %s, The reverse lookup zone is: %s",
instance_id, str(reversed_lookup_zone))
@@ -408,14 +536,28 @@ def lambda_handler(
return caller_response
# These are collections of zones in Route 53.
- hosted_zones = new_list_hosted_zones(route53, instance_id)
- LOGGER.debug("hosted_zones: %s", str(hosted_zones) + lineno())
- private_hosted_zones = get_private_hosted_zones(hosted_zones)
- LOGGER.debug("private_hosted_zones: %s", str(list(private_hosted_zones)) + lineno())
- private_hosted_zone_collection = get_private_hosted_zone_collection(
- private_hosted_zones)
- LOGGER.debug("private_hosted_zone_collection: %s",
- str(list(private_hosted_zone_collection)) + lineno())
+# hosted_zones = new_list_hosted_zones(route53, instance_id)
+# LOGGER.debug("hosted_zones for vpc_id %s: %s", vpc_id, str(hosted_zones) + lineno())
+ hosted_zones_by_vpc = new_list_hosted_zones_by_vpc(
+ route53, instance_id, vpc_id, region)
+ LOGGER.debug("hosted_zones_by_vpc for vpc_id %s: %s",
+ vpc_id, str(hosted_zones_by_vpc) + lineno())
+# private_hosted_zones = get_private_hosted_zones(hosted_zones)
+# LOGGER.debug("private_hosted_zones: %s", str(list(private_hosted_zones)) + lineno())
+ private_hosted_zones_by_vpc = get_private_hosted_zones_by_vpc(hosted_zones_by_vpc)
+ LOGGER.debug("private_hosted_zones_by_vpc: vpc_id: %s, %s",
+ vpc_id, str(list(private_hosted_zones_by_vpc)) + lineno())
+# private_hosted_zone_collection = get_private_hosted_zone_collection(
+# private_hosted_zones)
+# LOGGER.debug("private_hosted_zone_collection: %s",
+# str(list(private_hosted_zone_collection)) + lineno())
+ private_hosted_zone_collection_by_vpc = get_private_hosted_zone_collection_by_vpc(
+ private_hosted_zones_by_vpc)
+ LOGGER.debug("private_hosted_zone_collection_by_vpc: %s",
+ str(list(private_hosted_zone_collection_by_vpc)) + lineno())
+# key is zone name, name,zone_id,owner_account,is_amazon,enabled only if enabled (aka, not amazonaws.com for shared endpoints)
+ phz_collection_by_vpc = {
+ item['name']: item for item in private_hosted_zone_collection_by_vpc if item['enabled']}
# Check to see whether a reverse lookup zone for the instance
# already exists. If it does, check to see whether
@@ -423,39 +565,23 @@ def lambda_handler(
LOGGER.info("instance: %s, reversed_lookup_zone: %s",
instance_id, str(reversed_lookup_zone) + lineno())
reverse_zone = None
- for record in hosted_zones['HostedZones']:
- LOGGER.debug("record name: %s", str(record['Name']) + lineno())
- if record['Name'] == reversed_lookup_zone:
- reverse_zone = record['Name']
- break
+ reverse_zone_item = None
+ if reversed_lookup_zone in phz_collection_by_vpc:
+ reverse_zone = reversed_lookup_zone
+ reverse_zone_item = phz_collection_by_vpc[reverse_zone]
+ LOGGER.debug("by_vpc.reverse_zone: %s found, item: %s",
+ str(reverse_zone) + lineno(), reverse_zone_item)
if reverse_zone:
LOGGER.debug("Reverse lookup zone found: %s",
str(reversed_lookup_zone) + lineno())
- reverse_lookup_zone_id = get_zone_id(reversed_lookup_zone, hosted_zones)
+ reverse_lookup_zone_id = reverse_zone_item['zone_id']
+ reverse_lookup_owner_account = reverse_zone_item['owner_account']
+ reverse_lookup_is_mine = reverse_lookup_owner_account == account_id
LOGGER.debug("reverse_lookup_zone_id: %s", str(
reverse_lookup_zone_id) + lineno())
-
- reverse_hosted_zone_properties = new_get_hosted_zone_properties(
- route53, instance_id, reverse_lookup_zone_id)
-
- # need to check if the property is empty {}
- if reverse_hosted_zone_properties == {}:
- LOGGER.error("get_private_hosted_zone_properties returned no zone property",
- reverse_lookup_zone_id + lineno())
- reverse_zone_associated = False
- else:
- LOGGER.debug("reverse_hosted_zone_properties:"
- " %s", str(reverse_hosted_zone_properties) + lineno())
-
- if vpc_id in map(lambda x: x['VPCId'], reverse_hosted_zone_properties['VPCs']):
- LOGGER.info("instance: %s, Reverse lookup zone %s is associated with VPC %s %s",
- instance_id, reverse_lookup_zone_id, vpc_id, lineno())
- reverse_zone_associated = True
- else:
- LOGGER.info("instance: %s, Reverse lookup zone %s is NOT associated with VPC %s %s",
- instance_id, reverse_lookup_zone_id, vpc_id, lineno())
- reverse_zone_associated = False
-
+ LOGGER.debug("reverse_lookup_owner_account: is_mine %s, owner %s", str(reverse_lookup_is_mine), str(
+ reverse_lookup_owner_account) + lineno())
+ reverse_zone_associated = True
else:
LOGGER.info(
"instance: %s, No matching reverse lookup zone, PTR record will not be created %s", instance_id, lineno())
@@ -473,6 +599,8 @@ def lambda_handler(
LOGGER.debug("waiting random seconds, %s", str(wait_time) + lineno())
time.sleep(wait_time)
+ count['sleep.count'] += 1
+ count['sleep.time'] += wait_time
# Is there a DHCP option set?
# Get DHCP option set configuration
@@ -482,9 +610,9 @@ def lambda_handler(
LOGGER.debug("trying to get dhcp option set id %s", lineno())
dhcp_options_id = get_dhcp_option_set_id_for_vpc(compute, instance_id, vpc_id)
LOGGER.debug("dhcp_options_id: %s", str(dhcp_options_id) + lineno())
- dhcp_configurations = get_dhcp_configurations(
- compute, instance_id, dhcp_options_id)
- LOGGER.debug("dhcp_configurations: %s", str(get_dhcp_configurations) + lineno())
+# dhcp_configurations = get_dhcp_configurations(
+ dhcp_zone = get_dhcp_configurations(compute, instance_id, dhcp_options_id)
+ LOGGER.debug("dhcp_zone: %s", str(dhcp_zone) + lineno())
except BaseException as err:
LOGGER.error("instance: %s, No DHCP option set assigned to this VPC %s\n",
@@ -506,235 +634,184 @@ def lambda_handler(
# associated with the VPC. If so, it will set the zone name to be used later.
has_dhcp_dns_zone_associated_vpc = False
- # store verified valid dns zones so to speed up the script.
- valid_dns_zones = []
-
- for configuration in dhcp_configurations:
- LOGGER.debug("configuration: %s", str(configuration) + lineno())
- LOGGER.debug("private hosted zones: %s", str(
- private_hosted_zone_collection) + lineno())
-
- if configuration in private_hosted_zone_collection:
- private_hosted_zone_name = configuration
-
- # check if the zone is already validated, if not check
- if private_hosted_zone_name in valid_dns_zones:
- has_dhcp_dns_zone_associated_vpc = True
- LOGGER.debug("private_hosted_zone_name already valid: %s", str(
- private_hosted_zone_name) + lineno())
- elif is_valid_zone(route53, instance_id, private_hosted_zone_name, hosted_zones, vpc_id, private_hosted_zone_collection,):
- has_dhcp_dns_zone_associated_vpc = True
- valid_dns_zones.append(private_hosted_zone_name)
+ dhcp_zone_id = None
+ if dhcp_zone is not None:
+ LOGGER.debug("configuration dhcp_zone: %s", str(dhcp_zone) + lineno())
+
+ if dhcp_zone in phz_collection_by_vpc:
+ private_hosted_zone_name = dhcp_zone
+ private_hosted_zone_item = phz_collection_by_vpc[private_hosted_zone_name]
+ dhcp_zone_id = private_hosted_zone_item['zone_id']
+ has_dhcp_dns_zone_associated_vpc = True
+ LOGGER.debug("by_vpc.private_hosted_zone_name dhcp_zone: %s found, item: %s",
+ str(private_hosted_zone_name) + lineno(), private_hosted_zone_item)
+ else:
+ has_dhcp_dns_zone_associated_vpc = False
+ LOGGER.debug("by_vpc.private_hosted_zone_name dhcp_zone: %s found, not valid (not associated with vpc)",
+ str(dhcp_zone) + lineno())
- # Loop through the instance's tags
LOGGER.debug("iterating through tags %s", lineno())
- # validate the tag value and set certain boolean values
- has_valid_zone_tag = False
- has_valid_cname_tag = False
- has_valid_hostname_tag = False
- has_valid_Name_tag_hostname = False
- has_valid_Name_tag_zonename = False
-
- for tag in tags:
- LOGGER.debug("#### tag: %s", str(tag) + lineno())
-
- if tag.get('Key').lstrip().lower() == TAGKEY_ZONE.lower():
- LOGGER.debug("Zone Tag key: %s", tag.get('Key') + lineno())
-
- # pause 1s to spread out API calls
- # time.sleep(1)
- custom_zone_name = tag.get('Value').lstrip().lower()
-
- # add a trailing period if it does not have it.
- if custom_zone_name[-1] != '.':
- custom_zone_name = custom_zone_name + '.'
-
- LOGGER.debug("Checking if custom_zone_name is valid: %s",
- str(custom_zone_name) + lineno())
-
- # check if the zone is already validated, if not check
- if custom_zone_name in valid_dns_zones:
- LOGGER.debug("custom_zone_name already valid: %s", str(
- custom_zone_name) + lineno())
- zone_tag_hosted_zone_name = custom_zone_name
- has_valid_zone_tag = True
- elif is_valid_zone(route53, instance_id, custom_zone_name, hosted_zones, vpc_id, private_hosted_zone_collection):
- zone_tag_hosted_zone_name = custom_zone_name
- valid_dns_zones.append(zone_tag_hosted_zone_name)
- LOGGER.debug("zone_tag_hosted_zone_name: %s", str(
- zone_tag_hosted_zone_name) + lineno())
- has_valid_zone_tag = True
-
- elif tag.get('Key').lstrip().lower() == TAGKEY_CNAME.lower():
- LOGGER.debug("CNAME Tag key: %s", tag.get('Key') + lineno())
-
- # pause 1s to spread out API calls
- # time.sleep(1)
-
- if is_valid_hostname(tag.get('Value')):
-
- LOGGER.debug("CNAME hostname of %s is valid %s",
- str(tag.get('Value')), lineno())
-
- # convert the cname value to lower case and strip whitespace and newline characters
- icname = tag.get('Value').lstrip().lower()
-
- if icname[-1] != '.': # add a trailing period if it does not have it.
- icname = icname + '.'
-
- LOGGER.debug("icname: %s", str(icname) + lineno())
- # Gets the prefix for the cname
- cname_host_name = icname.split('.')[0]
- LOGGER.debug("cname_host_name: %s", str(cname_host_name) + lineno())
-
- # Gets suffix
- cname_domain_suffix = icname[icname.find('.') + 1:]
- LOGGER.debug("cname_domain_suffix: %s", str(
- cname_domain_suffix) + lineno())
-
- # check if the zone is already validated, if not check
- if cname_domain_suffix in valid_dns_zones:
- has_valid_cname_tag = True
- LOGGER.debug("cname_domain_suffix already valid: %s", str(
- cname_domain_suffix) + lineno())
- elif is_valid_zone(route53, instance_id, cname_domain_suffix, hosted_zones, vpc_id, private_hosted_zone_collection):
- LOGGER.debug("cname domain is valid: %s",
- cname_domain_suffix + lineno())
- valid_dns_zones.append(cname_domain_suffix)
- has_valid_cname_tag = True
- else:
- LOGGER.debug("cname domain is not valid: %s",
- cname_domain_suffix + lineno())
-
- elif tag.get('Key').lstrip().lower() == TAGKEY_HOSTNAME.lower():
- LOGGER.debug("Custom Hostname Tag key: %s", tag.get('Key') + lineno())
-
- # pause 1s to spread out API calls
- # time.sleep(1)
-
- if is_valid_hostname(tag.get('Value')):
- LOGGER.debug("Custom hostname of %s is valid %s",
- str(tag.get('Value')), lineno())
- # convert the hostname value to lower case and strip whitespace and newline characters
- hostname = tag.get('Value').lstrip().lower()
-
- LOGGER.debug("hostname: %s", str(hostname) + lineno())
- # Gets the prefix for the custom hostname
- custom_host_name = hostname.split('.')[0]
- LOGGER.debug("custom_host_name: %s", str(custom_host_name) + lineno())
-
- has_valid_hostname_tag = True
- else:
- LOGGER.debug("Custom hostname of %s is invalid %s",
- str(tag.get('Value')), lineno())
-
- elif tag.get('Key') == 'Name':
- LOGGER.debug("Name Tag key: %s", tag.get('Key') + lineno())
-
- # pause 1s to spread out API calls
- # time.sleep(1)
-
- # if name exist, split into hostname/domain
- if is_valid_hostname(tag.get('Value')):
- LOGGER.debug("Name of %s is valid %s",
- str(tag.get('Value')), lineno())
- # convert the hostname value to lower case and strip whitespace and newline characters
- name_value = tag.get('Value').lstrip().lower()
-
- # add a trailing period if it does not have it.
- if name_value[-1] != '.':
- name_value = name_value + '.'
-
- LOGGER.debug("Name Tag Value: %s", str(name_value) + lineno())
-
- # Gets the host and the zone name (split up based)
- name_host = name_value.split('.')[0]
- LOGGER.debug("name_host: %s", str(
- name_host) + lineno())
- name_domain_suffix = name_value[name_value.find('.') + 1:]
- LOGGER.debug("name_domain_suffix: %s", str(
- name_domain_suffix) + lineno())
-
- # recheck the host portion
- if is_valid_hostname(name_host):
- has_valid_Name_tag_hostname = True
-
- LOGGER.debug("has_valid_Name_tag_hostname: %s", str(
- has_valid_Name_tag_hostname) + lineno())
-
- # check if the zone is already validated, if not check
- if name_domain_suffix in valid_dns_zones:
- has_valid_Name_tag_zonename = True
- LOGGER.debug("name_domain_suffix already valid: %s", str(
- name_domain_suffix) + lineno())
- elif is_valid_zone(route53, instance_id, name_domain_suffix, hosted_zones, vpc_id, private_hosted_zone_collection):
- valid_dns_zones.append(name_domain_suffix)
- has_valid_Name_tag_zonename = True
-
- LOGGER.debug("has_valid_Name_tag_zonename: %s", str(
- has_valid_Name_tag_zonename) + lineno())
+ tags_dict = tags_to_dict(tags)
+ flags = process_tags_flags(tags)
+ LOGGER.debug("New flags structure: %s", str(pformat(flags)) + lineno())
+ LOGGER.info("Options flags: " + ' '.join([f"{x}={flags[x]}" for x in flags]))
+
+ hostname_tuple = namedtuple(
+ 'Hostname', ['zone_exists', 'valid', 'hostname', 'zonename', 'zone_id', 'name'])
+ tag_data = {}
+# tag_data_fields = ['defined', 'valid', 'hostname', 'zonename']
+# tag_data_tuple = namedtuple('TagData', tag_data_fields)
+# tag_data['option_cname'] = tag_data_tuple(*process_tags_option_cname(tags))
+# tag_data['option_zone'] = tag_data_tuple(*process_tags_option_zone(tags))
+# tag_data['option_name'] = tag_data_tuple(*process_tags_option_name(tags))
+# tag_data['option_ptrname'] = tag_data_tuple(*process_tags_option_ptrname(tags))
+# tag_data['dhcp_options'] = tag_data_tuple(True,
+# has_dhcp_dns_zone_associated_vpc, None, private_hosted_zone_name)
+# tag_data['ptr_entry'] = tag_data_tuple(
+# True, True, reversed_entry, reversed_lookup_zone)
+# tag_data['name'] = tag_data_tuple(*process_tags_name(tags))
+
+ tag_data['option_cname'] = process_tags_option_cname(tags)
+ tag_data['option_zone'] = process_tags_option_zone(tags)
+ tag_data['option_name'] = process_tags_option_name(tags)
+ tag_data['option_ptrname'] = process_tags_option_ptrname(tags)
+# tag_data['dhcp_options'] = hostname_tuple(has_dhcp_dns_zone_associated_vpc, False, None, private_hosted_zone_name, dhcp_zone_id, private_hosted_zone_name )
+ tag_data['dhcp_options'] = process_tags_value(dhcp_zone)
+ tag_data['ptr_entry'] = process_tags_value(reversed_entry)
+ tag_data['name'] = process_tags_name(tags)
+
+ LOGGER.info("New tag_data structure: %s", str(pformat(tag_data)) + lineno())
+
+ emr_status = discover_emr_cluster(tags_dict)
+ LOGGER.info(f"discover_emr instance: {instance_id} result {emr_status}")
+
+ default_hostname = '-'.join(['ip'] + private_ip.split('.'))
+ if tag_data['option_zone'].valid:
+ if tag_data['option_name'].valid:
+ LOGGER.info(
+ f"1.1 instance: {instance_id}, using tag_option.zone hostname {tag_data['option_name'].hostname} and tag_option.zone zone {tag_data['option_zone'].zonename}")
+ f_hostname = tag_data['option_name'].hostname
+ f_zonename = tag_data['option_zone'].zonename
+ elif tag_data['name'].valid:
+ LOGGER.info(
+ f"1.2 instance: {instance_id}, using tag_key.Name hostname {tag_data['name'].hostname} and tag_option.zone zone {tag_data['option_zone'].zonename}")
+ f_hostname = tag_data['name'].hostname
+ f_zonename = tag_data['option_zone'].zonename
+ else:
+ LOGGER.info(
+ f"1.3 instance: {instance_id}, using default ip-address {default_hostname} for hostname and tag_option.zone zone {tag_data['option_zone'].zonename}")
+ f_hostname = default_hostname
+ f_zonename = tag_data['option_zone'].zonename
- else:
- LOGGER.debug("Name of %s is invalid %s",
- str(tag.get('Value')), lineno())
+ else:
+ if tag_data['option_name'].valid:
+ LOGGER.info(
+ f"2.1 instance: {instance_id}, using tag_option.name hostname {tag_data['option_name'].hostname} and and tag_option.name zone {tag_data['option_name'].zonename}")
+ f_hostname = tag_data['option_name'].hostname
+ f_zonename = tag_data['option_name'].zonename
+# elif not tag_data['option_name'].valid and tag_data['option_name'].hostname and tag_data['option_name'].zonename and all([flags['noforward'], flags['forcename']]):
+# LOGGER.info(
+# f"2.2 instance: {instance_id}, using tag_option.name hostname {tag_data['option_name'].hostname} and and tag_option.name zone {tag_data['option_name'].zonename} [noforward,forcename]")
+# f_hostname = tag_data['option_name'].hostname
+# f_zonename = tag_data['option_name'].zonename
+ elif not tag_data['option_name'].valid and tag_data['option_name'].hostname and tag_data['dhcp_options'].valid:
+ LOGGER.info(
+ f"2.3 instance: {instance_id}, using tag_option.name hostname {tag_data['option_name'].hostname} and and tag_option.name zone {tag_data['dhcp_options'].zonename}")
+ f_hostname = tag_data['option_name'].hostname
+ f_zonename = tag_data['dhcp_options'].zonename
+ elif tag_data['name'].valid:
+ LOGGER.info(
+ f"2.4 instance: {instance_id}, using tag_key.Name hostname {tag_data['name'].hostname} and tag_key.Name zone {tag_data['name'].zonename}")
+ f_hostname = tag_data['name'].hostname
+ f_zonename = tag_data['name'].zonename
+ elif not tag_data['name'].valid and tag_data['dhcp_options'].valid:
+ LOGGER.info(
+ f"2.5 instance: {instance_id}, using default ip-addresss hostname {default_hostname} and dhcp_options zone {tag_data['dhcp_options'].zonename}")
+ f_hostname = default_hostname
+ f_zonename = tag_data['dhcp_options'].zonename
else:
- LOGGER.debug("Skipping Tag key: %s", tag.get('Key') + lineno())
+ LOGGER.info(f"3.1 instance: {instance_id}, no valid hostname or zone found")
+ f_hostname = None
+ f_zonename = None
+ LOGGER.error(
+ f"instance: {instance_id}, No DHCP Associated for VPC and no custom tags. Exiting Script")
+ caller_response.append(
+ f"No DHCP Associated for VPC and no custom tags. Exiting Script")
+ return caller_response
- # determine correct A/PTR record to be created based upon the boolean values from the tags above
- if has_valid_hostname_tag and has_valid_zone_tag:
+ if tag_data['option_cname'].valid:
LOGGER.info(
- "instance: %s, custom hostname tag and custom zone tag valid.", instance_id)
- final_private_hostname = custom_host_name
- final_hosted_zone_name = zone_tag_hosted_zone_name
- elif has_valid_hostname_tag and not (has_valid_zone_tag) and has_dhcp_dns_zone_associated_vpc: # 3
- LOGGER.info("instance: %s, custom hostname tag valid only.", instance_id)
- final_private_hostname = custom_host_name
- final_hosted_zone_name = private_hosted_zone_name
- elif has_valid_Name_tag_hostname and has_valid_Name_tag_zonename:
+ f"4.1 instance: {instance_id}, CNAME using tag_option.cname hostname {tag_data['option_cname'].hostname} and tag_option.cname {tag_data['option_cname'].zonename}")
+ cf_hostname = tag_data['option_cname'].hostname
+ cf_zonename = tag_data['option_cname'].zonename
+ elif not tag_data['option_cname'].valid and not tag_data['option_cname'].zonename and tag_data['option_cname'].name and f_zonename:
LOGGER.info(
- "instance: %s, Name tag hostname valid and Name tag zonename valid.", instance_id)
- final_private_hostname = name_host
- final_hosted_zone_name = name_domain_suffix
- elif has_valid_Name_tag_hostname and has_valid_zone_tag:
+ f"4.2 instance: {instance_id}, CNAME using tag_option.cname hostname {tag_data['option_cname'].name} and current zone {f_zonename}")
+ cf_hostname = tag_data['option_cname'].name
+ cf_zonename = f_zonename
+ elif not tag_data['option_cname'].valid and tag_data['option_cname'].zonename and tag_data['option_cname'].hostname:
LOGGER.info(
- "instance: %s, Name tag hostname valid and custom zone tag valid.", instance_id)
- final_private_hostname = name_host
- final_hosted_zone_name = zone_tag_hosted_zone_name
- elif has_valid_Name_tag_hostname and has_dhcp_dns_zone_associated_vpc:
+ f"4.3 instance: {instance_id}, CNAME NOT using invalid tag_option.cname hostname {tag_data['option_cname'].name} and tag_option.cname zone {tag_data['option_cname'].zonename}")
+# cf_hostname = tag_data['option_cname'].hostname
+# cf_zonename = tag_data['option_cname'].zonename
+ cf_hostname = None
+ cf_zonename = None
+ else:
LOGGER.info(
- "instance: %s, Name tag hostname valid and DHCP zone is valid.", instance_id)
- final_private_hostname = name_host
- final_hosted_zone_name = private_hosted_zone_name
- elif has_valid_zone_tag and not (has_valid_hostname_tag) and not(has_valid_Name_tag_hostname):
+ f"4.4 instance: {instance_id}, CNAME no valid hostname {tag_data['option_cname'].hostname} or zone {tag_data['option_cname'].zonename}|{f_zonename} found, skipping CNAME")
+ cf_hostname = None
+ cf_zonename = None
+ if cf_hostname and cf_zonename:
+ cf_fqdn = create_fqdn(cf_hostname, cf_zonename)
+ else:
+ cf_fqdn = ''
+
+ if tag_data['option_ptrname'].valid:
LOGGER.info(
- "instance: %s, custom zone tag valid but no custom hostname, using IP address.", instance_id)
- final_private_hostname = private_host_name
- final_hosted_zone_name = zone_tag_hosted_zone_name
- elif has_dhcp_dns_zone_associated_vpc:
- LOGGER.info("instance: %s, no custom tags - use default.", instance_id)
- final_private_hostname = private_host_name
- final_hosted_zone_name = private_hosted_zone_name
+ f"5.1 instance: {instance_id}, PTR using tag_option.ptrname hostname {tag_data['option_ptrname'].hostname} and tag_option.ptrname zone {tag_data['option_ptrname'].zonename}")
+ p_hostname = tag_data['option_ptrname'].hostname
+ p_zonename = tag_data['option_ptrname'].zonename
+ elif not tag_data['option_ptrname'].zone_exists and tag_data['option_ptrname'].zonename:
+ LOGGER.info(
+ f"5.2 instance: {instance_id}, PTR using name not-valid, forcing tag_option.ptrname hostname {tag_data['option_ptrname'].hostname} and tag_option.ptrname zone {tag_data['option_ptrname'].zonename}")
+ p_hostname = tag_data['option_ptrname'].hostname
+ p_zonename = tag_data['option_ptrname'].zonename
else:
- LOGGER.error(
- "instance: %s, No DHCP Associated for VPC and no custom tags. Exiting Script", instance_id)
- # nothing to do, exit out script
- caller_response.append(
- 'No DHCP Associated for VPC and no custom tags. Exiting Script')
- return caller_response
+ LOGGER.info(
+ f"5.3 instance: {instance_id}, PTR default using current hostname {f_hostname} and current zone {f_zonename}")
+ p_hostname = f_hostname
+ p_zonename = f_zonename
+ p_fqdn = create_fqdn(p_hostname, p_zonename)
+
+# note this will not continue and set a cname
- # put together the FQDN of the dns name...
- final_private_dns_name = final_private_hostname + '.' + final_hosted_zone_name
- LOGGER.info("instance: %s, final hostname for A and PTR record: %s",
- instance_id, str(final_private_dns_name) + lineno())
+ final_private_hostname = f_hostname if len(f_hostname) > 0 else default_hostname
+ final_hosted_zone_name = f_zonename
+# final_private_dns_name = '.'.join([f_hostname, f_zonename])
+ final_private_dns_name = create_fqdn(final_private_hostname, final_hosted_zone_name)
+ f_fqdn = create_fqdn(final_private_hostname, final_hosted_zone_name)
+
+ zone_data_fields = ['name', 'zone_id', 'owner_account', 'is_amazon', 'enabled']
+ zone_data_tuple = namedtuple('ZoneData', zone_data_fields)
+
+ LOGGER.info(
+ f"instance: {instance_id}, final names for A and PTR record host {f_hostname} zone {f_zonename} fqdn {final_private_dns_name} ptr-fqdn {p_fqdn} {lineno()}")
# Get the PHZ ID for the Zone
- final_hosted_zone_id = get_zone_id(final_hosted_zone_name, hosted_zones)
- LOGGER.debug("private_hosted_zone_id:"
- " %s", str(final_hosted_zone_id) + lineno())
+ zone_data_forward = zone_data_tuple(
+ *phz_collection_by_vpc[final_hosted_zone_name].values())
+ zone_data_reverse = zone_data_tuple(
+ *phz_collection_by_vpc[tag_data['ptr_entry'].zonename].values())
+# final_hosted_zone_item = phz_collection_by_vpc[final_hosted_zone_name]
+# final_hosted_zone_id = zone_data_forward.zone_id
+# final_hosted_zone_owner = final_hosted_zone_item['owner_account']
- LOGGER.debug("valid_dns_zones:"
- " %s", str(valid_dns_zones) + lineno())
+ LOGGER.info(f"zone_data_forward: {pformat(zone_data_forward)}")
+ LOGGER.info(f"zone_data_reverse: {pformat(zone_data_reverse)}")
+
+ LOGGER.debug(
+ f"private_hosted_zone: zone_id {zone_data_forward.zone_id} is_mine {zone_data_forward.owner_account==account_id} owner {zone_data_forward.owner_account}: {lineno()}")
# create the TXT heritage record
heritage = initialize_heritage(HERITAGE_TAG, VERSION,
@@ -751,300 +828,316 @@ def lambda_handler(
LOGGER.debug("heritage value:"
" %s", str(heritage_value) + lineno())
-
delete_records = True
- # get_rr = False
# Create OR Delete the A / PTR Record
if state == 'running':
- # create the records
- try:
- LOGGER.debug("Creating resource records %s", lineno())
- create_response = create_resource_record(
- route53,
- instance_id,
- final_hosted_zone_id,
- final_private_hostname,
- final_hosted_zone_name,
- 'A',
- private_ip
- )
- append_msg = 'A record in zone id: ' + \
- str(final_hosted_zone_id) + \
- ' for hosted zone ' + \
- str(final_private_hostname) + '.' + \
- str(final_hosted_zone_name) + \
- ' with value: ' + \
- str(private_ip)
- if create_response == 'success':
- LOGGER.info("instance: %s, Created %s",
- instance_id, append_msg + lineno())
- caller_response.append('Created ' + append_msg)
- else:
- caller_response.append(create_response)
- caller_response.append('Failed to create ' + append_msg)
- LOGGER.error('Failed to create A record: %s', create_response)
- except BaseException as err:
- LOGGER.error("instance: %s, unexpected error. %s\n",
- instance_id, str(err) + lineno())
-
- try:
- if len(heritage) > 0:
- LOGGER.debug("Creating heritage TXT resource records %s, with a value of %s",
- final_private_hostname, str(heritage_value) + lineno())
+ dns_data = []
+ if not flags['noforward']:
+ # create the records
+ try:
+ LOGGER.debug("Creating resource records %s", lineno())
create_response = create_resource_record(
route53,
instance_id,
- final_hosted_zone_id,
+ zone_data_forward.zone_id,
final_private_hostname,
- final_hosted_zone_name,
- 'TXT',
- heritage_value
- )
- append_msg = 'TXT record in zone id: ' + \
- str(final_hosted_zone_id) + \
- ' for hosted zone ' + \
- str(final_private_hostname) + '.' + \
- str(final_hosted_zone_name) + \
- ' with value: ' + \
- str(heritage_value)
-
- if create_response == 'success':
- LOGGER.info("instance: %s, Created %s",
- instance_id, append_msg + lineno())
- caller_response.append('Created ' + append_msg)
- else:
- caller_response.append(create_response)
- caller_response.append('Failed to create ' + append_msg)
- LOGGER.error('Failed to create TXT record: %s', create_response)
-
- except BaseException as err:
- LOGGER.error("instance: %s, unexpected error. %s\n",
- instance_id, str(err) + lineno())
-
- try:
- if reverse_zone_associated:
- create_response = create_resource_record(
- route53,
- instance_id,
- reverse_lookup_zone_id,
- reversed_ip_address,
- 'in-addr.arpa',
- 'PTR',
- final_private_dns_name
+ zone_data_forward.name,
+ 'A',
+ private_ip
)
- append_msg = 'PTR record in zone id: ' + \
- str(reverse_lookup_zone_id) + \
- ' for hosted zone ' + \
- str(reversed_ip_address) + \
- 'in-addr.arpa with value: ' + \
- str(final_private_dns_name)
+ append_msg = f"A record in zone id: {zone_data_forward.zone_id} owner {zone_data_forward.owner_account} for hostname {final_private_hostname} " + \
+ f"zone {final_hosted_zone_name} to value {private_ip}"
+ count[create_response] += 1
if create_response == 'success':
+ dns_data.append(dns_data_tuple(
+ zone_data_forward.zone_id, final_private_hostname, zone_data_forward.name, 'A', private_ip))
LOGGER.info("instance: %s, Created %s",
instance_id, append_msg + lineno())
caller_response.append('Created ' + append_msg)
else:
caller_response.append(create_response)
caller_response.append('Failed to create ' + append_msg)
- LOGGER.error('Failed to create PTR record: %s', create_response)
-
- except BaseException as err:
- LOGGER.error("instance: %s, unexpected error. %s\n",
- instance_id, str(err) + lineno())
-
- try:
- if reverse_zone_associated and len(heritage) > 0:
- LOGGER.debug("Creating heritage TXT resource records %s, with a value of %s",
- str(reversed_ip_address), str(heritage_value) + lineno())
- create_response = create_resource_record(
- route53,
- instance_id,
- reverse_lookup_zone_id,
- reversed_ip_address,
- 'in-addr.arpa',
- 'TXT',
- heritage_value
- )
- append_msg = 'TXT reverse record in zone id: ' + \
- str(reverse_lookup_zone_id) + \
- ' for hosted zone ' + \
- str(reversed_ip_address) + \
- 'in-addr.arpa with value: ' + \
- str(heritage_value)
-
- if create_response == 'success':
- LOGGER.info("instance: %s, Created %s",
- instance_id, append_msg + lineno())
- caller_response.append('Created ' + append_msg)
- else:
- caller_response.append(create_response)
- caller_response.append('Failed to create ' + append_msg)
- LOGGER.error('Failed to create TXT record: %s', create_response)
-
- except BaseException as err:
- LOGGER.error("instance: %s, unexpected error. %s\n",
- instance_id, str(err) + lineno())
-
- else: # not running so delete the records
-
- # Process and delete A record and associated TXT record
- process_response = process_delete_records(
- route53,
- instance_id,
- final_hosted_zone_id,
- final_private_hostname,
- final_hosted_zone_name,
- 'A',
- private_ip,
- heritage_value
- )
-
- # only true if existing delete_records and the delete_success from the subroutine is true
- delete_records = delete_records and process_response['delete_success']
- # append to the lsit
- caller_response = caller_response + process_response['msg']
-
- # Process and delete PTR record and associated TXT record
- process_response = process_delete_records(
- route53,
- instance_id,
- reverse_lookup_zone_id,
- reversed_ip_address,
- 'in-addr.arpa.',
- 'PTR',
- final_private_dns_name,
- heritage_value
- )
- # only true if existing delete_records and the delete_success from the subroutine is true
- delete_records = delete_records and process_response['delete_success']
- # append to the lsit
- caller_response = caller_response + process_response['msg']
-
- # Process the CNAME record only if it has passed the check
- if has_valid_cname_tag:
- LOGGER.debug("cname record is valid - creating CNAME record:"
- " %s", str(cname_host_name) + "." + str(cname_domain_suffix) + lineno())
-
- # Try and find the hosted zone with the cname suffix
- cname_domain_suffix_id = get_zone_id(cname_domain_suffix, hosted_zones)
- LOGGER.debug("cname_domain_suffix_id: %s", str(cname_domain_suffix_id))
+ LOGGER.error('Failed to create A record: %s', create_response)
+ except BaseException as err:
+ LOGGER.error("instance: %s, unexpected error. %s\n",
+ instance_id, str(err) + lineno())
- # create CNAME record in private zone
- if state == 'running':
try:
- LOGGER.debug("cname_host_name:"
- " %s", str(cname_host_name) + lineno())
- LOGGER.debug("cname_domain_suffix:"
- " %s", str(cname_domain_suffix) + lineno())
- LOGGER.debug("cname_domain_suffix_id:"
- " %s", str(cname_domain_suffix_id) + lineno())
-
- create_response = create_resource_record(
- route53,
- instance_id,
- cname_domain_suffix_id,
- cname_host_name,
- cname_domain_suffix,
- 'CNAME',
- final_private_dns_name
- )
- append_msg = 'CNAME record in zone id: ' + \
- str(cname_domain_suffix_id) + \
- ' for hosted zone ' + \
- str(cname_host_name) + '.' + \
- str(cname_domain_suffix) + \
- ' with value: ' + \
- str(final_private_dns_name)
-
- if create_response == 'success':
- LOGGER.info("instance: %s, Created %s",
- instance_id, append_msg + lineno())
- caller_response.append('Created ' + append_msg)
+ if not flags['noheritage']:
+ if len(heritage) > 0:
+ LOGGER.debug(
+ f"Creating heritage TXT resource records {final_private_hostname} with value {heritage_value}: {lineno()}")
+ create_response = create_resource_record(
+ route53,
+ instance_id,
+ zone_data_forward.zone_id,
+ final_private_hostname,
+ zone_data_forward.name,
+ 'TXT',
+ heritage_value
+ )
+ append_msg = f"TXT record in zone id: {zone_data_forward.zone_id} owner {zone_data_forward.owner_account} for hostname {final_private_hostname} " + \
+ f"zone {zone_data_forward.name} to value {heritage_value}"
+
+ count[create_response] += 1
+ if create_response == 'success':
+ dns_data.append(dns_data_tuple(
+ zone_data_forward.zone_id, final_private_hostname, zone_data_forward.name, 'TXT', heritage_value))
+ LOGGER.info("instance: %s, Created %s",
+ instance_id, append_msg + lineno())
+ caller_response.append('Created ' + append_msg)
+ else:
+ caller_response.append(create_response)
+ caller_response.append('Failed to create ' + append_msg)
+ LOGGER.error('Failed to create TXT record: %s',
+ create_response)
else:
- caller_response.append(create_response)
- caller_response.append('Failed to create ' + append_msg)
- LOGGER.error('Failed to create CNAME record: %s', create_response)
+ LOGGER.info(
+ f"flags=noheritage, not adding heritage TXT for A host {final_private_hostname} zone {zone_data_forward.name} value {private_ip}")
+
except BaseException as err:
LOGGER.error("instance: %s, unexpected error. %s\n",
instance_id, str(err) + lineno())
+ else:
+ LOGGER.info(
+ f"flags=noforward, not adding A and heritage TXT for host {final_private_hostname} zone {zone_data_forward.name} value {private_ip}")
+ if not flags['noptr']:
+ # fqdn = create_fqdn(final_private_hostname, final_hosted_zone_name)
try:
- if len(heritage) > 0:
- LOGGER.debug("Creating heritage TXT resource records %s, with value of %s",
- TXT_RR_PREFIX + '.' + cname_host_name, str(heritage_value) + lineno())
+ if reverse_zone_associated:
create_response = create_resource_record(
route53,
instance_id,
- cname_domain_suffix_id,
- TXT_RR_PREFIX + '.' + cname_host_name,
- cname_domain_suffix,
- 'TXT',
- heritage_value
+ zone_data_reverse.zone_id,
+ tag_data['ptr_entry'].hostname,
+ tag_data['ptr_entry'].zonename,
+ 'PTR',
+ p_fqdn
)
- append_msg = 'TXT for CNAME record in zone id: ' + \
- str(cname_domain_suffix_id) + \
- ' for hosted zone ' + \
- str(TXT_RR_PREFIX) + '.' + \
- str(cname_host_name) + '.' + \
- str(cname_domain_suffix) + \
- ' with value: ' + \
- str(heritage_value)
-
+ append_msg = f"PTR record in zone id: {zone_data_reverse.zone_id} owner {zone_data_reverse.owner_account} for hostname {tag_data['ptr_entry'].hostname} " + \
+ f"zone {tag_data['ptr_entry'].zonename} to value {p_fqdn}"
+ count[create_response] += 1
if create_response == 'success':
+ dns_data.append(dns_data_tuple(
+ zone_data_reverse.zone_id, tag_data['ptr_entry'].hostname, tag_data['ptr_entry'].zonename, 'PTR', p_fqdn))
LOGGER.info("instance: %s, Created %s",
instance_id, append_msg + lineno())
caller_response.append('Created ' + append_msg)
else:
caller_response.append(create_response)
caller_response.append('Failed to create ' + append_msg)
- LOGGER.error(
- 'Failed to create TXT fpr CNAME record: %s', create_response)
+ LOGGER.error('Failed to create PTR record: %s', create_response)
except BaseException as err:
LOGGER.error("instance: %s, unexpected error. %s\n",
instance_id, str(err) + lineno())
- # not running, so process delete CNAME and associated TXT record
+ try:
+ if not flags['noheritage']:
+ if reverse_zone_associated and len(heritage) > 0:
+ LOGGER.debug(
+ f"Creating heritage TXT resource records {tag_data['ptr_entry'].hostname} with value {heritage_value}: {lineno()}")
+ create_response = create_resource_record(
+ route53,
+ instance_id,
+ zone_data_reverse.zone_id,
+ tag_data['ptr_entry'].hostname,
+ tag_data['ptr_entry'].zonename,
+ 'TXT',
+ heritage_value
+ )
+ append_msg = f"TXT record in zone id: {zone_data_reverse.zone_id} owner {zone_data_reverse.owner_account} for hostname {tag_data['ptr_entry'].hostname} " + \
+ f"zone {tag_data['ptr_entry'].zonename} to value {heritage_value}"
+
+ count[create_response] += 1
+ if create_response == 'success':
+ dns_data.append(dns_data_tuple(
+ zone_data_reverse.zone_id, tag_data['ptr_entry'].hostname, tag_data['ptr_entry'].zonename, 'TXT', heritage_value))
+ LOGGER.info("instance: %s, Created %s",
+ instance_id, append_msg + lineno())
+ caller_response.append('Created ' + append_msg)
+ else:
+ caller_response.append(create_response)
+ caller_response.append('Failed to create ' + append_msg)
+ LOGGER.error('Failed to create TXT record: %s',
+ create_response)
+
+ else:
+ LOGGER.info(
+ f"flags=noheritage, not adding heritage TXT for PTR host {tag_data['ptr_entry'].hostname} zone {tag_data['ptr_entry'].zonename}")
+ except BaseException as err:
+ LOGGER.error("instance: %s, unexpected error. %s\n",
+ instance_id, str(err) + lineno())
else:
- # Process and delete CNAME record and associated TXT record
- process_response = process_delete_records(
- route53,
- instance_id,
- cname_domain_suffix_id,
- cname_host_name,
- cname_domain_suffix,
- 'CNAME',
- final_private_dns_name,
- heritage_value
- )
+ LOGGER.info(
+ f"flags=noptr, not adding PTR and heritage TXT for host {tag_data['ptr_entry'].hostname} zone {tag_data['ptr_entry'].zonename} value {final_private_dns_name}")
- # only true if existing delete_records and the delete_success from the subroutine is true
- delete_records = delete_records and process_response['delete_success']
- # append to the lsit
- caller_response = caller_response + process_response['msg']
+# else: # not running so delete the records. Note this may leave orphans around if the flags are set and then the host is shut down. We may want to remove no matter what.
+# go through the dns_data records, and delete them. dns_data contains the records that were added. It is possible the tags have changed
+# so using existing tag data will not be valid. nodelete_dns_data is written back to the ddb if there are entries which did not get removed
- # Clean up DynamoDB after deleting records
if state != 'running':
+ heritage_records = {}
+ o_dns_data = dns_data.copy()
+ nodelete_dns_data = []
+ for entry in dns_data:
+ if entry.rr_type == 'TXT' and "heritage=" in entry.rr_value:
+ heritage_records[entry.rr_name] = entry.rr_value
+ p_delete = True
+ for entry in dns_data:
+ if not (entry.rr_type == 'TXT' and "heritage=" in entry.rr_value):
+ process_response = new_process_delete_records(
+ instance_id, entry.zone_id, entry.rr_name, entry.zone_name, entry.rr_type, entry.rr_value, heritage_records.get(entry.rr_name, ''))
+ delete_records = delete_records and process_response['delete_success']
+ caller_response = caller_response + process_response['msg']
+ count[f"delete_success.{process_response.get('delete_success')}"] += 1
+ p_delete = process_response['delete_success']
+ if not p_delete:
+ nodelete_dns_data.append(entry)
+ dns_data = nodelete_dns_data
+
+ # Process the CNAME record only if it has passed the check
+ if cf_hostname:
+ LOGGER.debug(
+ f"cname record is valid - creating CNAME record host {cf_hostname} zone {cf_zonename}: {lineno()}")
+# cname_host_name = tag_data['option_cname'].hostname
+# cname_domain_suffix = tag_data['option_cname'].zonename
+# cname_domain_suffix_item = phz_collection_by_vpc[cname_domain_suffix]
+# cname_domain_suffix_id = cname_domain_suffix_item['zone_id']
+# LOGGER.debug("cname_domain_suffix_id: %s", str(cname_domain_suffix_id))
+ cf_zonename_id = phz_collection_by_vpc[cf_zonename]['zone_id']
+ LOGGER.debug(f"cname_domain_suffix_id: {cf_zonename_id}")
+
+ # create CNAME record in private zone
+ if state == 'running':
+ if not flags['nocname'] and (all([emr_status.is_cluster, emr_status.is_master]) or not emr_status.is_cluster):
+ try:
+ LOGGER.debug(f"cname_host_name: {cf_hostname} {lineno()}")
+ LOGGER.debug(f"cname_domain_suffix: {cf_zonename} {lineno()}")
+ LOGGER.debug(f"cname_domain_suffix_id: {cf_zonename_id} {lineno()}")
+ LOGGER.debug(f"cname_target: {final_private_dns_name} {lineno()}")
+ if emr_status.is_cluster:
+ LOGGER.info(
+ f"instance {instance_id}: is_cluster && is_master cluster_id {emr_status.cluster_id} setting CNAME {cf_hostname} in zone {cf_zonename} {lineno()}")
+
+ create_response = create_resource_record(
+ route53,
+ instance_id,
+ # cname_domain_suffix_id,
+ # cname_host_name,
+ # cname_domain_suffix,
+ cf_zonename_id,
+ cf_hostname,
+ cf_zonename,
+ 'CNAME',
+ final_private_dns_name
+ )
+ append_msg = f"CNAME record in zone id: {cf_zonename_id} owner {phz_collection_by_vpc[cf_zonename]['owner_account']} " + \
+ f"hostname {cf_hostname} in zone {cf_zonename} with value {final_private_dns_name}"
+
+ if create_response == 'success':
+ dns_data.append(dns_data_tuple(
+ cf_zonename_id, cf_hostname, cf_zonename, 'CNAME', final_private_dns_name))
+ LOGGER.info(
+ f"instance: {instance_id}, Created {append_msg} {lineno()}")
+ caller_response.append('Created ' + append_msg)
+ else:
+ caller_response.append(create_response)
+ caller_response.append('Failed to create ' + append_msg)
+ LOGGER.error('Failed to create CNAME record: %s',
+ create_response)
+ except BaseException as err:
+ LOGGER.error("instance: %s, unexpected error. %s\n",
+ instance_id, str(err) + lineno())
+
+ try:
+ if len(heritage) > 0:
+ cf_hostname_txt = TXT_RR_PREFIX + '.' + cf_hostname
+ LOGGER.debug(
+ f"Creating heritage TXT resource records host {cf_hostname_txt} zone {cf_zonename} value {heritage_value} {lineno()}")
+ create_response = create_resource_record(
+ route53,
+ instance_id,
+ # cname_domain_suffix_id,
+ # cname_host_name_txt,
+ # cname_domain_suffix,
+ cf_zonename_id,
+ cf_hostname_txt,
+ cf_zonename,
+ 'TXT',
+ heritage_value
+ )
+ append_msg = f"TXT for CNAME record in zone id: {cf_zonename_id} owner {phz_collection_by_vpc[cf_zonename]['owner_account']} " + \
+ f"hostname {cf_hostname} in zone {cf_zonename} with value {heritage_value}"
+
+ if create_response == 'success':
+ dns_data.append(dns_data_tuple(
+ cf_zonename_id, cf_hostname_txt, cf_zonename, 'TXT', heritage_value))
+ LOGGER.info(
+ f"instance: {instance_id}, Created {append_msg} {lineno()}")
+ caller_response.append('Created ' + append_msg)
+ else:
+ caller_response.append(create_response)
+ caller_response.append('Failed to create ' + append_msg)
+ LOGGER.error(
+ f"Failed to create TXT for CNAME record: {create_response}")
+
+ except BaseException as err:
+ LOGGER.error(
+ f"instance: {instance_id}, unexpected error: {err} {lineno()}")
+ else:
+ if emr_status.is_cluster:
+ LOGGER.info(
+ f"instance {instance_id}: is_cluster && not is_master cluster_id {emr_status.cluster_id} NOT setting CNAME {cf_hostname} in zone {cf_zonename} {lineno()}")
+
+#
+# update ddb entry to include dns entries written to be able to delete them properly
+ if state == 'running':
+ try:
+ instance['_DnsEntries'] = [dict(d._asdict()) for d in dns_data]
+ instance = remove_empty_from_dict(instance)
+ instance_dump = json.dumps(instance, default=json_serial)
+ instance_attributes = instance_dump
+ LOGGER.info(
+ f"Updating DDB Entry item for {instance_id} to include DNS entries {lineno()}")
+ put_item_in_dynamodb_table(
+ dynamodb_client, DDBNAME, instance_id, instance_attributes)
+ LOGGER.debug(f"Done updating item in dynamodb table {lineno()}")
+ except Exception as err:
+ LOGGER.error(
+ f"Error putting item in DDB table {instance_id} error {err} {lineno()}")
+
+# Clean up DynamoDB after deleting records
+ if state != 'running':
# only if all records were succesfully deleted
if delete_records:
delete_item_from_dynamodb_table(dynamodb_client, DDBNAME, instance_id)
LOGGER.info("instance: %s, deleted the item from DynamoDB: %s",
instance_id, DDBNAME + lineno())
caller_response.insert(0, 'Successfully removed recordsets')
- return caller_response
else:
LOGGER.info("instance: %s, not all records deleted, leaving item in DynamoDB: %s",
instance_id, DDBNAME + lineno())
caller_response.insert(
0, 'Failed to remove recordsets, leaving DynamoDB item for instance: ' + instance_id)
- return caller_response
else:
LOGGER.info("instance: %s, Successfully created recordsets. %s",
instance_id, lineno())
caller_response.insert(0, 'Successfully created recordsets')
- return caller_response
+
+ LOGGER.info(f"dns_data records written:\n{pformat(dns_data)}")
+
+ count['end'] = datetime.datetime.now()
+ count['elapsed_ms'] = (count['end'] - count['start']).total_seconds() * 1000.0
+ LOGGER.info(f"{APPNAME} stats: source={event_source} state={state} " +
+ ' '.join([f"{c}={count[c]}" for c in sorted(count.keys())]))
+ return caller_response
+
+# end lambda_handler
def get_cname_from_tags(tags):
@@ -1093,20 +1186,12 @@ def get_instances(client, instance_id):
LOGGER.info("instance: %s, describe_instances returned RequestLimitExceeded, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
+ count['retry'] += 1
return instance_data
-# def list_hosted_zones(client):
-# """
-# Get route53 hosted zones
-# :param client:
-# :return:
-# """
-# try:
-# return client.list_hosted_zones()
-# except ClientError as err:
-# LOGGER.info("unexpected error. %s\n", str(err) + lineno())
-
def new_list_hosted_zones(client, instance_id):
"""
@@ -1136,6 +1221,9 @@ def new_list_hosted_zones(client, instance_id):
LOGGER.info("instance: %s, list_hosted_zones returned error, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
+ count['retry'] += 1
if hosted_zones == {}:
LOGGER.error("instance: %s, list_hosted_zones returned error. Timed out. %s",
@@ -1158,6 +1246,65 @@ def new_list_hosted_zones(client, instance_id):
return hosted_zones
+def new_list_hosted_zones_by_vpc(client, instance_id, vpc_id, region):
+ """
+ Get route53 hosted zones
+ :param client:
+ :param instance_id:
+ :param vpc_id:
+ :param region:
+ :return:
+ """
+
+ i = 0
+ hosted_zones = {}
+ # retry to handle errors in the possible API call
+ while i < MAX_API_RETRY:
+ try:
+ hosted_zones = client.list_hosted_zones_by_vpc(
+ VPCId=vpc_id, VPCRegion=region)
+ LOGGER.debug(
+ "list_hosted_zones_by_vpc returned without error. %s", lineno())
+ break
+ except ClientError as err:
+ error_message = str(err)
+ if "(Throttling)" in str(err):
+ LOGGER.debug(
+ "list_hosted_zones_by_vpc throttled due to API limit, retrying: %s", str(err) + lineno())
+ else:
+ LOGGER.info("vpc: %s, instance: %s, unexpected error. %s\n",
+ vpc_id, instance_id, error_message + lineno())
+ i += 1
+ LOGGER.info("instance: %s, list_hosted_zones_by_vpc %v returned error, waiting before retry. %s",
+ vpc_id, instance_id, str(i) + lineno())
+ time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
+ count['retry'] += 1
+
+ if hosted_zones == {}:
+ LOGGER.error("vpc_id: %s, instance: %s, list_hosted_zones_by_vpc returned error. Timed out. %s",
+ vpc_id, instance_id, str(i) + lineno())
+ if SNS_ENABLE:
+ try:
+ sns_msg = {}
+ sns_msg['instance_id'] = instance_id
+ sns_msg['vpc_id'] = vpc_id
+ sns_msg['region'] = region
+ sns_msg['account_id'] = get_caller_account_id()
+ sns_msg['client'] = 'route53'
+ sns_msg['boto3_method'] = 'list_hosted_zones_by_vpc'
+ sns_msg['message'] = 'list_hosted_zones_by_vpc timed out'
+ publish_to_sns(get_sns_client(), json.dumps(sns_msg))
+ LOGGER.info("vpc_id: %s, instance: %s, sending sns message %s", vpc_id, instance_id,
+ json.dumps(sns_msg) + lineno())
+ except:
+ LOGGER.info("vpc_id: %s, instance: %s, error: %s", vpc_id, instance_id,
+ str(sys.exc_info()[0]) + lineno())
+
+ return hosted_zones
+
+
def list_tables(client):
"""
List the dynamodb tables
@@ -1277,6 +1424,31 @@ def get_private_hosted_zone_collection(private_hosted_zones):
LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+def get_private_hosted_zone_collection_by_vpc(private_hosted_zones):
+ """
+ Get private hosted zone collection
+ :param private_hosted_zones:
+ :return:
+ """
+ try:
+ private_hosted_zone_collection = []
+
+ for item in private_hosted_zones:
+ LOGGER.debug("item: %s", str(item) + lineno())
+ my_item = {
+ 'name': item['Name'],
+ 'zone_id': item['HostedZoneId'],
+ 'owner_account': item['Owner'].get('OwningAccount', ''),
+ 'is_amazon': 'amazonaws.com' in item['Name'],
+ 'enabled': 'amazonaws.com' not in item['Name']
+ }
+ private_hosted_zone_collection.append(my_item)
+
+ return private_hosted_zone_collection
+ except:
+ LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+
+
def get_private_hosted_zones(hosted_zones):
"""
Get private hosted zones
@@ -1297,6 +1469,27 @@ def get_private_hosted_zones(hosted_zones):
LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+# 'HostedZoneSummaries': [{'HostedZoneId': 'Z03702572CGNJTW5OPG68', 'Name': '0.192.10.in-addr.arpa.', 'Owner': {'OwningAccount': '057405694017'}}, {'HostedZoneId': 'Z07556021WTTHLOJ6S3C', 'Name':
+
+def get_private_hosted_zones_by_vpc(hosted_zones):
+ """
+ Get private hosted zones
+ :param hosted_zones:
+ :return:
+ """
+ try:
+ private_hosted_zones = []
+
+ for item in hosted_zones['HostedZoneSummaries']:
+ LOGGER.debug("item: %s", str(item) + lineno())
+
+ private_hosted_zones.append(item)
+
+ return private_hosted_zones
+ except:
+ LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+
+
def get_dhcp_option_set_id_for_vpc(client, instance_id, vpc_id):
"""
Get the dhcp option set from vpc
@@ -1333,6 +1526,9 @@ def get_dhcp_option_set_id_for_vpc(client, instance_id, vpc_id):
LOGGER.info("instance: %s, describe_vpcs returned RequestLimitExceeded, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
+ count['retry'] += 1
return option_set_for_vpc
@@ -1404,10 +1600,10 @@ def get_dynamodb_table(client, table_name):
# return 'Unexpected error: ' + str(err)
-def new_change_resource_recordset(client, instance_id, zone_id, host_name, hosted_zone_name, record_type, value):
+def new_change_resource_recordset(oclient, instance_id, zone_id, host_name, hosted_zone_name, record_type, value):
"""
Change resource recordset
- :param client:
+ :param oclient:
:param zone_id:
:param host_name:
:param hosted_zone_name:
@@ -1415,20 +1611,43 @@ def new_change_resource_recordset(client, instance_id, zone_id, host_name, hoste
:return:
"""
+# this ignores the client, and uses the session from sessions[account] with a new route53 client
+
+ global phz_collection_by_vpc
+ zone_item = phz_collection_by_vpc[hosted_zone_name]
+ LOGGER.debug("Using zone %s, zone item %s: %s", str(
+ hosted_zone_name), str(zone_item), lineno())
+ zone_account = zone_item['owner_account']
+ try:
+ LOGGER.debug("Calling get_session_assume_role() on account %s: %s",
+ zone_account, lineno())
+ this_session = get_session_assume_role(zone_account)
+ except Exception as err:
+ LOGGER.error("Unable to esablish assume_role session in account %s: %s",
+ str(zone_account), str(err) + lineno())
+ update_response = "AssumeRoleFailed"
+ return update_response
+
+ client = this_session.client('route53')
+
i = 0
update_response = {}
+
+ LOGGER.debug("Creating %s record %s in zone %s: %s",
+ record_type, host_name, hosted_zone_name, lineno())
+
# retry to handle errors in the possible API call
while i < MAX_API_RETRY:
try:
- LOGGER.debug("Creating %s record %s in zone %s"
- " %s", record_type, host_name, hosted_zone_name, lineno())
+ LOGGER.debug("Try %s Creating %s record %s in zone %s: %s", str(
+ i), record_type, host_name, hosted_zone_name, lineno())
change_batch = {
- "Comment": "Updated by Lambda DDNS",
+ "Comment": f"Updated by {APPNAME} v{VERSION} from {account_id} in {region}",
"Changes": [
{
"Action": "UPSERT",
"ResourceRecordSet": {
- "Name": host_name + hosted_zone_name,
+ "Name": create_fqdn(host_name, hosted_zone_name),
"Type": record_type,
"TTL": DNS_RR_TTL,
"ResourceRecords": [
@@ -1451,27 +1670,50 @@ def new_change_resource_recordset(client, instance_id, zone_id, host_name, hoste
LOGGER.debug("change_resource_record_sets UPSERT returned without error - response: %s",
str(update_response) + lineno())
break
- except ClientError as err:
- if 'NoSuchHostedZone' in str(err) and 'No hosted zone found with ID' in str(err):
- LOGGER.error("Hosted zone not found error: %s", str(err) + lineno())
- update_response = "NoSuchHostedZone"
- break
- elif 'InvalidChangeBatch' in str(err) and 'is not permitted in zone' in str(err):
- LOGGER.error(
- "Cannot create record - most likely wrong zone name specified: %s", str(err) + lineno())
- update_response = "InvalidChangeBatch-WrongZoneName"
- break
- elif "(Throttling)" in str(err):
- LOGGER.debug("change_resource_record_sets UPSERT throttled due to API limit, retrying: %s", str(
- err) + lineno())
- else:
- LOGGER.info("instance: %s, unexpected error. %s\n",
- instance_id, str(err) + lineno())
+# except ClientError as err:
+# LOGGER.info(
+# f"exception: {instance_id} err {err} sys.exc_info {sys.exc_info()[0]} {sys.exc_info()[1]} {lineno()}")
+ except Route53.Client.exceptions.NoSuchHostedZone as err:
+
+ # if 'NoSuchHostedZone' in str(err) and 'No hosted zone found with ID' in str(err):
+ # LOGGER.error("Hosted zone not found error: %s", str(err) + lineno())
+ LOGGER.info(
+ f"exception: NoSuchHostedZone {err} sys.exc_info {sys.exc_info()[0]} {sys.exc_info()[1]} {lineno()}")
+ update_response = "NoSuchHostedZone"
+ break
+ except Route53.Client.exceptions.InvalidChangeBatch as err:
+ # elif 'InvalidChangeBatch' in str(err) and 'is not permitted in zone' in str(err):
+ # LOGGER.error(
+ # "Cannot create record - most likely wrong zone name specified: %s", str(err) + lineno())
+ # update_response = "InvalidChangeBatch-WrongZoneName"
+ LOGGER.info(
+ f"exception: InvalidChangeBatch {err} sys.exc_info {sys.exc_info()[0]} {sys.exc_info()[1]} {lineno()}")
+ update_response = "InvalidChangeBatch"
+ break
+ except Route53.Client.exceptions.InvalidInput as err:
+ LOGGER.info(
+ f"exception: InvalidInput {err} sys.exc_info {sys.exc_info()[0]} {sys.exc_info()[1]} {lineno()}")
+ update_response = "InvalidInput"
+ break
+ except Route53.Client.exceptions.PriorRequestNotComplete as err:
+ # elif "(Throttling)" in str(err):
+ LOGGER.info(
+ f"exception: PriorRequestNotComplete {err} sys.exc_info {sys.exc_info()[0]} {sys.exc_info()[1]} {lineno()}")
+# LOGGER.debug("change_resource_record_sets UPSERT throttled due to API limit, retrying: %s", str(err) + lineno())
+# else:
+# LOGGER.info(
+# f"instance: {instance_id} unexpected error: {err} sys.exc_info {sys.exc_info()[0]} {sys.exc_info()[1]} {lineno()}")
+ except Exception as err:
+ LOGGER.info(
+ f"exception: Exception {err} sys.exc_info {sys.exc_info()[0]} {sys.exc_info()[1]} {lineno()}")
i += 1
LOGGER.info("instance: %s, change_resource_record_sets UPSERT returned error, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
+ count['retry'] += 1
if i >= MAX_API_RETRY:
LOGGER.error("instance: %s, change_resource_record_sets exceeded max retry of %s",
@@ -1514,11 +1756,8 @@ def create_resource_record(client, instance_id, zone_id, host_name, hosted_zone_
" %s hosted_zone_name: %s record_type: %s value: %s %s", zone_id,
host_name, hosted_zone_name, record_type, value, lineno())
try:
- if host_name[-1] != '.':
- host_name = host_name + '.'
-
LOGGER.debug(
- "Updating %s in zone %s%s to %s %s", record_type, host_name,
+ "Updating %s in name %s zone %s to %s %s", record_type, host_name,
hosted_zone_name, value, lineno())
# To prevent rate throttling
@@ -1600,10 +1839,10 @@ def create_resource_record(client, instance_id, zone_id, host_name, hosted_zone_
# return value
-def new_get_resource_record(client, instance_id, zone_id, host_name, hosted_zone_name, record_type, unused=None):
+def new_get_resource_record(oclient, instance_id, zone_id, host_name, hosted_zone_name, record_type, unused=None):
"""
- This function getts resource records from the hosted zone passed by the calling function.
- :param str client:
+ This function gets resource records from the hosted zone passed by the calling function.
+ :param str oclient:
:param str instance_id:
:param str zone_id:
:param str host_name:
@@ -1613,6 +1852,25 @@ def new_get_resource_record(client, instance_id, zone_id, host_name, hosted_zone
:return str value: Value of record if found, empty "' if not found
"""
+# this ignores the client, and uses the session from sessions[account] with a new route53 client
+
+ global phz_collection_by_vpc
+ zone_item = phz_collection_by_vpc[hosted_zone_name]
+ LOGGER.debug("Using zone %s, zone item %s: %s", str(
+ hosted_zone_name), str(zone_item), lineno())
+ zone_account = zone_item['owner_account']
+ try:
+ LOGGER.debug("Calling get_session_assume_role() on account %s: %s",
+ zone_account, lineno())
+ this_session = get_session_assume_role(zone_account)
+ except Exception as err:
+ LOGGER.error("Unable to esablish assume_role session in account %s: %s",
+ str(zone_account), str(err) + lineno())
+ update_response = "AssumeRoleFailed"
+ return update_response
+
+ client = this_session.client('route53')
+
i = 0
value = ''
@@ -1621,15 +1879,17 @@ def new_get_resource_record(client, instance_id, zone_id, host_name, hosted_zone
LOGGER.debug("Getting %s record type for %s",
record_type, host_name + lineno())
- if host_name[-1] != '.':
- host_name = host_name + '.'
+# if host_name[-1] != '.':
+# host_name = host_name + '.'
LOGGER.debug("list_resource_record_sets looking for record %s in zone %s",
str(host_name), str(hosted_zone_name) + lineno())
+ fqdn = create_fqdn(host_name, hosted_zone_name)
response = client.list_resource_record_sets(
HostedZoneId=zone_id,
- StartRecordName=host_name + hosted_zone_name,
+ # StartRecordName=host_name + hosted_zone_name,
+ StartRecordName=fqdn,
StartRecordType=record_type,
MaxItems='1')
@@ -1640,13 +1900,14 @@ def new_get_resource_record(client, instance_id, zone_id, host_name, hosted_zone
rr_name = rr_set['Name']
# check if the return value matches the record, if not ignore
# if the record isn't there, it returns the list_resource_record_sets returns the next record
- if rr_name == (host_name + hosted_zone_name):
+# if rr_name == (host_name + hosted_zone_name):
+ if rr_name == fqdn:
value = rr_set['ResourceRecords'][0]['Value']
- LOGGER.debug("list_resource_record_sets returned value. %s",
- str(value) + lineno())
+ LOGGER.debug(
+ f"list_resource_record_sets returned value {value}: {lineno()}")
else:
- LOGGER.debug("list_resource_record_sets returned different record ignoring. %s",
- str(rr_name) + lineno())
+ LOGGER.debug(
+ f"list_resource_record_sets returned different record ignoring, fqdn [{fqdn}] != rr_name [{rr_name}]: {lineno()}")
LOGGER.debug(
"list_resource_record_sets returned without error. %s", lineno())
@@ -1668,6 +1929,9 @@ def new_get_resource_record(client, instance_id, zone_id, host_name, hosted_zone
LOGGER.info("instance: %s, list_resource_record_sets returned error, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
+ count['retry'] += 1
if i >= MAX_API_RETRY:
LOGGER.error("instance: %s, list_resource_record_sets exceeded max retry of %s",
@@ -1690,63 +1954,10 @@ def new_get_resource_record(client, instance_id, zone_id, host_name, hosted_zone
return str(value)
-# def delete_resource_record(client, zone_id, host_name, hosted_zone_name, record_type, value):
-# """
-# This function deletes resource records from the hosted zone passed by the calling function.
-# :param client:
-# :param zone_id:
-# :param host_name:
-# :param hosted_zone_name:
-# :param record_type:
-# :param value:
-# :return:
-# """
-# try:
-# LOGGER.debug("Deleting %s record %s in zone %s"
-# " %s", record_type, host_name, hosted_zone_name, lineno())
-# if host_name[-1] != '.':
-# host_name = host_name + '.'
-# response = client.change_resource_record_sets(
-# HostedZoneId=zone_id,
-# ChangeBatch={
-# "Comment": "Updated by Lambda DDNS",
-# "Changes": [
-# {
-# "Action": "DELETE",
-# "ResourceRecordSet": {
-# "Name": host_name + hosted_zone_name,
-# "Type": record_type,
-# "TTL": DNS_RR_TTL,
-# "ResourceRecords": [
-# {
-# "Value": value
-# },
-# ]
-# }
-# }
-# ]
-# }
-# )
-
-# LOGGER.debug("delete record response: %s", str(response) + lineno())
-# return response
-
-# except ClientError as err:
-# if 'Not Found' in str(err):
-# LOGGER.debug("Record not found error: %s", str(err) + lineno())
-# return
-
-# if 'InvalidChangeBatch' in str(err) and 'it was not found' in str(err):
-# LOGGER.debug("Record not found error: %s", str(err) + lineno())
-# return
-
-# LOGGER.info("unexpected error. %s\n", str(err) + lineno())
-
-
-def new_delete_resource_record(client, instance_id, zone_id, host_name, hosted_zone_name, record_type, value):
+def new_delete_resource_record(oclient, instance_id, zone_id, host_name, hosted_zone_name, record_type, value):
"""
This function deletes resource records from the hosted zone passed by the calling function.
- :param client:
+ :param oclient:
:param instance_id:
:param zone_id:
:param host_name:
@@ -1756,24 +1967,48 @@ def new_delete_resource_record(client, instance_id, zone_id, host_name, hosted_z
:return:
"""
+# this ignores the client, and uses the session from sessions[account] with a new route53 client
+
+ global phz_collection_by_vpc
+ zone_item = phz_collection_by_vpc[hosted_zone_name]
+ LOGGER.debug("Using zone %s, zone item %s: %s", str(
+ hosted_zone_name), str(zone_item), lineno())
+ zone_account = zone_item['owner_account']
+ try:
+ LOGGER.debug("Calling get_session_assume_role() on account %s: %s",
+ zone_account, lineno())
+ this_session = get_session_assume_role(zone_account)
+ except Exception as err:
+ LOGGER.error("Unable to esablish assume_role session in account %s: %s",
+ str(zone_account), str(err) + lineno())
+ update_response = "AssumeRoleFailed"
+ return update_response
+
+ client = this_session.client('route53')
+
i = 0
delete_response = {}
+
+ LOGGER.debug("Deleting %s record %s in zone %s: %s",
+ record_type, host_name, hosted_zone_name, lineno())
+# if host_name[-1] != '.':
+# host_name = host_name + '.'
+# fqdn = host_name
+# fqdn += '.' + hosted_zone_name if hosted_zone_name in host_name else ''
+
# retry to handle errors in the possible API call
while i < MAX_API_RETRY:
try:
- LOGGER.debug("Deleting %s record %s in zone %s"
- " %s", record_type, host_name, hosted_zone_name, lineno())
- if host_name[-1] != '.':
- host_name = host_name + '.'
-
+ LOGGER.debug("Try %s Deleting %s record %s in zone %s: %s", str(
+ i), record_type, host_name, hosted_zone_name, lineno())
change_batch = {
- "Comment": "Updated by Lambda DDNS",
+ "Comment": f"Deleted by {APPNAME} v{VERSION} from {account_id} in {region}",
"Changes": [
{
"Action": "DELETE",
"ResourceRecordSet": {
- "Name": host_name + hosted_zone_name,
+ "Name": create_fqdn(host_name, hosted_zone_name),
"Type": record_type,
"TTL": DNS_RR_TTL,
"ResourceRecords": [
@@ -1823,6 +2058,8 @@ def new_delete_resource_record(client, instance_id, zone_id, host_name, hosted_z
LOGGER.info("instance: %s, change_resource_record_sets DELETE returned error, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
if i >= MAX_API_RETRY:
LOGGER.error("instance: %s, change_resource_record_sets exceeded max retry of %s",
@@ -1963,18 +2200,19 @@ def is_valid_zone(route53, instance_id, zonename, hosted_zones, vpc_id, private_
def get_dhcp_configurations(client, instance_id, dhcp_options_id):
"""
- This function returns the names of the zones/domains that are in the option set.
+ This function returns the names of the zone/domain that are in the option set. According to the boto3 [docs](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2/client/create_dhcp_options.html)
+ this is a single valuea. It includes the trailing dot in the return.
+
:param client:
:param dhcp_options_id:
:return:
"""
i = 0
-
while i < MAX_API_RETRY:
try:
- zone_names = []
+ zone_name = None
response = client.describe_dhcp_options(
DhcpOptionsIds=[
@@ -1988,8 +2226,9 @@ def get_dhcp_configurations(client, instance_id, dhcp_options_id):
if configuration['Key'] == 'domain-name': # only if the key is domain-name
for item in configuration['Values']:
LOGGER.debug("item: %s", str(item) + lineno())
- zone_names.append(str(item['Value']) + '.')
- LOGGER.debug("zone name: %s", str(zone_names) + lineno())
+ zone_name = str(item['Value']) + '.'
+ break
+ LOGGER.debug("zone name: %s", str(zone_name) + lineno())
break
except ClientError as err:
@@ -2003,96 +2242,116 @@ def get_dhcp_configurations(client, instance_id, dhcp_options_id):
LOGGER.info("instance: %s, describe_dhcp_options returned RequestLimitExceeded, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
- return zone_names
+ return zone_name
def new_reverse_list(ip_list):
"""
Reverses the order of the instance's IP address and helps construct the reverse lookup zone name.
:param str ip_list: IPv4 address to reverse
- :return str: Returns a string in the PTR format
+ :return str: Returns a string in the PTR format minus the PTR zone (in-addr.arpa)
"""
try:
ip = ipaddress.ip_address(ip_list)
- reversed_list = ip.reverse_pointer
+ reversed_list = '.'.join(ip.reverse_pointer.split('.')[:4])
LOGGER.debug("returning: %s", str(reversed_list) + lineno())
+ return reversed_list
except:
LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
return None
- np = '.'.join(p.split('.')[1:])
-
-def new_reverse_domain(ip_list):
+def new_reverse_entry(ip_list):
"""
Reverses the order of the instance's IP address and helps construct the reverse lookup zone name.
:param str ip_list: IPv4 address to reverse
- :return str: Returns a string in the PTR format of the subnet (/24 boundary)
+ :return str: Returns a string in the PTR format (with trailing dot)
"""
try:
ip = ipaddress.ip_address(ip_list)
- reversed_list = '.'.join(ip.reverse_pointer.split('.')[1:])
+ reversed_list = ip.reverse_pointer
+ reversed_list += '.' if reversed_list[-1] != '.' else ''
LOGGER.debug("returning: %s", str(reversed_list) + lineno())
+ return reversed_list
except:
LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
return None
-def reverse_list(ip_list):
+def new_reverse_domain(ip_list):
"""
- Reverses the order of the instance's IP address and
- helps construct the reverse lookup zone name.
- :param list:
- :return:
+ Reverses the order of the instance's IP address and helps construct the reverse lookup zone name.
+ :param str ip_list: IPv4 address to reverse
+ :return str: Returns a string in the PTR format of the subnet (/24 boundary) with the traling dot appended
"""
try:
- if (re.search(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}", ip_list)) or \
- (re.search(r"\d{1,3}.\d{1,3}.\d{1,3}\.", ip_list)) or \
- (re.search(r"\d{1,3}.\d{1,3}\.", ip_list)) or \
- (re.search(r"\d{1,3}\.", ip_list)):
- my_temp_list = str.split(str(ip_list), '.')
- LOGGER.debug("temp list: %s", str(my_temp_list) + lineno())
- my_list = []
- for item in my_temp_list:
- LOGGER.debug("item: %s", str(item) + lineno())
- if len(item) > 0:
- my_list.append(int(item))
-
- LOGGER.debug("list1: %s", str(my_list) + lineno())
- LOGGER.debug("type: %s", str(type(my_list)) + lineno())
-
- my_list.reverse()
- reversed_list = ''
- for item in my_list:
- reversed_list = reversed_list + str(item) + '.'
- LOGGER.debug("returning: %s", str(reversed_list) + lineno())
- return reversed_list
-
- LOGGER.error('Not a valid ip. %s', lineno())
- return None
-
+ ip = ipaddress.ip_address(ip_list)
+ reversed_list = '.'.join(ip.reverse_pointer.split('.')[1:])
+ reversed_list += '.' if reversed_list[-1] != '.' else ''
+ LOGGER.debug("returning: %s", str(reversed_list) + lineno())
+ return reversed_list
except:
- LOGGER.error("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
- return None
+ LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+ return None
-def get_reversed_domain_prefix(subnet_mask, private_ip):
- """
- Uses the mask to get the zone prefix for the reverse lookup zone
- :param subnet_mask:
- :param private_ip:
- :return:
- """
- try:
- LOGGER.debug("### Subnet mask: %s", str(subnet_mask) + lineno())
- LOGGER.debug("### Private ip: %s", str(private_ip) + lineno())
-
- third_octet = re.search(r"\d{1,3}.\d{1,3}.\d{1,3}.", private_ip)
- return third_octet.group(0)
- except:
- LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+# def reverse_list(ip_list):
+# """
+# Reverses the order of the instance's IP address and
+# helps construct the reverse lookup zone name.
+# :param list:
+# :return:
+# """
+# try:
+# if (re.search(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}", ip_list)) or \
+# (re.search(r"\d{1,3}.\d{1,3}.\d{1,3}\.", ip_list)) or \
+# (re.search(r"\d{1,3}.\d{1,3}\.", ip_list)) or \
+# (re.search(r"\d{1,3}\.", ip_list)):
+# my_temp_list = str.split(str(ip_list), '.')
+# LOGGER.debug("temp list: %s", str(my_temp_list) + lineno())
+# my_list = []
+# for item in my_temp_list:
+# LOGGER.debug("item: %s", str(item) + lineno())
+# if len(item) > 0:
+# my_list.append(int(item))
+#
+# LOGGER.debug("list1: %s", str(my_list) + lineno())
+# LOGGER.debug("type: %s", str(type(my_list)) + lineno())
+#
+# my_list.reverse()
+# reversed_list = ''
+# for item in my_list:
+# reversed_list = reversed_list + str(item) + '.'
+# LOGGER.debug("returning: %s", str(reversed_list) + lineno())
+# return reversed_list
+#
+# LOGGER.error('Not a valid ip. %s', lineno())
+# return None
+#
+# except:
+# LOGGER.error("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+# return None
+#
+# def get_reversed_domain_prefix(subnet_mask, private_ip):
+# """
+# Uses the mask to get the zone prefix for the reverse lookup zone
+# :param subnet_mask:
+# :param private_ip:
+# :return:
+# """
+# try:
+# LOGGER.debug("### Subnet mask: %s", str(subnet_mask) + lineno())
+# LOGGER.debug("### Private ip: %s", str(private_ip) + lineno())
+#
+# third_octet = re.search(r"\d{1,3}.\d{1,3}.\d{1,3}.", private_ip)
+# return third_octet.group(0)
+# except:
+# LOGGER.info("unexpected error. %s\n", str(sys.exc_info()[0]) + lineno())
+#
def json_serial(obj):
"""
@@ -2161,6 +2420,8 @@ def is_dns_hostnames_enabled(client, instance_id, vpc_id):
LOGGER.info("instance: %s, describe_vpc_attribute returned RequestLimitExceeded, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
return dns_hostname_enabled
@@ -2198,6 +2459,8 @@ def is_dns_support_enabled(client, instance_id, vpc_id):
LOGGER.info("instance: %s, describe_vpc_attribute returned RequestLimitExceeded, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
return dns_suppport_enabled
@@ -2259,6 +2522,8 @@ def new_get_hosted_zone_properties(client, instance_id, zone_id):
LOGGER.info("instance: %s, get_hosted_zone returned error, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
if hosted_zone_properties == {}:
LOGGER.error("instance: %s, get_hosted_zone exceeded max retry of %s",
@@ -2312,6 +2577,8 @@ def get_subnet_cidr_block(client, instance_id, subnet_id):
LOGGER.info("instance: %s, describe_subnets returned RequestLimitExceeded, waiting before retry. %s",
instance_id, str(i) + lineno())
time.sleep(i)
+ count['sleep.count'] += 1
+ count['sleep.time'] += i
return cidr_block
@@ -2535,6 +2802,14 @@ def publish_to_sns(client, message):
LOGGER.debug("No SNS Topic specified, ignoring")
+def new_process_delete_records(instance_id, zone_id, record_name, zone_name, record_type, record_value, heritage_value):
+ LOGGER.info(
+ f"new delete records: instance {instance_id}, zone_id {zone_id}, name {record_name} zone_name {zone_name} type {record_type} value {record_value} heritage {heritage_value}")
+ process_response = process_delete_records(
+ route53, instance_id, zone_id, record_name, zone_name, record_type, record_value, heritage_value)
+ return process_response
+
+
def process_delete_records(route53, instance_id, zone_id,
record_name, zone_name, record_type, record_value, heritage_value):
"""
@@ -2546,6 +2821,7 @@ def process_delete_records(route53, instance_id, zone_id,
:param zone_name:
:param record_type:
:param record_value:
+ :param heritage_value:
:return response: # dictionary of 'delete_success' and 'msg'
"""
@@ -2760,3 +3036,308 @@ def process_delete_records(route53, instance_id, zone_id,
response['msg'] = response_msg
return response
+
+
+def tags_to_dict(tags):
+ """
+ Process all tag key/value pairs into a dict
+
+ :param list(dict(string)) tags: tags from instance, list of dict of string
+ :return dict(string): flag settings in defaultdict for controlling which names are registered and when
+ """
+
+ tag_dict = {}
+ if len(tags) > 0:
+ tag_dict = {tag.get('Key', '').lstrip().rstrip() : tag.get('Value', '') for tag in tags}
+ return tag_dict
+
+
+def process_tags_flags(tags):
+ """
+ Process the DNS flags tags into for tags[key]=='boc:dns:flags'. Available flags:
+
+ - noforward: do not define A or AAAA (when available) or the associated heritage TXT record
+ - noptr: do not define a PTR or associated heritage TXT record with default or boc:dns:ptrname flag
+ - nocname: do not define a CNAME or associated heritage TXT record, even if specified in the boc:dns:cname flag
+ - noheritage: do not create a heritage TXT record, used to indicate which service created the entries
+
+ :param list(dict(string)) tags: tags from instance, list of dict of string. Keys and values turned to lowercase.
+ :return dict(string): flag settings in defaultdict for controlling which names are registered and when
+ """
+
+# tag_dict = {tag['Key'].lstrip().lower(): tag['Value'].lower() for tag in tags}
+ tag_dict = {k.lower(): v.lower() for k, v in tags_to_dict(tags).items()}
+ flags_dict = defaultdict(lambda: False)
+ flags = tag_dict.get(TAGKEY_FLAGS.lower(), '').split(',')
+ for flag in flags:
+ if flag != '':
+ LOGGER.debug("Setting 'flags' to True: %s", str(flag) + lineno())
+ flags_dict[flag] = True
+
+ return flags_dict
+
+
+def process_tags_value(name):
+ """
+ Process the name from one of the tags and return vald (true|false), hostname (the name if domainname is not found), domainname (if the domainname exists in a PHZ)
+
+ :param str: name (hostname or domainname) to check
+ :return tuple(bool,str,str): true|false if vaid, hostname, domainname
+ """
+
+ return parse_hostname_to_components(name)
+# if name != '':
+# return (True,) + parse_hostname_to_components(name)
+# components = parse_hostname_to_components(name)
+# if components:
+# return (True, components[0], components[1])
+# return (name != '', False, name, None)
+
+
+def process_tags_option_cname(tags):
+ """
+ Process the CNAME option tag 'boc:dns:cname', determine if name and zone are valid. This allows an unqualified name (i.e, no zones)
+ and if a zone is not found, it will mark it as not valid, but will be used later with the tag zone or dhcp zone.
+
+ : param list(dict(string)) tags: tags from instance, list of dict of string
+ : return tuple(bool, str, str): true | false if vaid, hostname, domainname
+ """
+
+# tag_dict = {tag['Key'].lstrip().lower(): tag['Value'] for tag in tags}
+ tag_dict = {k.lower(): v for k, v in tags_to_dict(tags).items()}
+# value = tag_dict.get(TAGKEY_CNAME.lower(), '').split(',')
+# need additional work to handle a comma-separated list
+ value = tag_dict.get(TAGKEY_CNAME.lower(), '')
+ return process_tags_value(value)
+
+
+def process_tags_option_zone(tags):
+ """
+ Process the Zone option tag 'boc:dns:zone', determine if zone are valid
+
+ : param list(dict(string)) tags: tags from instance, list of dict of string
+ : return tuple(bool, str, str): true | false if vaid, hostname, domainname
+ """
+
+# tag_dict = {tag['Key'].lstrip().lower(): tag['Value'] for tag in tags}
+ tag_dict = {k.lower(): v for k, v in tags_to_dict(tags).items()}
+ value = tag_dict.get(TAGKEY_ZONE.lower(), '')
+ return process_tags_value(value)
+
+
+def process_tags_option_name(tags):
+ """
+ Process the Hostname option tag 'boc:dns:name', determine if name if and zone are valid
+
+ : param list(dict(string)) tags: tags from instance, list of dict of string
+ : return:
+ """
+
+# tag_dict = {tag['Key'].lstrip().lower(): tag['Value'] for tag in tags}
+ tag_dict = {k.lower(): v for k, v in tags_to_dict(tags).items()}
+ value = tag_dict.get(TAGKEY_HOSTNAME.lower(), '')
+ return process_tags_value(value)
+
+
+def process_tags_option_ptrname(tags):
+ """
+ Process the Name option tag 'boc:dns:ptrname', determine if name if and zone are valid. This name is used to provide and
+ alternate target name for the PTR address. A warning is issued if the zone/domain does not exist, but it will still create the
+ name anyway. This is expected to be used in the case of a Windows host, where the hostname cannot be set in route53, but the PTR
+ can, and it has to point back to the real AD hostname. Using this involves two tags:
+
+ boc:dns:flags = noforward
+ boc:dns:ptrname = winhostname.ead.census.gov
+
+ : param list(dict(string)) tags: tags from instance, list of dict of string
+ : return:
+ """
+
+# tag_dict = {tag['Key'].lstrip().lower(): tag['Value'] for tag in tags}
+ tag_dict = {k.lower(): v for k, v in tags_to_dict(tags).items()}
+ value = tag_dict.get(TAGKEY_PTRNAME.lower(), '')
+ return process_tags_value(value)
+
+
+def process_tags_name(tags):
+ """
+ Process the Hostname from the Name tag to determine if name if and zone are valid
+
+ : param list(dict(string)) tags: tags from instance, list of dict of string
+ : return:
+ """
+
+# tag_dict = {tag['Key'].lstrip().lower(): tag['Value'] for tag in tags}
+ tag_dict = {k.lower(): v for k, v in tags_to_dict(tags).items()}
+ value = tag_dict.get('name', '')
+ return process_tags_value(value)
+
+
+def get_session_assume_role(account):
+ """
+ Return the session associated with the assumed role in the target account. If its been done once, no need to redo it.
+
+ : param str account: Account ID of the remote account(which may also be this account, no assume role is done)
+ : return: boto3.session corresonding to the assumed role
+ """
+
+ this_session = sessions.get(account, None)
+ try:
+ if this_session is None:
+ LOGGER.debug("Existing session not found for account %s: %s",
+ account, lineno())
+
+ role_arn = format(REMOTE_ROLE_ARN_FORMAT % (partition, account))
+ response = sts_client.assume_role(RoleArn=role_arn, RoleSessionName=APPNAME)
+ LOGGER.debug("Called sts:assumerole for arn %s: %s",
+ str(role_arn), lineno())
+ credentials = response['Credentials']
+ LOGGER.info(
+ f"Called assume_role for {account} ARN {role_arn}, got credentials with expiration {credentials['Expiration']}: {lineno()}")
+ count['assumed_role.new'] += 1
+ this_session = boto3.Session(
+ aws_access_key_id=credentials["AccessKeyId"],
+ aws_secret_access_key=credentials["SecretAccessKey"],
+ aws_session_token=credentials["SessionToken"],
+ region_name=region)
+ sessions[account] = this_session
+ LOGGER.debug("Crated new session for account %s: %s",
+ str(account), lineno())
+ else:
+ LOGGER.debug(f"Found existing session for account {account}: {lineno()}")
+ count['assumed_role.cached'] += 1
+ return this_session
+ except Exception as err:
+ LOGGER.error(
+ f"Unable assume_role session in account {account} error {err}: {lineno()}")
+ return None
+
+
+def parse_hostname_to_components(name):
+ """
+ This takes a hostname (FQDN) and parses out each dotted segment until it finds an existing PHZ. For example, if
+ the Name, boc:dns:cname or boc:dns:name tags contain myhost.db.common.edl.census.gov, and the zone is common.edl.census.gov,
+ simply checking at the first dot (host==myhost, domain==db.common.edl.census.gov) will fail. This will go through the
+ list of PHZs found for the VPC for each of the possible domains, and return the name components if a match is found.
+ That returned name/zone should be used for all settings. If the zone is not found it will parse into first part before dot, remainder.
+
+ This returns:
+ - zone_exists: if the PHZ is found associated to this VPC
+ - valid: the name conists of a hostname + domainname, where the domainname is > 1 item (alias.db is not valid, alias.db.zone is valid, though not really)
+ - hostname: hostname part of name. If not valid, it is the whole name passed in
+ - zonename: the domain part of the name. If not valid, it will be None
+ - zone_id: if zone exists, the zone id
+ - name: the original name from the arguments
+
+ :param str name: FQDN to parse and find existing defined PHZ
+ :return namedtuple: Tuple, see above
+ """
+
+ global phz_collection_by_vpc
+
+ hostname_tuple = namedtuple(
+ 'Hostname', ['zone_exists', 'valid', 'hostname', 'zonename', 'zone_id', 'name'])
+
+ if name == '' or name == None:
+ return hostname_tuple(False, False, None, None, None, None)
+ names = name.rstrip('.').split('.')
+ for i in range(len(names)):
+ host = '.'.join(names[0:i])
+ domain = '.'.join(names[i:]) + '.'
+ item = phz_collection_by_vpc.get(domain)
+ if item:
+ return hostname_tuple(True, True, host, domain, item['zone_id'], name)
+ LOGGER.info(
+ f"No PHZ found for any domain components of {name} associated with this vpc, returing host {host} domain {domain}: {lineno()}")
+ if len(names) > 2:
+ host = names[0]
+ domain = '.'.join(names[1:]) + '.'
+ LOGGER.debug(
+ f"zone does not exist but name {name} appears to be a legal domain (>2 components) setting hostname {host} and zone {domain}: {lineno()}")
+ return hostname_tuple(False, True, host, domain, None, name)
+ else:
+ host = '.'.join(names) + '.'
+ domain = ''
+ return hostname_tuple(False, False, host, domain, None, name)
+
+
+def create_fqdn(host, zone):
+ """
+ This takes a hostname (may or nay not be FQDN) and a zone, and returns the proper concatenation of the two, with a trailing dot.
+
+ :param str host: hostname (short or FQDN)
+ :param str zone: zone name
+ :return (str,str): Tuple containing hostname components (may include dot) and domain name for which a PHZ exists. None is returned if not found.
+ """
+
+ fqdn = host.replace(zone, '').rstrip('.') + '.' + zone
+ fqdn += '.' if fqdn[-1] != '.' else ''
+ return fqdn
+
+
+def discover_emr_cluster(tags):
+ """
+ This tags a dict of tags and determines if the appropriate EMR tags are set. For testing, you can set EmrTagPrefix to emr, and then set
+ your tags to emr: vs aws: as listed below (because you cannot create a tag that starts with aws:).
+
+ If set, this is a member of an EMR cluster with the cluster_id as the value:
+ - aws:elasticmapreduce:job-flow-id = j-xxxxxxx
+
+ If set, this is a master node of the cluster:
+ - aws:elasticmapreduce:instance-group-role == MASTER
+
+ :param dict tags: dict of tag
+ :return (bool,str): Tuple containing is_master, is_cluster, and cluster_id if it's a cluster. cluster_id will be empty if not a cluster.
+ """
+
+ cluster_tuple = namedtuple('EMRCluster', ['is_cluster', 'is_master', 'cluster_id'])
+ cluster_id = tags.get('{EMR_TAG_PREFIX}:elasticmapreduce:job-flow-id', '')
+ is_master = tags.get(
+ '{EMR_TAG_PREFIX}:elasticmapreduce:instance-group-role', '') == 'MASTER'
+ is_cluster = cluster_id != ''
+
+ result = cluster_tuple(is_cluster, is_master, cluster_id)
+ return result
+
+
+def evaluate_event_action(event):
+ """
+ This takes the EventBridge event and returns a DNS action to take (ADD|REMOVE) based on running or not-running (stopping, terminating). If not an ec2 event, it returns None.
+
+ :param dict(str) event: Event dict from handler
+ :return str: DNS action, either ADD (to create records) or REMOVE (to delete records)
+ """
+
+ e_source = event['source']
+ action = None
+ try:
+ if e_source == 'aws.ec2':
+ e_instance_id = event['detail']['instance-id']
+ e_state = event['detail']['state']
+ action = "ADD" if e_state == "running" else "DELETE"
+ LOGGER.info(
+ f"event_action ** {action} ** instance_id {e_instance_id} instance_state {e_state}")
+ else:
+ LOGGER.info(f"event_action unrecognized source {e_source}")
+ except:
+ LOGGER.error(f"event_action event structure cannot be parsed {lineno()}")
+ return action
+
+
+##
+# aws: elasticmapreduce: job - flow - id j - 8O514K6HPIYZ
+# A2 GenerateCertificate FALSE
+# A2 Webhook 1680546860
+# A2 StackName DAS - REL - groupii - dashboard - integration - docim
+# A2 CAMPAIGN_NAME ashen
+# A2 DontPatch true
+# A2 ProjectNumber fs0000000033
+# A2 NoBootstrap FALSE
+# A2 RunConfiguration NMFv73State
+# A2 ClusterSize Small2
+# A2 FRIENDLY_NAME docim
+# A2 CloudWatchEnable FALSE
+# A2 ModeOfOperation MODE2: Run Group II Safetab P
+# A2 LightsOut true
+# A2 POC Ryan Kane
+# A2 aws: elasticmapreduce: instance - group - role MASTER
diff --git a/code/ddns-lambda.zip b/code/ddns-lambda.zip
index 0e4a747..22e2872 100644
Binary files a/code/ddns-lambda.zip and b/code/ddns-lambda.zip differ
diff --git a/code/make-zip-file.tf b/code/make-zip-file.tf
index b4a1f39..6bc3f1a 100644
--- a/code/make-zip-file.tf
+++ b/code/make-zip-file.tf
@@ -7,17 +7,32 @@ locals {
# to make a new zip file or not
lambda_code_files_hashes = { for f in local.lambda_code_files : f => filesha256(f) }
lambda_files_hash = sha256(join(",", values(local.lambda_code_files_hashes)))
+ lambda_code_file = "ddns-lambda.py"
}
resource "null_resource" "zip" {
triggers = {
lambda_files_hash = local.lambda_files_hash
}
+}
- provisioner "local-exec" {
- command = "rm ${local.lambda_file}"
- }
- provisioner "local-exec" {
- command = "zip ${local.lambda_file} -j -r ${join(" ", local.lambda_code_files)}"
- }
+## resource "null_resource" "zip" {
+## triggers = {
+## lambda_files_hash = local.lambda_files_hash
+## }
+##
+## provisioner "local-exec" {
+## command = "rm ${local.lambda_file}"
+## }
+## provisioner "local-exec" {
+## command = "zip ${local.lambda_file} -j -r ${join(" ", local.lambda_code_files)}"
+## }
+#3 }
+
+data "archive_file" "zip" {
+ source_file = "${path.root}/${local.lambda_code_file}"
+ output_path = "${path.root}/${local.lambda_file}"
+ type = "zip"
+ depends_on = [null_resource.zip]
}
+
diff --git a/code/versions.tf b/code/versions.tf
new file mode 100644
index 0000000..866ee22
--- /dev/null
+++ b/code/versions.tf
@@ -0,0 +1,18 @@
+terraform {
+ required_version = ">= 0.13"
+ # required_version = ">= 1.0"
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = ">= 4.55.0"
+ }
+ null = {
+ source = "hashicorp/null"
+ version = ">= 1.0"
+ }
+ archive = {
+ source = "hashicorp/archive"
+ version = ">= 2.0"
+ }
+ }
+}
diff --git a/role.tf b/role.tf
index d612dc6..96a8e09 100644
--- a/role.tf
+++ b/role.tf
@@ -40,6 +40,8 @@ data "aws_iam_policy" "lambda_policies" {
name = each.key
}
+data "aws_organizations_organization" "org" {}
+
data "aws_iam_policy_document" "lambda_policy" {
statement {
sid = "AllowRoute53"
@@ -64,6 +66,17 @@ data "aws_iam_policy_document" "lambda_policy" {
actions = ["dynamodb:ListTables"]
resources = ["*"]
}
+ statement {
+ sid = "AssumeRoleInRemoteAccounts"
+ effect = "Allow"
+ actions = ["sts:assumeRole"]
+ resources = [format("arn:%v:iam::*:role/r-inf-dynamic-route53-actions", data.aws_arn.current.partition)]
+ condition {
+ test = "StringEquals"
+ variable = "aws:PrincipalOrgID"
+ values = [data.aws_organizations_organization.org.id]
+ }
+ }
statement {
sid = "DynamoDBTable"
effect = "Allow"
diff --git a/variables.tf b/variables.tf
index 9b9c410..c9ed4ac 100644
--- a/variables.tf
+++ b/variables.tf
@@ -47,12 +47,17 @@ variable "lambda_environment_variables" {
SleepTime = 60
SnsEnable = false
SnsTopicArn = ""
+ RemoteRoleArnFormat = "arn:%s:iam::%s:role/r-inf-dynamic-route53-actions"
+ EMRTagPrefix = "aws"
TagKeyCname = "boc:dns:cname"
TagKeyHostName = "boc:dns:name"
TagKeyZone = "boc:dns:zone"
+ TagKeyPtrname = "boc:dns:ptrname"
+ TagKeyFlags = "boc:dns:flags"
}
}
+
variable "lambda_environment_variables_override" {
description = "Map of lambda environment variables and values to override from the defaults"
type = map(string)
diff --git a/version.tf b/version.tf
index 374ba43..6b49608 100644
--- a/version.tf
+++ b/version.tf
@@ -1,3 +1,3 @@
locals {
- _module_version = "1.0.1"
+ _module_version = "2.0.0"
}