Add API configs and refactor lambda function code

README.md

@@ -67,6 +67,7 @@ All done! ✨ 🍰 ✨
 
 ## NOTES
 
+- A GitHub Personal Access Token (PAT) is required to access the Census GitHub Enterprise Server. The PAT should be securely stored in AWS Systems Manager Parameter Store.
 - This lambda function relies on [`git-lambda-layer`](https://github.com/lambci/git-lambda-layer), which must be uploaded to the S3 bucket specified in samconfig.toml prior to deployment.
 - The REST API for this Lambda function is configured to be accessed using an API key.
 - To access the Census GitHub Enterprise Server, a VPC with private subnets connected to the server must be attached.

eks_automation/app.py

@@ -2,7 +2,7 @@
 # Script
 # @usage python main.py
 ##################################################################
-from __future__ import print_function
+# from __future__ import print_function
 
 import os
 import stat
@@ -15,11 +15,11 @@
 import json
 from jinja2 import Environment, FileSystemLoader
 
-# pragma pylint: disable=E0401
+# pylint: disable=import-error
 from github import Github, Auth, GithubException
 from git import Repo
 
-# pragma pylint: enable=E0401
+# pylint: enable=import-error
 
 import boto3
 from botocore.exceptions import ClientError
@@ -29,112 +29,190 @@
 ORG_NAME = "SCT-Engineering"
 SECRET_NAME = "/dev/eks_automation_github_token"
 
-TEMPLATE_REPO_NAME = "platform-tg-infra"
+ORIG_REPO_NAME = "platform-tg-infra"
 NEW_REPO_NAME = "eks-automation-lambda-test1"
 
+TEMPLATE_FILE_NAME = "eks.hcl.j2"
+HCL_FILE_NAME = "eks.hcl"
+
+# DATA_FILE_NAME = "data.json"
+
 # Initialize the logger
 logger = logging.getLogger()
 logger.setLevel("INFO")
 
 
-# pylint: disable-next=W0613
+# pylint: disable=unused-argument
 def lambda_handler(event, context):
+    """
+    Main Lambda handler function
 
-    # personId = event['queryStringParameters']['personId']
-    operate_github()
+    Args:
+        event (dict): Dict containing the Lambda function event data.
+        context (dict): Lambda runtime context.
 
-    return {"statusCode": 200, "message": "Processed successfully"}
+    Returns:
+        dict: Dict containing status message.
+    """
+
+    # For test, load input data from a local file.
+    # input_data = ""
+    # with open(DATA_FILE_NAME, "r") as file:
+    #     input_data = json.load(file)
+
+    try:
+        input_data = json.loads(event["body"])
+        rendered = operate_github(NEW_REPO_NAME, input_data, HCL_FILE_NAME)
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        return {"statusCode": 400, "body": json.dumps({"error": str(e)})}
 
+    return {
+        "statusCode": 200,
+        "headers": {"Access-Control-Allow-Origin": "*"},
+        "body": json.dumps({"result": rendered}),
+    }
 
-def operate_github():
-    org, token = github_org(CENSUS_GITHUB_API, ORG_NAME)
-    repo_template = template_repo(org, TEMPLATE_REPO_NAME)
-    repo_new = new_repo(org, NEW_REPO_NAME)
 
-    if os.path.exists(NEW_REPO_NAME):
-        shutil.rmtree(NEW_REPO_NAME, ignore_errors=False, onerror=remove_readonly)
+def operate_github(new_repo_name, json_data, output_hcl):
+    """Clone a GitHub repo, add an EKS parameter file rendered
+    from a template and the input JSON dta, and push to a new repo.
 
+    Args:
+        new_repo_name (str): Name of the new GitHub repo.
+        json_data (json): Input JSON data with all the EKS parameter values.
+        output_hcl (str): Name of the EKS parameter file in HCL format.
+
+    Returns:
+        str: The rendered EKS parameter string.
+    """
+
+    # Get both the original repo and the new repo objects from GitHub.
+    # If the new repo doesn't exist, create it in GitHub.
+    token = github_token()
+    org = github_org(CENSUS_GITHUB_API, ORG_NAME, token)
+    repo_orig = get_repo(org, ORIG_REPO_NAME)
+    repo_new = get_repo(org, new_repo_name, create=True)
+
+    # In case the new repo already exists locally, delete it.
+    if os.path.exists(f"/tmp/{new_repo_name}"):
+        shutil.rmtree(new_repo_name, ignore_errors=False, onerror=remove_readonly)
+
+    # Since Census GitHub Enterprise server uses a private TLS certificate,
+    # the certificate veriification must be disabled.
+    # This Git command will save the setting into ".gitconfig" file locally in the $HOME directory.
+    # Because the only writable place in Lambda fucntion is "/tmp",
+    # The HOME environment must be set to there.
+    # This is done using the "Environment" attribute in the "template.yaml" file.
     cmd = ["git", "config", "--global", "http.sslVerify", "false"]
     subprocess.run(cmd, check=False)
 
-    repo_url_with_token = f"https://{token}@{repo_template.html_url.split('//')[1]}"
+    # Clone the original repo.
+    # Since the only writable directory is "/tmp", we store the cloned repo there.
+    repo_url_with_token = f"https://{token}@{repo_orig.html_url.split('//')[1]}"
+    cloned_repo = Repo.clone_from(repo_url_with_token, f"/tmp/{new_repo_name}")
 
-    cloned_repo = Repo.clone_from(repo_url_with_token, f"/tmp/{NEW_REPO_NAME}")
-    origin = cloned_repo.remotes.origin
+    # Change the remote URL of the local staging repo to the URL of the new repo.
     repo_url_with_token = f"https://{token}@{repo_new.html_url.split('//')[1]}"
-    origin.set_url(repo_url_with_token, allow_unsafe_protocols=True)
+    origin = cloned_repo.remotes.origin
+    origin.set_url(repo_url_with_token)
 
+    # If the default branch of the original repo is "master", rename it to "main".
     branch_name = cloned_repo.head.ref.name
     if branch_name == "master":
         current_branch = cloned_repo.heads.master
         current_branch.rename("main", force=True)
 
-    process_eks_data("data.json", "eks.hcl", "eks.hcl.j2")
+    # Render the j2 template using the input data.
+    rendered = render_j2_template(json_data, TEMPLATE_FILE_NAME)
+    # Write the renderd data to a file in the local staging repository root directory
+    with open(f"/tmp/{new_repo_name}/{output_hcl}", "w") as file:
+        file.write(rendered)
 
-    # os.chdir(NEW_REPO_NAME)
-    cloned_repo.index.add("eks.hcl")
-    commit_message = "Add a new file"
+    # Commit and push the changes.
+    cloned_repo.index.add(output_hcl)
+    commit_message = "Add the EKS paramter file by the Lambda function"
     cloned_repo.index.commit(commit_message)
     cloned_repo.git.push("--set-upstream", origin.name, "main", force=True)
 
-    return True
+    return rendered
 
 
-def template_repo(org, template_repo_name):
-    try:
-        repo_template = org.get_repo(template_repo_name)
-    except GithubException as e:
-        if e.status == 404:
-            logger.error("Repo: %s doesn't exist", template_repo_name)
-            raise
+def get_repo(org, repo_name, create=False):
+    """Retrieve a repository from GitHub Org.
 
-    return repo_template
+    Args:
+        org (obj): GitHub Organization object
+        repo_name (str): Name of the repository to retrieve
+        create (bool): Whether to create it if the named repository doesn't exist
 
-
-def new_repo(org, repo_name):
+    Returns:
+        obj: GitHub repository object
+    """
     try:
-        repo_new = org.get_repo(repo_name)
+        repo = org.get_repo(repo_name)
     except GithubException as e:
         if e.status == 404:
-            logger.info("Create repo: %s", repo_name)
-            repo_desc = "EKS Automation CI/CD Pipeline Repo"
-            repo_new = org.create_repo(
-                repo_name, description=repo_desc, visibility="internal", private=True
-            )
-
-    return repo_new
+            if create:
+                logger.info("Create repo: %s", repo_name)
+                repo_desc = "EKS Automation CI/CD Pipeline Repo"
+                repo = org.create_repo(
+                    repo_name,
+                    description=repo_desc,
+                    visibility="internal",
+                    private=True,
+                )
+            else:
+                logger.error("Repo: %s doesn't exist", repo_name)
+                raise
+
+    return repo
+
+
+def render_j2_template(json_data, j2_template, j2_template_dir="templates/"):
+    """Render the j2 template with the input JSON data
+
+    Args:
+        json_data (json): input data in JSON format.
+        j2_template (j2): Name of the template file to generate the output.
+        j2_template_dir (str, optional): The directory where the templates are stored. Defaults to "templates/".
+
+    Returns:
+        str: Rendered template string.
+    """
 
+    # Render template
+    jinja_env = Environment(loader=FileSystemLoader(j2_template_dir), trim_blocks=True)
+    template = jinja_env.get_template(j2_template)
 
-def process_eks_data(
-    json_fname, hcl_fname, j2_template_fname, j2_template_dir="templates/"
-):
-    # Open and read the JSON file
-    data = ""
-    with open(json_fname, "r") as file:
-        data = json.load(file)
+    return template.render(data=json_data)
 
-    jinja_env = Environment(loader=FileSystemLoader(j2_template_dir), trim_blocks=True)
-    template = jinja_env.get_template(j2_template_fname)
-    rendered = template.render(data=data)
 
-    with open(f"/tmp/{NEW_REPO_NAME}/{hcl_fname}", "w") as file_obj:
-        file_obj.write(rendered)
+def github_org(base_url, org_name, token):
+    """Get GitHub Organization Object
 
-    return True
+    Args:
+        base_url (str): Base URL of the GitHub Org.
+        org_name (str): name of the GitHub Org.
+        token (str): Access token to authenticated to the GitHub Org.
 
+    Returns:
+        obj: the GitHub Org.
+    """
 
-def github_org(base_url, org_name):
-    token = github_token()
     auth = Auth.Token(token)
+    # Since Census GitHub Enterprise server uses a private TLS certificate,
+    # the certificate veriification must be disabled.
     g = Github(auth=auth, base_url=base_url, verify=False)
 
-    return g.get_organization(org_name), token
+    return g.get_organization(org_name)
 
 
 def github_token():
+    """Retrieve GitHub access token from AWS SSM Parameter store
 
-    # session = boto3.session.Session(profile_name=PROFILE)
-    # ssm = session.client(service_name="ssm", region_name=REGION_NAME)
+    Returns:
+        str: The GitHub access token.
+    """
     ssm = boto3.client("ssm")
     try:
         token = ssm.get_parameter(Name=SECRET_NAME, WithDecryption=True)["Parameter"][
@@ -149,7 +227,8 @@ def github_token():
 
 def remove_readonly(func, path, _):
     """
-    Clear the readonly bit and reattempt the removal
+    Clear the readonly bit and reattempt the removal.
+    This function is used by `shutil.rmtree` function.
     """
     os.chmod(path, stat.S_IWRITE)
     func(path)

template.yaml

@@ -9,7 +9,41 @@ Resources:
     Properties:
       StageName: Prod
       Auth:
-        ApiKeyRequired: true # sets for all methods
+        ApiKeyRequired: true
+        UsagePlan:
+          UsagePlanName: eks-automation-usage-plan
+          CreateUsagePlan: PER_API
+          Description: Usage plan for EKS Automation API
+          Quota:
+            Limit: 5000
+            Period: MONTH
+          Throttle:
+            BurstLimit: 500
+            RateLimit: 100
+          Tags:
+            - Key: finops_project_name
+              Value: csvd_platformbaseline
+            - Key: finops_project_number
+              Value: fs0000000078
+            - Key: finops_project_role
+              Value: csvd_platformbaseline_app
+            - Key: environment
+              Value: development
+            - Key: environment_abbr
+              Value: dev
+            - Key: organization
+              Value: census:ocio:csvd
+      Cors:
+        AllowMethods: "'POST,OPTIONS'"
+        AllowHeaders: "'Content-Type,Authorization'"
+        AllowOrigin: "'*'"
+      Tags:
+        environment: development
+        environment_abbr: dev
+        organization: census:ocio:csvd
+        finops_project_name: csvd_platformbaseline
+        finops_project_number: fs0000000078
+        finops_project_role: csvd_platformbaseline_app
   GitLambdaLayer:
     Type: AWS::Serverless::LayerVersion
     Properties:
@@ -33,7 +67,7 @@ Resources:
           Properties:
             RestApiId: !Ref EKSAutomationApi
             Path: /EKSAutomation
-            Method: get
+            Method: post
             Auth:
               ApiKeyRequired: true
       Policies: