From 289682d5d6019b7b89c94fa53c56d312e7d24046 Mon Sep 17 00:00:00 2001 From: Delong Yang Date: Tue, 8 Apr 2025 22:50:29 -0400 Subject: [PATCH] Fist working version --- .gitignore | 2 + eks_automation/app.py | 161 ++++++++++++++++++++++++++++-- eks_automation/client/__init__.py | 0 eks_automation/client/client.py | 32 ------ eks_automation/main.py | 124 ----------------------- eks_automation/requirements.txt | 12 +-- samconfig.toml | 9 ++ template.yaml | 45 ++++++++- 8 files changed, 212 insertions(+), 173 deletions(-) delete mode 100644 eks_automation/client/__init__.py delete mode 100644 eks_automation/client/client.py delete mode 100644 eks_automation/main.py create mode 100644 samconfig.toml diff --git a/.gitignore b/.gitignore index 779d52c..2be8f1d 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,5 @@ cython_debug/ #.idea/ artifacts/ + +.aws-sam diff --git a/eks_automation/app.py b/eks_automation/app.py index cfbb039..9f53e14 100644 --- a/eks_automation/app.py +++ b/eks_automation/app.py @@ -1,14 +1,155 @@ -# import json -# # import requests +################################################################## +# Script +# @usage python main.py +################################################################## +from __future__ import print_function +import os +import stat +import subprocess +import shutil +import logging -# def lambda_handler(event, context): +# import requests -# personId = event['queryStringParameters']['personId'] +import json +from jinja2 import Environment, FileSystemLoader -# return { -# "statusCode": 200, -# "body": json.dumps({ -# "personId": personId + " from Lambda" , -# }), -# } +# pragma pylint: disable=E0401 +from github import Github, Auth, GithubException +from git import Repo + +# pragma pylint: enable=E0401 + +import boto3 +from botocore.exceptions import ClientError + + +CENSUS_GITHUB_API = "https://github.e.it.census.gov/api/v3" +ORG_NAME = "SCT-Engineering" +SECRET_NAME = "/dev/eks_automation_github_token" + +TEMPLATE_REPO_NAME = "platform-tg-infra" +NEW_REPO_NAME = "eks-automation-lambda-test1" + +# Initialize the logger +logger = logging.getLogger() +logger.setLevel("INFO") + + +# pylint: disable-next=W0613 +def lambda_handler(event, context): + + # personId = event['queryStringParameters']['personId'] + operate_github() + + return {"statusCode": 200, "message": "Processed successfully"} + + +def operate_github(): + org, token = github_org(CENSUS_GITHUB_API, ORG_NAME) + repo_template = template_repo(org, TEMPLATE_REPO_NAME) + repo_new = new_repo(org, NEW_REPO_NAME) + + if os.path.exists(NEW_REPO_NAME): + shutil.rmtree(NEW_REPO_NAME, ignore_errors=False, onerror=remove_readonly) + + cmd = ["git", "config", "--global", "http.sslVerify", "false"] + subprocess.run(cmd, check=False) + + repo_url_with_token = f"https://{token}@{repo_template.html_url.split('//')[1]}" + + cloned_repo = Repo.clone_from(repo_url_with_token, f"/tmp/{NEW_REPO_NAME}") + origin = cloned_repo.remotes.origin + repo_url_with_token = f"https://{token}@{repo_new.html_url.split('//')[1]}" + origin.set_url(repo_url_with_token, allow_unsafe_protocols=True) + + branch_name = cloned_repo.head.ref.name + if branch_name == "master": + current_branch = cloned_repo.heads.master + current_branch.rename("main", force=True) + + process_eks_data("data.json", "eks.hcl", "eks.hcl.j2") + + # os.chdir(NEW_REPO_NAME) + cloned_repo.index.add("eks.hcl") + commit_message = "Add a new file" + cloned_repo.index.commit(commit_message) + cloned_repo.git.push("--set-upstream", origin.name, "main", force=True) + + return True + + +def template_repo(org, template_repo_name): + try: + repo_template = org.get_repo(template_repo_name) + except GithubException as e: + if e.status == 404: + logger.error("Repo: %s doesn't exist", template_repo_name) + raise + + return repo_template + + +def new_repo(org, repo_name): + try: + repo_new = org.get_repo(repo_name) + except GithubException as e: + if e.status == 404: + logger.info("Create repo: %s", repo_name) + repo_desc = "EKS Automation CI/CD Pipeline Repo" + repo_new = org.create_repo( + repo_name, description=repo_desc, visibility="internal", private=True + ) + + return repo_new + + +def process_eks_data( + json_fname, hcl_fname, j2_template_fname, j2_template_dir="templates/" +): + # Open and read the JSON file + data = "" + with open(json_fname, "r") as file: + data = json.load(file) + + jinja_env = Environment(loader=FileSystemLoader(j2_template_dir), trim_blocks=True) + template = jinja_env.get_template(j2_template_fname) + rendered = template.render(data=data) + + with open(f"/tmp/{NEW_REPO_NAME}/{hcl_fname}", "w") as file_obj: + file_obj.write(rendered) + + return True + + +def github_org(base_url, org_name): + token = github_token() + auth = Auth.Token(token) + g = Github(auth=auth, base_url=base_url, verify=False) + + return g.get_organization(org_name), token + + +def github_token(): + + # session = boto3.session.Session(profile_name=PROFILE) + # ssm = session.client(service_name="ssm", region_name=REGION_NAME) + ssm = boto3.client("ssm") + try: + token = ssm.get_parameter(Name=SECRET_NAME, WithDecryption=True)["Parameter"][ + "Value" + ] + except ClientError: + logger.error("Error occured when retrieving GitHub token from SSM Parameter") + raise + + return token + + +def remove_readonly(func, path, _): + """ + Clear the readonly bit and reattempt the removal + """ + os.chmod(path, stat.S_IWRITE) + func(path) diff --git a/eks_automation/client/__init__.py b/eks_automation/client/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/eks_automation/client/client.py b/eks_automation/client/client.py deleted file mode 100644 index 653d54c..0000000 --- a/eks_automation/client/client.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -Boto client -TODO: change client to assume role -""" - -import boto3 -from botocore.exceptions import ClientError -from loguru import logger - - -class BotoClient: - def __init__(self, profile, region, service): - """ - BotoClient constructor - """ - self.profile = profile - self.region = region - self.service = service - - def create_client(self): - """ - creates a boto session and returns a client for a given profile - """ - try: - session = boto3.session.Session(profile_name=self.profile) - client = session.client(service_name=self.service, region_name=self.region) - - return client - - except ClientError as error: - logger.critical(f"error creating client {error}") - return None diff --git a/eks_automation/main.py b/eks_automation/main.py deleted file mode 100644 index c714fb6..0000000 --- a/eks_automation/main.py +++ /dev/null @@ -1,124 +0,0 @@ -################################################################## -# Script -# @usage python main.py -################################################################## -# import os -import sys -import json -from loguru import logger -from jinja2 import Environment, FileSystemLoader -from github import Github, Auth - -# import boto3 -from botocore.exceptions import ClientError -from .client.client import BotoClient - -# import pygit2 - -CENSUS_GITHUB_API = "https://github.e.it.census.gov/api/v3" -REGION_NAME = "us-gov-east-1" -PROFILE = "224384469011-lab-gov-dev-nonprod" -ORG_NAME = "SCT-Engineering" - - -def get_github_token(): - - bc = BotoClient(PROFILE, REGION_NAME, "secretsmanager") - - if bc is None: - logger.critical("error connecting to AWS") - logger.critical("exiting...bye") - sys.exit() - - client = bc.create_client() - - secret_name = "dev/eks_automation_github_token" - - try: - get_secret_value_response = client.get_secret_value(SecretId=secret_name) - except ClientError as e: - raise e - - secret = get_secret_value_response["SecretString"] - - return secret - - -def operate_github(): - - token = get_github_token() - - auth = Auth.Token(token) - g = Github(auth=auth, base_url=CENSUS_GITHUB_API) - user = g.get_user() - print(user.login) - - # print(g) - # org = g.get_organization(ORG_NAME) - - # repo = org.get_repo("platform-tg-infra") - # print(repo) - - # #create the new repository - # repo = org.create_repo(projectName, description = projectDescription ) - - # #create some new files in the repo - # repo.create_file("/README.md", "init commit", readmeText) - - # #Clone the newly created repo - # repoClone = pygit2.clone_repository(repo.git_url, '/path/to/clone/to') - - # #put the files in the repository here - - # #Commit it - # repoClone.remotes.set_url("origin", repo.clone_url) - # index = repoClone.index - # index.add_all() - # index.write() - # author = pygit2.Signature("your name", "your email") - # commiter = pygit2.Signature("your name", "your email") - # tree = index.write_tree() - # oid = repoClone.create_commit( - # 'refs/heads/master', - # author, - # commiter, - # "init commit", - # tree, - # [repoClone.head.get_object().hex] - # ) - # remote = repoClone.remotes["origin"] - # credentials = pygit2.UserPass(userName, password) - # remote.credentials = credentials - - # callbacks=pygit2.RemoteCallbacks(credentials=credentials) - - # remote.push(['refs/heads/master'],callbacks=callbacks) - - -def main(): - """ - main entry routine - """ - # Open and read the JSON file - data = "" - with open("data.json", "r") as file: - data = json.load(file) - - jinja_env = Environment(loader=FileSystemLoader("templates/"), trim_blocks=True) - template = jinja_env.get_template("eks.hcl.j2") - rendered = template.render(data=data) - - print(rendered) - - # with open(template_location, "w") as file_obj: - # file_obj.write(rendered) - - logger.info("EKS CI/CD pipeline payload has been created!") - - operate_github() - - return True - - -if __name__ == "__main__": - main() diff --git a/eks_automation/requirements.txt b/eks_automation/requirements.txt index 45e1b90..b7f4404 100644 --- a/eks_automation/requirements.txt +++ b/eks_automation/requirements.txt @@ -1,9 +1,9 @@ -pylint -black -pre-commit +# pylint +# black +# pre-commit -loguru -Jinja2 +jinja2 +boto3 requests pygithub -boto3 +gitpython diff --git a/samconfig.toml b/samconfig.toml new file mode 100644 index 0000000..82ceb55 --- /dev/null +++ b/samconfig.toml @@ -0,0 +1,9 @@ +version = 0.1 +[default.deploy.parameters] +stack_name = "eks-automation" +s3_bucket = "eks-automation-s3-bucket" +region = "us-gov-east-1" +profile = "229685449397-csvd-dev-gov" +confirm_changeset = false +capabilities = "CAPABILITY_IAM" +image_repositories = [] diff --git a/template.yaml b/template.yaml index f4ce153..ed47b24 100644 --- a/template.yaml +++ b/template.yaml @@ -4,19 +4,62 @@ Description: > eks-automation-lambda Resources: + GitLambdaLayer: + Type: AWS::Serverless::LayerVersion + Properties: + LayerName: git-lambda-layer + Description: Git Lambda Layer + ContentUri: s3://eks-automation-s3-bucket/layer.zip + CompatibleRuntimes: + - python3.9 + - python3.10 + - python3.11 EKSAutomationFunction: Type: AWS::Serverless::Function Properties: CodeUri: eks_automation/ Handler: app.lambda_handler Runtime: python3.11 + Timeout: 30 Events: EKSAutomation: Type: Api Properties: Path: /EKSAutomation Method: get - + Policies: + - AWSLambdaVPCAccessExecutionRole + - Statement: + - Sid: SSMDescribeParametersPolicy + Effect: Allow + Action: + - ssm:DescribeParameters + Resource: '*' + - Sid: SSMGetParameterPolicy + Effect: Allow + Action: + - ssm:GetParameters + - ssm:GetParameter + Resource: '*' + VpcConfig: + SecurityGroupIds: + - sg-03cbf2a626ed55c7e + SubnetIds: + - subnet-05192178ac094f639 + - subnet-022370a5a03585376 + PropagateTags: True + Tags: + environment: development + environment_abbr: dev + organization: census:ocio:csvd + finops_project_name: csvd_platformbaseline + finops_project_number: fs0000000078 + finops_project_role: csvd_platformbaseline_app + Layers: + - !Ref GitLambdaLayer + Environment: + Variables: + HOME: /tmp Outputs: EKSAutomationApi: Description: "API Gateway endpoint URL for Prod stage for EKS Automation function"