From dbcc1979e948201ca9843b0a05bb1ae3d8c16c27 Mon Sep 17 00:00:00 2001 From: Delong Yang Date: Fri, 28 Mar 2025 08:11:08 -0400 Subject: [PATCH] First draft --- .pylintrc | 549 ++++++++++++++++++ README.md | 8 - __init__.py | 0 eks_automation/__init__.py | 0 eks_automation/app.py | 14 + eks_automation/client/__init__.py | 0 eks_automation/client/client.py | 32 + eks_automation/data.json | 23 + eks_automation/main.py | 124 ++++ .../requirements.txt | 2 + eks_automation/templates/eks.hcl.j2 | 10 + 11 files changed, 754 insertions(+), 8 deletions(-) create mode 100644 .pylintrc create mode 100644 __init__.py create mode 100644 eks_automation/__init__.py create mode 100644 eks_automation/app.py create mode 100644 eks_automation/client/__init__.py create mode 100644 eks_automation/client/client.py create mode 100644 eks_automation/data.json create mode 100644 eks_automation/main.py rename requirements.txt => eks_automation/requirements.txt (76%) create mode 100644 eks_automation/templates/eks.hcl.j2 diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..25e6537 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,549 @@ +[MAIN] + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +init-hook='import sys; sys.path.append(".")' + +# Files or directories to be skipped. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths= + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. +ignore-patterns=^\.# + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + pylint.extensions.check_elif, + pylint.extensions.bad_builtin, + pylint.extensions.docparams, + pylint.extensions.for_any_all, + pylint.extensions.set_membership, + pylint.extensions.code_style, + pylint.extensions.overlapping_exceptions, + pylint.extensions.typing, + pylint.extensions.redefined_variable_type, + pylint.extensions.comparison_placement, + pylint.extensions.broad_try_clause, + pylint.extensions.dict_init_mutate, + pylint.extensions.consider_refactoring_into_while_condition, + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-allow-list= + +# Minimum supported python version +py-version = 3.8.0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# Specify a score threshold under which the program will exit with error. +fail-under=10.0 + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint in +# a server-like mode. +clear-cache-post-run=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +# confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + use-symbolic-message-instead, + useless-suppression, + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" + +disable= + attribute-defined-outside-init, + invalid-name, + missing-docstring, + protected-access, + too-few-public-methods, + # handled by black + format, + # We anticipate #3512 where it will become optional + fixme, + consider-using-assignment-expr, + unspecified-encoding + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables 'fatal', 'error', 'warning', 'refactor', 'convention' +# and 'info', which contain the number of messages in each category, as +# well as 'statement', which is the total number of statements analyzed. This +# score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + +# Activate the evaluation score. +score=yes + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=6 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Maximum number of lines in a module +max-module-lines=2000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,}$ + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. +#class-const-rgx= + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,}$ + +# Regular expression matching correct type variable names +#typevar-rgx= + +# Regular expression which should only match function or class names that do +# not require a docstring. Use ^(?!__init__$)_ to also check __init__. +no-docstring-rgx=__.*__ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# List of decorators that define properties, such as abc.abstractproperty. +property-classes=abc.abstractproperty + + +[TYPECHECK] + +# Regex pattern to define which classes are considered mixins if ignore-mixin- +# members is set to 'yes' +mixin-class-rgx=.*MixIn + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent,argparse.Namespace + +# List of decorators that create context managers from functions, such as +# contextlib.contextmanager. +contextmanager-decorators=contextlib.contextmanager + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# List of comma separated words that should be considered directives if they +# appear and the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:,pragma:,# noinspection + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file=.pyenchant_pylint_custom_dict.txt + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=2 + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args = 9 + +# Maximum number of locals for function / method body +max-locals = 19 + +# Maximum number of return / yield for function / method body +max-returns=11 + +# Maximum number of branch for function / method body +max-branches = 20 + +# Maximum number of statements in function / method body +max-statements = 50 + +# Maximum number of attributes for a class (see R0902). +max-attributes=11 + +# Maximum number of statements in a try-block +max-try-statements = 7 + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp,__post_init__ + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=builtins.Exception + + +[TYPING] + +# Set to ``no`` if the app / library does **NOT** need to support runtime +# introspection of type annotations. If you use type annotations +# **exclusively** for type checking of an application, you're probably fine. +# For libraries, evaluate if some users what to access the type hints at +# runtime first, e.g., through ``typing.get_type_hints``. Applies to Python +# versions 3.7 - 3.9 +runtime-typing = no + + +[DEPRECATED_BUILTINS] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,input + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=10 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[CODE_STYLE] + +# Max line length for which to sill emit suggestions. Used to prevent optional +# suggestions which would get split by a code formatter (e.g., black). Will +# default to the setting for ``max-line-length``. +#max-line-length-suggestions= diff --git a/README.md b/README.md index eacfba4..315e0f6 100644 --- a/README.md +++ b/README.md @@ -62,11 +62,3 @@ reformatted main.py All done! ✨ 🍰 ✨ 1 files reformatted, 1 files left unchanged. ``` - -### How to run the application? - -Run `main.py`: - -```sh -python main.py -``` diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eks_automation/__init__.py b/eks_automation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eks_automation/app.py b/eks_automation/app.py new file mode 100644 index 0000000..cfbb039 --- /dev/null +++ b/eks_automation/app.py @@ -0,0 +1,14 @@ +# import json +# # import requests + + +# def lambda_handler(event, context): + +# personId = event['queryStringParameters']['personId'] + +# return { +# "statusCode": 200, +# "body": json.dumps({ +# "personId": personId + " from Lambda" , +# }), +# } diff --git a/eks_automation/client/__init__.py b/eks_automation/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eks_automation/client/client.py b/eks_automation/client/client.py new file mode 100644 index 0000000..653d54c --- /dev/null +++ b/eks_automation/client/client.py @@ -0,0 +1,32 @@ +""" +Boto client +TODO: change client to assume role +""" + +import boto3 +from botocore.exceptions import ClientError +from loguru import logger + + +class BotoClient: + def __init__(self, profile, region, service): + """ + BotoClient constructor + """ + self.profile = profile + self.region = region + self.service = service + + def create_client(self): + """ + creates a boto session and returns a client for a given profile + """ + try: + session = boto3.session.Session(profile_name=self.profile) + client = session.client(service_name=self.service, region_name=self.region) + + return client + + except ClientError as error: + logger.critical(f"error creating client {error}") + return None diff --git a/eks_automation/data.json b/eks_automation/data.json new file mode 100644 index 0000000..debd447 --- /dev/null +++ b/eks_automation/data.json @@ -0,0 +1,23 @@ +{ + "attrs": { + "account_name": "lab-dev-ew", + "aws_region": "us-gov-east-1", + "cluster_mailing_list": "matthew.c.morgan@census.gov", + "cluster_name": "csvd-platform-lab-mcm", + "eks_instance_disk_size": 100, + "eks_ng_desired_size": 2, + "eks_ng_max_size": 10, + "eks_ng_min_size": 2, + "environment": "development", + "environment_abbr": "dev", + "organization": "census:ocio:csvd", + "finops_project_name": "csvd_platformbaseline", + "finops_project_number": "fs0000000078", + "finops_project_role": "csvd_platformbaseline_app", + "vpc_domain_name": "dev.lab.csp2.census.gov", + "vpc_name": "vpc3-lab-dev" + }, + "tags" : { + "slim:schedule": "8:00-17:00" + } +} diff --git a/eks_automation/main.py b/eks_automation/main.py new file mode 100644 index 0000000..c714fb6 --- /dev/null +++ b/eks_automation/main.py @@ -0,0 +1,124 @@ +################################################################## +# Script +# @usage python main.py +################################################################## +# import os +import sys +import json +from loguru import logger +from jinja2 import Environment, FileSystemLoader +from github import Github, Auth + +# import boto3 +from botocore.exceptions import ClientError +from .client.client import BotoClient + +# import pygit2 + +CENSUS_GITHUB_API = "https://github.e.it.census.gov/api/v3" +REGION_NAME = "us-gov-east-1" +PROFILE = "224384469011-lab-gov-dev-nonprod" +ORG_NAME = "SCT-Engineering" + + +def get_github_token(): + + bc = BotoClient(PROFILE, REGION_NAME, "secretsmanager") + + if bc is None: + logger.critical("error connecting to AWS") + logger.critical("exiting...bye") + sys.exit() + + client = bc.create_client() + + secret_name = "dev/eks_automation_github_token" + + try: + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + except ClientError as e: + raise e + + secret = get_secret_value_response["SecretString"] + + return secret + + +def operate_github(): + + token = get_github_token() + + auth = Auth.Token(token) + g = Github(auth=auth, base_url=CENSUS_GITHUB_API) + user = g.get_user() + print(user.login) + + # print(g) + # org = g.get_organization(ORG_NAME) + + # repo = org.get_repo("platform-tg-infra") + # print(repo) + + # #create the new repository + # repo = org.create_repo(projectName, description = projectDescription ) + + # #create some new files in the repo + # repo.create_file("/README.md", "init commit", readmeText) + + # #Clone the newly created repo + # repoClone = pygit2.clone_repository(repo.git_url, '/path/to/clone/to') + + # #put the files in the repository here + + # #Commit it + # repoClone.remotes.set_url("origin", repo.clone_url) + # index = repoClone.index + # index.add_all() + # index.write() + # author = pygit2.Signature("your name", "your email") + # commiter = pygit2.Signature("your name", "your email") + # tree = index.write_tree() + # oid = repoClone.create_commit( + # 'refs/heads/master', + # author, + # commiter, + # "init commit", + # tree, + # [repoClone.head.get_object().hex] + # ) + # remote = repoClone.remotes["origin"] + # credentials = pygit2.UserPass(userName, password) + # remote.credentials = credentials + + # callbacks=pygit2.RemoteCallbacks(credentials=credentials) + + # remote.push(['refs/heads/master'],callbacks=callbacks) + + +def main(): + """ + main entry routine + """ + # Open and read the JSON file + data = "" + with open("data.json", "r") as file: + data = json.load(file) + + jinja_env = Environment(loader=FileSystemLoader("templates/"), trim_blocks=True) + template = jinja_env.get_template("eks.hcl.j2") + rendered = template.render(data=data) + + print(rendered) + + # with open(template_location, "w") as file_obj: + # file_obj.write(rendered) + + logger.info("EKS CI/CD pipeline payload has been created!") + + operate_github() + + return True + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/eks_automation/requirements.txt similarity index 76% rename from requirements.txt rename to eks_automation/requirements.txt index 6723c82..45e1b90 100644 --- a/requirements.txt +++ b/eks_automation/requirements.txt @@ -5,3 +5,5 @@ pre-commit loguru Jinja2 requests +pygithub +boto3 diff --git a/eks_automation/templates/eks.hcl.j2 b/eks_automation/templates/eks.hcl.j2 new file mode 100644 index 0000000..40c4feb --- /dev/null +++ b/eks_automation/templates/eks.hcl.j2 @@ -0,0 +1,10 @@ +locals { + {% for key, value in data['attrs'] | items -%} + {{ key }} = "{{ value }}" + {% endfor -%} + tags = { + {% for key, value in data['tags'] | items -%} + {{ key }} = "{{ value }}" + {% endfor -%} + } +}