Feature/al/add generation of scaffold (#45)

* 🎉 Added a new feature

* 🔧 Renamed command name

* 🎉 Added templating feature

* 📝 Updated REAMDE

* 📝 Updated REAMDE

* 📝 Updated shortcuts

* 🔧 Reorganized the folder structure

* 🐛 Fixed issues in the template scripts

* 🔧 Renamed file

* 🔧 Updated deployment scripts

* 🔧 Added config file to cibuild

* 🔧 Updated find command to look for config instead of executable scripts

* 🔧 Updated docker-compose file

* 🔧 Added ACCOUNT/REGION/ENVIRONMENT to the infra script

* 🔧 Added scripts into scaffold

---------

Co-authored-by: Alex Lubneuski <[email protected]>

README.md

@@ -180,6 +180,24 @@ Igor will retrieve secrets for a specific application.
 igor secrets --app kf-keycloak --environment qa --region us-east-1 --account kf-strides
 ~~~
 
+generate
+--------
+
+Igor will generate a standard template and will replace variables.
+
+***Usage***
+
+~~~
+igor generate --project route_53_healthchecks --environment qa --account kf-strides --region us-east-1 --template simple-tf
+~~~
+
+In this example, "igor" will generate files based on the "simple-tf" template. It will replace placeholders for account, environment, project, region with values supplied values. Additionally, igor will replace values in templates that found in accounts_info.json file. 
+In our case it will look up a bucket for terraform state files. 
+You can add more templates here:
+
+~~~
+d3b_cli_igor/utils/templates
+~~~
 
 Developing Igor
 ===============

d3b_cli_igor/common.py

@@ -4,6 +4,7 @@
 import colorlog
 import numpy
 import boto3
+import os,json
 from pathlib import Path
 
 logger = logging.getLogger()
@@ -20,6 +21,14 @@
 RESULT_ABORTED = {"Result": "aborted"}
 AVAILABLE_OPERATIONS = ["delete_objects", "delete_bucket", "md5check", "thaw_objects"]
 
+path = os.path.dirname(__file__)
+config_file = "config/account_info.json"
+
+def get_account_info():
+    account_information={}
+    with open(path + "/" + config_file ) as json_file:
+        account_information = json.load(json_file)
+    return account_information
 
 def setup(client_name=""):
     client = boto3.client(client_name)

d3b_cli_igor/config/account_info.json

@@ -0,0 +1,115 @@
+{
+   "chopd3b": {
+       "state_files_bucket": "chopawsd3b-remote-state-files",
+       "institution_naming_convention": "chopd3b",
+       "account_id"	   : "538745987955",
+       "organization"      : "kf",
+       "dev_cidr"          : "[ \"10.10.0.0/16\"]",
+       "qa_cidr"           : "[\"10.11.0.0/16\"]",
+       "prd_cidr"          : "[\"0.0.0.0/0\"]",
+       "service_cidr"      : "[\"10.0.0.0/8\"]",
+       "instance_type"     : "db.t2.small",
+       "domain_external"   : "kidsfirstdrc.org",
+       "domain_internal"   : "kids-first.io",
+       "vpc_prefix"        : "apps",
+       "subnet_prefix"     : "apps",
+       "azs"               : "[\"a\",\"b\",\"c\",\"e\"]"
+   },
+   "chopd3bPrdHippa":
+   {
+       "state_files_bucket"    : "chopd3bprdhippa-remote-state-files",
+       "institution_naming_convention": "research-27200-7225140000",
+       "account_id"	       : "515011955314",
+       "organization"          : "d3b",
+       "dev_cidr"              : "[\"10.10.0.0/16\"]",
+       "qa_cidr"               : "[\"10.11.0.0/16\"]",
+       "prd_cidr"              : "[\"10.12.0.0/16\"]",
+       "service_cidr"          : "[\"10.0.0.0/16\"]",
+       "instance_type"         : "db.t2.small",
+       "domain_external"       : "research-27200-7225140000.aws.cloud.chop.edu",
+       "domain_internal"       : "research-27200-7225140000.aws.cloud.chop.edu",
+       "vpc_prefix"            : "apps",
+       "subnet_prefix"         : "apps",
+       "azs"                   : "[\"a\",\"b\"]"
+    },
+   "ml-research":
+   {
+       "state_files_bucket"    : "-remote-state-files",
+       "institution_naming_convention": "aws-research-7225140000-d3b-ml-research01-dev",
+       "account_id"	       : "381376188863",
+       "organization"          : "d3b",
+       "dev_cidr"              : "[\"172.22.32.192/26\"]",
+       "instance_type"         : "db.t2.small",
+       "domain_external"       : "aws-research-7225140000-d3b-ml-research01-dev.aws.cloud.chop.edu",
+       "domain_internal"       : "aws-research-7225140000-d3b-ml-research01-dev.aws.cloud.chop.edu",
+       "vpc_prefix"            : "apps",
+       "subnet_prefix"         : "apps",
+       "azs"                   : "[\"a\",\"b\"]"
+    },
+   "sandbox":
+   {
+       "state_files_bucket"    : "sandbox-remote-state-files",
+       "institution_naming_convention": "aws-research-7225140000-d3b-sandbox01-dev",
+       "account_id"	       : "452734729332",
+       "organization"          : "d3b",
+       "dev_cidr"              : "[\"172.22.30.128/26\"]",
+       "instance_type"         : "db.t2.small",
+       "domain_external"       : "aws-research-7225140000-d3b-sandbox01-dev.aws.cloud.chop.edu",
+       "domain_internal"       : "aws-research-7225140000-d3b-sandbox01-dev.aws.cloud.chop.edu",
+       "vpc_prefix"            : "baseline-primary-vpc",
+       "subnet_prefix"         : "baseline-primary-subnet",
+       "azs"                   : "[\"a\",\"b\"]"
+    },
+   "include":
+   {
+       "state_files_bucket"    : "include-remote-state-files",
+       "institution_naming_convention": "",
+       "account_id"	       : "373997854230",
+       "organization"          : "include",
+       "dev_cidr"               : "[\"10.10.0.0/16\"]",
+       "qa_cidr"               : "[\"10.11.0.0/16\"]",
+       "prd_cidr"              : "[\"0.0.0.0/0\"]",
+       "service_cidr"          : "[\"10.32.0.0/16\"]",
+       "instance_type"         : "db.t2.small",
+       "domain_external"       : "373997854230.d3b.io",
+       "domain_internal"       : "373997854230.d3b.io",
+       "vpc_prefix"            : "apps",
+       "subnet_prefix"         : "apps",
+       "azs"                   : "[\"a\",\"b\"]"
+    },
+   "chopd3bPrd":
+   {
+       "state_files_bucket": "chopd3bprd-remote-state-files",
+       "institution_naming_convention": "chop-d3b-prod",
+       "account_id"	       : "684194535433",
+       "organization"      : "d3b",
+       "dev_cidr"          : "[\"10.39.0.0/16\"]",
+       "qa_cidr"           : "[\"10.39.0.0/16\"]",
+       "prd_cidr"          : "[\"0.0.0.0/0\"]",
+       "service_cidr"      : "[\"10.0.0.0/8\"]",
+       "instance_type"     : "db.t2.small",
+       "domain_external"   : "d3b.io",
+       "domain_internal"   : "d3b.io",
+       "vpc_prefix"        : "dc",
+       "subnet_prefix"     : "dc",
+       "azs"               : "[\"a\",\"b\"]"
+    },
+   "kf-strides":
+   {
+       "state_files_bucket": "kf-strides-remote-state-files",
+       "institution_naming_convention": "chop-d3b-center",
+       "account_id"	       : "232196027141",
+       "organization"      : "kf-strides",
+       "dev_cidr"          : "[ \"10.10.0.0/16\"]",
+       "qa_cidr"           : "[\"10.11.0.0/16\"]",
+       "prd_cidr"          : "[\"0.0.0.0/0\"]",
+       "service_cidr"      : "[\"10.0.0.0/8\"]",
+       "instance_type"     : "db.t2.small",
+       "domain_external"   : "kf-strides.org",
+       "domain_internal"   : "kf-strides.org",
+       "vpc_prefix"        : "apps",
+       "subnet_prefix"     : "apps",
+       "azs"               : "[\"a\",\"b\",\"c\",\"e\"]"
+    } 
+ }
+

d3b_cli_igor/utils/config/shortcuts.yaml

@@ -20,3 +20,4 @@ github-kidsfirst:
 aws-accounts:
         name: "https://www.notion.so/d3b/AWS-Environments-ddc99e01cad746ff8ed03705e058a140"
         description: "AWS Accounts Information Notion Page"
+

d3b_cli_igor/utils/generate_tf.py

@@ -0,0 +1,52 @@
+import sys, pathlib
+import os as os
+import click
+import stat
+import d3b_cli_igor.common
+import jinja2, json, boto3
+from jinja2 import ChoiceLoader, FileSystemLoader
+
+logger = d3b_cli_igor.common.get_logger(
+    __name__, testing_mode=False, log_format="detailed"
+)
+
+path = os.path.dirname(__file__)
+config_file = "config/account_info.json"
+
+#Get files in the folder
+def get_files(path):
+    filelist = []
+    for root, dirs, files in os.walk(path):
+        for folder in dirs:
+            print(os.path.join(root,folder))
+            os.makedirs(folder, exist_ok=True)
+        for file in files:
+            print(file)
+            filelist.append(os.path.join(root,file))
+    filelist_modified=[]
+    for file in filelist:
+        index = file.find("template")
+        filelist_modified.append(file[index:len(file)])
+    return filelist
+
+#Generate TF scaffold using different templates
+def generate(project, region, account, environment, template_name):
+    templateEnv = jinja2.Environment(
+        loader=FileSystemLoader(str(pathlib.Path(__file__).parent.absolute()))
+    )
+    path = os.getcwd()
+    logger.info("Checking for " + config_file)
+
+    for file in get_files(str(pathlib.Path(__file__).parent.absolute())+"/templates/"+template_name+"/"):
+        #Generate template
+        print(file)
+        template_path = file[file.find("templates"):len(file)]
+        #Create folder structure
+        #template_path_structure=template_path[len("templates/"+template_name+"/"):]
+        #if(not os.path.exists(os.path.dirname(template_path_structure))):
+        #    print(os.path.dirname(template_path_structure))
+            #os.makedirs(os.path.dirname(template_path_structure))
+        template = templateEnv.get_template(template_path)
+        output = template.render(project=project, region=region, account=account, environment=environment, state_files_bucket=d3b_cli_igor.common.get_account_info()[account]["state_files_bucket"], account_id=d3b_cli_igor.common.get_account_info()[account]["account_id"],azs=d3b_cli_igor.common.get_account_info()[account]["azs"])
+        with(open(template_path[len("templates/"+template_name+"/"):len(template_path)], "w")) as f:
+            f.write(output)

d3b_cli_igor/utils/templates/simple-tf/docker-compose.ci.yml

@@ -0,0 +1,6 @@
+services:
+  {{ project }}:
+    image: {{ project }}:${GIT_COMMIT:-latest}
+    build:
+        context: .
+        dockerfile: Dockerfile

d3b_cli_igor/utils/templates/simple-tf/scripts/cibuild

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+config_file=$(find . -name "config" | grep scripts)
+source $config_file
+
+if [[ -n "${ETL_DEBUG}" ]]; then
+    set -x
+fi
+
+function usage() {
+    echo -n \
+        "Usage: $(basename "$0")
+Build application for staging or a release.
+"
+}
+
+if [[ -n "${GIT_COMMIT}" ]]; then
+    GIT_COMMIT="${GIT_COMMIT:0:7}"
+else
+    GIT_COMMIT="$(git rev-parse --short HEAD)"
+fi
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+    if [[ "${1:-}" == "--help" ]]; then
+        usage
+    else
+        # Build tagged container images
+        GIT_COMMIT="${GIT_COMMIT}" docker compose \
+            -f docker-compose.ci.yml \
+            build $project_name 
+    fi
+fi

d3b_cli_igor/utils/templates/simple-tf/scripts/cipublish

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+config_file=$(find . -name "config" | grep scripts)
+source $config_file
+
+function usage() {
+    echo -n \
+        "Usage: $(basename "$0")
+Publish container images to Elastic Container Registry (ECR).
+"
+}
+
+if [[ -n "${GIT_COMMIT}" ]]; then
+    GIT_COMMIT="${GIT_COMMIT:0:7}"
+else
+    GIT_COMMIT="$(git rev-parse --short HEAD)"
+fi
+
+function amazon_ecr_login() {
+    # Retrieves a temporary authorization token that can be used to access
+    # Amazon ECR, along with the registry URL.
+    read -r AUTHORIZATION_TOKEN ECR_REGISTRY \
+        <<<"$(aws ecr get-authorization-token \
+            --output "text" \
+            --query "authorizationData[0].[authorizationToken, proxyEndpoint]")"
+
+    # The authorization token is base64 encoded, and we need to strip the
+    # protocol from the registry URL.
+    AUTHORIZATION_TOKEN="$(echo "${AUTHORIZATION_TOKEN}" | base64 --decode)"
+    ECR_REGISTRY="${ECR_REGISTRY##*://}"
+
+    # Authenticate to the ECR registry. The authorization token is presented in
+    # the format user:password.
+    echo "${AUTHORIZATION_TOKEN##*:}" |
+        docker login \
+            --username "${AUTHORIZATION_TOKEN%%:*}" \
+            --password-stdin "${ECR_REGISTRY}"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+    if [[ "${1:-}" == "--help" ]]; then
+        usage
+    else
+        amazon_ecr_login
+        docker tag "${project_name}:${GIT_COMMIT}" \
+            "${ECR_REGISTRY}/${project_name}:${GIT_COMMIT}"
+        docker push "${ECR_REGISTRY}/${project_name}:${GIT_COMMIT}"
+    fi
+fi

d3b_cli_igor/utils/templates/simple-tf/scripts/config

@@ -0,0 +1 @@
+project_name="{{ project }}"