From 40d188629e5d9e1b5e7f94dc058e62fd4d1f17b7 Mon Sep 17 00:00:00 2001 From: vggonzal <9Tcostoamm> Date: Sat, 5 Aug 2023 15:12:54 -0700 Subject: [PATCH] update terraform to provision endpoints - change docker script --- docker/.dockerignore | 1 - docker/Dockerfile | 19 +++------ docker/README.md | 48 ++++++++++++++++++++++ docker/build-docker.sh | 24 +++-------- docker/docker-start-command | 27 ------------- docker/push-docker-artifactory.sh | 66 ------------------------------- docker/push-docker-ecr.sh | 10 ++--- terraform/README.md | 12 +----- 8 files changed, 63 insertions(+), 144 deletions(-) delete mode 100644 docker/.dockerignore create mode 100644 docker/README.md delete mode 100755 docker/docker-start-command delete mode 100755 docker/push-docker-artifactory.sh diff --git a/docker/.dockerignore b/docker/.dockerignore deleted file mode 100644 index b512c09..0000000 --- a/docker/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -node_modules \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 8f51dde..e5a21ea 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,16 +1,7 @@ -FROM node:10 -LABEL org.opencontainers.image.source="https://github.com/podaac/hydrocron-api" -RUN npm install forever -g +FROM public.ecr.aws/lambda/python:3.8 -ENV project_dir /project -ENV app_dir ${project_dir}/app -ENV config_dir ${project_dir}/config +COPY $SOURCE . +RUN pip3 install -t /var/task --force ./$SOURCE -RUN mkdir ${project_dir} ${app_dir} ${config_dir} -WORKDIR ${app_dir} - -COPY package*.json ./ -#RUN npm install -COPY . . - -CMD ${app_dir}/docker/docker-start-command \ No newline at end of file +# Run the lambda +#CMD ["/var/task/hydrocronapi/controllers/hydrocron_controller.lambda_handler"] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..c70bb67 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,48 @@ +# FTS API Docker Image + +This directory contains the `Dockerfile` used to build the Docker image capable of running FTS API as a lambda. + +It includes a number of helper scripts to be run by the CI/CD pipeline but can also be run locally to build the image. + +## Building + +Building the FTS API docker image depends on a tar file version of the project. This can be built using `poetry build` or by downloading a previously built version of the project as a tar. + +### Building from tar + +`build-docker.sh` script can be used to build the docker image from the +local tar file. There are two required arguments that must be set: + +1. service-name: The name of the service being built (from pyproject.toml) +2. service-version: The version of the service being built (also from pyproject.toml) + +The docker tag of the built image will be returned from the script. + +Example: + +```shell script +./docker/build-docker.sh -n podaac-fts -v 1.0.0-alpha.3 +``` + +## Running + +The Docker image can be run directly using the `docker run` command. + +See [Testing Lambda container images locally](https://docs.aws.amazon.com/lambda/latest/dg/images-test.html) for details. + +## Pushing to ECR + +The `push-docker-ecr.sh` script can be used to push a docker image to AWS ECR. There are two required arguments: + +1. tf-venue: The target venue for uploading (sit, uat, or ops). +2. docker-tag: The docker tage of the image being pushed + +The easiest way to use the `push-docker-ecr.sh` script is to first call `build-docker.sh` and save the output to the +`docker_tag` environment variable. Then call `push-docker-ecr.sh`. + +Example: + +```shell script +export docker_tag=$(./docker/build-docker.sh -n podaac-fts -v 1.0.0-alpha.3) +./docker/push-docker-ecr.sh -v sit -t $docker_tag +``` \ No newline at end of file diff --git a/docker/build-docker.sh b/docker/build-docker.sh index 497ccd3..36a2f0f 100755 --- a/docker/build-docker.sh +++ b/docker/build-docker.sh @@ -1,11 +1,9 @@ #!/usr/bin/env bash -# This script is intended to be run by the CI/CD pipeline to build a specific version of this application. +# This script is intended to be run by the CI/CD pipeline to build a specific version of the Hydrocron API. set -Eeo pipefail -LOCAL_BUILD=false - POSITIONAL=() while [[ $# -gt 0 ]] do @@ -22,10 +20,6 @@ case $key in shift # past argument shift # past value ;; - --local) - LOCAL_BUILD=true - shift # past argument - ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument @@ -34,7 +28,7 @@ esac done set -- "${POSITIONAL[@]}" # restore positional parameters -USAGE="USAGE: build-docker.sh -n|--service-name service_name -v|--service-version service_version [--local]" +USAGE="USAGE: build-docker.sh -n|--service-name service_name -v|--service-version service_version" # shellcheck disable=SC2154 if [[ -z "${service_name}" ]]; then @@ -54,21 +48,13 @@ set -u SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" PROJECT_DIR="$(dirname "${SCRIPTPATH}")" -DIST_PATH="dist/" -app_path=$( cd ${SCRIPTPATH}/.. && echo $(pwd) ) - -repositoryName=ghcr.io/podaac/${service_name} +repositoryName=podaac/podaac-cloud/${service_name} # Docker tags can't include '+' https://github.com/docker/distribution/issues/1201 dockerTagVersion=$(echo "${service_version}" | tr "+" _) -# Build the image -if [ "$LOCAL_BUILD" = true ] ; then - wheel_filename="$(echo "${service_name}" | tr "-" _)-${service_version}-py3-none-any.whl" - docker build -t "${repositoryName}":"${dockerTagVersion}" --build-arg DIST_PATH="${DIST_PATH}" --build-arg SOURCE="${DIST_PATH}${wheel_filename}" -f "$SCRIPTPATH"/Dockerfile "$PROJECT_DIR" 1>&2 -else - docker build -t "${repositoryName}":"${dockerTagVersion}" --build-arg SOURCE="${service_name}==${service_version}" -f "$SCRIPTPATH"/Dockerfile "$app_path" 1>&2 -fi +tar_filename="${service_name}-${service_version}.tar.gz" +docker build -t "${repositoryName}":"${dockerTagVersion}" --build-arg SOURCE="dist/${tar_filename}" -f "$SCRIPTPATH"/Dockerfile "$PROJECT_DIR" 1>&2 echo "${repositoryName}":"${dockerTagVersion}" \ No newline at end of file diff --git a/docker/docker-start-command b/docker/docker-start-command deleted file mode 100755 index 10430d6..0000000 --- a/docker/docker-start-command +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -outside_config_dir=/config -work_dir=/work -app_dir=/app - -if [ -e "/config/server.crt" ]; then - echo "server.crt is here" - cp /config/server.crt /project/config/server.crt -fi; - -if [ -e "/config/server.key" ]; then - echo "server.key is here" - cp /config/server.key /project/config/server.key -fi; - -if [ -e "/config/config.js" ]; then - echo "config.js is here" - cp /config/config.js /project/config/config.js -fi; - -if [ -e "/config/private-config.js" ]; then - echo "private-config.js is here" - cp /config/private-config.js /project/config/private-config.js -fi; - -forever server/server.js diff --git a/docker/push-docker-artifactory.sh b/docker/push-docker-artifactory.sh deleted file mode 100755 index c84d2cd..0000000 --- a/docker/push-docker-artifactory.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -# This script is intended to be run by the CI/CD pipeline to push a docker tag previously built by build-docker.sh - -set -Eeo pipefail - -POSITIONAL=() -while [[ $# -gt 0 ]] -do -key="$1" - -case $key in - -t|--docker-tag) - docker_tag="$2" - shift # past argument - shift # past value - ;; - -r|--registry) - ARTIFACTORY_DOCKER_REGISTRY="$2" - shift # past argument - shift # past value - ;; - -u|--artifactory-username) - ARTIFACTORY_USER="$2" - shift # past argument - shift # past value - ;; - -p|--artifactory-password) - ARTIFACTORY_PASSWORD="$2" - shift # past argument - shift # past value - ;; - *) # unknown option - POSITIONAL+=("$1") # save it in an array for later - shift # past argument - ;; -esac -done -set -- "${POSITIONAL[@]}" # restore positional parameters - -USAGE="push-docker-artifactory.sh -t|--docker-tag docker_tag -u|--artifactory-username ARTIFACTORY_USER -p|--artifactory-password ARTIFACTORY_PASSWORD" - -# shellcheck disable=SC2154 -if [[ -z "${docker_tag}" ]]; then - echo "docker_tag required." >&2 - echo "$USAGE" >&2 - exit 1 -fi - -# shellcheck disable=SC2154 -if [[ -z "${ARTIFACTORY_USER}" ]]; then - echo "ARTIFACTORY_USER required." >&2 - echo "$USAGE" >&2 - exit 1 -fi - -# shellcheck disable=SC2154 -if [[ -z "${ARTIFACTORY_PASSWORD}" ]]; then - echo "ARTIFACTORY_PASSWORD required." >&2 - echo "$USAGE" >&2 - exit 1 -fi - -echo "${ARTIFACTORY_PASSWORD}" | docker login --username "${ARTIFACTORY_USER}" --password-stdin "${ARTIFACTORY_DOCKER_REGISTRY}" -docker tag "${docker_tag}" "${ARTIFACTORY_DOCKER_REGISTRY}/${docker_tag}" -docker push "${ARTIFACTORY_DOCKER_REGISTRY}/${docker_tag}" \ No newline at end of file diff --git a/docker/push-docker-ecr.sh b/docker/push-docker-ecr.sh index 54ab71c..ae0f385 100755 --- a/docker/push-docker-ecr.sh +++ b/docker/push-docker-ecr.sh @@ -18,7 +18,7 @@ case $key in -v|--tf-venue) tf_venue="$2" case $tf_venue in - ngap-service-sit|ngap-service-uat|ngap-service-ops|ngap-cumulus-swot-sit|ngap-cumulus-sit|ngap-cumulus-swot-uat|ngap-cumulus-uat|ngap-cumulus-ops|ngap-cumulus-sndbx) ;; + sit|uat|ops) ;; *) echo "tf_venue must be sit, uat, or ops" exit 1;; @@ -53,7 +53,7 @@ fi set -u repositoryName=$(echo "${docker_tag}" | awk -F':' '{print $1}') -tf_profile="${tf_venue}" +tf_profile="ngap-service-${tf_venue}" # Get the AWS Account ID for this venue/profile # shellcheck disable=SC2154 @@ -63,9 +63,10 @@ aws_acct=$(aws sts get-caller-identity --profile "$tf_profile" | python -c "impo aws ecr create-repository --repository-name "${repositoryName}" --profile "$tf_profile" || echo "No need to create, repository ${repositoryName} already exists" # Login to ECR +echo "aws ecr get-login-password --region us-west-2 --profile \"$tf_profile\" | docker login --username AWS --password-stdin \"$aws_acct\".dkr.ecr.us-west-2.amazonaws.com" set +x $(aws ecr get-login --no-include-email --region us-west-2 --profile "$tf_profile" 2> /dev/null) || \ -docker login --username AWS --password "$(aws ecr get-login-password --region us-west-2 --profile "$tf_profile")" "$aws_acct".dkr.ecr.us-west-2.amazonaws.com + docker login --username AWS --password "$(aws ecr get-login-password --region us-west-2 --profile "$tf_profile")" "$aws_acct".dkr.ecr.us-west-2.amazonaws.com set -x # Tag the image for this venue's ECR @@ -73,6 +74,3 @@ docker tag "${docker_tag}" "$aws_acct".dkr.ecr.us-west-2.amazonaws.com/"${docker # Push the tag docker push "$aws_acct".dkr.ecr.us-west-2.amazonaws.com/"${docker_tag}" - -# Clean up docker -docker rmi "$aws_acct".dkr.ecr.us-west-2.amazonaws.com/"${docker_tag}" || true diff --git a/terraform/README.md b/terraform/README.md index b68faa2..8b01b07 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -37,17 +37,7 @@ Follow the instructions in the [docker README](../docker/README.md) to build the ## Build and deploy the application We use a pre-built docker container to do the deployment (Please do not use local terraform!) -From the project root directory: -``` -export tf_venue=sit -docker run -v ~/.aws:/home/dockeruser/.aws:ro -v ${PWD}:/home/dockeruser -w /home/dockeruser/terraform cae-artifactory.jpl.nasa.gov:16003/podaac/service/deploy-terraform-1.0.3:latest bash bin/deploy.sh -v ${tf_venue} -t ${docker_tag} -``` - ## Destroying the Application Similarly, use the pre-built docker container to do the destroy (Please do not use local terraform!) -From the project root directory: -``` -docker run -v ~/.aws:/home/dockeruser/.aws:ro -v ${PWD}:/home/dockeruser cae-artifactory.jpl.nasa.gov:16003/podaac/service/deploy-terraform-1.0.3:latest bash bin/destroy.sh -v ${tf_venue} -t ${docker_tag} -``` -This will take anywhere from 3-10 minutes. +