Merge branch 'deploy-for-2024' of github.com:codeforsanjose/e-immigra…

…te into deploy-for-2024

.github/scripts/argocd_deploy.py

@@ -0,0 +1,132 @@
+import logging
+import typing as t
+from argparse import ArgumentParser, Namespace
+from json import loads
+from logging import Formatter, StreamHandler 
+from math import floor
+from os import environ, getenv
+from subprocess import run as subproccess_run
+from sys import exit, stdout
+
+
+# Wrapper for ArgoCD CLI
+# https://argo-cd.readthedocs.io/en/stable/user-guide/commands/argocd/
+class ArgoCDeployer:
+    def __init__(self, application: str, docker_tag: str) -> None:
+        self.application = application or getenv('ARGO_CD_APPLICATION', environ['DEPLOYMENT'])
+        self.docker_tag = docker_tag or environ['DOCKER_TAG']
+        self.image_tag_parameter = getenv('IMAGE_TAG_PARAMETER', 'global.image.tag=')
+        self.argo_cd_credentials = [
+            f'--username={environ["ARGO_CD_USERNAME"]}',
+            f'--password={environ["ARGO_CD_PASSWORD"]}',
+        ]
+
+    def _execute_cli(self, command: t.List[str], return_stdout: bool = False) -> t.Optional[str]:
+        command = ['argocd'] + command
+        LOG.info(' '.join(command).replace(environ['ARGO_CD_PASSWORD'], 'redacted'))
+        output = subproccess_run(command, capture_output=True, encoding='utf-8')
+        if output.returncode != 0:
+            LOG.fatal(f'ArgoCD CLI failed {command=} returncode={output.returncode} standard_error={output.stderr}')
+        if return_stdout:
+            return output.stdout
+
+    def login(self):
+        self._execute_cli(
+            command=[
+                'login',
+                '--grpc-web',
+                environ['ARGO_CD_SERVER'],
+                *self.argo_cd_credentials,
+            ]
+        )
+
+    def update_image_tag(self):
+        args = []
+        parameters = [
+            f'{self.image_tag_parameter}{self.docker_tag}',
+        ]
+        for parameter in parameters:
+            args.extend(['--parameter', parameter])
+        self._execute_cli(command=['app', 'set', self.application, *args])
+
+    def sync_app(self, preview: bool = False, timeout: int = 1200):
+        args = [
+            '--assumeYes',
+            '--prune',
+            '--retry-backoff-max-duration',
+            f'{floor(timeout/60)}m',
+            '--timeout',
+            str(timeout + 30),
+        ]
+        if preview:
+            args.append('--dry-run')
+        self._execute_cli(command=['app', 'sync', self.application, *args])
+
+    def preview_deploy(self):
+        self.sync_app(preview=True)
+
+    def wait_health(self):
+        args = [
+            '--health',
+            '--sync',
+            '--timeout',
+            '600',
+        ]
+        self._execute_cli(command=['app', 'wait', self.application, *args])
+
+    def verify_health(self):
+        args = [
+            '--output',
+            'json',
+        ]
+        info = self._execute_cli(command=['app', 'get', self.application, *args], return_stdout=True)
+        info = loads(info)
+        status = info['status']
+        if status['health']['status'] == 'Degraded':
+            LOG.critical(
+                f'Application is unhealthy, aborting deploy\nCheck Application https://ARGO_CD_SERVER/applications/argocd/{self.application}?view=network&resource='
+            )
+            exit(1)
+        sync_status = status['sync']['status']
+        if sync_status not in ['Synced', 'OutofSync']:
+            LOG.info('Application non-deployable SyncStatus={sync_status} - Will try to wait before aborting')
+            self.wait_health()
+
+
+def Logger(logger_name):
+    logger = logging.getLogger(logger_name)
+
+    logger.setLevel(logging.INFO)
+
+    standard_output = StreamHandler(stdout)
+    standard_output.setFormatter(
+        Formatter(
+            '{asctime} {name} {levelname:8} {message}',
+            '%Y-%m-%d %H:%M:%S',
+            '{',
+        )
+    )
+    logger.addHandler(standard_output)
+    return logger
+
+
+if __name__ == '__main__':
+    LOG = Logger('argocd.deploy')
+
+    parser = ArgumentParser(
+        prog='ArgoCDeployer',
+        description='Deploy ArgoCD Applications utilizing argocd CLI',
+    )
+    # TODO: Convert arguments to non-positional args and enforce required=True
+    # Requires CICD changes where command is utilized
+    parser.add_argument('--application', help='ArgoCD Application Name i.e., foobar-dev')
+    parser.add_argument('--docker_tag', help='Docker Tag to be deployed, usually the full Git SHA')
+    args: Namespace = parser.parse_args()
+
+    argocd = ArgoCDeployer(application=args.application, docker_tag=args.docker_tag)
+    argocd.login()
+    argocd.verify_health()
+    argocd.update_image_tag()
+    argocd.sync_app()
+
+    LOG.info(f'Deploy succeeded {args.application} docker_tag={args.docker_tag}')

.github/scripts/requirements.in

@@ -0,0 +1,3 @@
+boto3
+boto3-stubs[ecr]
+botocore

.github/scripts/requirements.txt

@@ -0,0 +1,39 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+boto3==1.34.7
+    # via -r requirements.in
+boto3-stubs[ecr]==1.34.7
+    # via -r requirements.in
+botocore==1.34.7
+    # via
+    #   -r requirements.in
+    #   boto3
+    #   s3transfer
+botocore-stubs==1.34.7
+    # via boto3-stubs
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+mypy-boto3-ecr==1.34.0
+    # via boto3-stubs
+python-dateutil==2.8.2
+    # via botocore
+s3transfer==0.10.0
+    # via boto3
+six==1.16.0
+    # via python-dateutil
+types-awscrt==0.20.0
+    # via botocore-stubs
+types-s3transfer==0.10.0
+    # via boto3-stubs
+typing-extensions==4.9.0
+    # via
+    #   boto3-stubs
+    #   mypy-boto3-ecr
+urllib3==2.0.7
+    # via botocore

.github/workflows/ci.yaml

@@ -1,54 +1,54 @@
 ---
 name: ci
-run-name: ci ${{ github.event_name}} ${{ github.ref_name }} by ${{ github.actor }}
+run-name: ci ${{ github.event_name }} ${{ github.ref_name }}
 on:
-  schedule:
-    - cron: '20 11 * * *'
   pull_request:
   push:
-    branches: [master]
+    branches: ['master']
   workflow_dispatch:
     inputs:
       env:
         type: environment
       ref:
         description: 'Branch, Tag, or SHA'
         required: true
-
 concurrency:
-  group: test-${{ github.head_ref || github.run_id }}
+  group: ci-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
-permissions:
-  id-token: write
-  contents: read
-
 env:
   REFERENCE: ${{ github.event.pull_request.head.sha || github.event.push.head_commit.id }}
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        image: [build, backend]
     steps:
-      - uses: aws-actions/configure-aws-credentials@v2
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ env.REFERENCE }}
+      - uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions
           role-session-name: gha
           aws-region: us-west-2
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ env.REFERENCE }}
-      - uses: docker/setup-qemu-action@v2
-      - uses: docker/setup-buildx-action@v2
-        with:
-          install: true
-          version: latest
+      - uses: docker/setup-buildx-action@v3
       - id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
-      - run: make build
+        uses: aws-actions/amazon-ecr-login@v2
+        with:
+          mask-password: 'true'
+      - run: make build-${{ matrix.image }}
         env:
           AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
-      # - uses: docker/build-push-action@v4
-      #   with:
-      #     push: true
-      #     tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.us-west-2.amazonaws.com/eimmigrate:${{ env.REFERENCE }}
+          # REACT_APP_GRAPHQL_URL: ${{ vars.REACT_APP_GRAPHQL_URL }}
+          # PUBLIC_URL: ${{ vars.PUBLIC_URL }}
+          # REACT_APP_JWT_ISSUER: ${{ secrets.REACT_APP_JWT_ISSUER }}
+          # REACT_APP_RECAPTCHAS_SITE_KEY: ${{ secrets.REACT_APP_RECAPTCHAS_SITE_KEY }}
+          # REACT_APP_MICROSOFT_ID: ${{ secrets.REACT_APP_MICROSOFT_ID }}
+          # REACT_APP_GOOGLE_ID: ${{ secrets.REACT_APP_GOOGLE_ID }}
+          # MICROSOFT_CLIENT_ID: ${{ secrets.MICROSOFT_CLIENT_ID }}

.github/workflows/deploy.yaml

@@ -1,11 +1,12 @@
 ---
 name: deploy
-run-name: ${{ github.event.inputs.env }} deploy ${{ github.ref_name }} ${{ github.event_name }} by ${{ github.actor }}
+run-name: ${{ github.event.inputs.env }} deploy ${{ github.ref_name }}
 on:
   workflow_dispatch:
     inputs:
       env:
         type: environment
+        description: 'ArgoCD Deployed environment'
       ref:
         description: 'Branch, Tag, or Full SHA'
         required: true
@@ -14,26 +15,37 @@ on:
 concurrency:
   group: ${{ github.event.inputs.env }}
 
-permissions:
-  id-token: write
-  contents: read
-
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
     environment:
       name: ${{ github.event.inputs.env }}
+      url: ${{ vars.PUBLIC_URL }}
     env:
-      DEPLOYMENT: ${{ github.event.inputs.env }}
-      AWS_ECR_REPOSITORY: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.us-west-2.amazonaws.com/eimmigrate
+      ENVIRONMENT: ${{ github.event.inputs.env }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: aws-actions/configure-aws-credentials@v2
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.ref }}
+      - uses: clowdhaus/argo-cd-action/@main
+        with:
+          version: 2.10.0
+          command: version
+          options: --client
+      - uses: actions/setup-python@v5
         with:
-          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions
-          role-session-name: gha
-          aws-region: us-west-2
-      - id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
-      - run: make deploy
-      - run: make deploy-status
+          python-version: '3.12'
+          cache: 'pip'
+      - run: pip install -r .github/scripts/requirements.txt
+      - name: deploy
+        run: |-
+          python .github/scripts/argocd_deploy.py \
+          --application=${{ env.ENVIRONMENT }} \
+          --docker_tag=$(git rev-parse HEAD)
+        env:
+          ARGO_CD_SERVER: ${{ secrets.ARGO_CD_SERVER }}
+          ARGO_CD_USERNAME: ${{ secrets.ARGO_CD_USERNAME }}
+          ARGO_CD_PASSWORD: ${{ secrets.ARGO_CD_PASSWORD }}

Makefile

@@ -21,7 +21,11 @@ build: aws-cli-creds
 build: ## Build Docker image
 	docker buildx bake --print
 	docker buildx bake
-
+build-%: aws-cli-creds
+build-%: ## Build Docker specific image
+	docker buildx bake --file docker-bake.hcl --print $*
+	docker buildx bake --file docker-bake.hcl $*
+
 run: ## Run the app
 	docker compose up -d
 

backend/src/index.ts

@@ -26,7 +26,7 @@ app.use(cors());
 app.use(express.json());
 app.use(fileUpload({ limits: { fileSize: MAX_EXCEL_FILE_SIZE } }));
 
-function getPort(defaultPort = 5000) {
+function getPort(defaultPort = 3001) {
     const port = process.env.PORT;
     if (port == null) return defaultPort;
     else if (typeof port === 'number') return port;

backend/src/scripts/uploadQuestionnaireFromExcel.ts

@@ -22,7 +22,7 @@ async function generateQuestionnaires() {
         }
 
         await sendRequest({
-            url: 'http://localhost:5000/api/questionnaires/add',
+            url: 'http://localhost:3001/api/questionnaires/add',
             method: 'POST',
             body: JSON.stringify({
                 title: WorkshopTitle,

backend/src/scripts/uploadTranslatedContentFromExcel.ts

@@ -93,7 +93,7 @@ async function generateLanguageContent() {
         // });
         // submit them locally
         await sendRequest({
-            url: 'http://localhost:5000/api/translatedContent/add',
+            url: 'http://localhost:3001/api/translatedContent/add',
             method: 'POST',
             body: JSON.stringify({
                 title: WorkshopTitle,

deployments/eimmigrate-prod/values.yaml

@@ -15,7 +15,7 @@ web:
   memory:
     requests: 256Mi
     limit: 2048Mi
-  port: 5000
+  port: 3001
 
 serviceaccount:
   create: false