Merge pull request #1092 from sillsdev/staging

Merge staging into master for production build/deploy flow testing purposes

Author: billy clark

.github/workflows/build-and-deploy-images.yml

@@ -55,12 +55,6 @@ jobs:
           echo "BUILD_VERSION=$(echo ${GITHUB_REF:11})" >> "$GITHUB_ENV"
           echo "BUILD_VERSION_LATEST=latest" >> "$GITHUB_ENV"
 
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
-
       - name: Build and tag app
         run: docker build -t $IMAGE_NAME:$BUILD_VERSION -t $IMAGE_NAME:$BUILD_VERSION_LATEST --build-arg BUILD_VERSION=${BUILD_VERSION} -f docker/app/Dockerfile .
         env:
@@ -69,6 +63,12 @@ jobs:
       - name: Reveal images
         run: docker images
 
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
+
       - name: Publish all tagged images to Docker Hub
         run: |
           docker push -a sillsdev/web-languageforge

README.md

@@ -143,10 +143,6 @@ If you edit files in the `src` or `data` folders of the test container, these ch
 
 After a minute or two, your source or test changes should be applied and you should see the result of your changes when you run the E2E tests again.
 
-### Viewing logs
-
-1. `make logs` will show logs from any running containers, results can be filtered by simply grepping, e.g., `make logs | grep lf-ui-builder`
-
 ### Cleanup
 
 1. `make clean` is the most common, it shuts down and cleans up running containers
@@ -157,9 +153,9 @@ After a minute or two, your source or test changes should be applied and you sho
 
 1. `make dev` will start the app in development mode, i.e. changes to source code will immediately be reflected in the locally running app.
 
-### Building for production
+### Building for deployment
 
-1. `make prod` will build a production version of the app.
+1. Refer to `/.github/workflows/build-and-deploy-images.yml` for build commands.
 
 ### Visual Studio Code ###
 
@@ -228,25 +224,35 @@ To debug the tests:
 - To debug in VSCode, select the "Node debugger" debug configuration and run it.
 
 ## Application deployment ##
-Language Forge is built to run in a containerized environment.  For now, Kubernetes is the chosen runtime platform.
+Language Forge is built to run in a containerized environment.  For now, Kubernetes is the chosen runtime platform.  Deployments are not currently automated and must be manually run with the appropriate credentials or from within our CD platform, TeamCity at this time.  Deployment scripts for k8s can be found in `docker/deployment`
 
 ### Staging (QA) ###
-Deployments are not currently automated and must be manually run with the appropriate credentials or from within our CD platform, TeamCity at this time.
-
-Deployment scripts for k8s can be found in `docker/deployment` and staging deployments can be run via `VERSION=<some-docker-tag-or-semver> make deploy-staging` from within the same directory.
+Staging deployments can be run with `VERSION=<some-docker-tag-or-semver> make deploy-staging`.
 
 Current workflow:
-1. move the `staging` branch to the appropriate commit on `master`
+1. merge commits into or make commits on `staging` branch
 1. this will kick off the GHA (`.github/workflows/build-and-deploy-images.yml`) to build and publish the necessary images to Docker Hub (https://hub.docker.com/r/sillsdev/web-languageforge/tags)
 1. then the deployment scripts can be run either manually or via the TeamCity deploy job
 
 ### Production ###
-WIP:  `docker/deployment/Makefile` currently has a `VERSION=<some-docker-tag-or-semver> make deploy-prod` that can be run locally with the right cluster credenitals.
+Production deployments can be run with `VERSION=<some-docker-tag-or-semver> make deploy-prod`.
+
+Current workflow:
+1. merge from `staging` into `master`
+1. tag the desired commit on `master` with a `v#.#.#` format and push the tag
+1. this will kick off the GHA (`.github/workflows/build-and-deploy-images.yml`) to build and publish the necessary images to Docker Hub (https://hub.docker.com/r/sillsdev/web-languageforge/tags)
+1. then the deployment scripts can be run either manually or via the TeamCity deploy job
 
-- [ ] build prod images
-- [ ] publish prod images via GHA workflow
-- [x] parameterize image reference in `app-deployment.yaml`
-- [ ] set up CD for prod
+### Backup/Restore ###
+Backups will be established automatically by LTOps and utilized by LF through the `storageClassName` property in a Persistent Volume Claim.  This storage class provided by LTOps establishes both a frequency and retention for a backup.  Any time a restoration is needed, the LF team will need to coordinate the effort with LTOps.  The process of restoring from a point in time will require the application be brought down for maintenance.  The process will roughly follow these steps:
+1. Notify LTOps of the need to restore a backup (App team)
+1. Coordinate a time to bring the app down for maintenance (LTOps/App team)
+1. Scale the app down (LTOps/App team)
+1. Initiate the Backup restore (LTOps)
+1. Notify app team of the restoration completion (LTOps)
+1. Scale the app up (LTOps/App team)
+1. Test the app (App team)
+1. Communicate maintenance completion
 
 ## Libraries Used ##
 

docker/Makefile

@@ -37,14 +37,6 @@ endif
 build:
 	docker-compose build mail app ld-api
 
-.PHONY: deployable-app
-deployable-app:
-	docker-compose build --build-arg BUILD_VERSION=${BUILD_VERSION} app
-
-.PHONY: logs
-logs:
-	docker-compose logs
-
 .PHONY: clean
 clean:
 	docker-compose down

docker/app/Dockerfile

@@ -1,5 +1,5 @@
 # ENVIRONMENT value can be either "production" or "development"
-ARG ENVIRONMENT=production
+ARG ENVIRONMENT=${ENVIRONMENT:-'production'}
 
 # UI-BUILDER
 FROM node:14.16.1-alpine3.11 AS ui-builder-base

docker/app/entrypoint.sh

@@ -1,22 +1,10 @@
 #!/bin/sh
 
-if [ "x$ENVIRONMENT" = "xdevelopment" ]; then
-    /wait
-    RETCODE=$?
-    if [ "$RETCODE" -gt 0 ]; then
-        exit $RETCODE
-    fi
-fi
-
 # rsyslog needs to run so that lfmerge can log to /var/log/syslog
 /etc/init.d/rsyslog start
 
 # run lfmergeqm on startup to clear out any failed send/receive sessions from previous container
-if which /lfmergeqm-background.sh > /dev/null
-then
-    # MUST be run as a background process as it kicks off an infinite loop to run every 24 hours
-    /lfmergeqm-background.sh &
-fi
+/lfmergeqm-background.sh & # MUST be run as a background process as it kicks off an infinite loop to run every 24 hours
 
 # Now chain to Docker entrypoint from base php:apache image
 exec docker-php-entrypoint "$@"

docker/deployment/Makefile

@@ -28,15 +28,13 @@ deploy-staging: deploy-db deploy-mail-staging deploy-app-staging
 deploy-mail-staging:
 	sed -e s/{{SERVER_HOSTNAME}}/qa.languageforge.org/ mail-deployment.yaml | kubectl apply -f -
 deploy-app-staging:
-	sed -e s/{{ENVIRONMENT}}/development/ app-deployment.yaml \
-  | sed -e s/{{WEBSITE}}/qa.languageforge.org/ \
+	sed -e s/{{WEBSITE}}/qa.languageforge.org/ app-deployment.yaml \
   | sed -e s/{{VERSION}}/$(VERSION)/ | kubectl apply -f -
 deploy-prod: deploy-db deploy-mail-prod deploy-app-prod
 deploy-mail-prod:
 	sed -e s/{{SERVER_HOSTNAME}}/beta.languageforge.org/ mail-deployment.yaml | kubectl apply -f -
 deploy-app-prod:
-	sed -e s/{{ENVIRONMENT}}/production/ app-deployment.yaml \
-  | sed -e s/{{WEBSITE}}/beta.languageforge.org/ \
+	sed -e s/{{WEBSITE}}/beta.languageforge.org/ app-deployment.yaml \
   | sed -e s/{{VERSION}}/$(VERSION)/ | kubectl apply -f -
 deploy-db:
 	kubectl apply -f db-deployment.yaml

docker/deployment/app-deployment.yaml

@@ -22,6 +22,10 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: languageforge-app
+  annotations:
+    # https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#custom-max-body-size
+    # Added this to allow large file uploads, this setting should match the php custom config, i.e., upload_max_filesize, found in the app's customizations.php.ini
+    nginx.ingress.kubernetes.io/proxy-body-size: 60M
 spec:
   rules:
   - host: {{WEBSITE}}
@@ -51,7 +55,7 @@ spec:
     - ReadWriteMany
   resources:
     requests:
-      storage: 7Gi
+      storage: 10Gi
   storageClassName: weekly-snapshots-retain-4 # provided by LTOps
 
 ---
@@ -66,7 +70,7 @@ spec:
     - ReadWriteMany
   resources:
     requests:
-      storage: 10Gi
+      storage: 35Gi
   storageClassName: weekly-snapshots-retain-4 # provided by LTOps
 
 ---
@@ -122,7 +126,7 @@ spec:
           - name: DATABASE
             value: scriptureforge
           - name: ENVIRONMENT
-            value: {{ENVIRONMENT}}
+            value: production
           - name: WEBSITE
             value: {{WEBSITE}}
           - name: MAIL_HOST

docker/deployment/db-deployment.yaml

@@ -23,7 +23,7 @@ spec:
     - ReadWriteOnce
   resources:
     requests:
-      storage: 3Gi
+      storage: 10Gi
   storageClassName: weekly-snapshots-retain-4 # provided by LTOps
 
 ---

docker/deployment/scripts/kubeRsyncAssets.sh

@@ -5,4 +5,6 @@
 # -p 55555 this port is being forwarded to 22 on the container
 # -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" these are a hack to bypass security checks for "localhost".  localhost isn't a real host so I don't care about "host key verification failed" errors when doing localhost port forwarding
 
-ssh -A -p 55555 -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" root@localhost "rsync -avzhP [email protected]:/var/www/languageforge.org/htdocs/assets/lexicon/ /var/www/html/assets/lexicon/"
+USERANDHOSTNAME=user@hostname
+
+ssh -A -p 55555 -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" root@localhost "rsync -avzhP $USERANDHOSTNAME:/var/www/languageforge.org/htdocs/assets/lexicon/ /var/www/html/assets/lexicon/"

docker/deployment/scripts/kubeRsyncMongo.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+echo Mongo Dump
+ssh mongo-xf mongodump
+
+echo Remove Scripture Forge data from dump
+ssh mongo-xf rm -r dump/xforge
+
+echo Create tar file of dump
+ssh mongo-xf tar -czvf mongodump.tgz dump
+
+echo Copy tar file to local filesystem
+scp mongo-xf:mongodump.tgz .
+
+echo Clean up on remote
+ssh mongo-xf rm -r dump
+ssh mongo-xf rm mongodump.tgz
+
+MONGOPOD=$(kubectl get pods --selector='app=db' -o name | sed -e s'/pod\///')
+
+echo Copying mongodump.tgz into $MONGOPOD/data/db
+kubectl cp -c db mongodump.tgz $MONGOPOD:/data/db
+
+echo Untarring mongodump.tgz
+kubectl exec -c db $MONGOPOD -- bash -c "cd /data/db \
+    && tar -xvzf mongodump.tgz"
+
+echo MongoRestore
+kubectl exec -c db $MONGOPOD -- bash -c "cd /data/db \
+    && mongorestore --drop dump"
+
+echo Clean Up
+kubectl exec -c db $MONGOPOD -- bash -c "rm -r /data/db/dump && rm /data/db/mongodump.tgz"
+rm mongodump.tgz
+
+echo Run Mongo migration
+kubectl exec -c db $MONGOPOD -- bash -c 'mongo scriptureforge --eval "db.projects.updateMany({}, {"\$unset": {userProperties: 1}});"'

docker/deployment/scripts/kubeSetup.sh

@@ -1,14 +1,25 @@
 #!/bin/bash
 MONGO=deploy/db
-POD=$(kubectl get pods --selector='app=app' -o name | sed -e s'/pod\///')
+APPPOD=$(kubectl get pods --selector='app=app' -o name | sed -e s'/pod\///')
+MONGOPOD=$(kubectl get pods --selector='app=db' -o name | sed -e s'/pod\///')
 
 # install SSH and rsync; also start the SSH service
-echo Installing SSH on pod $POD
-kubectl exec -c app $POD -- bash -c "apt-get update \
+echo Installing SSH on pod $APPPOD
+kubectl exec -c app $APPPOD -- bash -c "apt-get update \
 && apt-get install -y openssh-server openssh-client rsync \
 && mkdir -p -m 700 /root/.ssh \
 && service ssh start"
 
 # add my public key to the container so I can "ssh root@localhost"
 # --no-preserve=true keeps the permissions intact for root on the container side
-kubectl cp -c app ~/.ssh/authorized_keys $POD:/root/.ssh/authorized_keys --no-preserve=true
+kubectl cp -c app ~/.ssh/authorized_keys $APPPOD:/root/.ssh/authorized_keys --no-preserve=true
+
+echo Installing SSH on pod $MONGOPOD
+kubectl exec -c db $MONGOPOD -- bash -c "apt-get update \
+&& apt-get install -y openssh-server openssh-client rsync \
+&& mkdir -p -m 700 /root/.ssh \
+&& service ssh start"
+
+# add my public key to the container so I can "ssh root@localhost"
+# --no-preserve=true keeps the permissions intact for root on the container side
+kubectl cp -c db ~/.ssh/authorized_keys $MONGOPOD:/root/.ssh/authorized_keys --no-preserve=true

docker/docker-compose.yml

@@ -49,6 +49,7 @@ services:
       - FACEBOOK_CLIENT_SECRET=bogus-development-token
       - REMEMBER_ME_SECRET=bogus-development-key
       - LANGUAGE_DEPOT_API_TOKEN=bogus-development-token
+    command: sh -c "/wait && apache2-foreground"
     extra_hosts:
       - "host.docker.internal:host-gateway"
     volumes:

docker/test-e2e/Dockerfile

@@ -1,8 +1,5 @@
-# ENVIRONMENT value can be either "production" or "development"
-ARG ENVIRONMENT=development
-
 # UI-BUILDER
-FROM node:14.16.1-alpine3.11 AS ui-builder-base
+FROM node:14.16.1-alpine3.11
 
 # install npm globally
 RUN npm config set unsafe-perm true && npm install -g [email protected]
@@ -28,15 +25,8 @@ COPY src/sass ./src/sass
 COPY src/service-worker ./src/service-worker
 COPY src/Site/views ./src/Site/views
 
-FROM ui-builder-base AS production-ui-builder
-ENV NPM_BUILD_SUFFIX=prd
-
-FROM ui-builder-base AS development-ui-builder
-ENV NPM_BUILD_SUFFIX=dev
-
-FROM ${ENVIRONMENT}-ui-builder AS ui-builder
 # artifacts built to /data/src/dist
-RUN npm run build:${NPM_BUILD_SUFFIX}
+RUN npm run build:dev
 
 # make wait available for container ochestration
 COPY --from=sillsdev/web-languageforge:wait-latest /wait /wait