Setup Integration Environment #524
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Setup Integration Environment | |
| on: | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: '0 4 * * *' | |
| concurrency: integration_environment | |
| jobs: | |
| environment-cleanup: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: sefaria-integrations | |
| steps: | |
| - name: Setup base utils | |
| run: | | |
| export DEBIAN_FRONTEND=noninteractive | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends curl unzip | |
| - name: setup helm | |
| uses: azure/setup-helm@v3 | |
| - name: cleanup | |
| run: | | |
| helm list -n ${{ secrets.DEV_SANDBOX_NAMESPACE }} | grep integration | cut -f 1 -d " " | xargs -I env -n 1 helm delete env -n ${{ secrets.DEV_SANDBOX_NAMESPACE }} --debug --timeout 10m0s | |
| database-setup: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: sefaria-integrations | |
| steps: | |
| - name: Setup base utils | |
| run: | | |
| export DEBIAN_FRONTEND=noninteractive | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends curl | |
| - name: Set up yq | |
| run: | | |
| curl -L https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O | |
| sudo mv yq_linux_amd64 /usr/bin/yq | |
| chmod +x /usr/bin/yq | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| with: | |
| version: '>= 363.0.0' | |
| - name: Setup kubectl | |
| uses: azure/setup-kubectl@v3 | |
| - name: Setup mongodb-tools | |
| run: | | |
| curl -S https://fastdl.mongodb.org/tools/db/$MONGO_DEB -O | |
| sudo apt install ./$MONGO_DEB | |
| mongorestore --version | |
| env: | |
| MONGO_DEB: mongodb-database-tools-ubuntu2204-x86_64-100.6.1.deb | |
| - name: Get latest Backup | |
| run: gsutil -q cp gs://sefaria-mongo-backup/dump.tar.gz dump.tar.gz | |
| - name: Extract Backup | |
| run: tar xzvf dump.tar.gz -C . | |
| - name: Restore Backup | |
| run: | | |
| . <( kubectl get secret local-settings-secrets -n default -o yaml | yq -o=shell '.data | map_values(@base64d)') | |
| echo "::add-mask::$MONGO_HOST" | |
| echo "::add-mask::$MONGO_REPLICASET_NAME" | |
| echo "::add-mask::$SEFARIA_DB_USER" | |
| echo "::add-mask::$SEFARIA_DB_PASSWORD" | |
| if [[ -z "$MONGO_HOST" ]]; then | |
| echo "Mongo Host not specified" | |
| exit 1 | |
| fi | |
| URI="mongodb://" | |
| if [[ ! -z "$SEFARIA_DB_USER" ]]; then | |
| URI="${URI}${SEFARIA_DB_USER}" | |
| if [[ ! -z "$SEFARIA_DB_PASSWORD" ]]; then | |
| URI="${URI}:${SEFARIA_DB_PASSWORD}" | |
| fi | |
| URI="${URI}@" | |
| fi | |
| URI="${URI}${MONGO_HOST}/?ssl=false" | |
| if [[ ! -z "$SEFARIA_DB_USER" ]]; then | |
| URI="${URI}&authSource=admin" | |
| fi | |
| if [[ ! -z "MONGO_REPLICASET_NAME" ]]; then | |
| URI="${URI}&replicaSet=${MONGO_REPLICASET_NAME}" | |
| fi | |
| mongorestore --drop --uri="$URI" -v -d sefaria-integration --dir=dump/sefaria |