Build Mongodb-backup Image and store backup in aws s3 and azure storage

.github/workflows/backup-sealed-secrets-keys.yml

@@ -0,0 +1,46 @@
+name: Build backup-sealed-secrets-keys
+
+on:
+  push:
+    branches: [ "*" ]
+    paths: 'backup-sealed-secrets-keys/**'
+    tags: [ "v*" ]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Generate container metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ghcr.io/obmondo/backup-sealed-secrets-keys
+          tags: |
+            type=ref,event=branch,enable=${{ !endsWith(github.ref, github.event.repository.default_branch) }}
+            type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }}
+            type=ref,event=tag
+          flavor: |
+            latest=false
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build & push container image
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          file: "./backup-sealed-secrets-keys/Dockerfile"
+          context: .
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/mongodb-backup.yml

@@ -0,0 +1,46 @@
+name: Build mongodb backup
+
+on:
+  push:
+    branches: [ "*" ]
+    paths: 'mongodb-backup/**'
+    tags: [ "v*" ]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Generate container metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ghcr.io/obmondo/mongodb-backup
+          tags: |
+            type=ref,event=branch,enable=${{ !endsWith(github.ref, github.event.repository.default_branch) }}
+            type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }}
+            type=ref,event=tag
+          flavor: |
+            latest=false
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build & push container image
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          file: "./mongodb-backup/Dockerfile"
+          context: .
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/opensearch-curator.yml

@@ -0,0 +1,46 @@
+name: Build opensearch-curator
+
+on:
+  push:
+    branches: [ "*" ]
+    paths: 'opensearch-curator/**'
+    tags: [ "v*" ]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Generate container metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ghcr.io/obmondo/opensearch-curator
+          tags: |
+            type=ref,event=branch,enable=${{ !endsWith(github.ref, github.event.repository.default_branch) }}
+            type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }}
+            type=ref,event=tag
+          flavor: |
+            latest=false
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build & push container image
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          file: "./opensearch-curator/Dockerfile"
+          context: .
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

mongodb-backup/Dockerfile

@@ -0,0 +1,25 @@
+FROM ubuntu:22.04@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
+
+RUN apt-get -y update && apt-get -y install --no-install-recommends python3 python3-pymongo curl wget awscli gcc musl-dev python3-dev libffi-dev cargo make pip
+
+RUN wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.6.1.deb -O /tmp/mongodb-tools.deb && \
+    dpkg -i /tmp/mongodb-tools.deb && \
+    rm /tmp/mongodb-tools.deb
+
+RUN pip install --upgrade pip
+RUN pip install azure-cli
+RUN mkdir /backup /tiw
+
+ENV S3_PATH=mongodb AWS_DEFAULT_REGION=us-east-1
+
+COPY ./mongodb-backup/script/entrypoint.sh /usr/local/bin/entrypoint
+COPY ./mongodb-backup/script/backup.sh /usr/local/bin/backup
+COPY ./mongodb-backup/script/mongouri.py /usr/local/bin/mongouri
+
+RUN chmod 775 /usr/local/bin/entrypoint
+RUN chmod 775 /usr/local/bin/backup
+RUN chmod 775 /usr/local/bin/mongouri
+
+VOLUME /backup
+
+CMD /usr/local/bin/entrypoint

mongodb-backup/script/backup.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env sh
+
+OPTIONS=`python3 /usr/local/bin/mongouri`
+OPTIONS="$OPTIONS $EXTRA_OPTIONS"
+DEFAULT_BACKUP_NAME="$(date -u +%Y-%m-%d_%H-%M-%S)_UTC.gz"
+BACKUP_NAME=${BACKUP_NAME:-$DEFAULT_BACKUP_NAME}
+LOCAL_BACKUP_ROOT_FOLDER="/backup"
+LOCAL_DUMP_LOCATION="$LOCAL_BACKUP_ROOT_FOLDER/dump"
+
+notify() {
+  if [ "${SLACK_URI}" ]; then
+    message="$BACKUP_NAME has been backed up  at s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}"
+    if [ "${1}" != "0" ]; then
+      message="Unable to backup $BACKUP_NAME at s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}. See Logs."
+    fi
+    curl -X POST --data-urlencode "payload={\"text\": \"$message\"}" $SLACK_URI
+  fi
+}
+
+# Run backup
+mongodump ${OPTIONS} -o "${LOCAL_DUMP_LOCATION}"
+status=$?
+if [ "${status}" -eq "1" ]; then
+  echo "ERROR: Mongodump failed."
+  notify 1
+  exit 1
+fi
+
+# Compress backup
+tar -cvzf "${LOCAL_BACKUP_ROOT_FOLDER}/${BACKUP_NAME}" "${LOCAL_DUMP_LOCATION}"
+
+# If the AWS_S3_ENDPOINT variable isn't empty, then populate the --endpoint-url parameter to use a custom S3 compatable endpoint
+if [ ! -z "$AWS_S3_ENDPOINT" ]; then
+    ENDPOINT="--endpoint-url=$AWS_S3_ENDPOINT"
+fi
+
+# Upload backup
+
+if [ $BACKUP_PROVIDER = "s3" ]; then
+  aws $ENDPOINT s3 cp "${LOCAL_BACKUP_ROOT_FOLDER}/${BACKUP_NAME}" "s3://${S3_BUCKET}/${S3_PATH}/${BACKUP_NAME}"
+  status=$?
+  echo $status
+  if [ "${status}" != "0" ]; then
+    echo "ERROR: AWS Upload failed."
+    notify 1
+    exit 1
+  fi
+elif [ $BACKUP_PROVIDER = "az" ]; then
+
+  az storage blob upload --file "${LOCAL_BACKUP_ROOT_FOLDER}/${BACKUP_NAME}" --account-name "${AZURE_STORAGE_ACCOUNT_NAME}" --account-key "${AZURE_STORAGE_ACCOUNT_KEY}" -c "${AZURE_STORAGE_CONTAINER}" --name "${AZ_BACKUP_PATH}/${BACKUP_NAME}"
+  status=$?
+  echo $status
+  if [ "${status}" != "0" ]; then
+    echo "ERROR: AZURE Upload failed."
+    notify 1
+    exit 1
+  fi
+else 
+  echo "No logical backup provider is given"
+fi
+
+notify 0
+
+# Delete temp files
+rm -rf "${LOCAL_DUMP_LOCATION}"
+
+# Delete backup files
+if [ -n "${MAX_BACKUPS}" ]; then
+  while [ $(ls ${LOCAL_BACKUP_ROOT_FOLDER} -w 1 | wc -l) -gt ${MAX_BACKUPS} ];
+  do
+    BACKUP_TO_BE_DELETED=$(ls /backup -w 1 | sort | head -n 1)
+    rm -rf ${LOCAL_BACKUP_ROOT_FOLDER}/${BACKUP_TO_BE_DELETED}
+  done
+else
+  rm -rf ${LOCAL_BACKUP_ROOT_FOLDER}/*
+fi

mongodb-backup/script/entrypoint.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env sh
+
+set +e
+
+if [ -z ${BACKUP_CRON_SCHEDULE+x} ]; then
+  /usr/local/bin/backup
+else
+  BACKUP_CRON_SCHEDULE=${BACKUP_CRON_SCHEDULE}
+  echo "${BACKUP_CRON_SCHEDULE} /usr/local/bin/backup" > /etc/crontabs/root
+  # Starting cron
+  crond -f -d 0
+fi

mongodb-backup/script/mongouri.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+import os
+import pymongo
+
+uri = os.environ['MONGO_URI']
+data = pymongo.uri_parser.parse_uri(uri)
+
+if os.environ.get('MONGO_COMPLETE') is None:
+    dbname = '-d %s' % data['database']
+    else:
+        dbname = ''
+        options = '%s %s' % (dbname, uri)
+
+        print(options)

opensearch-curator/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3-slim-buster
 
-RUN pip install curator-opensearch==0.0.7
+RUN pip install curator-opensearch==0.0.10
 
 WORKDIR /opt/bitnami/python/bin/
 RUN cp /usr/local/bin/curator $(pwd)