feat: initial version

.github/workflows/publish.yml

@@ -0,0 +1,53 @@
+name: Build and Publish Docker Image
+
+on:
+  push:
+    branches:
+      - develop
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - develop
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -0,0 +1,34 @@
+ARG PROMETHEUS_VERSION=v3.0.1
+ARG SUPERCRONIC_VERSION=0.2.33
+
+FROM ghcr.io/lumeweb/promster:develop AS promster
+FROM debian:bookworm-slim AS certs
+RUN apt-get update && apt-get install -y curl ca-certificates
+RUN mkdir -p /rootfs/bin /rootfs/usr/bin /rootfs/etc/ssl/certs /rootfs/lib/x86_64-linux-gnu && \
+    cp /usr/bin/curl /rootfs/bin/ && \
+    cp -r /etc/ssl/certs/* /rootfs/etc/ssl/certs/ && \
+    cp -L /lib/x86_64-linux-gnu/lib*.so* /rootfs/lib/x86_64-linux-gnu/
+
+FROM prom/prometheus:${PROMETHEUS_VERSION}
+COPY --from=promster /bin/promster /usr/bin/promster
+COPY --from=certs /rootfs/ /
+
+RUN echo $PATH && \
+    ls -l /bin/curl && \
+    /bin/curl -Lo /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && \
+    chmod +x /usr/local/bin/mc && \
+    /bin/curl -Lo /usr/local/bin/supercronic https://github.com/aptible/supercronic/releases/download/v${SUPERCRONIC_VERSION}/supercronic-linux-amd64 && \
+    chmod +x /usr/local/bin/supercronic
+
+COPY entrypoint.sh /entrypoint.sh
+COPY bin/config-validator /bin/config-validator
+COPY bin/backup-manager /bin/backup-manager
+COPY bin/storage-manager /bin/storage-manager
+
+RUN chmod +x /entrypoint.sh \
+    /bin/config-validator \
+    /bin/backup-manager \
+    /bin/storage-manager
+
+USER root
+ENTRYPOINT [ "/entrypoint.sh" ]

bin/backup-manager

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Perform backup
+backup() {
+  # Use mc CLI to upload TSDB files to S3
+  mc --endpoint=${AWS_S3_ENDPOINT} mirror --overwrite /data ${AWS_BUCKET_NAME}/
+}
+
+# Restore from S3
+restore() {
+  # Check available disk space
+  available_disk_space=$(df -h "/data" | awk '{print $4}' | sed 's/%//g')
+  available_disk_space_bytes=$(echo "$available_disk_space * 1024 * 1024" | bc)
+
+  # Calculate total size of data to be restored
+  total_size=$(mc --endpoint=${AWS_S3_ENDPOINT} du --recursive ${AWS_BUCKET_NAME}/ | awk '{print $1}')
+
+  # Calculate max safe threshold
+  max_safe_threshold=$(echo "$available_disk_space_bytes * $MAX_DISK_USAGE_PERCENT / 100" | bc)
+
+  # Check if there is enough disk space
+  if [ $total_size -gt $max_safe_threshold ]; then
+    # Restore only the data that fits within the available disk space
+    mc --endpoint=${AWS_S3_ENDPOINT} mirror --overwrite ${AWS_BUCKET_NAME}/ /data --max-size $max_safe_threshold
+  else
+    # Restore all data
+    mc --endpoint=${AWS_S3_ENDPOINT} mirror --overwrite ${AWS_BUCKET_NAME}/ /data
+  fi
+}
+
+# Verify backup integrity
+verify() {
+  # Use mc CLI to verify backup integrity
+  mc --endpoint=${AWS_S3_ENDPOINT} ls ${AWS_BUCKET_NAME}/
+}
+
+# List available backups
+list() {
+  # Use mc CLI to list available backups
+  mc --endpoint=${AWS_S3_ENDPOINT} ls ${AWS_BUCKET_NAME}/
+}
+
+# Show backup system status
+status() {
+  # mc CLI does not support get-bucket-status command
+  echo "mc CLI does not support get-bucket-status command"
+}
+
+case $1 in
+  --backup)
+    backup
+    ;;
+  --restore)
+    restore
+    ;;
+  --verify)
+    verify
+    ;;
+  --list)
+    list
+    ;;
+  --status)
+    status
+    ;;
+  *)
+    echo "Usage: backup-manager <option>"
+    exit 1
+    ;;
+esac

bin/config-validator

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Validate environment variables
+validate_env() {
+  required_vars=("AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY" "AWS_REGION" "AWS_BUCKET_NAME" "AWS_S3_ENDPOINT" "PROMSTER_SCRAPE_ETCD_URL" "PROMSTER_ETCD_USERNAME" "PROMSTER_ETCD_PASSWORD")
+  for var in "${required_vars[@]}"; do
+    if [ -z "${!var}" ]; then
+      echo "Error: ${var} is not set"
+      exit 1
+    fi
+  done
+}
+
+# Validate S3 connection
+validate_s3() {
+  # Test S3 connection using mc CLI
+  if ! mc --endpoint=${AWS_S3_ENDPOINT} ls ${AWS_BUCKET_NAME} > /dev/null; then
+    echo "Error: Unable to connect to S3"
+    exit 1
+  fi
+}
+
+# Validate file system
+validate_paths() {
+  required_paths=("/data" "/etc/crontab" "/prometheus.yml")
+  for path in "${required_paths[@]}"; do
+    if [ ! -e "${path}" ]; then
+      echo "Error: ${path} does not exist"
+      exit 1
+    fi
+  done
+}
+
+case $1 in
+  --check-env)
+    validate_env
+    ;;
+  --check-s3)
+    validate_s3
+    ;;
+  --check-paths)
+    validate_paths
+    ;;
+  *)
+    echo "Usage: config-validator <option>"
+    exit 1
+    ;;
+esac

bin/storage-manager

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Remove old backups
+cleanup() {
+  # Use mc CLI to remove old backups
+  mc --endpoint=${AWS_S3_ENDPOINT} rm --recursive --force ${AWS_BUCKET_NAME}/
+}
+
+# Calculate restore size
+calculate_restore_size() {
+  # Use mc CLI to calculate restore size
+  mc --endpoint=${AWS_S3_ENDPOINT} du --recursive ${AWS_BUCKET_NAME}/
+}
+
+# Compact TSDB
+compact_tsdb() {
+  # Use Prometheus CLI to compact TSDB
+  prometheus --config.file=/prometheus.yml --storage.tsdb.path=/data --compact
+}
+
+case $1 in
+  --cleanup)
+    cleanup
+    ;;
+  --calculate)
+    calculate_restore_size
+    ;;
+  --optimize)
+    compact_tsdb
+    ;;
+  *)
+    echo "Usage: storage-manager <option>"
+    exit 1
+    ;;
+esac

entrypoint.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -euo pipefail
+
+# Set default environment variables
+export PROMETHEUS_CONFIG_FILE=${PROMETHEUS_CONFIG_FILE:-/prometheus.yml}
+export PROMETHEUS_DATA_DIR=${PROMETHEUS_DATA_DIR:-/data}
+export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
+export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
+export AWS_REGION=${AWS_REGION:-}
+export AWS_BUCKET_NAME=${AWS_BUCKET_NAME:-}
+export AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT:-}
+export BACKUP_SCHEDULE=${BACKUP_SCHEDULE:-0 0 * * *}
+export RETENTION_DAYS=${RETENTION_DAYS:-30}
+export MAX_DISK_USAGE_PERCENT=${MAX_DISK_USAGE_PERCENT:-80}
+export PROMSTER_LOG_LEVEL=${PROMSTER_LOG_LEVEL:-info}
+export PROMSTER_REGISTER_ETCD_PATH=${PROMSTER_REGISTER_ETCD_PATH:-/promster}
+export PROMSTER_SCRAPE_PATHS=${PROMSTER_SCRAPE_PATHS:-/metrics}
+export PROMSTER_SCRAPE_INTERVAL=${PROMSTER_SCRAPE_INTERVAL:-30s}
+export PROMSTER_SCRAPE_TIMEOUT=${PROMSTER_SCRAPE_TIMEOUT:-30s}
+export PROMSTER_EVALUATION_INTERVAL=${PROMSTER_EVALUATION_INTERVAL:-30s}
+export PROMSTER_SCHEME=${PROMSTER_SCHEME:-http}
+export PROMSTER_TLS_INSECURE=${PROMSTER_TLS_INSECURE:-false}
+export PROMSTER_ETCD_TIMEOUT=${PROMSTER_ETCD_TIMEOUT:-30}
+export PROMSTER_REGISTER_TTL=${PROMSTER_REGISTER_TTL:-60}
+
+# 1. Validate environment
+config-validator --check-env
+
+# 2. Initialize system
+config-validator --init
+
+# 3. Check if data directory is empty or corrupted
+if [ ! -d "${PROMETHEUS_DATA_DIR}" ] || [ -z "$(ls -A ${PROMETHEUS_DATA_DIR})" ] || [ -f "${PROMETHEUS_DATA_DIR}/CORRUPTED" ]; then
+  echo "Data directory is empty or corrupted, triggering recovery"
+  backup-manager --restore
+fi
+
+# 4. Start monitoring
+/bin/promster &
+/bin/prometheus --config.file=${PROMETHEUS_CONFIG_FILE} --storage.tsdb.path=${PROMETHEUS_DATA_DIR} &
+
+# 5. Configure backup schedule
+echo "${BACKUP_SCHEDULE} backup-manager --backup" >> /etc/crontab
+echo "0 0 * * * storage-manager --optimize" >> /etc/crontab
+supercronic /etc/crontab