repo-s3-mirror

#!/bin/sh
# -*- sh-basic-offset: 2 -*-

help () {
  cat <<EOF
usage: repo-s3-mirror [-h] [-b BUCKET] [-p PREFIX] [-r REPO]

Script to create a mirror of alibuild repository in an S3 bucket.

  -b BUCKET, --bucket BUCKET   the S3 bucket to upload packages to
  -p PREFIX, --prefix PREFIX   the local path where the repository is stored
  -r REPO, --repo REPO         the part of the repository to sync
  -h, --help                   display this help message and exit
EOF
  exit "$1"
}

bucket=${BUCKET:-alibuild-repo}
prefix=${PREFIX:-/build/reports/repo}
repo=${REPO:-TARS/slc7_x86-64}

while [ $# -gt 0 ]; do
  case "$1" in
    -b|--bucket) bucket=$2; shift 2;;
    -p|--prefix) prefix=$2; shift 2;;
    -r|--repo) repo=$2; shift 2;;
    -h|--help) help 0;;
    *) echo "$(basename "$0"): unknown option: $1" >&2; help 1 >&2;;
  esac
done

rclone_defaults () {
  # Simple wrapper for rclone, specifying common default options.
  rclone --config /secrets/alibuild_rclone_config --transfers=10 --progress --delete-before "$@"
}

find_dirs () {
  # Find directories in $prefix/$repo, with extra find predicates passed as args
  # to this function. Exclude store/ completely (it doesn't contain symlinks, so
  # ignoring it avoids pointless disk accesses).
  find "$prefix/$repo" -path "$prefix/$repo/store" -prune -o -type d "$@" -print0
}

# Create a separate filesystem tree, with symlinks only. However, as S3 doesn't
# support real symlinks, make them regular files containing the path they point
# to instead.
symlink_tree=/tmp/repo-symlinks
printf 'Replicating symlinks to %s' "$symlink_tree" >&2
if
  # Check if $symlink_tree exists, to avoid an error message from find.
  # If it doesn't, this is the first run, and we want to sync everything.
  ! [ -d "$symlink_tree" ] ||
    # We only copy symlinks in directories that have changed since the last run
    # to save on disk accesses.
    ! find_dirs -newer "$symlink_tree"
then
  # If this is the first run on this machine, recreate the entire hierarchy.
  # (In that case, $symlink_tree won't exist yet.)
  find_dirs -true
fi |

  # Find and read symlinks under each path found above. New child directories
  # are passed separately, so we don't look for symlinks in child directories.
  # %p is the symlink's path including $prefix/$repo/..., %l is its target,
  # which for the RPM repo will be a relative path (so we can use it directly).
  xargs -0rI {} find {} -mindepth 1 -maxdepth 1 -type l -printf '%p\t%l\n' |

  # Create a pseudo-symlink in $symlink_tree for each symlink found above.
  while IFS='	' read -r symlink target; do
    # Remove the starting point from the symlink path. We can't do this using %P
    # in find as we might get any "starting point" under $prefix/$repo, and we
    # only want to remove $prefix/$repo.
    symlink=${symlink#$prefix/$repo}
    mkdir -p "$(dirname "$symlink_tree/$symlink")"
    # A pseudo-symlink is just a file containing the symlink target.
    printf %s "$target" > "$symlink_tree/$symlink"
    # Show progress by printing a dot for each symlink processed. Printing names
    # would be much too much output.
    printf . >&2
  done

printf ' done.\n' >&2
# Sync the pseudo-symlinks to S3. Package tarballs are synced separately, below.
rclone_defaults move --delete-empty-src-dirs "local:$symlink_tree/" "rpms3:$bucket/$repo/"

# Sync tarballs from the repo to S3.
rclone_defaults sync "local:$prefix/$repo/store/" "rpms3:$bucket/$repo/store/"