[ci] Mirror third-party images and hailgenetics images on deploy (#12818)

- On *deploys*, makes sure that whatever is in our third-party images is
in our private registry before starting builds like hail-ubuntu that
might depend on those images. This means that we can update our ubuntu
base image without the australians needing to deploy any images by hand.
However, this does not run in PRs because I 1) didn't want to add that
kind of latency for PRs and 2) we don't do any kind of namespacing for
our images so if we did include this for a PR that ultimately wasn't
merged we would have to manually remove the image anyway so why not
manually add it if you're going to PR it… I think point 2 is a little
weak but I recall this being what we agreed on a couple months back when
we discussed this. I'm wondering if we should just eat the minute or so
latency at the beginning of PRs to be safe but it also feels like a
shame for something that changes so infrequently.

- Again on deploys, upload the hailgenetics/* images to the private
registry if they don't already exist there. This way any deployments
that aren't hail team's GCP deployment can get these images
automatically when they deploy a new SHA instead of uploading them
manually. It won't backfill skipped versions, but we decided that was
ok. This seems less relevant for testing on PRs as it will get triggered
on releases and we can easily dev deploy to rectify the image if this
breaks.

Author: daniel-goldstein

build.yaml

@@ -33,20 +33,6 @@ steps:
         to: /git_version
     dependsOn:
       - git_make_bash_image
-  - kind: buildImage2
-    name: hail_ubuntu_image
-    dockerFile: /io/hail-ubuntu/Dockerfile
-    contextPath: /io/hail-ubuntu
-    publishAs: hail-ubuntu
-    resources:
-      storage: 10Gi
-      cpu: "2"
-      memory: standard
-    inputs:
-      - from: /repo/docker/hail-ubuntu
-        to: /io/hail-ubuntu
-    dependsOn:
-      - merge_code
   - kind: createNamespace
     name: default_ns
     namespaceName: default
@@ -73,6 +59,55 @@ steps:
       - name: batch-worker-ssh-public-key
         clouds:
           - azure
+  - kind: runImage
+    name: copy_third_party_images
+    image: quay.io/skopeo/stable:v1.11.1
+    script: |
+      set -ex
+
+      REGISTRY={{ global.docker_prefix.split('/')[0] }}
+
+      {% if global.cloud == "gcp" %}
+      cat /registry-push-credentials/credentials.json | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
+      {% elif global.cloud == "azure" %}
+      dnf install -y jq
+      USERNAME=$(cat /registry-push-credentials/credentials.json | jq -jr '.appId')
+      cat /registry-push-credentials/credentials.json | jq -jr '.password' | skopeo login -u $USERNAME --password-stdin $REGISTRY
+      {% else %}
+      echo "unknown cloud {{ global.cloud }}"
+      exit 1
+      {% endif %}
+
+      cd /io/docker/third-party
+      DOCKER_PREFIX={{ global.docker_prefix }} bash copy_images.sh
+    inputs:
+      - from: /repo/docker
+        to: /io/docker
+    secrets:
+      - name: registry-push-credentials
+        namespace:
+          valueFrom: default_ns.name
+        mountPath: /registry-push-credentials
+    scopes:
+      - deploy
+    dependsOn:
+      - default_ns
+      - merge_code
+  - kind: buildImage2
+    name: hail_ubuntu_image
+    dockerFile: /io/hail-ubuntu/Dockerfile
+    contextPath: /io/hail-ubuntu
+    publishAs: hail-ubuntu
+    resources:
+      storage: 10Gi
+      cpu: "2"
+      memory: standard
+    inputs:
+      - from: /repo/docker/hail-ubuntu
+        to: /io/hail-ubuntu
+    dependsOn:
+      - merge_code
+      - copy_third_party_images
   - kind: deploy
     name: deploy_batch_sa
     namespace:
@@ -655,6 +690,7 @@ steps:
         to: /io/repo/ci
     dependsOn:
       - merge_code
+      - copy_third_party_images
   - kind: runImage
     name: build_hail_jar_and_wheel_only
     image:
@@ -1563,6 +1599,7 @@ steps:
       - default_ns
       - base_image
       - create_certs
+      - copy_third_party_images
   - kind: deploy
     name: deploy_prometheus
     namespace:
@@ -1574,6 +1611,7 @@ steps:
     dependsOn:
       - default_ns
       - create_certs
+      - copy_third_party_images
   - kind: runImage
     name: create_dummy_oauth2_client_secret
     image:
@@ -3141,6 +3179,49 @@ steps:
       - make_docs
     clouds:
       - gcp
+  - kind: runImage
+    name: mirror_hailgenetics_images
+    image: quay.io/skopeo/stable:v1.11.1
+    script: |
+      set -ex
+
+      REGISTRY={{ global.docker_prefix.split('/')[0] }}
+
+      set +x
+      {% if global.cloud == "gcp" %}
+      cat /registry-push-credentials/credentials.json | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
+      {% elif global.cloud == "azure" %}
+      dnf install -y jq
+      USERNAME=$(cat /registry-push-credentials/credentials.json | jq -jr '.appId')
+      cat /registry-push-credentials/credentials.json | jq -jr '.password' | skopeo login -u $USERNAME --password-stdin $REGISTRY
+      {% else %}
+      echo "unknown cloud {{ global.cloud }}"
+      exit 1
+      {% endif %}
+      set -x
+
+      cd /io/docker/hailgenetics
+      export HAIL_PIP_VERSION=$(cat /io/hail_pip_version)
+      export DOCKER_PREFIX={{ global.docker_prefix }}
+      bash mirror_images.sh
+    inputs:
+      - from: /hail_pip_version
+        to: /io/hail_pip_version
+      - from: /repo/docker/hailgenetics/mirror_images.sh
+        to: /io/docker/hailgenetics/mirror_images.sh
+      - from: /repo/docker/copy_image.sh
+        to: /io/docker/copy_image.sh
+    secrets:
+      - name: registry-push-credentials
+        namespace:
+          valueFrom: default_ns.name
+        mountPath: /registry-push-credentials
+    scopes:
+      - deploy
+      - dev
+    dependsOn:
+      - default_ns
+      - merge_code
   - kind: buildImage2
     name: website_image
     dockerFile: /io/repo/website/Dockerfile

docker/copy_image.sh

@@ -4,6 +4,21 @@ set -ex
 
 source ../copy_image.sh
 
+copy_if_not_present() {
+    src_image=$1
+    dest_image=$2
+    if ! skopeo inspect "docker://docker.io/$1";
+    then
+        echo "$1 does not exist yet, doing nothing"
+    elif skopeo inspect "docker://$2";
+    then
+      echo "$2 already exists, doing nothing"
+    else
+      echo "$2 does not exist, copying $1 to $2"
+      copy_image $1 $2
+    fi
+}
+
 if [[ -z "${DOCKER_PREFIX}" ]];
 then
     echo "Env variable DOCKER_PREFIX must be set"
@@ -30,5 +45,5 @@ images=(
 )
 for image in "${images[@]}"
 do
-    copy_image "hailgenetics/${image}" "${DOCKER_PREFIX}/hailgenetics/${image}"
+    copy_if_not_present "hailgenetics/${image}" "${DOCKER_PREFIX}/hailgenetics/${image}"
 done