Merge pull request #1 from epics-containers/dev

Update CI to manage IOC versions

.github/workflows/bl_buiild.yml

@@ -1,4 +1,4 @@
-name:  Build and publish a beamline's ioc helm charts
+name: Build and publish a beamline's ioc helm charts
 
 on:
   push:
@@ -7,6 +7,7 @@ on:
 env:
   HELM_VERSION_TO_INSTALL: 3.8.2 # version of HELM to install
   GHCR_ROOT: ghcr.io/epics-containers
+  CR_PAT: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   build-and-push-helm-charts:
@@ -27,34 +28,18 @@ jobs:
 
       - name: push each ioc helm chart
         run: |
-          echo ${{ secrets.GITHUB_TOKEN }} | helm registry login ${{ env.GHCR_ROOT }} --username ${{ github.repository_owner }} --password-stdin
-
           # tar up any config folders that require it
           ./tar_config_src.sh
 
-          # only publish if this is a tagged release
-          if [[ ${GITHUB_REF} = refs/tags/* ]] ; then
-
-            TAG=${GITHUB_REF#refs/tags/}
-            ioc_dirs=$(ls -d iocs/*/)
-
-            # Update all chart dependencies.
-            for ioc in ${ioc_dirs}; do helm dependency update ${ioc}; done
-
-            # update the helm chart versions with the tag
-            sed -e "s/^version: .*$/version: ${TAG}/g" -e "s/^appVersion: .*$/appVersion: ${TAG}/g" -i iocs/*/Chart.yaml
-
-            # push all ioc chart packages to the registry
-            for ioc in ${ioc_dirs}
-            do
-                for THIS_TAG in latest ${TAG}
-                do
-                    URL="${GHCR_ROOT}/$(basename $ioc):${THIS_TAG}"
-                    echo saving ${ioc} to "${URL}" ...
-                    helm chart save ${ioc} "${URL}"
-                    echo push to "${URL}" ...
-                    helm chart push "${URL}"
-                done
-            done
-
-          fi
+          # helm tags must be SemVar. Use 0.0.0-b0 for testing the latest non-tagged build
+          if [ "${GITHUB_REF_TYPE}" == "tag" ] ; then
+            TAG=${GITHUB_REF_NAME}
+          else
+            TAG="0.0.0-b0"
+          fi
+
+          # All IOCS that have changes since last pushed to the helm registry
+          # will have version ${TAG} pushed 
+          for ioc_root in $(ls -d iocs/*/) ; do
+            ./helm-push.sh ${ioc_root} ${TAG}
+          done

helm-push.sh

@@ -1,58 +1,63 @@
-# TODO UNDER DEVELOPMENT
-# WARNING - this needs genericizing
-# all real beamlines should have CI to push to their private repos
-# this is an example of how to do it to ghcr.io/epics-containers/
-
-
-helm version
+#!/bin/bash
 
+# NOTE: all beamlines should have CI to push their IOC helm charts
+# WARNING: use this script for testing but not for deploying 
+# production IOCs. Instead use the CI by pushing a tag to 
+# the main branch (this makes the source for the chart discoverable)
 set -e
 
-# Helm Registry in diamond GCR
-HELM_REPO=ghcr.io/epics-containers/
+IOC_ROOT="$(realpath ${1})"
+# determine the tag to use based on date or argument 2
+TAG=${2:-$(date +%Y.%-m.%-d-%-H%M)}
 
-# log in to the registry
-if [ -z ${CI_BUILD_ID} ]
-then
-    # this was run locally - get creds from gcloud and push to work repo
-    echo "LOCAL deploy to helm repo"
-    echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin
-else
-    # running under Gitlab CI - get creds from /etc/gcp/config.json
-    cat /etc/gcp/config.json | helm registry login -u _json_key \
-      --password-stdin ${HELM_REPO}
+if [ -z "${IOC_ROOT}" ] ; then
+  echo "usage: helm-push.sh <ioc root folder> <semvar tag>"
 fi
 
-if [ -z "${CI_COMMIT_TAG}" ]
-then
-    # untagged builds go into the work repo instead of beamline repo
-    CI_PROJECT_NAME="work"
-    # determine the tag to use based on date
-    TAG=$(date +%Y.%-m.%-d-work%-H%M)
-else
-    # determine the tag to use based on date
-    TAG=$(date +%Y.%-m.%-d-%-H%M)
-fi
+# Helm Registry in GHCR
+HELM_REPO=ghcr.io/epics-containers
+
+# extract name from the Chart.yaml
+NAME=$(sed -n '/^name: */s///p' "${IOC_ROOT}/Chart.yaml")
+CHART=oci://${HELM_REPO}/${NAME}
+
+# Before calling this script: set CR_PAT to a github personal access token 
+# see https://github.com/settings/tokens
+echo $CR_PAT | helm registry login -u USERNAME --password-stdin $HELM_REPO
 
-ioc_dirs=$(ls -d iocs/*/)
+echo "Verifying deploy of ${IOC_ROOT} to helm repo as version ${TAG} ..."
+helm package "${IOC_ROOT}" -u --app-version ${TAG} --version ${TAG}
+PACKAGE=$(realpath ${NAME}-${TAG}.tgz)
 
-# Update all chart dependencies.
-for ioc in ${ioc_dirs}; do helm dependency update $ioc; done
+# Temp dir for testing if the helm chart has changed
+TMPDIR=$(mktemp -d); cd ${TMPDIR}
 
-# udate the helm chart versions with the tag
-sed -e "s/^version: .*$/version: ${TAG}/g" -e "s/^appVersion: .*$/appVersion: ${TAG}/g" -i iocs/*/Chart.yaml
+# extract the latest version to a temporary folder
+helm pull ${CHART} > out.txt
+LATEST_VERSION=$(sed -n '/^Pulled:/s/.*://p' out.txt)
+echo "latest version of ${CHART} is ${LATEST_VERSION}"
+mkdir latest_ioc this_ioc
+tar -xf *tgz -C latest_ioc &> tar1.out
 
-# push all ioc chart packages to the registry
-for ioc in ${ioc_dirs}
-do
-    for THIS_TAG in latest ${TAG}
-    do
-        URL="${HELM_REPO}/$(basename $ioc):${THIS_TAG}"
-        echo saving ${ioc} to "${URL}" ...
-        helm chart save ${ioc} "${URL}"
-        echo push to "${URL}" ...
-        helm chart push "${URL}"
-    done
-done
+# repackage the new IOC with same version as above for direct comparison
+# (packaging reformats the Chart.yaml file so this is the simplest approach)
+helm package "${IOC_ROOT}" --app-version ${LATEST_VERSION} --version ${LATEST_VERSION}
+tar -xf *tgz -C this_ioc &> tar2.out
 
-echo Done
+# compare the packages and push the new package if it has changed
+if diff -r latest_ioc this_ioc &> diff.out ; then 
+    echo "not pushing unchanged IOC ${CHART} version ${LATEST_VERSION}"   
+else
+    echo "pushing ${CHART} version ${TAG}"   
+    helm push "${PACKAGE}" oci://${HELM_REPO}
+fi
+
+cat diff.out
+
+# clean up
+if [ -z ${HELM_PUSH_DEBUG} ] ; then
+    rm "${PACKAGE}"
+    rm -r ${TMPDIR}
+else
+    echo "helm-push.sh comparison output is in ${TMPDIR}"
+fi

iocs/bl45p-ea-ioc-02/values.yaml

@@ -1,15 +1,2 @@
-beamline: bl45p
-namespace: bl45p
+name: bl45p-ea-ioc-02
 base_image: ghcr.io/epics-containers/ioc-adpandablocks:4.14r1.0.run
-
-# root folder of generic ioc source - not expected to change
-iocFolder: /epics/ioc
-
-# useAffinity - only run on nodes with label beamline:bl45p
-useAffinity: true
-# use default resource limits
-memory:
-cpu:
-
-# when autosave is true: create PVC and mount at /autosave
-autosave: true

iocs/bl45p-mo-ioc-01/Chart.yaml

@@ -6,8 +6,8 @@ description: |
 type: application
 
 # chart and app version - these are updated automatically by CI
-version: 2021.2.0
-appVersion: 2021.2.0
+version: 0.0.0-b0
+appVersion: 0.0.0-b0
 
 dependencies:
   - name: bl45p-chart

iocs/bl45p-mo-ioc-01/config/start.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-#
+########################################
 # generic kubernetes IOC startup script
-#
+########################################
 
 this_dir=$(realpath $(dirname $0))
 TOP=$(realpath ${this_dir}/..)
@@ -23,7 +23,7 @@ boot=${config_dir}/ioc.boot
 
 # Update the boot script to work in the directory it resides in
 # using msi MACRO substitution.
-# Output to /tmp for guarenteed writability
+# Output to /tmp for guarenteed writability.
 msi -MTOP=${TOP},THIS_DIR=${config_dir} ${boot} > /tmp/ioc.boot
 
 exec ${IOC}/bin/linux-x86_64/ioc /tmp/ioc.boot