Merge pull request #199 from fscheiner/switchover-to-new-src-repo

CI changes
gridcf · Nov 9, 2022 · aed61cc · aed61cc
2 parents 26eb5a4 + e32c9d3
commit aed61cc
Show file tree

Hide file tree

Showing 6 changed files with 139 additions and 39 deletions.
diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml
@@ -13,6 +13,8 @@ jobs:
   run-scripts:
     runs-on: ubuntu-latest
     strategy:
+      # Don't cancel the remaining running jobs if some job(s) fail(s)
+      fail-fast: false
       matrix:
         image: ['centos:centos7', 'rockylinux:8', 'quay.io/centos/centos:stream8']
         components: ['udt,myproxy,ssh', 'gram5']
@@ -22,6 +24,10 @@ jobs:
             components: 'myproxy,ssh'
           - image: 'quay.io/centos/centos:stream9'
             components: 'gram5'
+          - image: 'rockylinux:9'
+            components: 'myproxy,ssh'
+          - image: 'rockylinux:9'
+            components: 'gram5'
     steps:
       - uses: actions/checkout@v2
         with:
@@ -35,26 +41,29 @@ jobs:
         run: travis-ci/setup_tasks.sh
 
       - name: build source tarballs and srpms
-        # Only run this step for the centos:centos7 case and for only one component selection
+        # Run this step for all OS cases but for only one component selection each,
+        # it still builds **all** source tarballs and SRPMs
         if: |
-          contains(matrix.image , 'centos:centos7') &&
-          contains(matrix.components , 'udt,myproxy,ssh')
+          contains(matrix.components , 'gram5')
         env:
-          IMAGE: centos:centos7
+          IMAGE: ${{ matrix.image }}
           TASK: srpms
         run: travis-ci/setup_tasks.sh
 
       # SSH key recipe from https://www.webfactory.de/blog/use-ssh-key-for-private-repositories-in-github-actions
       - name: Establish ssh and upload source tarballs
-        # Only run this step for the centos:centos7 case
+        # Only run this step for the centos:centos7 case and
+        # for only one component selection (must be the same as for the previous step!) and
+        # only when a tag was created
         if: |
           contains(matrix.image , 'centos:centos7') &&
+          contains(matrix.components , 'gram5') &&
           contains(github.ref , 'refs/tags/')
         env:
           SSH_AUTH_SOCK: /tmp/ssh_agent.sock
         run: |
           mkdir -p ~/.ssh
           ssh-keyscan github.com >> ~/.ssh/known_hosts
           ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
-          ssh-add - <<< "${{ secrets.ID_GCTUPLOADER }}"
+          ssh-add - <<< "${{ secrets.ID_GRIDCF_UPLOADER }}"
           travis-ci/upload_source_tarballs.sh ${{ github.repository_owner }}
diff --git a/.travis.yml b/.travis.yml
@@ -5,21 +5,29 @@
 
 # Also, the following gram tests failed for all fedora versions:
 #   nonblocking-register-test.pl, register-callback-test.pl, register-test.pl
-arch:
+os: linux
+dist: focal
+language: ruby
+# As per [1] explicitly included builds inherit the first value in an array.
+# [1]: https://docs.travis-ci.com/user/multi-cpu-architectures#example-multi-architecture-build-matrix
+# Hence the desired arch needs to be specified for each job additionally
+# below.
+#arch:
   # Disabled for now
   #- amd64
   # Handled via partner queue, uncharged
-  - ppc64le
+  #- ppc64le
   #- arm64
 jobs:
   include:
 
     - &run_tests
       # Template; subsequent uses modify 'env'
+      arch:
+        - ppc64le
       env:
         - IMAGE=centos:centos7 TASK=tests COMPONENTS=udt,myproxy,ssh
       stage: test
-      sudo: required
       services:
         - docker
 
@@ -30,6 +38,19 @@ jobs:
 
       script:
       - travis-ci/setup_tasks.sh
+
+    - <<: *run_tests
+      arch:
+        - arm64
+
+    - <<: *run_tests
+      arch:
+        - ppc64le
+      env:
+        - IMAGE=centos:centos7 TASK=tests COMPONENTS=gram5
+
     - <<: *run_tests
+      arch:
+        - arm64
       env:
         - IMAGE=centos:centos7 TASK=tests COMPONENTS=gram5
diff --git a/travis-ci/make_rpms.sh b/travis-ci/make_rpms.sh
@@ -16,6 +16,14 @@ usage()
 
 umask 022
 
+case $(</etc/redhat-release) in
+    CentOS*\ 7*) OS=centos7 ;;
+    CentOS\ Stream*\ 8*) OS=centos-stream-8;;
+    CentOS\ Stream*\ 9*) OS=centos-stream-9;;
+    Rocky\ Linux*\ 8*) OS=rockylinux8 ;;
+    Rocky\ Linux*\ 9*) OS=rockylinux9 ;;
+    *) OS=unknown ;;
+esac
 
 root=$(git rev-parse --show-toplevel)
 packagingdir=$root/packaging
@@ -81,8 +89,17 @@ cat <<EOF >> "$topdir/.rpmmacros"
 %_excludedocs 0
 EOF
 
+# Limit package list according to OS possibilities
+all_packages=( $(grep -v '^#' $fedoradir/ORDERING | grep '[^[:space:]]') )
 
-packages=( $(grep -v '^#' $fedoradir/ORDERING | grep '[^[:space:]]') )
+if [[ $OS != *9 ]]; then
+
+    packages=( ${all_packages[@]} )
+else
+    # Not building globus-xio-udt-driver on *9
+    packages_9=( ${all_packages[@]/globus-xio-udt-driver/} )
+    packages=( ${packages_9[@]} )
+fi
 
 cp -f "$tarballdir"/*.tar.gz "$topdir"/SOURCES
 

diff --git a/travis-ci/make_source_tarballs.sh b/travis-ci/make_source_tarballs.sh
@@ -10,6 +10,15 @@
 
 set -eu
 
+case $(</etc/redhat-release) in
+    CentOS*\ 7*) OS=centos7 ;;
+    CentOS\ Stream*\ 8*) OS=centos-stream-8;;
+    CentOS\ Stream*\ 9*) OS=centos-stream-9;;
+    Rocky\ Linux*\ 8*) OS=rockylinux8 ;;
+    Rocky\ Linux*\ 9*) OS=rockylinux9 ;;
+    *) OS=unknown ;;
+esac
+
 root=$(git rev-parse --show-toplevel)
 cd "$root"
 

diff --git a/travis-ci/run_task_inside_docker.sh b/travis-ci/run_task_inside_docker.sh
@@ -9,10 +9,36 @@ COMPONENTS=${3-}
 set -e
 
 case $(</etc/redhat-release) in
-    CentOS*\ 7*) OS=centos7 ;;
-    CentOS\ Stream*\ 8*) OS=centos-stream-8;;
-    CentOS\ Stream*\ 9*) OS=centos-stream-9;;
-    Rocky\ Linux*\ 8*) OS=rockylinux8 ;;
+    CentOS*\ 7*)
+
+        OS=centos7
+        release_ver=el7
+        ;;
+
+    CentOS\ Stream*\ 8*)
+
+        OS=centos-stream-8
+        release_ver=el8
+        ;;
+
+    CentOS\ Stream*\ 9*)
+
+        OS=centos-stream-9
+        release_ver=el9
+        ;;
+
+    Rocky\ Linux*\ 8*)
+
+        OS=rockylinux8
+        release_ver=el8
+        ;;
+
+    Rocky\ Linux*\ 9*)
+
+        OS=rockylinux9
+        release_ver=el9
+        ;;
+
     *) OS=unknown ;;
 esac
 
@@ -26,6 +52,11 @@ case $OS in
               dnf config-manager --set-enabled powertools
               dnf -y install epel-release
               ;;
+    rockylinux9)
+              dnf -y install dnf-plugins-core
+              dnf config-manager --set-enabled crb
+              dnf -y install epel-release
+              ;;
     centos-stream-8)
               dnf -y install dnf-plugins-core
               dnf config-manager --set-enabled powertools
@@ -48,11 +79,11 @@ packages=(gcc gcc-c++ make autoconf automake libtool \
           'perl(URI)' file sudo bison patch curl \
           pam pam-devel libedit libedit-devel)
 
-if [[ $OS != centos7 ]]; then
+if [[ $OS != *7 ]]; then
 
     # provides `cmp` used by `packaging/git-dirt-filter`
     packages+=(diffutils)
-    if [[ $OS == centos-stream-9 ]]; then
+    if [[ $OS == *9 ]]; then
 
         # also install "zlib zlib-devel" because it's needed for `configure`ing
         # "gridftp/server/src"
@@ -78,22 +109,42 @@ elif [[ $TASK == *rpms ]]; then
     # for globus-gridftp-server:
     packages+=(fakeroot)
     # for globus-xio-udt-driver:
-    packages+=(udt udt-devel glib2-devel libnice-devel gettext-devel libffi-devel)
+    if [[ $OS == *9 ]]; then
+
+        # libnice-devel is not available for CentOS Stream 9 / Rocky Linux 9.
+        #
+        # make_rpms.sh was also updated in this regard.
+        :
+    else
+        packages+=(udt udt-devel glib2-devel libnice-devel gettext-devel libffi-devel)
+    fi
     # for globus-gram-job-manager:
     packages+=(libxml2-devel)
     # for myproxy:
     packages+=(pam-devel voms-devel cyrus-sasl-devel openldap-devel voms-clients initscripts)
     # for globus-net-manager:
-    packages+=(python-devel)
+    if [[ $OS == *7 ]]; then
+
+        packages+=(python-devel)
+    else
+        packages+=(python3-devel)
+    fi
     # for globus-gram-audit:
     packages+=('perl(DBI)')
     # for globus-scheduler-event-generator:
-    packages+=(redhat-lsb-core)
+    if [[ $OS == *9 ]]; then
+
+        # redhat-lsb-core is not available for CentOS Stream 9 / Rocky Linux 9.
+        # But the default is also to not use LSB, so can be ignored.
+        :
+    else
+        packages+=(redhat-lsb-core)
+    fi
     # for gsi-openssh
     packages+=(pam libedit libedit-devel)
 fi
 
-if [[ $OS == centos7 ]]; then
+if [[ $OS == *7 ]]; then
 	yum -y -d1 install "${packages[@]}"
 else
 	dnf --allowerasing -y -d1 install "${packages[@]}"
@@ -153,7 +204,8 @@ case $TASK in
     srpms)
         make_tarballs
         echo '==========================================================================================='
-        make_srpms .gct
+        # NOTICE: No dashes in the dist string!
+        make_srpms .gct.$release_ver
 
         # copy all the files we want to deploy into one directory b/c
         # can't specify multiple directories in travis
@@ -165,14 +217,9 @@ case $TASK in
     rpms)
         make_tarballs
         echo '==========================================================================================='
-        if [[ $OS == centos6 ]]; then
-            # doesn't support rpmbuild --nocheck
-            nocheck=
-        else
-            # -C = skip unit tests
-            nocheck=-C
-        fi
-        make_rpms $nocheck .gct.$OS
+        # NOTICE: No dashes in the dist string!
+        # -C = skip unit tests
+        make_rpms -C .gct.$release_ver
 
         # copy all the files we want to deploy into one directory b/c
         # can't specify multiple directories in travis
@@ -182,12 +229,10 @@ case $TASK in
         # raise an error).
         # `overwrite: true` in .travis.yml ought to fix that, but doesn't
         # appear to.
-        if [[ $OS == centos6 ]]; then
-            cp -f packaging/rpmbuild/SRPMS/*.rpm package-output/*.tar.gz  \
-                travis_deploy/
-        fi
+        cp -f packaging/rpmbuild/SRPMS/*.rpm package-output/*.tar.gz  \
+              travis_deploy/
         cp -f packaging/rpmbuild/RPMS/noarch/*.rpm packaging/rpmbuild/RPMS/x86_64/*.rpm  \
-            travis_deploy/
+              travis_deploy/
         ;;
     *)
         echo "*** INVALID TASK '$TASK' ***"

diff --git a/travis-ci/upload_source_tarballs.sh b/travis-ci/upload_source_tarballs.sh
@@ -7,12 +7,11 @@ set -eux
 
 repo_owner=$1
 
-# also hosts repo.gridcf.org; can't use repo.gridcf.org directly because
-# CloudFlare apparently doesn't properly handle SSH
-upload_server=hcc-osg-repo.unl.edu
+# hosts repo.gridcf.org
+upload_server="repo-gridcf.redir.ops.egi.eu"
 
-# obtained by running "ssh-keyscan hcc-osg-repo.unl.edu"
-hostsig="hcc-osg-repo.unl.edu ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2AIWAVx2KY+GhDab9SdxLTvjjzTiNa4pfHe7TvRZ5O+qZNc4c8sBlsG7OZGZvDLMRjGTKFyjJx3jDVUwaf14DwzQi9rgZxEZgBsRFffLATZqz+DyVN1H9uw215pah9Wh6yzaqMn51y6kqg0kk/ip62cYcXFgLKUNkzV0yz5WFugm5ziROZn01v5o74VdCABTAdlZhviUoObCn+bycXoUGGETY5GZ3muAW6y5LydDTD+2S97qJWGdSW7JBIfcmU7n5dl8MrtYKYwGswOgdUDrLtCp6CdZt/Evr+3NyLp35IhLnwxdkBBlKHPY0jXrGHyemsXa0Hq0PG/Ih5d0M8RMp"
+# obtained by running "repo-gridcf.redir.ops.egi.eu"
+hostsig="repo-gridcf.redir.ops.egi.eu ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4jPJDFLpljf6pSai/W2yh1/bgvRkfpLf9vvHs6ETYb"
 
 
 echo "$hostsig" > ~/.ssh/known_hosts