Add working spack build with build cache. (#695)

* Add working spack build with build cache.

* Fix spacke => spack typo.

* Also exclude curl from build.

* Also exclude openssl.

* Use custom spack for with fix.

* Change how spack is configured.

* Fix configuring spack command.

* Fix cache path and TMPDIR.

* Add output of full log on error and add verbosity to HiOp build.

* Force openblas on all builds

* Update spack_build.yml

* Update spack_build.yml with checkout in right place

* Load openmpi, and use git.version spack version

* Update spack_build.yml with correct Spack commit SHA

* Update spack_build.yml with correct spack vesion syntax

* Update spack_build.yml with `--sh` for loading openmpi

* Update spack_build.yml

* Load MPI through printed variables from spack.

* Update .github/workflows/spack_build.yml

* Update spack_build.yml

.github/workflows/spack_build.yml

@@ -0,0 +1,185 @@
+# https://spack.readthedocs.io/en/latest/binary_caches.html#spack-build-cache-for-github-actions
+name: Spack Builds (Ubuntu x86_64 Buildcache)
+
+on: [pull_request]
+
+env:
+  SPACK_COLOR: always
+  REGISTRY: ghcr.io/llnl
+  SPACK_CACHE: /opt/spack-cache
+  tempdir: /opt/spack-cache
+  TMP: /opt/spack-cache
+  TMPDIR: /opt/spack-cache
+  # Our repo name contains upper case characters, so we can't use ${{ github.repository }}
+  IMAGE_NAME: hiop
+  USERNAME: hiop-bot
+  BASE_VERSION: ubuntu-24.04-fortran
+
+jobs:
+  base_image_build:
+    runs-on: ubuntu-24.04
+    permissions:
+      packages: write
+      contents: read
+
+    name: Build Custom Base Image
+    steps:
+      # No GHCR base image with skopeo, so this will do...
+      - name: "Set up skopeo"
+        uses: warjiang/[email protected]
+        with:
+          version: latest
+
+      # Use skopeo to check for image for convenience
+      - name: Check for existing base images
+        run: |
+          set -e
+          CONTAINER_TAG=${{ env.BASE_VERSION }}
+          OCI_URL="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }}"
+          echo Checking for CONTAINER_TAG $CONTAINER_TAG
+          skopeo inspect \
+              docker://$OCI_URL \
+              --raw \
+              --creds "${{ env.USERNAME }}:${{ secrets.GITHUB_TOKEN }}" \
+            > /dev/null && echo "Image already exists. Please bump version." && exit 0
+          echo "IMAGE_EXISTS=false" >> $GITHUB_ENV
+
+      # Need to build custom base image with gfortran
+      - name: Create Dockerfile heredoc
+        if: ${{ env.IMAGE_EXISTS == 'false' }}
+        run: |
+          cat << EOF > Dockerfile
+          FROM ubuntu:24.04
+          RUN apt-get update && \
+              apt-get install -y --no-install-recommends \
+                software-properties-common \
+                gpg-agent \
+                openssh-client \
+                openssh-server \
+              && rm -rf /var/lib/apt/lists/*
+          RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \
+              apt-get install -y --no-install-recommends \
+                gfortran \
+                gcc \
+                libstdc++6 \
+              && rm -rf /var/lib/apt/lists/*
+          EOF
+
+      # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USERNAME }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        if: ${{ env.IMAGE_EXISTS == 'false' }}
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          labels: org.opencontainers.image.version=${{ env.BASE_VERSION }}
+
+      - name: Build and push Docker base image
+        if: ${{ env.IMAGE_EXISTS == 'false' }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  hiop_spack_builds:
+    needs: base_image_build
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      matrix:
+        spack_spec:
+          - hiop@develop+mpi~raja~shared~kron~sparse ^openblas ^openmpi ^libevent~openssl
+          - hiop@develop~mpi~raja~shared~kron~sparse ^openblas ^libevent~openssl
+          - hiop@develop~mpi+raja~shared~kron~sparse ^openblas ^libevent~openssl
+
+            # We will need coinhsl for this, but what are the rules for using
+            # a coinhsl tarball?
+            # - hiop@develop~mpi~raja~shared~kron+sparse
+
+    name: Build HiOp with Spack
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Once we move submodule deps into spack, we can do some more builds
+          # Also need to change build script to use spack from base image
+          submodules: true
+
+      - name: Clone Spack
+        run: git clone https://github.com/spack/spack.git
+
+      - name: Setup Spack
+        run: echo "$PWD/spack/bin" >> "$GITHUB_PATH"
+
+      - name: Create heredoc spack.yaml
+        run: |
+          spack debug report
+          cat << EOF > spack.yaml
+          spack:
+            specs:
+            - ${{ matrix.spack_spec }} target=x86_64_v2
+            concretizer:
+              reuse: dependencies
+            config:
+              source_cache: $SPACK_CACHE/source_cache
+              misc_cache: $SPACK_CACHE/misc_cache
+              build_stage: $SPACK_CACHE/build_stage
+              install_tree:
+                root: /opt/spack
+                padded_length: False
+            mirrors:
+              local-buildcache: oci://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+            # spack: https://binaries.spack.io/develop
+            packages:
+              all:
+                require: "%gcc"
+          EOF
+
+      - name: Configure GHCR mirror
+        run: spack -e . mirror set --oci-username ${{ env.USERNAME }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
+
+      - name: Trust keys
+        run: spack -e . buildcache keys --install --trust
+
+      - name: Find external packages
+        run: spack -e . external find --all --exclude python --exclude curl --exclude openssl
+
+      - name: Spack develop HiOp
+        run: spack -e . develop --path=$(pwd) hiop@git."${{ github.head_ref || github.ref_name }}"=develop
+
+      - name: Concretize
+        run: spack -e . concretize --fresh
+
+      - name: Install Dependencies
+        run: spack -e . install --no-check-signature --fail-fast --show-log-on-error --verbose --only dependencies
+
+      - name: Install HiOp
+        run: |
+          ls -al
+          spack -d -e . install  --keep-stage --verbose --show-log-on-error --only package --no-cache
+
+      - name: Test Build
+        run: |
+          # Not all pipelines have `+mpi`, so this command might fail
+          # Command also only prints shell commands, need to source
+          spack -e . load --sh openmpi > env.txt || [ rm env.txt || true ]
+          source env.txt || true
+          cd $(spack -e . location --build-dir hiop@develop)
+          ctest -VV
+
+      - name: Push binaries to buildcache
+        run: |
+          spack -e . buildcache push --force --base-image ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }} --unsigned --update-index local-buildcache
+        if: ${{ !cancelled() }}