Skip to content

Commit c2bf588

Browse files
madbaronmadbaron
authored
Merge pull request #17 from madbaron/master
Re-work of mucoll-spack to enable build of releases fully on github
2 parents de1dac6 + 22b9739 commit c2bf588

File tree

20 files changed

+569
-230
lines changed

20 files changed

+569
-230
lines changed

‎.cherry-pick

+7-8
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
# root: fix for +opengl, remove when https://github.com/spack/spack/pull/45057 is merged
2-
curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/4505742844.diff | patch -p1
1+
# py-onnx: add a patch for the standard, remove after 1.15.0
2+
git cherry-pick bbd205543b09139f2f57f21099eb366c9f183d76 -X theirs --no-commit
33

4-
# protobuf: updated patch path, remove when https://github.com/spack/spack/pull/44210
5-
# and https://github.com/spack/spack/pull/44443 are merged
6-
curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/44210.diff | patch -p1
7-
curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/44443.diff | patch -p1
4+
# add gaudi 39
5+
git cherry-pick 5ebf45861f688337dddcdee2242d4f479a832d06 -X theirs --no-commit
86

9-
# dd4hep: update to version 1.29, remove when https://github.com/spack/spack/pull/44652 is merged
10-
curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/44652.diff | patch -p1
7+
# openloops: Fix application of cmodel to compilation for aarch64.
8+
git cherry-pick a14f10e8825fd25e0615095feaa548b1938fffac -X theirs --no-commit
9+
git cherry-pick e76f8fda2d959fdf7a262eb539a4002a6a0c900f -X theirs --no-commit

‎.github/workflows/concretize-template.yaml

+7-5
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,21 @@ jobs:
1313
fail-fast: false
1414
steps:
1515
- name: Checkout Spack
16-
uses: actions/checkout@v3
16+
uses: actions/checkout@v4
1717
with:
1818
repository: spack/spack
1919
path: spack
20+
ref: 6cb16c39ab85fbc211e50be804fa7a15f24ccebc
2021

2122
- name: Checkout Key4hep
22-
uses: actions/checkout@v3
23+
uses: actions/checkout@v4
2324
with:
2425
repository: key4hep/key4hep-spack
2526
path: key4hep-spack
27+
ref: 334aa25cf90cbbaf693ac29509d4d7b790effecb
2628

2729
- name: Checkout this repo
28-
uses: actions/checkout@v3
30+
uses: actions/checkout@v4
2931
with:
3032
path: mucoll-spack
3133

@@ -42,7 +44,7 @@ jobs:
4244
spack spec -Nt > ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log
4345
4446
- name: Store artifacts
45-
uses: actions/upload-artifact@v3
47+
uses: actions/upload-artifact@v4
4648
with:
4749
name: spec-list
48-
path: ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log
50+
path: ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log

‎.github/workflows/concretize.yaml

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: concretize-ubuntu
22

3-
on: [push, pull_request]
3+
on: [pull_request]
44

55
jobs:
66
concretize:
@@ -10,5 +10,4 @@ jobs:
1010
target: [common, release, release-debug]
1111
uses: ./.github/workflows/concretize-template.yaml
1212
with:
13-
target: ${{ matrix.target }}
14-
13+
target: ${{ matrix.target }}

‎.github/workflows/full-rebuild.yml

+164
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
name: Re-build and publish All
2+
on: [push, workflow_dispatch]
3+
4+
env:
5+
REGISTRY: ghcr.io
6+
7+
permissions:
8+
contents: read
9+
packages: write
10+
id-token: write
11+
attestations: write
12+
13+
jobs:
14+
build-spack:
15+
runs-on: ubuntu-latest
16+
strategy:
17+
matrix:
18+
os: [{dir: AlmaLinux9,suffix: alma9}]
19+
env:
20+
IMAGE_NAME: mucoll-spack
21+
steps:
22+
- name: Checkout
23+
uses: actions/checkout@v4
24+
with:
25+
fetch-depth: 0
26+
- name: Set up Docker buildx
27+
uses: docker/setup-buildx-action@v3
28+
- name: Login to container registry
29+
uses: docker/login-action@v3
30+
with:
31+
registry: ${{ env.REGISTRY }}
32+
username: ${{ github.actor }}
33+
password: ${{ secrets.GITHUB_TOKEN }}
34+
- name: Make owner name lowercase
35+
run: |
36+
echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
37+
env:
38+
OWNER: '${{ github.repository_owner }}'
39+
- name: Extract metadata for Docker
40+
id: meta
41+
uses: docker/metadata-action@v5
42+
with:
43+
images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy
44+
- name: Build and Push
45+
id: docker_build
46+
uses: docker/build-push-action@v6
47+
with:
48+
context: ./${{matrix.os.dir}}
49+
file: ./${{matrix.os.dir}}/Dockerfile-spack
50+
push: true
51+
build-args: |
52+
VERSION=${{ steps.meta.outputs.version }}
53+
REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}}
54+
tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}}
55+
labels: ${{ steps.meta.outputs.labels }}
56+
build-externals:
57+
runs-on: ubuntu-latest
58+
needs: build-spack
59+
strategy:
60+
matrix:
61+
os: [{dir: AlmaLinux9,suffix: alma9}]
62+
env:
63+
IMAGE_NAME: mucoll-externals
64+
steps:
65+
- name: Free Disk Space (Ubuntu)
66+
uses: endersonmenezes/free-disk-space@v2
67+
with:
68+
remove_android: true
69+
remove_dotnet: true
70+
remove_haskell: true
71+
remove_tool_cache: true
72+
remove_swap: true
73+
remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*"
74+
remove_packages_one_command: true
75+
remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell"
76+
testing: false
77+
- name: Checkout
78+
uses: actions/checkout@v4
79+
with:
80+
fetch-depth: 0
81+
- name: Set up Docker Buildx
82+
uses: docker/setup-buildx-action@v3
83+
- name: Login to DockerHub
84+
uses: docker/login-action@v3
85+
with:
86+
registry: ${{ env.REGISTRY }}
87+
username: ${{ github.actor }}
88+
password: ${{ secrets.GITHUB_TOKEN }}
89+
- name: lowercase owner name
90+
run: |
91+
echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
92+
env:
93+
OWNER: '${{ github.repository_owner }}'
94+
- name: Docker Meta
95+
id: meta
96+
uses: docker/metadata-action@v5
97+
with:
98+
images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy
99+
- name: Build and Push
100+
id: docker_build
101+
uses: docker/build-push-action@v6
102+
with:
103+
push: true
104+
context: ./${{matrix.os.dir}}
105+
file: ./${{matrix.os.dir}}/Dockerfile-externals
106+
build-args: |
107+
VERSION=${{ steps.meta.outputs.version }}
108+
REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}}
109+
tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}}
110+
labels: ${{ steps.meta.outputs.labels }}
111+
build-sim:
112+
runs-on: ubuntu-latest
113+
needs: build-externals
114+
strategy:
115+
matrix:
116+
os: [{dir: AlmaLinux9,suffix: alma9}]
117+
steps:
118+
- name: Free Disk Space (Ubuntu)
119+
uses: endersonmenezes/free-disk-space@v2
120+
with:
121+
remove_android: true
122+
remove_dotnet: true
123+
remove_haskell: true
124+
remove_tool_cache: true
125+
remove_swap: true
126+
remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*"
127+
remove_packages_one_command: true
128+
remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell"
129+
testing: false
130+
- name: Checkout
131+
uses: actions/checkout@v4
132+
with:
133+
fetch-depth: 0
134+
- name: Set up Docker Buildx
135+
uses: docker/setup-buildx-action@v3
136+
- name: Login to DockerHub
137+
uses: docker/login-action@v3
138+
with:
139+
registry: ${{ env.REGISTRY }}
140+
username: ${{ github.actor }}
141+
password: ${{ secrets.GITHUB_TOKEN }}
142+
- name: lowercase owner name
143+
run: |
144+
echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
145+
env:
146+
OWNER: '${{ github.repository_owner }}'
147+
- name: Docker Meta
148+
id: meta
149+
uses: docker/metadata-action@v5
150+
with:
151+
images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy
152+
- name: Build and Push
153+
id: docker_build
154+
uses: docker/build-push-action@v6
155+
with:
156+
push: true
157+
context: ./${{matrix.os.dir}}
158+
file: ./${{matrix.os.dir}}/Dockerfile-sim
159+
build-args: |
160+
VERSION=${{ steps.meta.outputs.version }}
161+
REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}}
162+
GITHUB_REPOSITORY=${{ github.repository }}
163+
MUCOLL_SHA=${{ github.sha }}
164+
tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}}

‎.key4hep-commit

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
dc0b2bc4aaad8f3ef5f3c7f62ff5b0277a554eee
1+
4b46ee141dabde811a047ca635ca73605b704144

‎.latest-commit

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
f5946c4621035dd466953c8d2664ff5f82f38138
1+
6cb16c39ab85fbc211e50be804fa7a15f24ccebc

‎AlmaLinux9/Dockerfile-externals

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
###############################################################################
2+
# Repository: ${REPOSITORY}/key4hep-base
3+
# Tag: ${VERSION}-alma9
4+
###############################################################################
5+
6+
ARG VERSION=main
7+
ARG REPOSITORY=madbaron
8+
FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9
9+
10+
# Picking specific key4hep commit
11+
ARG KEY4HEP_COMMIT=4b46ee141dabde811a047ca635ca73605b704144
12+
13+
RUN source /opt/setup_spack.sh && \
14+
REPOPATH=${SPACK_ROOT}/var/key4hep-spack && \
15+
git clone -b main --single-branch https://github.com/key4hep/key4hep-spack ${REPOPATH} && \
16+
if [ -n "${KEY4HEP_COMMIT}" ]; then \
17+
cd ${REPOPATH}; \
18+
git checkout ${KEY4HEP_COMMIT}; \
19+
fi
20+
21+
# Apply our patches to spack
22+
COPY apply_patches.sh ./apply_patches.sh
23+
24+
RUN source /opt/setup_spack.sh && \
25+
./apply_patches.sh ${SPACK_ROOT}/var/key4hep-spack
26+
27+
# Add the package repositories
28+
RUN source /opt/setup_spack.sh && \
29+
spack repo add --scope system ${SPACK_ROOT}/var/key4hep-spack
30+
31+
# Create the release environment
32+
RUN source /opt/setup_spack.sh && \
33+
cd ${SPACK_ROOT}/var/key4hep-spack/environments/key4hep-dev-external && \
34+
spack env activate . && \
35+
cd - && \
36+
echo "source /opt/setup_spack.sh" > ${HOME}/setup_env.sh && \
37+
echo "cd ${SPACK_ROOT}/var/key4hep-spack/environments/key4hep-dev-external" >> ${HOME}/setup_env.sh && \
38+
echo "spack env activate ." >> ${HOME}/setup_env.sh && \
39+
echo "cd -" >> ${HOME}/setup_env.sh && \
40+
echo "spack env status" >> ${HOME}/setup_env.sh
41+
42+
# Concretizing the stack reusing system packages as external
43+
RUN source ${HOME}/setup_env.sh && \
44+
spack concretize --reuse
45+
46+
# Installing fragments of dependency tree in separate layers for cached debugging
47+
ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast"
48+
49+
RUN source ${HOME}/setup_env.sh && \
50+
spack spec -NIt && \
51+
spack install ${SPACK_INSTALL_OPTS} && \
52+
spack clean -a
53+
54+
RUN source ${HOME}/setup_env.sh && \
55+
echo "source ${SPACK_ENV}/.spack-env/view/setup.sh" > /opt/setup_k4_externals.sh && \
56+
echo "alias setup_k4_externals=\"source /opt/setup_k4_externals.sh\"" >> /etc/profile.d/aliases.sh

‎AlmaLinux9/Dockerfile-sim

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
###############################################################################
2+
# Repository: ${IMAGE}/mucoll-sim
3+
# Tag: ${VERSION}-alma9
4+
###############################################################################
5+
6+
ARG VERSION=master
7+
ARG REPOSITORY=madbaron
8+
FROM ${REPOSITORY}/mucoll-externals:${VERSION}-alma9
9+
10+
ARG MUCOLL_SHA=e49a0639ff987a84fa0c6263e068776f9d7b6f67
11+
ARG GITHUB_REPOSITORY=kkrizka/mucoll-spack
12+
13+
# Adding repository: MuColl
14+
RUN source /opt/setup_spack.sh && \
15+
REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \
16+
git clone https://github.com/${GITHUB_REPOSITORY} ${REPOPATH} && \
17+
cd ${REPOPATH} && \
18+
git checkout ${MUCOLL_SHA} && \
19+
cd -
20+
21+
# Apply our patches to spack
22+
COPY apply_patches.sh ./apply_patches.sh
23+
24+
RUN source /opt/setup_spack.sh && \
25+
./apply_patches.sh ${SPACK_ROOT}/var/mucoll-spack
26+
27+
# Add the package repositories
28+
RUN source /opt/setup_spack.sh && \
29+
spack repo add --scope system ${SPACK_ROOT}/var/mucoll-spack
30+
31+
# Create the release environment
32+
RUN source /opt/setup_spack.sh && \
33+
cd ${SPACK_ROOT}/var/mucoll-spack/environments/mucoll-release && \
34+
spack env activate . && \
35+
cd - && \
36+
echo "source /opt/setup_spack.sh" > ${HOME}/setup_env.sh && \
37+
echo "cd ${SPACK_ROOT}/var/mucoll-spack/environments/mucoll-release" >> ${HOME}/setup_env.sh && \
38+
echo "spack env activate ." >> ${HOME}/setup_env.sh && \
39+
echo "cd -" >> ${HOME}/setup_env.sh && \
40+
echo "spack env status" >> ${HOME}/setup_env.sh
41+
42+
# Concretizing the MuColl stack reusing system packages as external
43+
RUN source ${HOME}/setup_env.sh && \
44+
spack concretize --reuse
45+
46+
# Installing fragments of dependency tree in separate layers for cached debugging
47+
ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast"
48+
49+
RUN source ${HOME}/setup_env.sh && \
50+
spack spec -NIt && \
51+
spack install ${SPACK_INSTALL_OPTS} && \
52+
spack clean -a
53+
54+
RUN source ${HOME}/setup_env.sh && \
55+
echo "source ${SPACK_ENV}/.spack-env/view/setup.sh" > /opt/setup_mucoll.sh && \
56+
echo "alias setup_mucoll=\"source /opt/setup_mucoll.sh\"" >> /etc/profile.d/aliases.sh
57+
58+
# Setting up users to be used for the local environment setup
59+
RUN useradd --shell /bin/bash --create-home mucoll
60+
USER mucoll
61+
WORKDIR /home/mucoll
62+
63+
ENTRYPOINT ["/bin/bash", "--login"]
64+

0 commit comments

Comments
 (0)