Skip to content

Commit df4c8b5

Browse files
phwissmannphwissmann
authored
Update archiver to uv (#153)
A couple of maintenance items: - Introduce UV also in the archiver in order to aligned with the rest of the repository ~- Use specific and configurable user in docker container that writes to/reads from the LTS; these are the permission flags set on the files in the LTS and therefore need to be consistent~ - Improved CI by adding test report and run steps conditionally
2 parents b677492 + b455060 commit df4c8b5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+3609
-3459
lines changed

‎.github/workflows/build-docker-images.yml

+62-27
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,14 @@ jobs:
3434
filters: |
3535
backend:
3636
- 'backend/**'
37+
api:
38+
- 'backend/api/**'
39+
archiver:
40+
- 'backend/archiver/**'
41+
prefect:
42+
- 'backend/prefect/prefect/**'
43+
44+
3745
3846
# test_api_server:
3947
# runs-on: ubuntu-latest
@@ -60,38 +68,66 @@ jobs:
6068
# if: ${{ always() }}
6169
#
6270

63-
test_archiver_flows:
71+
test_flows:
6472
runs-on: ubuntu-latest
6573
needs: detect_changes
74+
if: ${{ needs.detect_changes.outputs.archiver == 'true' }} || github.event_name != 'pull_request'
75+
defaults:
76+
run:
77+
working-directory: ./backend/archiver
6678
steps:
6779
- uses: actions/checkout@v4
68-
- name: Set up Python
80+
81+
- name: Install uv
82+
uses: astral-sh/setup-uv@v5
83+
with:
84+
version: "0.6.1"
85+
86+
- name: "Set up Python"
6987
uses: actions/setup-python@v5
7088
with:
71-
python-version: "3.11"
72-
- name: Install dependencies and run tests
73-
working-directory: ./backend/archiver
89+
python-version-file: backend/archiver/pyproject.toml
90+
91+
- name: Install the project
92+
run: |
93+
uv sync --all-extras --dev
94+
uv add pytest-md-report
95+
96+
- name: Run tests
97+
env:
98+
REPORT_OUTPUT: md_report.md
99+
shell: bash
74100
run: |
75-
python -m pip install --upgrade pip
76-
python -m pip install pipenv
77-
pipenv sync -d
78-
pipenv run pytest --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html
79-
- name: Upload pytest test results
80-
uses: actions/upload-artifact@v4
101+
echo "REPORT_FILE=${REPORT_OUTPUT}" >> "$GITHUB_ENV"
102+
uv run pytest --md-report --md-report-flavor gfm --md-report-exclude-outcomes passed skipped xpassed --md-report-output "$REPORT_OUTPUT" --cov=. --cov-report=xml --cov-report=html
103+
104+
- name: Render the report to the PR when tests fail
105+
uses: marocchino/sticky-pull-request-comment@v2
106+
if: failure()
81107
with:
82-
name: archiver-test-results
83-
path: ./junit/test-results.xml
84-
# Use always() to always run this step to publish test results when there are test failures
85-
if: ${{ always() }}
108+
header: test-report
109+
recreate: true
110+
path: ${{ env.REPORT_FILE }}
86111

87-
build_runtime_image:
112+
- name: Output reports to the job summary when tests fail
113+
shell: bash
114+
run: |
115+
if [ -f "$REPORT_FILE" ]; then
116+
echo "<details><summary>Failed Test Report</summary>" >> $GITHUB_STEP_SUMMARY
117+
echo "" >> $GITHUB_STEP_SUMMARY
118+
cat "$REPORT_FILE" >> $GITHUB_STEP_SUMMARY
119+
echo "" >> $GITHUB_STEP_SUMMARY
120+
echo "</details>" >> $GITHUB_STEP_SUMMARY
121+
fi
122+
123+
build_flows_image:
88124
runs-on: ubuntu-latest
89125
permissions:
90126
packages: write
91127
contents: read
92128

93-
needs: test_archiver_flows
94-
if: ${{ needs.detect_changes.outputs.backend == 'true' }} || github.event_name != 'pull_request'
129+
needs: test_flows
130+
if: ${{ needs.detect_changes.outputs.archiver == 'true' }} || github.event_name != 'pull_request'
95131

96132
steps:
97133
- uses: actions/checkout@v4
@@ -114,10 +150,9 @@ jobs:
114150
id: build-and-push
115151
uses: docker/build-push-action@v6
116152
with:
117-
context: ./
118-
file: ./backend/prefect-runtime.Dockerfile
153+
context: ./backend/archiver
154+
file: ./backend/prefect/runtime.Dockerfile
119155
push: ${{ github.event_name != 'pull_request' }}
120-
build-args: PREFECT_VERSION=3.0.4-python3.11
121156
tags: ${{ steps.meta.outputs.tags }}
122157
labels: ${{ steps.meta.outputs.labels }}
123158

@@ -127,8 +162,8 @@ jobs:
127162
packages: write
128163
contents: read
129164

130-
needs: test_archiver_flows
131-
if: ${{ needs.detect_changes.outputs.backend == 'true' }} || github.event_name != 'pull_request'
165+
needs: test_flows
166+
if: ${{ needs.detect_changes.outputs.prefect == 'true' }} || github.event_name != 'pull_request'
132167

133168
steps:
134169
- uses: actions/checkout@v4
@@ -152,20 +187,20 @@ jobs:
152187
uses: docker/build-push-action@v6
153188
with:
154189
context: ./backend/
155-
file: ./backend/prefect-config.Dockerfile
190+
file: ./backend/prefect/config.Dockerfile
156191
push: ${{ github.event_name != 'pull_request' }}
157-
build-args: PREFECT_VERSION=3.0.4-python3.11
192+
build-args: PREFECT_VERSION=3.1.11-python3.11
158193
tags: ${{ steps.meta.outputs.tags }}
159194
labels: ${{ steps.meta.outputs.labels }}
160195

161-
build_openem_backend:
196+
build_service_api_image:
162197
runs-on: ubuntu-latest
163198

164199
permissions:
165200
packages: write
166201
contents: read
167202

168-
# needs: test_api_server
203+
needs: detect_changes
169204
if: ${{ needs.detect_changes.outputs.backend == 'true' }} || github.event_name != 'pull_request'
170205

171206
steps:

‎archiver-service.docker-compose.yml

+18-10
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ services:
108108
container_name: prefect-config
109109
build:
110110
context: ./backend/
111-
dockerfile: ./prefect-config.Dockerfile
111+
dockerfile: ./prefect/config.Dockerfile
112112
args:
113113
PREFECT_VERSION: ${PREFECT_VERSION}
114114
tags:
@@ -130,12 +130,12 @@ services:
130130
- SCICAT_PASSWORD_FILE=/run/secrets/scicat_pass
131131
volumes:
132132
- type: bind
133-
source: ./backend/prefect-vars.toml
134-
target: /var/local/prefect-vars.toml
133+
source: ./backend/prefect/vars.toml
134+
target: /var/local/vars.toml
135135
- type: bind
136-
source: ./backend/concurrency-limits.toml
136+
source: ./backend/prefect/concurrency-limits.toml
137137
target: /var/local/concurrency-limits.toml
138-
command: "-v /var/local/prefect-vars.toml -s MINIO_USER_FILE MINIO_PASSWORD_FILE GITHUB_USER_FILE GITHUB_PASSWORD_FILE SCICAT_USER_FILE SCICAT_PASSWORD_FILE -l /var/local/concurrency-limits.toml"
138+
command: "-v /var/local/vars.toml -s MINIO_USER_FILE MINIO_PASSWORD_FILE GITHUB_USER_FILE GITHUB_PASSWORD_FILE SCICAT_USER_FILE SCICAT_PASSWORD_FILE -l /var/local/concurrency-limits.toml"
139139
depends_on:
140140
prefect-server:
141141
condition: service_healthy
@@ -147,7 +147,14 @@ services:
147147
- scicat_user
148148
- scicat_pass
149149
prefect-flows-deployment:
150-
image: prefecthq/prefect:${PREFECT_VERSION}
150+
# Uses the runtime image: prefect will import the code and there it needs all packages installed
151+
image: ${PREFECT_RUNTIME_IMAGE_NAME}:${OPENEM_IMAGE_TAG:-latest}
152+
build:
153+
context: ./backend/archiver
154+
dockerfile: ../prefect/runtime.Dockerfile
155+
args:
156+
LTS_ROOT_FOLDER: ${LTS_ROOT_FOLDER}
157+
PREFECT_VERSION: ${PREFECT_VERSION}
151158
container_name: prefect-flows-deployment
152159
networks:
153160
- scopemarchiver_network
@@ -161,12 +168,13 @@ services:
161168
- PREFECT_WORKER_LTS_VOLUME_NAME=scopemarchiver_${PREFECT_WORKER_LTS_VOLUME_NAME}
162169
- LTS_ROOT_FOLDER=${LTS_ROOT_FOLDER}
163170
- PREFECT_NETWORK=scopemarchiver_network
164-
command: prefect --no-prompt deploy --all --prefect-file /home/dev/backend/prefect.yaml
165-
working_dir: /home/dev/
171+
- PREFECT_RUNTIME_IMAGE=${PREFECT_RUNTIME_IMAGE_NAME}:${OPENEM_IMAGE_TAG:-latest}
172+
command: prefect --no-prompt deploy --all --prefect-file /prefect.yaml
173+
working_dir: /app/backend
166174
volumes:
167175
- type: bind
168-
source: ./
169-
target: /home/dev/
176+
source: ./backend/prefect/prefect.yaml
177+
target: /prefect.yaml
170178
read_only: true
171179
- /usr/bin/docker:/usr/bin/docker
172180
- /var/run/docker.sock:/var/run/docker.sock

‎backend/.vscode/launch.json

+1-18
Original file line numberDiff line numberDiff line change
@@ -20,24 +20,7 @@
2020
},
2121
"module": "archiver.flows",
2222
"jinja": true
23-
},
24-
{
25-
"name": "Prefect Config",
26-
"type": "debugpy",
27-
"request": "launch",
28-
"cwd": "${workspaceFolder}",
29-
"env": {
30-
"PREFECT_API_URL": "http://localhost:4200/api",
31-
"MINIO_PASSWORD_FILE": "${workspaceFolder}/../.secrets/miniopass.txt"
32-
},
33-
"program": "prefect-config.py",
34-
"args": [
35-
"-s",
36-
"MINIO_PASSWORD_FILE",
37-
"-l",
38-
"${workspaceFolder}/concurrency-limits.toml"
39-
]
40-
},
23+
}
4124
],
4225
"compounds": [
4326
{

‎backend/api/.dockerignore

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.venv
2+
__pycache__
3+
.pytest_cache
4+
.vscode

‎backend/api/Dockerfile

+6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
44
# Change the working directory to the `app` directory
55
WORKDIR /app
66

7+
# Enable bytecode compilation
8+
ENV UV_COMPILE_BYTECODE=1
9+
10+
# Copy from the cache instead of linking since it's a mounted volume
11+
ENV UV_LINK_MODE=copy
12+
713
# Install dependencies
814
RUN --mount=type=cache,target=/root/.cache/uv \
915
--mount=type=bind,source=uv.lock,target=uv.lock \

‎backend/archiver/.vscode/launch.json

-18
Original file line numberDiff line numberDiff line change
@@ -20,30 +20,12 @@
2020
},
2121
"module": "flows",
2222
"jinja": true
23-
},
24-
{
25-
"name": "Prefect Config",
26-
"type": "debugpy",
27-
"request": "launch",
28-
"cwd": "${workspaceFolder}",
29-
"env": {
30-
"PREFECT_API_URL": "http://localhost:4200/api",
31-
"MINIO_PASSWORD_FILE": "${workspaceFolder}/../.secrets/miniopass.txt"
32-
},
33-
"program": "prefect-config.py",
34-
"args": [
35-
"-s",
36-
"MINIO_PASSWORD_FILE",
37-
"-l",
38-
"${workspaceFolder}/concurrency-limits.toml"
39-
]
4023
}
4124
],
4225
"compounds": [
4326
{
4427
"name": "Archiver Service",
4528
"configurations": [
46-
"Backend API",
4729
"Prefect Flows"
4830
],
4931
"stopAll": true

‎backend/archiver/Pipfile

-12
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,8 @@ verify_ssl = true
44
name = "pypi"
55

66
[packages]
7-
prefect = "==3.1.11"
8-
requests = "*"
9-
strenum = "*"
10-
pytest-asyncio = "*"
11-
requests-mock = "*"
12-
prefect-docker = "*"
13-
boto3 = "*"
14-
moto = {extras = ["s3"], version = "*"}
157

168
[dev-packages]
17-
autopep8 = "*"
18-
pytest = "*"
19-
requests-mock = "*"
20-
pytest-cov = "*"
219

2210
[requires]
2311
python_version = "3.11"

0 commit comments

Comments
 (0)