-
Notifications
You must be signed in to change notification settings - Fork 24
166 lines (150 loc) · 9.37 KB
/
Example.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
name: Examples
on:
workflow_dispatch:
pull_request:
push:
branches:
- main
paths:
- 'src/**'
- 'examples/**'
- '.github/workflows/Example.yml'
- 'Project.toml'
jobs:
# we need to provide all examples seperately, as they are computed in parallel by this action
jupyter:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [windows-latest] # ubuntu-latest
file-name: [inputs, manipulation, modelica_conference_2021, multiple_instances, multiprocessing, multithreading, parameterize, simulate, parameter_optimization]
julia-version: ['1.10']
julia-arch: [x64]
experimental: [false]
steps:
- name: "Check out repository"
uses: actions/checkout@v4
- name: "Set up Julia"
uses: julia-actions/setup-julia@v2
with:
version: ${{ matrix.julia-version }}
arch: ${{ matrix.julia-arch }}
- name: "Install dependencies"
run: julia --project=examples/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
- name: "Install packages"
run: pip install jupyter nbconvert
- name: "Execute notebook"
env:
FILE: examples/jupyter-src/${{ matrix.file-name }}.ipynb
run: jupyter nbconvert --ExecutePreprocessor.kernel_name="julia-${{ matrix.julia-version }}" --to notebook --inplace --execute ${{ env.FILE }}
# required, as newer versions of nbconvert create multiple ouput cells for graphics outputs (eg. plots).
# "--to markdown" fails to select the correct way to include those afterwards and prints the svg contents directly into the md file instead of linking them as !(svg)[...]
# if the text/html output of the plot is removed, "--to markdown" links the svgs correctly afterwards
- name: "Fix SVGs" # on notebook (json) level: removes all cells, that have the tag "text/html" AND have "<?xml " within their content
run: julia -e 'using Pkg; Pkg.add("JSON"); using JSON; data = JSON.parsefile(joinpath("examples", "jupyter-src", "${{ matrix.file-name }}.ipynb"); use_mmap=false); function search_and_remove!(json, tag, content=""); if typeof(json) == Vector{Any}; for e in json; search_and_remove!(e, tag, content); end; elseif typeof(json) == Dict{String, Any}; for (t, c) in json; if contains(t, tag) && (content == "" || any([contains(line, content) for line in c])); pop!(json, t); else; search_and_remove!(c, tag, content); end; end; end; end; search_and_remove!(data, "text/html", "<?xml "); open(joinpath("examples", "jupyter-src", "tmp_${{ matrix.file-name }}.ipynb"),"w") do f; JSON.print(f, data, 1); end; mv(joinpath("examples", "jupyter-src", "tmp_${{ matrix.file-name }}.ipynb"), joinpath("examples", "jupyter-src", "${{ matrix.file-name }}.ipynb"); force=true);'
- name: "Export notebook to jl and md"
env:
FILE: examples/jupyter-src/${{ matrix.file-name }}.ipynb
run: |
jupyter nbconvert --to script ${{ env.FILE }}
jupyter nbconvert --to markdown ${{ env.FILE }}
# required, as nbconvert does not fail, if the notbook contains errors in the julia cells. We do not want broken examples to push and be reported as successful
- name: "run generated jl script to determine success of example building"
run: julia --project=examples/ examples/jupyter-src/${{ matrix.file-name }}.jl
- name: "auto-commit (retry on merge)"
# only push to examples branch, if we execute on the current main branch of this repo and the example build successfully!
if: success() && github.event_name != 'pull_request' && github.ref_name == 'main'
# retrying is required, as a race condition can occur: all parallel example builds try to push to the same branch. Instead of merging, retrying is the cleanest solution
# successful termination is guaranteed, if max_attemts is >= the number of jupyter-examples plus one (pluto-slider-server) as each race condition ends with one example successfully pushing, as long as ONLY ONE PUSHING ACTION-RUN IS RUNNING AT THE SAME TIME (only pushing for runs on main branch)
# successful termination is still likely, even if max_attemts is < number of examples, as long as runtimes of most examples do not match eachother. This is also why there is no prevention of simultaneous runs of this action implemented.
uses: nick-fields/retry@v3
env:
CI_COMMIT_MESSAGE: jupyter-example-${{ matrix.file-name }}-${{ matrix.os }}-${{ matrix.julia-version }}-${{ matrix.julia-arch }}-${{ matrix.experimental }}[${{ github.ref_name }}]
CI_COMMIT_AUTHOR: github-actions[bot]
EXAMPLES_PATH: examples/jupyter-src
# Fetch all and clear the stash. Include all files from the examples folder (just generated those) to the stash and switch the branch.
# Reset the branch and remove all old files of "this" example in the examples folder.
# Checkout the last stash to restore the new notebooks and apply the stash index to restore all other new files in the folder.
# Commit changes and try to push. If another example was faster and pushed just before us, we undo our commit to not have to deal with the merge, but keep our generated files and just retry (fetching the commit from the faster push at the beginning).
with:
timeout_minutes: 999
max_attempts: 10
warning_on_retry: false
shell: bash
command: |
git fetch --all
git stash clear
git stash --include-untracked -- ${{ env.EXAMPLES_PATH }}/${{ matrix.file-name }}*
git switch examples
git reset --hard origin/examples
rm -r ${{ env.EXAMPLES_PATH }}/${{ matrix.file-name }}*
git checkout stash -f -- ${{ env.EXAMPLES_PATH }}
git stash apply --index
git stash drop
git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}"
git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com"
git config --global core.autocrlf false
git pull
git reset
git add ${{ env.EXAMPLES_PATH }}/${{ matrix.file-name }}*
git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
git push origin examples || (git reset --soft HEAD~1 && (exit 1))
# we do not need to provide all examples seperately, as pluto slider server just compiles everything in the pluto-src folder
pluto:
runs-on: ubuntu-latest
steps:
- name: "Check out repository"
uses: actions/checkout@v4
- name: "Set up Julia"
uses: julia-actions/setup-julia@v2
with:
version: '1.10'
- run: julia -e 'using Pkg; Pkg.add("PlutoSliderServer");'
- run: julia -e 'using PlutoSliderServer; PlutoSliderServer.export_directory("examples/pluto-src")'
- name: "auto-commit (retry on merge)"
# only push to examples branch, if we execute on the current main branch of this repo and the examples build successfully!
if: success() && github.event_name != 'pull_request' && github.ref_name == 'main'
# retrying is required, as a race condition can occur: all parallel example builds (one pluto and multiple jupyter) try to push to the same branch. Instead of merging, retrying is the cleanest solution
uses: nick-fields/retry@v3
env:
CI_COMMIT_MESSAGE: pluto-examples[${{ github.ref_name }}]
CI_COMMIT_AUTHOR: github-actions[bot]
EXAMPLES_PATH: examples/pluto-src
# Fetch all and clear the stash list. Include all files from the examples folder to the stash and switch the branch.
# Reset the branch and remove all current files in the examples folder.
# Checkout the last stash and apply the stash index to restore all new files in the folder.
with:
timeout_minutes: 999
max_attempts: 10
warning_on_retry: false
shell: bash
command: |
git fetch --all
git stash clear
git stash --include-untracked -- ${{ env.EXAMPLES_PATH }}
git switch examples
git reset --hard origin/examples
rm -r ${{ env.EXAMPLES_PATH }}/*
git checkout stash -f -- ${{ env.EXAMPLES_PATH }}
git stash apply --index
git stash drop
git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}"
git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com"
git config --global core.autocrlf false
git pull
git reset
git add ${{ env.EXAMPLES_PATH }}
git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
git push origin examples || (git reset --soft HEAD~1 && (exit 1))
call-docu:
needs: [jupyter, pluto]
# only trigger docu if examples branch contents could have changes (see jupyter and pluto job "auto-commit" restrictions)
if: github.event_name != 'pull_request' && github.ref_name == 'main'
runs-on: ubuntu-latest
steps:
# Trigger an repoisitory dispath event
- name: Repository Dispatch
uses: peter-evans/repository-dispatch@v3
with:
event-type: trigger-docu