-
Notifications
You must be signed in to change notification settings - Fork 44
180 lines (170 loc) · 8.4 KB
/
odh-notebooks-sync.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
# The aim of this GitHub workflow is to update the pipfile to sync with Codeflare-SDK release.
name: Sync ODH-notebooks with codeflare-sdk release
on:
workflow_dispatch:
inputs:
upstream-repository-organization:
required: true
description: "Owner of target upstream notebooks repository used to open a PR against"
default: "opendatahub-io"
codeflare-repository-organization:
required: true
description: "Owner of origin notebooks repository used to open a PR"
default: "project-codeflare"
codeflare_sdk_release_version:
required: true
description: "Provide version of the Codeflare-SDK release"
env:
BRANCH_NAME: main
CODEFLARE_RELEASE_VERSION: ${{ github.event.inputs.codeflare_sdk_release_version }}
UPDATER_BRANCH: odh-sync-updater-${{ github.run_id }}
UPSTREAM_OWNER: ${{ github.event.inputs.upstream-repository-organization }}
REPO_OWNER: ${{ github.event.inputs.codeflare-repository-organization }}
REPO_NAME: notebooks
GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }}
MINIMUM_SUPPORTED_PYTHON_VERSION: 3.9
jobs:
build:
runs-on: ubuntu-22.04-8core
steps:
- name: Clone repository and Sync
run: |
git clone https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git $REPO_NAME
cd $REPO_NAME
git remote add upstream https://github.com/$UPSTREAM_OWNER/$REPO_NAME.git
git config --global user.email "[email protected]"
git config --global user.name "codeflare-machine-account"
git remote -v
git pull upstream main && git push origin main
- name: Setup Python environment
uses: actions/setup-python@v4
with:
python-version: |
3.9
- name: Install pipenv and pip-versions
run: pip install pipenv pip-versions
- name: Update Pipfiles in accordance with Codeflare-SDK latest release
run: |
package_name=codeflare-sdk
available_python_versions=("3.9") # add space separated python versions according to 'python-versions' specified in 'Setup Python Environment' step
install_package_using_pipenv(){
if ! pipenv install ${package_name}~="${CODEFLARE_RELEASE_VERSION}"; then
echo "Failed to install ${package_name} with version ${CODEFLARE_RELEASE_VERSION} in $dir"
exit 1
fi
# Lock dependencies, ensuring pre-release are included and clear previous state
if ! pipenv lock --pre --clear ; then
echo "Failed to lock dependencies"
exit 1
fi
# remove virtual env and clear cache
if ! pipenv --rm --clear ; then
echo "Failed to remove virtual environment"
exit 1
fi
}
# Get the list of available versions for the package
if ! versions=$(pipenv run pip-versions list $package_name);then
echo "Failed to retrieve versions for $package_name"
exit 1
fi
# Check if the desired version exists in the list
if echo "$versions" | grep -q "${CODEFLARE_RELEASE_VERSION}"; then
echo "Version ${CODEFLARE_RELEASE_VERSION} is available for $package_name"
# list all Pipfile paths having Codeflare-SDK listed
paths+=($(grep -rl "${package_name} = \"~=.*\""))
# Extracting only directories from file paths, excluding a `.gitworkflow` directory
directories=()
exclude_directories=(
".git/objects/pack"
".github/workflows/",
)
for path in "${paths[@]}"; do
current_dir=$(dirname "$path")
#Check if current_dir is not in exclude_directories list
if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then
#Check if Pipfile exists in current_dir
if ls "$current_dir"/Pipfile* 1> /dev/null 2>&1;then
directories+=("$current_dir")
fi
fi
done
# Remove duplicates
directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
# Print the directories for verification
echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):"
for dir in "${directories[@]}"; do
echo "- $dir"
done
# iterate over the directories and update Pipfile
counter=0
total=${#directories[@]}
for dir in "${directories[@]}"; do
counter=$((counter+1))
echo "--Processing directory $counter '$dir' of total $total"
cd "$dir"
minimum_supported_python_version_major=$(echo "${MINIMUM_SUPPORTED_PYTHON_VERSION}" | awk -F '.' '{print $1}') #integer of MINIMUM_SUPPORTED_PYTHON_VERSION env variable
minimum_supported_python_version_minor=$(echo "${MINIMUM_SUPPORTED_PYTHON_VERSION}" | awk -F '.' '{print $2}') #decimal of MINIMUM_SUPPORTED_PYTHON_VERSION env variable
if ! [ -f "Pipfile" ]; then
if [ -f "Pipfile.cpu" ]; then
pipfile_python_version=$(grep -E '^python_version' ./Pipfile.cpu | cut -d '"' -f 2) # extracted from pipfile.cpu
fi
else
pipfile_python_version=$(grep -E '^python_version' ./Pipfile | cut -d '"' -f 2) # extracted from pipfile
fi
pipfile_python_version_major=$(echo "$pipfile_python_version" | awk -F '.' '{print $1}')
pipfile_python_version_minor=$(echo "$pipfile_python_version" | awk -F '.' '{print $2}')
if [[ " ${available_python_versions[@]} " =~ " ${pipfile_python_version} " && "$pipfile_python_version_major" -ge "$minimum_supported_python_version_major" && "$pipfile_python_version_minor" -ge "$minimum_supported_python_version_minor" ]]; then
if ! [ -f "Pipfile" ]; then
if [ -f "Pipfile.cpu" ]; then
mv Pipfile.cpu Pipfile
mv Pipfile.lock.cpu Pipfile.lock
#install specified package
install_package_using_pipenv
mv Pipfile.lock Pipfile.lock.cpu
mv Pipfile Pipfile.cpu
fi
if [ -f "Pipfile.gpu" ]; then
mv Pipfile.gpu Pipfile
mv Pipfile.lock.gpu Pipfile.lock
#install specified package
install_package_using_pipenv
mv Pipfile.lock Pipfile.lock.gpu
mv Pipfile Pipfile.gpu
fi
else
#install specified package
install_package_using_pipenv
fi
else
echo "Skipped installation of ${package_name} with version ${CODEFLARE_RELEASE_VERSION} in $dir"
fi
cd -
echo "$((total-counter)) directories remaining.."
done
else
versions_list=$(echo "$versions" | tr '\n' ' ' | sed 's/, $//')
versions="${versions_list%,}"
echo "Version '${CODEFLARE_RELEASE_VERSION}' is not available for $package_name"
echo "Available versions for $package_name: $versions"
exit 1
fi
- name: Push changes
run: |
cd $REPO_NAME
git add . && git status && git checkout -b ${{ env.UPDATER_BRANCH }} && \
git commit -am "Updated notebooks via ${{ env.UPDATER_BRANCH }} GitHub action" --signoff &&
git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git
git push origin ${{ env.UPDATER_BRANCH }}
- name: Create Pull Request
run: |
gh pr create --repo $UPSTREAM_OWNER/$REPO_NAME \
--title "$pr_title" \
--body "$pr_body" \
--head $REPO_OWNER:$UPDATER_BRANCH \
--base $BRANCH_NAME
env:
pr_title: "[Codeflare Action] Update notebook's pipfile to sync with Codeflare-SDK release ${{ env.CODEFLARE_RELEASE_VERSION }}"
pr_body: |
:rocket: This is an automated Pull Request.
This PR updates the `Pipfile` to sync with latest Codeflare-SDK release.