-
Notifications
You must be signed in to change notification settings - Fork 23
/
Copy pathos-autoinst-obs-auto-submit
executable file
·207 lines (189 loc) · 8.19 KB
/
os-autoinst-obs-auto-submit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
#!/bin/bash
set -euo pipefail
export LC_ALL=C
src_project=${src_project:-devel:openQA}
dst_project=${dst_project:-${src_project}:tested}
staging_project=${staging_project:-${src_project}:testing}
submit_target="${submit_target:-"openSUSE:Factory"}"
submit_target_escaped=$(echo -n "$submit_target" | sed -e 's|\:|_|g') # avoid, e.g. "repoid 'openSUSE:Factory' is illegal"
dry_run="${dry_run:-"0"}"
osc_poll_interval="${osc_poll_interval:-2}"
osc_build_start_poll_tries="${osc_build_start_poll_tries:-30}"
throttle_days=${throttle_days:-2}
XMLSTARLET=$(command -v xmlstarlet || true)
[[ -n $XMLSTARLET ]] || (echo "Need xmlstarlet" && exit 1)
# shellcheck source=/dev/null
. "$(dirname "$0")"/_common
factory_request() {
local package=$1
# writing xpath in url encoding is not for beginners, so don't stare at it too long :)
local states='(state%2F%40name%3D%27declined%27%20or%20state%2F%40name%3D%27new%27%20or%20state%2F%40name%3D%27review%27)'
local actions="(action%2Ftarget%2F%40project%3D%27openSUSE%3AFactory%27%20and%20action%2Fsource%2F%40project%3D%27devel%3AopenQA%3Atested%27%20and%20action%2Ftarget%2F%40package%3D%27$package%27)"
$osc api "/search/request/id?match=$states%20and%20$actions" | grep 'request id' | sed -e 's,.*id=.\([0-9]*\)[^[0-9]*$,\1,' | head -n 1
}
reenable_buildtime_services() {
sed -i -e 's,mode="buildtime" mode="disabled",mode="buildtime",' _service
}
wait_for_package_build() {
local osc_query="https://api.opensuse.org/public/build/$dst_project/_result?repository=$submit_target_escaped&package=$package"
local wip_states='\(unknown\|blocked\|scheduled\|dispatching\|building\|signing\|finished\)'
local bad_states='\(failed\|unresolvable\|broken\)'
local attempts="$osc_build_start_poll_tries"
while ! curl -sS "$osc_query" | grep -e "$wip_states"; do
if [[ $attempts -le 0 ]]; then
echo "warning: Re-build of $package has not been considered in time (or package has been re-built so fast that we've missed it)"
break
fi
echo "Waiting for re-build of $package to be considered (attempts left: $attempts)"
sleep "$osc_poll_interval"
attempts=$((attempts - 1))
done
while curl -sS "$osc_query" | grep -e "$wip_states"; do
echo "Waiting while $package is in progress"
sleep "$osc_poll_interval"
done
if curl -sS "$osc_query" | grep -e "$bad_states"; then
echo "Building $package has failed, skipping submission"
return 1
fi
}
update_package() {
package=$1
xmlstarlet ed -L -i "/services/service" --type attr -n mode --value 'disabled' _service
sed -i -e 's,mode="disabled" mode="disabled",mode="disabled",' _service
reenable_buildtime_services
# ignore those updates
rm -f _service:*-test.changes
cp -v .osc/*-test.changes . 2> /dev/null || :
for file in _service:*; do
# shellcheck disable=SC2001
mv -v "$file" "$(echo "$file" | sed -e 's,.*:,,')"
done
version=$(sed -n 's/^Version:\s*//p' ./*.spec || sed -n 's/^version:\s*//p' ./*.obsinfo)
rm -f ./*rpmlintrc _servicedata node_modules.sums package-lock.json
$osc addremove
sed -i '/rpmlintrc/d' ./*.spec || true
local ci_args=""
if [[ "$package" == "os-autoinst-distri-opensuse-deps" || "$package" =~ "container" ]]; then
ci_args="--noservice"
fi
# We would get an empty changelog when there are no changes to files contained
# in this package/container. So add some text.
sed -i -e 's/^ \* $/ * Update to latest openQA version/' ./*.changes
$osc ci -m "Offline generation of ${version}" $ci_args
wait_for_package_build
cmd="$osc sr"
req=$(factory_request "$package" || :)
# TODO: check if it's a different revision than HEAD
if test -n "$req"; then
cmd="$cmd -s $req"
fi
$cmd -m "Update to ${version}" "$submit_target"
}
last_revision() {
project=$1
package=$2
file=$package.changes
service=obs_scm
if [[ $project != "$submit_target" ]]; then
file=_service:$service:$file
fi
local line
# prevent SIGPIPE triggering osc cat to fail with the command group piping
# exceeding output to /dev/null
line=$($osc cat "$project/$package/$file" | {
grep -m1 'Update to version'
cat > /dev/null
})
# shellcheck disable=SC2001
echo "$line" | sed -e 's,.*\.\([^.]*\):$,\1,'
}
sync_changesrevision() {
dir=$1
package=$2
factory_rev=$3
path="$dir/$package"
$prefix xmlstarlet ed -L -u "//param[@name='changesrevision']" -v "$factory_rev" "$path"/_servicedata
if ! diff -u "$path"/.osc/_servicedata "$path"/_servicedata; then
$osc up "$path"
$osc cat "$submit_target" "$package" "$package".changes > "$path/$package".changes
$osc ci -m "sync changesrevision with $submit_target" "$path"
$osc up --server-side-source-service-files "$path"
fi
}
get_spec_changes_in_requirements() {
local package=$1
local project=$2
{ diff -u "$package-factory.spec" "$project/$package/_service:obs_scm:$package.spec" || :; } | grep '^[+-]Requires'
}
generate_os-autoinst-distri-opensuse-deps_changelog() {
dir=$1
package=$2
$osc cat "$submit_target" "$package" "$package".changes > "$package"-factory.changes
{
echo "-------------------------------------------------------------------"
echo "$(LANG=c date -u) - Dominik Heidler <[email protected]>"
echo
get_spec_changes_in_requirements "$package" "$dir" \
| sed -e 's/Requires:\s*/dependency /g' -e 's/^-/- Removed /g' -e 's/^+/- Added /g'
echo
cat "$package"-factory.changes
} > "$dir/$package/$package".changes
}
handle_auto_submit() {
package=$1
$osc service wait "$src_project" "$package"
$osc co --server-side-source-service-files "$src_project"/"$package"
if [[ "$package" == "os-autoinst-distri-opensuse-deps" ]]; then
$osc cat "$submit_target" "$package" "$package.spec" > "$package-factory.spec"
if get_spec_changes_in_requirements "$package" "$src_project"; then
# dependency added or removed
generate_os-autoinst-distri-opensuse-deps_changelog "$src_project" "$package"
else
echo "No dependency changes for $package"
return
fi
else
factory_rev=$(last_revision "$submit_target" "$package")
sync_changesrevision "$src_project" "$package" "$factory_rev"
if [[ "$factory_rev" == "$(last_revision "$src_project" "$package")" ]]; then
echo "Latest revision already in $submit_target"
return
fi
fi
$osc service wait "$dst_project" "$package"
$osc co "$dst_project"/"$package"
rm "$dst_project"/"$package"/*
cp -v "$src_project"/"$package"/* "$dst_project"/"$package"/
(
cd "$dst_project"/"$package"
update_package "$package"
)
}
prefix="${prefix:-""}"
[ "$dry_run" = "1" ] && prefix="echo"
osc="${osc:-"$prefix retry -e -- osc"}"
# submit from staging project if specified
[[ $staging_project && $staging_project != none && $staging_project != "$src_project" ]] && src_project=$staging_project from_staging=1
if [[ -e job_post_skip_submission ]]; then
echo "Skipping submission, reason: "
cat job_post_skip_submission
exit 0
fi
# throttle number of submissions to allow only one SR within a certain number of days
# note: Avoid using `grep --quiet` here to keep consuming input so osc does not run into "BrokenPipeError: [Errno 32] Broken pipe".
if [[ $throttle_days != 0 ]] && $osc request list --project "$dst_project" --type submit --state new,review --mine --days "$throttle_days" | grep 'Created by' > /dev/null; then
echo "Skipping submission, there is still a pending SR younger than $throttle_days days."
exit 0
fi
TMPDIR=${TMPDIR:-$(mktemp -d -t os-autoinst-obs-auto-submit-XXXX)}
trap 'rm -rf "$TMPDIR"' EXIT
(
cd "$TMPDIR"
auto_submit_packages=${packages:-$($osc ls "$dst_project" | grep -v '\-test$')}
for package in $auto_submit_packages; do
handle_auto_submit "$package"
# delete package from staging project
[[ $from_staging ]] && $osc rdelete -m "Cleaning up $package from $staging_project for next submission" "$staging_project" "$package"
done
)