Skip to content

Commit

Permalink
[2.0 backport] Port the release script from master
Browse files Browse the repository at this point in the history
This would allow us to use the release script as-is from the tag instead of from master. This would help reproducibility
  • Loading branch information
kit-ty-kate committed Jul 9, 2024
1 parent f75d371 commit 49ef89c
Show file tree
Hide file tree
Showing 5 changed files with 244 additions and 91 deletions.
36 changes: 20 additions & 16 deletions release/Dockerfile.in
Original file line number Diff line number Diff line change
@@ -1,29 +1,33 @@
FROM multiarch/debian-debootstrap:%TARGET_TAG%
FROM multiarch/alpine:%TARGET_TAG%
# May need configuration on the host:
# docker run --rm --privileged multiarch/qemu-user-static:register --reset
LABEL Description="opam release builds" Vendor="OCamlPro" Version="1.0"

RUN apt-get update && apt-get install bzip2 g++ make patch wget libltdl-dev --yes && apt-get clean --yes
RUN useradd -U --create-home opam
RUN apk add gcc g++ make coreutils openssl

ADD https://caml.inria.fr/pub/distrib/ocaml-4.04/ocaml-4.04.2.tar.gz /root/
RUN addgroup -S opam && adduser -S opam -G opam -s /bin/sh

ADD https://github.com/ocaml/ocaml/archive/refs/tags/%OCAMLV%.tar.gz /root/

WORKDIR /root
RUN tar xzf ocaml-4.04.2.tar.gz
WORKDIR ocaml-4.04.2
RUN tar xzf %OCAMLV%.tar.gz
WORKDIR ocaml-%OCAMLV%
RUN sed -i 's/gnueabi/*eabi/' configure
RUN sed -i 's/musl/musl*/' configure
RUN sed -i -e 's/NGROUPS_MAX/65536/' otherlibs/unix/getgroups.c
RUN ./configure %CONF% -prefix /usr/local
RUN make world opt.opt
RUN make install
RUN rm -rf /root/ocaml-4.04.2 /root/ocaml-4.04.2.tar.gz
RUN make "-j$(nproc)" && make install && rm -rf /root/ocaml-%OCAMLV% /root/ocaml-%OCAMLV%.tar.gz

RUN apk add patch

ENV PATH /usr/local/bin:/usr/bin:/bin
USER opam
VOLUME /src
WORKDIR /home/opam/
CMD tar xzf /src/opam-full-${VERSION}.tar.gz && \
cd opam-full-${VERSION} && \
echo "(${LINKING})" > src/client/linking.sexp && \
CMD tar xz >&2 && \
cd opam-full-${VERSION} >&2 && \
./configure --with-mccs && \
make lib-ext opam && \
strip opam && \
cp opam /src/opam-${VERSION}-${TARGET}
make lib-ext && \
echo "(${LINKING})" > src/client/linking.sexp && \
make opam >&2 && \
strip opam >&2 && \
cat opam
138 changes: 85 additions & 53 deletions release/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,71 +3,80 @@ VERSION = $(shell git describe $(TAG))
OPAM_VERSION = $(subst -,~,$(VERSION))
GIT_URL = ..

FULL_ARCHIVE_URL = https://github.com/ocaml/opam/releases/download/$(VERSION)/opam-full-$(VERSION).tar.gz

TARGETS = x86_64-linux i686-linux armhf-linux arm64-linux
# todo: x86_64-darwin

OCAMLV = 4.10.2
OCAMLV = 4.14.2
# currently hardcoded in Dockerfile.in
OCAML_URL = https://caml.inria.fr/pub/distrib/ocaml-$(basename $(OCAMLV))/ocaml-$(OCAMLV).tar.gz
OCAML_URL = https://github.com/ocaml/ocaml/archive/refs/tags/$(OCAMLV).tar.gz

HOST_OS = $(shell uname -s | tr A-Z a-z)
HOST_OS = $(shell uname -s | tr A-Z a-z | sed -e 's/_.*//' -e 's/darwin/macos/')
HOST = $(shell uname -m | sed 's/amd64/x86_64/')-$(HOST_OS)
OUTDIR = out/$(TAG)

# The equivalent of "which <cmd>". Taken from the GNU Make documentation
pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(PATH)))))

x86_64-linux: $(OUTDIR)/opam-$(VERSION)-x86_64-linux
i686-linux: $(OUTDIR)/opam-$(VERSION)-i686-linux
armhf-linux: $(OUTDIR)/opam-$(VERSION)-armhf-linux
arm64-linux: $(OUTDIR)/opam-$(VERSION)-arm64-linux
ppc64le-linux: $(OUTDIR)/opam-$(VERSION)-ppc64le-linux
s390x-linux: $(OUTDIR)/opam-$(VERSION)-s390x-linux

all: $(patsubst %,out/opam-$(VERSION)-%,$(TARGETS))

out/opam-full-$(VERSION).tar.gz:
mkdir -p out
cd out && curl -OfL $(FULL_ARCHIVE_URL) || { \
git clone $(GIT_URL) -b $(TAG) --depth 1 opam-full-$(VERSION); \
sed 's/^AC_INIT(opam,.*)/AC_INIT(opam,$(OPAM_VERSION))/' > \
opam-full-$(VERSION)/configure.ac.tmp; \
mv opam-full-$(VERSION)/configure.ac.tmp \
opam-full-$(VERSION)/configure.ac; \
$(MAKE) -C opam-full-$(VERSION) configure download-ext; \
tar cz --exclude-vcs opam-full-$(VERSION) -f $(notdir $@); \
rm -rf opam-full-$(VERSION); \
}
$(OUTDIR)/opam-full-$(VERSION).tar.gz:
mkdir -p "$(OUTDIR)"
git clone $(GIT_URL) -b $(TAG) "$(OUTDIR)/opam-full-$(VERSION)"
$(MAKE) -C "$(OUTDIR)/opam-full-$(VERSION)" OCAML=$(call pathsearch,ocaml) download-ext
cd "$(OUTDIR)" && tar cz -f $(notdir $@) --exclude .git opam-full-$(VERSION)
rm -rf "$(OUTDIR)/opam-full-$(VERSION)"

build/Dockerfile.x86_64-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/amd64-jessie/g' $^ | sed 's/%CONF%//g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/x86_64-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.i686-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/i386-jessie/g' $^ | sed 's/%CONF%/-host i686-linux/g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/x86-v3.13/g' -e 's/%CONF%/-build i586-alpine-linux-musl/g' $^ >$@

# Need to lie about gnueabihf instead of musleabihf, because of a ./configure bug
build/Dockerfile.armhf-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/armhf-jessie/g' $^ | sed 's/%CONF%//g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/armv7-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.arm64-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/arm64-jessie/g' $^ | sed 's/%CONF%//g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/arm64-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.ppc64le-linux: Dockerfile.in
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/ppc64le-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.s390x-linux: Dockerfile.in
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/s390x-v3.13/g' -e 's/%CONF%//g' $^ >$@


build/%.image: build/Dockerfile.%
docker build -t opam-build-$* -f $^ build
touch $@

# Actually, this is for debian 8 jessie, and varies wildly
# Actually, this is for alpine 3.13, and varies
CLINKING_linux = \
-Wl,-Bstatic \
-lunix -lmccs_stubs -lmccs_glpk_stubs \
-lstdc++ \
-Wl,-Bdynamic \
-static-libgcc
-static-libgcc \
-static
# -Wl,-Bdynamic

CLINKING_darwin = \
CLINKING_macos = \
-lunix -lmccs_stubs -lmccs_glpk_stubs \
-lstdc++

CLINKING_openbsd = $(CLINKING_darwin)
CLINKING_openbsd = $(CLINKING_macos)
CLINKING_freebsd = $(CLINKING_macos)

LINKING = (-noautolink $(patsubst %,-cclib %,$(CLINKING_$(1))))

EXPORTS_openbsd = \
CPATH=/usr/local/include: \
LIBRARY_PATH=/usr/local/lib: \

EXPORTS_freebsd = \
CPATH=/usr/local/include: \
LIBRARY_PATH=/usr/local/lib: \

%: opam-$(VERSION)-%

opam-$(VERSION)-%: out/opam-$(VERSION)-%
opam-$(VERSION)-%: $(OUTDIR)/opam-$(VERSION)-%
ln -sf $^ $@

# host: opam-$(VERSION)-$(HOST)
Expand All @@ -76,40 +85,44 @@ opam-$(VERSION)-%: out/opam-$(VERSION)-%
build/$(HOST).env:
mkdir -p build/$(HOST)
cd build/$(HOST) && curl -OL $(OCAML_URL)
cd build/$(HOST) && tar xzf ocaml-$(OCAMLV).tar.gz
cd build/$(HOST) && tar xzf $(OCAMLV).tar.gz
cd build/$(HOST)/ocaml-$(OCAMLV) && \
./configure -prefix $(shell pwd)/build/$(HOST) && \
$(MAKE) world opt.opt && \
./configure --prefix "$(shell pwd)/build/$(HOST)" \
--disable-debug-runtime --disable-debugger --disable-instrumented-runtime \
--disable-ocamldoc --disable-stdlib-manpages --disable-ocamltest && \
$(MAKE) -j$(JOBS) && \
$(MAKE) install
rm -rf build/$(HOST)/ocaml-$(OCAMLV) build/$(HOST)/ocaml-$(OCAMLV).tar.gz
rm -rf build/$(HOST)/ocaml-$(OCAMLV) build/$(HOST)/$(OCAMLV).tar.gz
touch $@

# Actually builds out/opam-$(VERSION)-$(HOST), but we don't want to override the
# Actually builds $(OUTDIR)/opam-$(VERSION)-$(HOST), but we don't want to override the
# rule that goes through a container
host: out/opam-full-$(VERSION).tar.gz build/$(HOST).env
host: $(OUTDIR)/opam-full-$(VERSION).tar.gz build/$(HOST).env
rm -rf build/opam-full-$(VERSION)
cd build && tar xzf ../$<
( export \
PATH=$(shell pwd)/build/$(HOST)/bin:$$PATH \
MAKE=$(MAKE) \
$(EXPORTS_$(HOST_OS)); \
cd build/opam-full-$(VERSION) && \
./configure && \
./configure --with-mccs && \
$(MAKE) lib-ext && \
echo "$(call LINKING,$(HOST_OS))" >src/client/linking.sexp && \
$(MAKE) lib-ext DUNE_ARGS="--root=`pwd`"; \
$(MAKE) opam DUNE_ARGS="--root=`pwd`"; \
$(MAKE) opam; \
)
cp build/opam-full-$(VERSION)/opam out/opam-$(VERSION)-$(HOST)
strip out/opam-$(VERSION)-$(HOST)
cp build/opam-full-$(VERSION)/opam $(OUTDIR)/opam-$(VERSION)-$(HOST)
strip $(OUTDIR)/opam-$(VERSION)-$(HOST)
$(OUTDIR)/opam-$(VERSION)-$(HOST) --version
rm -rf build/opam-full-$(VERSION)

# Containerised builds
out/opam-$(VERSION)-%-linux: build/%-linux.image out/opam-full-$(VERSION).tar.gz
docker run --rm -v `pwd`/out:/src \
$(OUTDIR)/opam-$(VERSION)-%-linux: $(OUTDIR)/opam-full-$(VERSION).tar.gz build/%-linux.image
docker run --rm -i \
-e "VERSION=$(VERSION)" \
-e "TARGET=$*-linux" \
-e "LINKING=$(call LINKING,$(HOST_OS))" \
opam-build-$*-linux
-e "LINKING=$(call LINKING,linux)" \
opam-build-$*-linux \
<$< >$@

clean:
rm -rf build
Expand All @@ -119,9 +132,28 @@ distclean: clean

REMOTE_DIR = /tmp/opam-release
REMOTE_MAKE = make
remote: out/opam-full-$(VERSION).tar.gz
ssh "$(REMOTE)" "mkdir -p $(REMOTE_DIR)/out"
scp Makefile "$(REMOTE):$(REMOTE_DIR)/"
scp "$^" "$(REMOTE):$(REMOTE_DIR)/$^"
ssh "$(REMOTE)" 'sh -c "cd $(REMOTE_DIR) && ulimit -s 8192 && $(REMOTE_MAKE) host TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)"'
scp "$(REMOTE):$(REMOTE_DIR)/out/opam-$(VERSION)*" out/
REMOTE_SHELL = /bin/sh
SSH = ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
SCP = scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
remote: $(OUTDIR)/opam-full-$(VERSION).tar.gz
$(SSH) "$(REMOTE)" "mkdir -p $(REMOTE_DIR)/$(OUTDIR)"
$(SCP) Makefile "$(REMOTE):$(REMOTE_DIR)/"
$(SCP) "$^" "$(REMOTE):$(REMOTE_DIR)/$^"
$(SSH) "$(REMOTE)" 'sh -c "cd $(REMOTE_DIR) && ulimit -s 8192 && $(REMOTE_MAKE) host TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)"'
$(SCP) "$(REMOTE):$(REMOTE_DIR)/$(OUTDIR)/opam-$(VERSION)*" "$(OUTDIR)/"

SSH_USER = root
ULIMIT = ulimit -s 8192 &&
qemu: $(OUTDIR)/opam-full-$(VERSION).tar.gz
$(SSH) -p "$(QEMU_PORT)" $(SSH_USER)@localhost "$(REMOTE_SHELL) -c \"mkdir -p $(REMOTE_DIR)/$(OUTDIR)\""
$(SCP) -P "$(QEMU_PORT)" Makefile "$(SSH_USER)@localhost:$(REMOTE_DIR)/"
$(SCP) -P "$(QEMU_PORT)" "$^" "$(SSH_USER)@localhost:$(REMOTE_DIR)/$^"
$(SSH) -p "$(QEMU_PORT)" $(SSH_USER)@localhost "$(REMOTE_SHELL) -c \"cd $(REMOTE_DIR) && $(ULIMIT) $(REMOTE_MAKE) host JOBS=$(JOBS) TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)\""
$(SCP) -P "$(QEMU_PORT)" "$(SSH_USER)@localhost:$(REMOTE_DIR)/$(OUTDIR)/opam-$(VERSION)*" "$(OUTDIR)/"

macos-local: $(OUTDIR)/opam-full-$(VERSION).tar.gz
LOCAL_RELEASE_DIR=$(shell mktemp -d); \
mkdir -p "$${LOCAL_RELEASE_DIR}/${OUTDIR}" && \
cp Makefile "$^" "$${LOCAL_RELEASE_DIR}" && \
cd "$${LOCAL_RELEASE_DIR}" && ulimit -s 8192 && arch -arch $(MACOS_ARCH) make host JOBS=$(JOBS) TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV) && \
cp $${LOCAL_RELEASE_DIR}/$(OUTDIR)/opam-$(VERSION)* "$(GIT_URL)/release/$(OUTDIR)/"
54 changes: 54 additions & 0 deletions release/readme.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Steps to follow for each release

## Finalise opam code for release
* update version in all the opam files and in configure.ac
* run `make configure` to regenerate `./configure` [checked by github actions]
* update copyright headers
* if you're releasing the first final release of a new branch (e.g. 2.2.0): make sure `root_version` in OpamFile.ml is set to the final release number (e.g. for 2.2.0, root_version should be 2.2). Make sure that opamFormatUpgrade.ml also contains an upgrade function from the previous version (that function will most likely be empty)
* run `make tests`, `opam-rt` [checked by github actions]
* update the CHANGE file: take `master_changes.md` content to fill it

## Github release

[ once bump version & changes PRs merged ]
* tag the release (git tag -am 2.2.0 2.2.0; git push origin 2.2.0)
* /!\ Once the tag pushed, it can be updated [different commit] only in case of severe issue
* create a release (or prerelease if intermediate release) draft on github based on your tag (https://github.com/ocaml/opam/releases/new)
* fetch locally the tag
* launch docker using the Docker GUI macOS app
* generate opam artifacts, using `release/release.sh <tag>` from a macOS/arm64 machine, it requires to have Docker and QEMU installed (see below device requirements)
* generate the signatures using `release/sign.sh <tag>`
* add releases notes (content of `master_changes.md`) in the release draft
* upload everything from `release/out/<tag>`
* finalise the release (publish)

## Publish the release

* add hashes in `install.sh` (and check signatures)
* publish opam packages in opam-repository (and add `flags: avoid-version` and `available: opam-version >= "2.1.0"` to each packages if this is not a stable version)
* update versions (and messages, if necessary) in https://github.com/ocaml/opam-repository/blob/master/repo

## Announce!

* a blog entry in opam.ocaml.org
* a announcement in discuss.ocaml.org
* copy the blog entry from opam.ocaml.org for https://github.com/ocaml/ocaml.org/tree/main/data/changelog/opam
* announce the release on the OCaml Discord server


## After release

* Bump the version with a `~dev` at the end (e.g. `2.2.0~alpha~dev`)
* Check if reftests needs an update
* Bring the changes to the changelog (CHANGES) from the branch of the release to the `master` branch

### On a release candidate
* create a branch to a `x.y` for rc's and the final release

---

## Device requirements
* Mac M1 or above with Rosetta2
* brew dependencies: git, gpg, qemu>=8.1.0, docker>=24.0.0, md5sha1sum
* opam repo with the tag fetched
* Have the secret key available
75 changes: 53 additions & 22 deletions release/release.sh
Original file line number Diff line number Diff line change
@@ -1,32 +1,63 @@
#!/usr/bin/env bash
set -uex

# This script is expected to run on Linux with docker available, and to have
# three remotes "some-osx-x86", "some-osx-arm" and "some-openbsd", with the
# corresponding OSes, ocaml deps installed
unset $(env | cut -d= -f1 | grep -Fvx HOME | grep -Fvx PWD | grep -Fvx USER | grep -Fvx SHELL | grep -Fvx TERM)
export PATH=/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin:/usr/local/bin

LC_ALL=C
DIR=$(dirname $0)
cd "$DIR"
if [[ $# -eq 0 || "x$1" =~ "x-" ]]; then
echo "Usage: $0 TAG [archive|builds]"
usage() {
echo "Usage: $0 TAG"
exit 1
}

if [[ $# -lt 1 || $# -gt 1 || "x$1" =~ "x-" ]]; then
usage
fi

TAG="$1"; shift
TAG="$1"
shift

if [[ $# -eq 0 || " $* " =~ " archive " ]]; then
make TAG="$TAG" GIT_URL="https://github.com/ocaml/opam.git" "out/opam-full-$TAG.tar.gz"
( cd out && git-upload-release ocaml opam "$TAG" "opam-full-$TAG.tar.gz"; )
if test "$(uname -s)" != Darwin -o "$(uname -m)" != arm64; then
echo "This script is required to be run on macOS/arm64"
exit 1
fi

if [[ $# -eq 0 || " $* " =~ " builds " ]]; then
make TAG="$TAG" all &
make TAG="$TAG" remote REMOTE=some-osx-x86 REMOTE_DIR=opam-release &
make TAG="$TAG" remote REMOTE=some-osx-arm REMOTE_DIR=opam-release &
make TAG="$TAG" remote REMOTE=some-openbsd REMOTE_MAKE=gmake REMOTE_DIR=opam-release &
wait
cd out && for f in opam-$TAG-*; do
git-upload-release ocaml opam "$TAG" $f
done
fi
DIR=$(dirname $0)
cd "$DIR"

LC_ALL=C
CWD=$(pwd)
JOBS=$(sysctl -n hw.ncpu)
SSH="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

OUTDIR="out/$TAG"
mkdir -p "$OUTDIR"

qemu_build() {
local port=$1
local image=$2
local install=$3
local make=$4
local arch=$5

if ! ${SSH} -p "${port}" root@localhost true; then
qemu-img convert -O raw "./qemu-base-images/${image}.qcow2" "./qemu-base-images/${image}.raw"
"qemu-system-${arch}" -drive "file=./qemu-base-images/${image}.raw,format=raw" -nic "user,hostfwd=tcp::${port}-:22" -m 2G -smp "${JOBS}" &
sleep 60
fi
${SSH} -p "${port}" root@localhost "${install}"
make TAG="$TAG" JOBS="${JOBS}" qemu QEMU_PORT="${port}" REMOTE_MAKE="${make}" REMOTE_DIR="opam-release-$TAG"
${SSH} -p "${port}" root@localhost "shutdown -p now"
}

make JOBS="${JOBS}" TAG="$TAG" "${OUTDIR}/opam-full-$TAG.tar.gz"
make JOBS="${JOBS}" TAG="$TAG" x86_64-linux
make JOBS="${JOBS}" TAG="$TAG" i686-linux
make JOBS="${JOBS}" TAG="$TAG" armhf-linux
make JOBS="${JOBS}" TAG="$TAG" arm64-linux
make JOBS="${JOBS}" TAG="$TAG" ppc64le-linux
make JOBS="${JOBS}" TAG="$TAG" s390x-linux
[ -f "${OUTDIR}/opam-$TAG-x86_64-macos" ] || make TAG="$TAG" JOBS="${JOBS}" macos-local MACOS_ARCH=x86_64 REMOTE_DIR=opam-release-$TAG GIT_URL="$CWD/.."
[ -f "${OUTDIR}/opam-$TAG-arm64-macos" ] || make TAG="$TAG" JOBS="${JOBS}" macos-local MACOS_ARCH=arm64 REMOTE_DIR=opam-release-$TAG GIT_URL="$CWD/.."
[ -d ./qemu-base-images ] || git clone https://gitlab.com/kit-ty-kate/qemu-base-images.git
[ -f "${OUTDIR}/opam-$TAG-x86_64-openbsd" ] || qemu_build 9999 OpenBSD-7.4-amd64 "pkg_add gmake curl bzip2" gmake x86_64
[ -f "${OUTDIR}/opam-$TAG-x86_64-freebsd" ] || qemu_build 9998 FreeBSD-13.2-RELEASE-amd64 "pkg install -y gmake curl bzip2" gmake x86_64
Loading

0 comments on commit 49ef89c

Please sign in to comment.