[2.1 backport] Port the release script from master

release/Dockerfile.in

@@ -1,29 +1,33 @@
-FROM multiarch/debian-debootstrap:%TARGET_TAG%
+FROM multiarch/alpine:%TARGET_TAG%
 # May need configuration on the host:
 #    docker run --rm --privileged multiarch/qemu-user-static:register --reset
 LABEL Description="opam release builds" Vendor="OCamlPro" Version="1.0"
 
-RUN apt-get update && apt-get install bzip2 g++ make patch wget libltdl-dev --yes && apt-get clean --yes
-RUN useradd -U --create-home opam
+RUN apk add gcc g++ make coreutils openssl
 
-ADD https://caml.inria.fr/pub/distrib/ocaml-4.07/ocaml-4.07.1.tar.gz /root/
+RUN addgroup -S opam && adduser -S opam -G opam -s /bin/sh
+
+ADD https://github.com/ocaml/ocaml/archive/refs/tags/%OCAMLV%.tar.gz /root/
 
 WORKDIR /root
-RUN tar xzf ocaml-4.07.1.tar.gz
-WORKDIR ocaml-4.07.1
+RUN tar xzf %OCAMLV%.tar.gz
+WORKDIR ocaml-%OCAMLV%
+RUN sed -i 's/gnueabi/*eabi/' configure
+RUN sed -i 's/musl/musl*/' configure
+RUN sed -i -e 's/NGROUPS_MAX/65536/' otherlibs/unix/getgroups.c
 RUN ./configure %CONF% -prefix /usr/local
-RUN make world opt.opt
-RUN make install
-RUN rm -rf /root/ocaml-4.07.1 /root/ocaml-4.07.1.tar.gz
+RUN make "-j$(nproc)" && make install && rm -rf /root/ocaml-%OCAMLV% /root/ocaml-%OCAMLV%.tar.gz
+
+RUN apk add patch
 
 ENV PATH /usr/local/bin:/usr/bin:/bin
 USER opam
-VOLUME /src
 WORKDIR /home/opam/
-CMD tar xzf /src/opam-full-${VERSION}.tar.gz && \
-    cd opam-full-${VERSION} && \
-    echo "(${LINKING})" > src/client/linking.sexp && \
+CMD tar xz >&2 && \
+    cd opam-full-${VERSION} >&2 && \
     ./configure --with-mccs && \
-    make lib-ext opam && \
-    strip opam && \
-    cp opam /src/opam-${VERSION}-${TARGET}
+    make lib-ext && \
+    echo "(${LINKING})" > src/client/linking.sexp && \
+    make opam >&2 && \
+    strip opam >&2 && \
+    cat opam

release/Makefile

@@ -3,71 +3,80 @@ VERSION = $(shell git describe $(TAG))
 OPAM_VERSION = $(subst -,~,$(VERSION))
 GIT_URL = ..
 
-FULL_ARCHIVE_URL = https://github.com/ocaml/opam/releases/download/$(VERSION)/opam-full-$(VERSION).tar.gz
-
-TARGETS = x86_64-linux i686-linux armhf-linux arm64-linux
-# todo: x86_64-darwin
-
-OCAMLV = 4.10.2
+OCAMLV = 4.14.2
 # currently hardcoded in Dockerfile.in
-OCAML_URL = https://caml.inria.fr/pub/distrib/ocaml-$(basename $(OCAMLV))/ocaml-$(OCAMLV).tar.gz
+OCAML_URL = https://github.com/ocaml/ocaml/archive/refs/tags/$(OCAMLV).tar.gz
 
-HOST_OS = $(shell uname -s | tr A-Z a-z | sed 's/darwin/macos/')
+HOST_OS = $(shell uname -s | tr A-Z a-z | sed -e 's/_.*//' -e 's/darwin/macos/')
 HOST = $(shell uname -m | sed 's/amd64/x86_64/')-$(HOST_OS)
+OUTDIR = out/$(TAG)
+
+# The equivalent of "which <cmd>". Taken from the GNU Make documentation
+pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(PATH)))))
+
+x86_64-linux: $(OUTDIR)/opam-$(VERSION)-x86_64-linux
+i686-linux: $(OUTDIR)/opam-$(VERSION)-i686-linux
+armhf-linux: $(OUTDIR)/opam-$(VERSION)-armhf-linux
+arm64-linux: $(OUTDIR)/opam-$(VERSION)-arm64-linux
+ppc64le-linux: $(OUTDIR)/opam-$(VERSION)-ppc64le-linux
+s390x-linux: $(OUTDIR)/opam-$(VERSION)-s390x-linux
 
-all: $(patsubst %,out/opam-$(VERSION)-%,$(TARGETS))
-
-out/opam-full-$(VERSION).tar.gz:
-	mkdir -p out
-	cd out && curl -OfL $(FULL_ARCHIVE_URL) || { \
-	  git clone $(GIT_URL) -b $(TAG) --depth 1 opam-full-$(VERSION); \
-	  sed 's/^AC_INIT(opam,.*)/AC_INIT(opam,$(OPAM_VERSION))/' opam-full-$(VERSION)/configure.ac > \
-	    opam-full-$(VERSION)/configure.ac.tmp; \
-	  mv opam-full-$(VERSION)/configure.ac.tmp \
-	    opam-full-$(VERSION)/configure.ac; \
-	  $(MAKE) -C opam-full-$(VERSION) configure download-ext; \
-	  tar cz --exclude-vcs opam-full-$(VERSION) -f $(notdir $@); \
-	  rm -rf opam-full-$(VERSION); \
-	}
+$(OUTDIR)/opam-full-$(VERSION).tar.gz:
+	mkdir -p "$(OUTDIR)"
+	git clone $(GIT_URL) -b $(TAG) "$(OUTDIR)/opam-full-$(VERSION)"
+	$(MAKE) -C "$(OUTDIR)/opam-full-$(VERSION)" OCAML=$(call pathsearch,ocaml) download-ext
+	cd "$(OUTDIR)" && tar cz -f $(notdir $@) --exclude .git opam-full-$(VERSION)
+	rm -rf "$(OUTDIR)/opam-full-$(VERSION)"
 
 build/Dockerfile.x86_64-linux: Dockerfile.in
-	mkdir -p build && sed 's/%TARGET_TAG%/amd64-jessie/g' $^ | sed 's/%CONF%//g' >$@
+	mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/x86_64-v3.13/g' -e 's/%CONF%//g' $^ >$@
 build/Dockerfile.i686-linux: Dockerfile.in
-	mkdir -p build && sed 's/%TARGET_TAG%/i386-jessie/g' $^ | sed 's/%CONF%/-host i686-linux/g' >$@
+	mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/x86-v3.13/g' -e 's/%CONF%/-build i586-alpine-linux-musl/g' $^ >$@
+
+# Need to lie about gnueabihf instead of musleabihf, because of a ./configure bug
 build/Dockerfile.armhf-linux: Dockerfile.in
-	mkdir -p build && sed 's/%TARGET_TAG%/armhf-jessie/g' $^ | sed 's/%CONF%//g' >$@
+	mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/armv7-v3.13/g' -e 's/%CONF%//g' $^ >$@
 build/Dockerfile.arm64-linux: Dockerfile.in
-	mkdir -p build && sed 's/%TARGET_TAG%/arm64-stretch/g' $^ | sed 's/%CONF%//g' >$@
+	mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/arm64-v3.13/g' -e 's/%CONF%//g' $^ >$@
+build/Dockerfile.ppc64le-linux: Dockerfile.in
+	mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/ppc64le-v3.13/g' -e 's/%CONF%//g' $^ >$@
+build/Dockerfile.s390x-linux: Dockerfile.in
+	mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/s390x-v3.13/g' -e 's/%CONF%//g' $^ >$@
 
 
 build/%.image: build/Dockerfile.%
 	docker build -t opam-build-$* -f $^ build
 	touch $@
 
-# Actually, this is for debian 8 jessie, and varies wildly
+# Actually, this is for alpine 3.13, and varies
 CLINKING_linux = \
 -Wl,-Bstatic \
 -lunix -lmccs_stubs -lmccs_glpk_stubs \
 -lstdc++ \
--Wl,-Bdynamic \
--static-libgcc
+-static-libgcc \
+-static
+# -Wl,-Bdynamic
 
 CLINKING_macos = \
 -lunix -lmccs_stubs -lmccs_glpk_stubs \
 -lstdc++
 
 CLINKING_openbsd = $(CLINKING_macos)
+CLINKING_freebsd = $(CLINKING_macos)
 
 LINKING = (-noautolink $(patsubst %,-cclib %,$(CLINKING_$(1))))
 
 EXPORTS_openbsd = \
 CPATH=/usr/local/include: \
 LIBRARY_PATH=/usr/local/lib: \
 
+EXPORTS_freebsd = \
+CPATH=/usr/local/include: \
+LIBRARY_PATH=/usr/local/lib: \
 
 %: opam-$(VERSION)-%
 
-opam-$(VERSION)-%: out/opam-$(VERSION)-%
+opam-$(VERSION)-%: $(OUTDIR)/opam-$(VERSION)-%
 	ln -sf $^ $@
 
 # host: opam-$(VERSION)-$(HOST)
@@ -76,40 +85,44 @@ opam-$(VERSION)-%: out/opam-$(VERSION)-%
 build/$(HOST).env:
 	mkdir -p build/$(HOST)
 	cd build/$(HOST) && curl -OL $(OCAML_URL)
-	cd build/$(HOST) && tar xzf ocaml-$(OCAMLV).tar.gz
+	cd build/$(HOST) && tar xzf $(OCAMLV).tar.gz
 	cd build/$(HOST)/ocaml-$(OCAMLV) && \
-	  ./configure -prefix $(shell pwd)/build/$(HOST) && \
-	  $(MAKE) world opt.opt && \
+	  ./configure --prefix "$(shell pwd)/build/$(HOST)" \
+	    --disable-debug-runtime --disable-debugger --disable-instrumented-runtime \
+	    --disable-ocamldoc --disable-stdlib-manpages --disable-ocamltest && \
+	  $(MAKE) -j$(JOBS) && \
 	  $(MAKE) install
-	rm -rf build/$(HOST)/ocaml-$(OCAMLV) build/$(HOST)/ocaml-$(OCAMLV).tar.gz
+	rm -rf build/$(HOST)/ocaml-$(OCAMLV) build/$(HOST)/$(OCAMLV).tar.gz
 	touch $@
 
-# Actually builds out/opam-$(VERSION)-$(HOST), but we don't want to override the
+# Actually builds $(OUTDIR)/opam-$(VERSION)-$(HOST), but we don't want to override the
 # rule that goes through a container
-host: out/opam-full-$(VERSION).tar.gz build/$(HOST).env
+host: $(OUTDIR)/opam-full-$(VERSION).tar.gz build/$(HOST).env
 	rm -rf build/opam-full-$(VERSION)
 	cd build && tar xzf ../$<
 	( export \
 	    PATH=$(shell pwd)/build/$(HOST)/bin:$$PATH \
 	    MAKE=$(MAKE) \
 	    $(EXPORTS_$(HOST_OS)); \
 	  cd build/opam-full-$(VERSION) && \
-	  ./configure && \
+	  ./configure --with-mccs && \
+	  $(MAKE) lib-ext && \
 	  echo "$(call LINKING,$(HOST_OS))" >src/client/linking.sexp && \
-	  $(MAKE) lib-ext DUNE_ARGS="--root=`pwd`"; \
-	  $(MAKE) opam DUNE_ARGS="--root=`pwd`"; \
+	  $(MAKE) opam; \
 	)
-	cp build/opam-full-$(VERSION)/opam out/opam-$(VERSION)-$(HOST)
-	strip out/opam-$(VERSION)-$(HOST)
+	cp build/opam-full-$(VERSION)/opam $(OUTDIR)/opam-$(VERSION)-$(HOST)
+	strip $(OUTDIR)/opam-$(VERSION)-$(HOST)
+	$(OUTDIR)/opam-$(VERSION)-$(HOST) --version
 	rm -rf build/opam-full-$(VERSION)
 
 # Containerised builds
-out/opam-$(VERSION)-%-linux: build/%-linux.image out/opam-full-$(VERSION).tar.gz
-	docker run --rm -v `pwd`/out:/src \
+$(OUTDIR)/opam-$(VERSION)-%-linux: $(OUTDIR)/opam-full-$(VERSION).tar.gz build/%-linux.image
+	docker run --rm -i \
 	  -e "VERSION=$(VERSION)" \
 	  -e "TARGET=$*-linux" \
-	  -e "LINKING=$(call LINKING,$(HOST_OS))" \
-	  opam-build-$*-linux
+	  -e "LINKING=$(call LINKING,linux)" \
+	  opam-build-$*-linux \
+	  <$< >$@
 
 clean:
 	rm -rf build
@@ -119,9 +132,28 @@ distclean: clean
 
 REMOTE_DIR = /tmp/opam-release
 REMOTE_MAKE = make
-remote: out/opam-full-$(VERSION).tar.gz
-	ssh "$(REMOTE)" "mkdir -p $(REMOTE_DIR)/out"
-	scp Makefile "$(REMOTE):$(REMOTE_DIR)/"
-	scp "$^" "$(REMOTE):$(REMOTE_DIR)/$^"
-	ssh "$(REMOTE)" 'sh -c "cd $(REMOTE_DIR) && ulimit -s 8192 && $(REMOTE_MAKE) host TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)"'
-	scp "$(REMOTE):$(REMOTE_DIR)/out/opam-$(VERSION)*" out/
+REMOTE_SHELL = /bin/sh
+SSH = ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
+SCP = scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
+remote: $(OUTDIR)/opam-full-$(VERSION).tar.gz
+	$(SSH) "$(REMOTE)" "mkdir -p $(REMOTE_DIR)/$(OUTDIR)"
+	$(SCP) Makefile "$(REMOTE):$(REMOTE_DIR)/"
+	$(SCP) "$^" "$(REMOTE):$(REMOTE_DIR)/$^"
+	$(SSH) "$(REMOTE)" 'sh -c "cd $(REMOTE_DIR) && ulimit -s 8192 && $(REMOTE_MAKE) host TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)"'
+	$(SCP) "$(REMOTE):$(REMOTE_DIR)/$(OUTDIR)/opam-$(VERSION)*" "$(OUTDIR)/"
+
+SSH_USER = root
+ULIMIT = ulimit -s 8192 &&
+qemu: $(OUTDIR)/opam-full-$(VERSION).tar.gz
+	$(SSH) -p "$(QEMU_PORT)" $(SSH_USER)@localhost "$(REMOTE_SHELL) -c \"mkdir -p $(REMOTE_DIR)/$(OUTDIR)\""
+	$(SCP) -P "$(QEMU_PORT)" Makefile "$(SSH_USER)@localhost:$(REMOTE_DIR)/"
+	$(SCP) -P "$(QEMU_PORT)" "$^" "$(SSH_USER)@localhost:$(REMOTE_DIR)/$^"
+	$(SSH) -p "$(QEMU_PORT)" $(SSH_USER)@localhost "$(REMOTE_SHELL) -c \"cd $(REMOTE_DIR) && $(ULIMIT) $(REMOTE_MAKE) host JOBS=$(JOBS) TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)\""
+	$(SCP) -P "$(QEMU_PORT)" "$(SSH_USER)@localhost:$(REMOTE_DIR)/$(OUTDIR)/opam-$(VERSION)*" "$(OUTDIR)/"
+
+macos-local: $(OUTDIR)/opam-full-$(VERSION).tar.gz
+	LOCAL_RELEASE_DIR=$(shell mktemp -d); \
+          mkdir -p "$${LOCAL_RELEASE_DIR}/${OUTDIR}" && \
+          cp Makefile "$^" "$${LOCAL_RELEASE_DIR}" && \
+          cd "$${LOCAL_RELEASE_DIR}" && ulimit -s 8192 && arch -arch $(MACOS_ARCH) make host JOBS=$(JOBS) TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV) && \
+          cp $${LOCAL_RELEASE_DIR}/$(OUTDIR)/opam-$(VERSION)* "$(GIT_URL)/release/$(OUTDIR)/"

release/readme.md

@@ -1,29 +1,54 @@
 # Steps to follow for each release
 
 ## Finalise opam code for release
-* update version in opam files, configure.ac
+* update version in all the opam files and in configure.ac
 * run `make configure` to regenerate `./configure` [checked by github actions]
 * update copyright headers
-* run `make tests`, `opam-rt` [checked by github actions & appveyor]
-* update the CHANGE file: take `master_changes.md` content to fil it
+* if you're releasing the first final release of a new branch (e.g. 2.2.0): make sure `root_version` in OpamFile.ml is set to the final release number (e.g. for 2.2.0, root_version should be 2.2). Make sure that opamFormatUpgrade.ml also contains an upgrade function from the previous version (that function will most likely be empty)
+* run `make tests`, `opam-rt` [checked by github actions]
+* update the CHANGE file: take `master_changes.md` content to fill it
 
 ## Github release
 
 [ once bump version & changes PRs merged ]
-* tag the release (git tag -a 2.2.0; git push origin 2.2.0)
-* /!\ Once the tag pushed, it can be updated only in case of severe issue
+* tag the release (git tag -am 2.2.0 2.2.0; git push origin 2.2.0)
+* /!\ Once the tag pushed, it can be updated [different commit] only in case of severe issue
 * create a release (or prerelease if intermediate release) draft on github based on your tag (https://github.com/ocaml/opam/releases/new)
-* generate opam artifacts, using `shell/release.sh`, it requires to have Docker install with several remotes, the different arches
-* add releases notes (content of `master_changes.md`) in the release
-* upload signature of artefacts
+* Make sure the repository is in the correct state: `git switch --detach <tag>`
+* launch docker using the Docker GUI macOS app
+* generate opam artifacts, using `release/release.sh <tag>` from a macOS/arm64 machine, it requires to have Docker and QEMU installed (see below device requirements)
+* generate the signatures using `release/sign.sh <tag>`
+* add releases notes (content of `master_changes.md`) in the release draft
+* upload everything from `release/out/<tag>`
 * finalise the release (publish)
 
 ## Publish the release
 
 * add hashes in `install.sh` (and check signatures)
-* publish opam packages in opam-repository
+* publish opam packages in opam-repository (and add `flags: avoid-version` and `available: opam-version >= "2.1.0"` to each packages if this is not a stable version)
+* update versions (and messages, if necessary) in https://github.com/ocaml/opam-repository/blob/master/repo
 
 ## Announce!
 
 * a blog entry in opam.ocaml.org
 * a announcement in discuss.ocaml.org
+* copy the blog entry from opam.ocaml.org for https://github.com/ocaml/ocaml.org/tree/main/data/changelog/opam
+* announce the release on the OCaml Discord server
+
+
+## After release
+
+* Bump the version with a `~dev` at the end (e.g. `2.2.0~alpha~dev`)
+* Check if reftests needs an update
+* Bring the changes to the changelog (CHANGES) from the branch of the release to the `master` branch
+
+### On a release candidate
+* create a branch to a `x.y` for rc's and the final release
+
+---
+
+## Device requirements
+* Mac M1 or above with Rosetta2
+* brew dependencies: git, gpg, qemu>=8.1.0, docker>=24.0.0, md5sha1sum
+* opam repo with the tag fetched
+* Have the secret key available