From 25e8353781fe722e4cb5d926f49a2c799456d22c Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 21 Dec 2023 15:26:47 -0800 Subject: [PATCH] Finally parse upstream releases data properly This ports our `versions.sh` script to finally actually parse https://github.com/ruby/www.ruby-lang.org/raw/master/_data/releases.yml directly instead of scraping the data from the webpages generated from it. Additionally, this *should* allow us to automatically pick up GA releases after the pre-release (we can find out whether it worked next week!) --- .gitignore | 1 + 3.3-rc/alpine3.18/Dockerfile | 6 +- 3.3-rc/alpine3.19/Dockerfile | 6 +- 3.3-rc/bookworm/Dockerfile | 6 +- 3.3-rc/bullseye/Dockerfile | 6 +- 3.3-rc/slim-bookworm/Dockerfile | 6 +- 3.3-rc/slim-bullseye/Dockerfile | 6 +- Dockerfile.template | 15 +++- apply-templates.sh | 5 ++ generate-stackbrew-library.sh | 7 +- versions.json | 153 ++++++++++++++++++++++++++----- versions.sh | 155 ++++++++++++++++++-------------- 12 files changed, 267 insertions(+), 105 deletions(-) diff --git a/.gitignore b/.gitignore index d548f66de0..da4ffd7830 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .jq-template.awk +.yq* diff --git a/3.3-rc/alpine3.18/Dockerfile b/3.3-rc/alpine3.18/Dockerfile index 12730d35c3..6e65161149 100644 --- a/3.3-rc/alpine3.18/Dockerfile +++ b/3.3-rc/alpine3.18/Dockerfile @@ -26,8 +26,10 @@ RUN set -eux; \ } >> /usr/local/etc/gemrc ENV LANG C.UTF-8 -ENV RUBY_MAJOR 3.3-rc + +# https://www.ruby-lang.org/en/news/2023/12/11/ruby-3-3-0-rc1-released/ ENV RUBY_VERSION 3.3.0-rc1 +ENV RUBY_DOWNLOAD_URL https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0-rc1.tar.xz ENV RUBY_DOWNLOAD_SHA256 051815637f1fa75a1edf2c54b66d5d5b69563daad777da8dc39543b7754b5290 # some of ruby's build scripts are written in ruby @@ -86,7 +88,7 @@ RUN set -eux; \ cargo --version; \ fi; \ \ - wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz"; \ + wget -O ruby.tar.xz "$RUBY_DOWNLOAD_URL"; \ echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum --check --strict; \ \ mkdir -p /usr/src/ruby; \ diff --git a/3.3-rc/alpine3.19/Dockerfile b/3.3-rc/alpine3.19/Dockerfile index 05358035e7..13557e7995 100644 --- a/3.3-rc/alpine3.19/Dockerfile +++ b/3.3-rc/alpine3.19/Dockerfile @@ -26,8 +26,10 @@ RUN set -eux; \ } >> /usr/local/etc/gemrc ENV LANG C.UTF-8 -ENV RUBY_MAJOR 3.3-rc + +# https://www.ruby-lang.org/en/news/2023/12/11/ruby-3-3-0-rc1-released/ ENV RUBY_VERSION 3.3.0-rc1 +ENV RUBY_DOWNLOAD_URL https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0-rc1.tar.xz ENV RUBY_DOWNLOAD_SHA256 051815637f1fa75a1edf2c54b66d5d5b69563daad777da8dc39543b7754b5290 # some of ruby's build scripts are written in ruby @@ -86,7 +88,7 @@ RUN set -eux; \ cargo --version; \ fi; \ \ - wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz"; \ + wget -O ruby.tar.xz "$RUBY_DOWNLOAD_URL"; \ echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum --check --strict; \ \ mkdir -p /usr/src/ruby; \ diff --git a/3.3-rc/bookworm/Dockerfile b/3.3-rc/bookworm/Dockerfile index 7993f281a6..12727ffe31 100644 --- a/3.3-rc/bookworm/Dockerfile +++ b/3.3-rc/bookworm/Dockerfile @@ -15,8 +15,10 @@ RUN set -eux; \ } >> /usr/local/etc/gemrc ENV LANG C.UTF-8 -ENV RUBY_MAJOR 3.3-rc + +# https://www.ruby-lang.org/en/news/2023/12/11/ruby-3-3-0-rc1-released/ ENV RUBY_VERSION 3.3.0-rc1 +ENV RUBY_DOWNLOAD_URL https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0-rc1.tar.xz ENV RUBY_DOWNLOAD_SHA256 051815637f1fa75a1edf2c54b66d5d5b69563daad777da8dc39543b7754b5290 # some of ruby's build scripts are written in ruby @@ -55,7 +57,7 @@ RUN set -eux; \ cargo --version; \ fi; \ \ - wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz"; \ + wget -O ruby.tar.xz "$RUBY_DOWNLOAD_URL"; \ echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum --check --strict; \ \ mkdir -p /usr/src/ruby; \ diff --git a/3.3-rc/bullseye/Dockerfile b/3.3-rc/bullseye/Dockerfile index c73d4e0f46..dd3f4a6e21 100644 --- a/3.3-rc/bullseye/Dockerfile +++ b/3.3-rc/bullseye/Dockerfile @@ -15,8 +15,10 @@ RUN set -eux; \ } >> /usr/local/etc/gemrc ENV LANG C.UTF-8 -ENV RUBY_MAJOR 3.3-rc + +# https://www.ruby-lang.org/en/news/2023/12/11/ruby-3-3-0-rc1-released/ ENV RUBY_VERSION 3.3.0-rc1 +ENV RUBY_DOWNLOAD_URL https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0-rc1.tar.xz ENV RUBY_DOWNLOAD_SHA256 051815637f1fa75a1edf2c54b66d5d5b69563daad777da8dc39543b7754b5290 # some of ruby's build scripts are written in ruby @@ -55,7 +57,7 @@ RUN set -eux; \ cargo --version; \ fi; \ \ - wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz"; \ + wget -O ruby.tar.xz "$RUBY_DOWNLOAD_URL"; \ echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum --check --strict; \ \ mkdir -p /usr/src/ruby; \ diff --git a/3.3-rc/slim-bookworm/Dockerfile b/3.3-rc/slim-bookworm/Dockerfile index 8c68665b83..17bb114152 100644 --- a/3.3-rc/slim-bookworm/Dockerfile +++ b/3.3-rc/slim-bookworm/Dockerfile @@ -29,8 +29,10 @@ RUN set -eux; \ } >> /usr/local/etc/gemrc ENV LANG C.UTF-8 -ENV RUBY_MAJOR 3.3-rc + +# https://www.ruby-lang.org/en/news/2023/12/11/ruby-3-3-0-rc1-released/ ENV RUBY_VERSION 3.3.0-rc1 +ENV RUBY_DOWNLOAD_URL https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0-rc1.tar.xz ENV RUBY_DOWNLOAD_SHA256 051815637f1fa75a1edf2c54b66d5d5b69563daad777da8dc39543b7754b5290 # some of ruby's build scripts are written in ruby @@ -81,7 +83,7 @@ RUN set -eux; \ cargo --version; \ fi; \ \ - wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz"; \ + wget -O ruby.tar.xz "$RUBY_DOWNLOAD_URL"; \ echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum --check --strict; \ \ mkdir -p /usr/src/ruby; \ diff --git a/3.3-rc/slim-bullseye/Dockerfile b/3.3-rc/slim-bullseye/Dockerfile index 9bb0bf1124..e87831cdc7 100644 --- a/3.3-rc/slim-bullseye/Dockerfile +++ b/3.3-rc/slim-bullseye/Dockerfile @@ -29,8 +29,10 @@ RUN set -eux; \ } >> /usr/local/etc/gemrc ENV LANG C.UTF-8 -ENV RUBY_MAJOR 3.3-rc + +# https://www.ruby-lang.org/en/news/2023/12/11/ruby-3-3-0-rc1-released/ ENV RUBY_VERSION 3.3.0-rc1 +ENV RUBY_DOWNLOAD_URL https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0-rc1.tar.xz ENV RUBY_DOWNLOAD_SHA256 051815637f1fa75a1edf2c54b66d5d5b69563daad777da8dc39543b7754b5290 # some of ruby's build scripts are written in ruby @@ -81,7 +83,7 @@ RUN set -eux; \ cargo --version; \ fi; \ \ - wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz"; \ + wget -O ruby.tar.xz "$RUBY_DOWNLOAD_URL"; \ echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum --check --strict; \ \ mkdir -p /usr/src/ruby; \ diff --git a/Dockerfile.template b/Dockerfile.template index c81cc16bdb..4cb9f1ea37 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -50,9 +50,17 @@ RUN set -eux; \ } >> /usr/local/etc/gemrc ENV LANG C.UTF-8 +{{ if .version | . == "3.0.6" or . == "3.1.4" or . == "3.2.2" then ( -}} ENV RUBY_MAJOR {{ env.version }} ENV RUBY_VERSION {{ .version }} -ENV RUBY_DOWNLOAD_SHA256 {{ .sha256 }} +ENV RUBY_DOWNLOAD_SHA256 {{ .sha256.xz }} +{{ ) else ( -}} + +# https://www.ruby-lang.org/{{ .post | ltrimstr("/") }} +ENV RUBY_VERSION {{ .version }} +ENV RUBY_DOWNLOAD_URL {{ .url.xz }} +ENV RUBY_DOWNLOAD_SHA256 {{ .sha256.xz }} +{{ ) end -}} # some of ruby's build scripts are written in ruby # we purge system ruby later to make sure our final image uses what we just built @@ -189,7 +197,12 @@ RUN set -eux; \ fi; \ {{ ) else "" end -}} \ +{{ if .version | . == "3.0.6" or . == "3.1.4" or . == "3.2.2" then ( -}} +{{ if .url.xz != "https://cache.ruby-lang.org/pub/ruby/\(env.version | rtrimstr("-rc"))/ruby-\(.version).tar.xz" then error("url for \(.version) is not as expected!") else "" end -}} wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz"; \ +{{ ) else ( -}} + wget -O ruby.tar.xz "$RUBY_DOWNLOAD_URL"; \ +{{ ) end -}} echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum --check --strict; \ \ mkdir -p /usr/src/ruby; \ diff --git a/apply-templates.sh b/apply-templates.sh index 7f9da43e19..9ed389b588 100755 --- a/apply-templates.sh +++ b/apply-templates.sh @@ -32,6 +32,11 @@ for version; do rm -rf "$version/" + if jq -e '.[env.version] | not' versions.json > /dev/null; then + echo "deleting $version ..." + continue + fi + variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)" eval "variants=( $variants )" diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 398f7d4f36..4e3679a087 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -75,11 +75,14 @@ join() { for version; do export version + + if ! fullVersion="$(jq -er '.[env.version] | if . then .version else empty end' versions.json)"; then + continue + fi + variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)" eval "variants=( $variants )" - fullVersion="$(jq -r '.[env.version].version' versions.json)" - versionAliases=( $fullVersion $version diff --git a/versions.json b/versions.json index 942667adaa..99ffa1b602 100644 --- a/versions.json +++ b/versions.json @@ -1,17 +1,70 @@ { "3.0": { - "sha256": "b5cbee93e62d85cfb2a408c49fa30a74231ae8409c2b3858e5f5ea254d7ddbd1", + "version": "3.0.6", + "date": "2023-03-30", + "post": "/en/news/2023/03/30/ruby-3-0-6-released/", + "url": { + "gz": "https://cache.ruby-lang.org/pub/ruby/3.0/ruby-3.0.6.tar.gz", + "xz": "https://cache.ruby-lang.org/pub/ruby/3.0/ruby-3.0.6.tar.xz", + "zip": "https://cache.ruby-lang.org/pub/ruby/3.0/ruby-3.0.6.zip" + }, + "size": { + "gz": 21315725, + "xz": 15864560, + "zip": 25694359 + }, + "sha1": { + "gz": "1052441f0abbb0302fb9f1481d2db99dfb4d4c29", + "xz": "7880c34d7193224e967163b12f33bf7aaf7304f6", + "zip": "e75d1bc14dd89c176145dc3968774e30f3a17652" + }, + "sha256": { + "gz": "6e6cbd490030d7910c0ff20edefab4294dfcd1046f0f8f47f78b597987ac683e", + "xz": "b5cbee93e62d85cfb2a408c49fa30a74231ae8409c2b3858e5f5ea254d7ddbd1", + "zip": "428d518d12f09df4146fc31dbed47c8d7e10fcccd2426948e5c0862d9321480d" + }, + "sha512": { + "gz": "d596bfd374ae777717379b409afe8ee1655ade0c0539ada7a10af4780b818efe25a28aa50a2a7226741d1776d744e10ad916641f9d12fb31c7444b0a01d0e0cc", + "xz": "abbf883cd9f3ddbd171df8f8c3cd35d930623c4c01a5e01387de0aee9811cca7604b82163e18e04f809773bf1ca5a450f13f62f3db14f191f610e116ae4fa6f8", + "zip": "576d11c668acac57cf4952228b148d17f16ab1dc491145355a4f2068b15f6cab8a4007a84d9d1eda4c1b62837675c82be99ebe6379c314f46c6ebbbf89677b5e" + }, "variants": [ "bullseye", "slim-bullseye", "buster", "slim-buster", "alpine3.16" - ], - "version": "3.0.6" + ] }, "3.1": { - "sha256": "1b6d6010e76036c937b9671f4752f065aeca800a6c664f71f6c9a699453af94f", + "version": "3.1.4", + "date": "2023-03-30", + "post": "/en/news/2023/03/30/ruby-3-1-4-released/", + "url": { + "gz": "https://cache.ruby-lang.org/pub/ruby/3.1/ruby-3.1.4.tar.gz", + "xz": "https://cache.ruby-lang.org/pub/ruby/3.1/ruby-3.1.4.tar.xz", + "zip": "https://cache.ruby-lang.org/pub/ruby/3.1/ruby-3.1.4.zip" + }, + "size": { + "gz": 20917933, + "xz": 15316604, + "zip": 25241255 + }, + "sha1": { + "gz": "38eddfc5a7536b6c8133183563009a4ed9bbe6db", + "xz": "2e2fbf43b7db6f24280548a3544912535bed8212", + "zip": "1061632623caa82a68a04a35777ed8f1797a9f8f" + }, + "sha256": { + "gz": "a3d55879a0dfab1d7141fdf10d22a07dbf8e5cdc4415da1bde06127d5cc3c7b6", + "xz": "1b6d6010e76036c937b9671f4752f065aeca800a6c664f71f6c9a699453af94f", + "zip": "1fce1ab3d61d10a857dc821dab6e77fa41d0663c5dbbfaa5d9b9c2bdec5ce303" + }, + "sha512": { + "gz": "41cf1561dd7eb249bb2c2f5ea958884880648cc1d11da9315f14158a2d0ff94b2c5c7d75291a67e57e1813d2ec7b618e5372a9f18ee93be6ed306f47b0d3199a", + "xz": "a627bb629a10750b8b2081ad451a41faea0fc85d95aa1e267e3d2a0f56a35bb58195d4a8d13bbdbd82f4197a96dae22b1cee1dfc83861ec33a67ece07aef5633", + "zip": "3a334302df97c2c7fec3c2d05d19a40b1ec6f95fef52c85d397196ce62fac4834f96783f0ac7fcba6e2a670f004bcc275db6f1810ace6c68a594e7d2fd9b297b" + }, "variants": [ "bookworm", "slim-bookworm", @@ -19,10 +72,45 @@ "slim-bullseye", "alpine3.19", "alpine3.18" - ], - "version": "3.1.4" + ] }, "3.2": { + "version": "3.2.2", + "date": "2023-03-30", + "post": "/en/news/2023/03/30/ruby-3-2-2-released/", + "url": { + "gz": "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.2.tar.gz", + "xz": "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.2.tar.xz", + "zip": "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.2.zip" + }, + "size": { + "gz": 20467023, + "xz": 15118856, + "zip": 24615317 + }, + "sha1": { + "gz": "670fce00d83771a1349b116e56a8a3b0ad323769", + "xz": "087af286b70b0e17f88c9c4469b471eca2010161", + "zip": "a1b6d57019d41dca269b4b16a80784755d34b81d" + }, + "sha256": { + "gz": "96c57558871a6748de5bc9f274e93f4b5aad06cd8f37befa0e8d94e7b8a423bc", + "xz": "4b352d0f7ec384e332e3e44cdbfdcd5ff2d594af3c8296b5636c710975149e23", + "zip": "cc216ecb4f49064d8f44e10ecf9218cfd7b28cf4168bb79ecdf171e321db4af1" + }, + "sha512": { + "gz": "bcc68f3f24c1c8987d9c80b57332e5791f25b935ba38daf5addf60dbfe3a05f9dcaf21909681b88e862c67c6ed103150f73259c6e35c564f13a00f432e3c1e46", + "xz": "a29f24cd80f563f6368952d06d6273f7241a409fa9ab2f60e03dde2ac58ca06bee1750715b6134caebf4c061d3503446dc37a6059e19860bb0010eef34951935", + "zip": "569a68d89cc9a646cd0319d7cb8d57df3a55c0ac2c64f1f61607cc9c06b3aa8415eb8d38f7893ab3dbf072da9e919fbc454a9338e924c20a6a5110a1fa301d52" + }, + "variants": [ + "bookworm", + "slim-bookworm", + "bullseye", + "slim-bullseye", + "alpine3.19", + "alpine3.18" + ], "rust": { "version": "1.74.1" }, @@ -54,8 +142,44 @@ } }, "version": "1.26.0" + } + }, + "3.3": null, + "3.3-rc": { + "version": "3.3.0-rc1", + "date": "2023-12-11", + "post": "/en/news/2023/12/11/ruby-3-3-0-rc1-released/", + "tag": "v3_3_0_rc1", + "stats": { + "files_changed": 5414, + "insertions": 306141, + "deletions": 183575 + }, + "url": { + "gz": "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0-rc1.tar.gz", + "zip": "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0-rc1.zip", + "xz": "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0-rc1.tar.xz" + }, + "size": { + "gz": 21783575, + "zip": 26735443, + "xz": 16163884 + }, + "sha1": { + "gz": "c75a860e06f27b7f69b874757417277c21d1d3f4", + "zip": "35583453a7734216b08829ef0ec9ea1bc0d4ae7f", + "xz": "26503f9bdc7d0a05aaa9836f54d3aa9e74a9ead9" + }, + "sha256": { + "gz": "c4ff82395a90ef76c7f906b7687026e0ab96b094dcf3a532d9ab97784a073222", + "zip": "56dd82e1dd714f2889ca975ae7befbe005675de08839c2cc4a484de2ae65201c", + "xz": "051815637f1fa75a1edf2c54b66d5d5b69563daad777da8dc39543b7754b5290" + }, + "sha512": { + "gz": "265fb2ffe55af47d8349edffcebe749cc170291227cef55529fe4e67363e4e84b88daa34ffb5364a99c8a0e32110266a34c9a11d62f3bd6f6d47fa76eca641f4", + "zip": "7fbe414c230aedc9f364512bcbc213038f8f6e4268929a559d2527e2f3e32a140b394e37098ab7e59161236eca8b89cc9e52d73a3be8d7bd44faa91681483f5d", + "xz": "82f4acfaad1dc47db27ee3eb952481a95b3474a98059e9e9f5ceb035b690d1faabe99f2ec52371c4089ed1615eb10c395f029088418fec4d26399b65b4f259b9" }, - "sha256": "4b352d0f7ec384e332e3e44cdbfdcd5ff2d594af3c8296b5636c710975149e23", "variants": [ "bookworm", "slim-bookworm", @@ -64,9 +188,6 @@ "alpine3.19", "alpine3.18" ], - "version": "3.2.2" - }, - "3.3-rc": { "rust": { "version": "1.74.1" }, @@ -98,16 +219,6 @@ } }, "version": "1.26.0" - }, - "sha256": "051815637f1fa75a1edf2c54b66d5d5b69563daad777da8dc39543b7754b5290", - "variants": [ - "bookworm", - "slim-bookworm", - "bullseye", - "slim-bullseye", - "alpine3.19", - "alpine3.18" - ], - "version": "3.3.0-rc1" + } } } diff --git a/versions.sh b/versions.sh index 061ee394a1..6b8b185335 100755 --- a/versions.sh +++ b/versions.sh @@ -12,9 +12,26 @@ else fi versions=( "${versions[@]%/}" ) -releasesPage="$(curl -fsSL 'https://www.ruby-lang.org/en/downloads/releases/' | grep -A 2 'Ruby')" # very wide grep to cut down on "set -x" output when debugging (should match the one later) -newsPage="$(curl -fsSL 'https://www.ruby-lang.org/en/news/' | grep 'Released')" # occasionally, releases don't show up on the Releases page (see https://github.com/ruby/www.ruby-lang.org/blob/master/_data/releases.yml) -# TODO consider parsing https://github.com/ruby/www.ruby-lang.org/blob/master/_data/downloads.yml as well +yq='./.yq' +# https://github.com/mikefarah/yq/releases +# TODO detect host architecture +yqUrl='https://github.com/mikefarah/yq/releases/download/v4.40.5/yq_linux_amd64' +yqSha256='0d6aaf1cf44a8d18fbc7ed0ef14f735a8df8d2e314c4cc0f0242d35c0a440c95' +if command -v yq &> /dev/null; then + # TODO verify that the "yq" in PATH is https://github.com/mikefarah/yq, not the python-based version you'd get from "apt-get install yq" somehow? maybe they're compatible enough for our needs that it doesn't matter? + yq='yq' +elif [ ! -x "$yq" ] || ! sha256sum <<<"$yqSha256 *$yq" --quiet --strict --check; then + wget -qO "$yq.new" "$yqUrl" + sha256sum <<<"$yqSha256 *$yq.new" --quiet --strict --check + chmod +x "$yq.new" + "$yq.new" --version + mv "$yq.new" "$yq" +fi + +releases="$( + wget -qO- 'https://github.com/ruby/www.ruby-lang.org/raw/master/_data/releases.yml' \ + | "$yq" -r '@json' # this *should* work on both the Go-based "yq" we download and the Python-based "yq" available from Debian's APT repo +)" for version in "${versions[@]}"; do rcGrepV='-v' @@ -24,75 +41,70 @@ for version in "${versions[@]}"; do fi export version rcVersion - IFS=$'\n'; allVersions=( $( - curl -fsSL --compressed "https://cache.ruby-lang.org/pub/ruby/$rcVersion/" \ - | grep -oE '["/]ruby-'"$rcVersion"'.[^"]+\.tar\.xz' \ - | sed -r 's!^["/]ruby-([^"]+)[.]tar[.]xz!\1!' \ - | grep $rcGrepV -E 'preview|rc' \ - | sort -ruV - ) ); unset IFS - - fullVersion= - shaVal= - for tryVersion in "${allVersions[@]}"; do - if \ - { - versionReleasePage="$(grep "Ruby $tryVersion" -A 2 <<<"$releasesPage" | awk -F '"' '$1 == "Ruby '"$tryVersion"' Released' <<<"$newsPage" | cut -d'"' -f2)" \ - && [ -n "$versionReleasePage" ] \ - && shaVal="$(curl -fsL "https://www.ruby-lang.org/$versionReleasePage" | grep "ruby-$tryVersion.tar.xz" -A 5)" \ - && shaVal="$(awk <<<"$shaVal" '$1 == "SHA256:" { print $2; exit }')" \ - && [ -n "$shaVal" ] - } \ - ; then - fullVersion="$tryVersion" - break - fi - done + doc="$(jq <<<"$releases" -c ' + map( + select( + .version + # exact versions ("3.1.0-preview1") should match exactly but "X.Y" or "X.Y-rc" should fuzzy match appropriately + | . == env.version or ( + ( + startswith(env.rcVersion + ".") + or startswith(env.rcVersion + "-") + ) and ( + contains("preview") or contains("rc") + | if env.version == env.rcVersion then not else . end + ) + ) + ) + ) + | first // empty + ')" - if [ -z "$fullVersion" ]; then - echo >&2 "error: cannot determine sha for $version (tried all of ${allVersions[*]})" - exit 1 + if [ -z "$doc" ]; then + echo >&2 "warning: skipping/removing '$version' (does not appear to exist upstream)" + json="$(jq <<<"$json" -c '.[env.version] = null')" + continue fi - echo "$version: $fullVersion; $shaVal" + fullVersion="$(jq <<<"$doc" -r '.version')" + echo "$version: $fullVersion" - export fullVersion shaVal - doc="$(jq -nc ' - { - version: env.fullVersion, - sha256: env.shaVal, - variants: [ - ( - # https://bugs.ruby-lang.org/issues/18658 - # https://github.com/docker-library/ruby/pull/392#issuecomment-1329896174 - if "3.0" == env.version then - "bullseye", - "buster" - else - "bookworm", - "bullseye", - empty # trailing comma hack - end - | ., "slim-" + .), # https://github.com/docker-library/ruby/pull/142#issuecomment-320012893 - ( - # Alpine 3.17+ defaults to OpenSSL 3 which is not supported by Ruby 3.0 - # https://bugs.ruby-lang.org/issues/18658 - # https://github.com/docker-library/ruby/pull/392#issuecomment-1329896174 - if "3.0" == env.version then "3.16" else - "3.19", - "3.18", - empty # trailing comma hack - end - | "alpine" + .) - ], - } + if [ "$rcVersion" != "$version" ] && gaFullVersion="$(jq <<<"$json" -er '.[env.rcVersion] | if . then .version else empty end')"; then + # Ruby pre-releases have only been for .0 since ~2011, so if our pre-release now has a relevant GA, it should go away 👀 + # just in case, we'll also do a version comparison to make sure we don't have a pre-release that's newer than the relevant GA + latestVersion="$({ echo "$fullVersion"; echo "$gaFullVersion"; } | sort -V | tail -1)" + if [[ "$fullVersion" == "$gaFullVersion"* ]] || [ "$latestVersion" = "$gaFullVersion" ]; then + # "x.y.z-rc1" == x.y.z* + json="$(jq <<<"$json" -c 'del(.[env.rcVersion])')" + continue + fi + fi + + doc="$(jq <<<"$doc" -c ' + .variants = [ + ( + # https://bugs.ruby-lang.org/issues/18658 + # https://github.com/docker-library/ruby/pull/392#issuecomment-1329896174 + if "3.0" == env.version then + "bullseye", + "buster" + else + "bookworm", + "bullseye", + empty # trailing comma hack + end + | ., "slim-" + .), # https://github.com/docker-library/ruby/pull/142#issuecomment-320012893 + ( + # Alpine 3.17+ defaults to OpenSSL 3 which is not supported by Ruby 3.0 + # https://bugs.ruby-lang.org/issues/18658 + # https://github.com/docker-library/ruby/pull/392#issuecomment-1329896174 + if "3.0" == env.version then "3.16" else + "3.19", + "3.18", + empty # trailing comma hack + end + | "alpine" + .) + ] ')" case "$rcVersion" in @@ -113,6 +125,11 @@ for version in "${versions[@]}"; do esac json="$(jq <<<"$json" -c --argjson doc "$doc" '.[env.version] = $doc')" + + # make sure pre-release versions have a placeholder for GA + if [ "$version" != "$rcVersion" ]; then + json="$(jq <<<"$json" -c '.[env.rcVersion] //= null')" + fi done -jq <<<"$json" -S . > versions.json +jq <<<"$json" 'to_entries | sort_by(.key) | from_entries' > versions.json