From 605ca80d131d1304b3d833cc4c33b0f6d62ab474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20F=C3=A4rnstrand?= Date: Tue, 14 Nov 2023 17:30:25 +0100 Subject: [PATCH 1/2] Add script for preparing RPM repositories on build server --- ci/buildserver-build.sh | 11 +++++-- ci/buildserver-config.sh | 2 ++ ci/prepare-rpm-repository.sh | 57 ++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) create mode 100755 ci/prepare-rpm-repository.sh diff --git a/ci/buildserver-build.sh b/ci/buildserver-build.sh index db172031eb85..f9d8af60fa01 100755 --- a/ci/buildserver-build.sh +++ b/ci/buildserver-build.sh @@ -46,10 +46,15 @@ esac function publish_linux_repositories { local artifact_dir=$1 local version=$2 - local deb_repo_dir="$SCRIPT_DIR/deb/$version" + local deb_repo_dir="$SCRIPT_DIR/deb/$version" + echo "Preparing Apt repository in $deb_repo_dir" "$SCRIPT_DIR/prepare-apt-repository.sh" "$artifact_dir" "$version" "$deb_repo_dir" + local rpm_repo_dir="$SCRIPT_DIR/rpm/$version" + echo "Preparing RPM repository in $rpm_repo_dir" + "$SCRIPT_DIR/prepare-rpm-repository.sh" "$artifact_dir" "$version" "$rpm_repo_dir" + "$SCRIPT_DIR/publish-linux-repositories.sh" --dev "$version" "$deb_repo_dir" # If this is a release build, also push to staging. # Publishing to production is done manually. @@ -226,7 +231,9 @@ function build_ref { fi fi - publish_linux_repositories "$artifact_dir" "$version" + if [[ "$(uname -s)" == "Linux" ]]; then + publish_linux_repositories "$artifact_dir" "$version" + fi (cd "$artifact_dir" && upload "$version") || return 1 # shellcheck disable=SC2216 yes | rm -r "$artifact_dir" diff --git a/ci/buildserver-config.sh b/ci/buildserver-config.sh index 789a9b98d21b..64cca6b33686 100644 --- a/ci/buildserver-config.sh +++ b/ci/buildserver-config.sh @@ -12,6 +12,8 @@ SUPPORTED_DEB_CODENAMES=("sid" "testing" "bookworm" "bullseye") SUPPORTED_DEB_CODENAMES+=("jammy" "focal" "lunar") export SUPPORTED_DEB_CODENAMES +export SUPPORTED_RPM_ARCHITECTURES=("x86_64" "aarch64") + # Servers to upload Linux deb/rpm repositories to export DEV_LINUX_REPOSITORY_SERVERS=("se-got-cdn-001.devmole.eu" "se-got-cdn-002.devmole.eu") export STAGING_LINUX_REPOSITORY_SERVERS=("se-got-cdn-001.stagemole.eu" "se-got-cdn-002.stagemole.eu") diff --git a/ci/prepare-rpm-repository.sh b/ci/prepare-rpm-repository.sh new file mode 100755 index 000000000000..6ed73742f7f7 --- /dev/null +++ b/ci/prepare-rpm-repository.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Usage: ./prepare-rpm-repository.sh +# +# Will create an rpm repository in and add all .rpm files from +# matching to the repository. + +set -eu + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +source "$SCRIPT_DIR/buildserver-config.sh" + +artifact_dir=$1 +version=$2 +repo_dir=$3 + +function generate_repository_configuration { + echo -e "[mullvad-rpm] +name=Mullvad VPN +baseurl=https://repository.mullvad.net/rpm/\$basearch +type=rpm +enabled=1 +gpgcheck=1 +gpgkey=https://repository.mullvad.net/rpm/mullvad-keyring.asc" +} + +function create_repository { + local arch_repo_dir=$1 + local rpm_path=$2 + + mkdir -p "$arch_repo_dir" + + # Copy RPM file into repository + cp "$rpm_path" "$arch_repo_dir"/ + + # Generate repository metadata files (containing among other things checksums + # for the above artifact) + createrepo_c "$arch_repo_dir" + + # Sign repository metadata (created by createrepo_c above) + # --yes is passed to automatically overwrite existing files + # in the case where the build server re-builds something we already + # have built. + gpg --detach-sign --armor --yes "$arch_repo_dir/repodata/repomd.xml" +} + +for arch in "${SUPPORTED_RPM_ARCHITECTURES[@]}"; do + rpm_path="$artifact_dir"/MullvadVPN-"$version"_"$arch".rpm + if [[ ! -e "$rpm_path" ]]; then + echo "RPM at $rpm_path does not exist" >&2 + exit 1 + fi + create_repository "$repo_dir/$arch" "$rpm_path" +done + +generate_repository_configuration > "$repo_dir/mullvad.repo" From 16e642a52048d5e41ccd8a1d5fd36c1bf4b2bdf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20F=C3=A4rnstrand?= Date: Tue, 14 Nov 2023 17:36:26 +0100 Subject: [PATCH 2/2] Upload RPMs to repository server on builds --- ci/buildserver-build.sh | 8 +++- ci/buildserver-config.sh | 4 ++ ci/prepare-rpm-repository.sh | 12 ----- ci/publish-linux-repositories.sh | 75 +++++++++++++++++++++++++++----- 4 files changed, 73 insertions(+), 26 deletions(-) diff --git a/ci/buildserver-build.sh b/ci/buildserver-build.sh index f9d8af60fa01..2b9d2a49048e 100755 --- a/ci/buildserver-build.sh +++ b/ci/buildserver-build.sh @@ -55,11 +55,15 @@ function publish_linux_repositories { echo "Preparing RPM repository in $rpm_repo_dir" "$SCRIPT_DIR/prepare-rpm-repository.sh" "$artifact_dir" "$version" "$rpm_repo_dir" - "$SCRIPT_DIR/publish-linux-repositories.sh" --dev "$version" "$deb_repo_dir" + "$SCRIPT_DIR/publish-linux-repositories.sh" --dev "$version" \ + --deb "$deb_repo_dir" \ + --rpm "$rpm_repo_dir" # If this is a release build, also push to staging. # Publishing to production is done manually. if [[ $version != *"-dev-"* ]]; then - "$SCRIPT_DIR/publish-linux-repositories.sh" --staging "$version" "$deb_repo_dir" + "$SCRIPT_DIR/publish-linux-repositories.sh" --staging "$version" \ + --deb "$deb_repo_dir" \ + --rpm "$rpm_repo_dir" fi } diff --git a/ci/buildserver-config.sh b/ci/buildserver-config.sh index 64cca6b33686..11e6cfeb820f 100644 --- a/ci/buildserver-config.sh +++ b/ci/buildserver-config.sh @@ -19,6 +19,10 @@ export DEV_LINUX_REPOSITORY_SERVERS=("se-got-cdn-001.devmole.eu" "se-got-cdn-002 export STAGING_LINUX_REPOSITORY_SERVERS=("se-got-cdn-001.stagemole.eu" "se-got-cdn-002.stagemole.eu") export PRODUCTION_LINUX_REPOSITORY_SERVERS=("se-got-cdn-111.mullvad.net" "se-mma-cdn-101.mullvad.net") +export DEV_LINUX_REPOSITORY_PUBLIC_URL="https://repository.devmole.eu" +export STAGING_LINUX_REPOSITORY_PUBLIC_URL="https://repository.stagemole.eu" +export PRODUCTION_LINUX_REPOSITORY_PUBLIC_URL="https://repository.mullvad.net" + # What container volumes cargo should put caches in. # Specify differently if running multiple builds in parallel on one machine, # so they don't use the same cache. diff --git a/ci/prepare-rpm-repository.sh b/ci/prepare-rpm-repository.sh index 6ed73742f7f7..bb89dce2bc21 100755 --- a/ci/prepare-rpm-repository.sh +++ b/ci/prepare-rpm-repository.sh @@ -15,16 +15,6 @@ artifact_dir=$1 version=$2 repo_dir=$3 -function generate_repository_configuration { - echo -e "[mullvad-rpm] -name=Mullvad VPN -baseurl=https://repository.mullvad.net/rpm/\$basearch -type=rpm -enabled=1 -gpgcheck=1 -gpgkey=https://repository.mullvad.net/rpm/mullvad-keyring.asc" -} - function create_repository { local arch_repo_dir=$1 local rpm_path=$2 @@ -53,5 +43,3 @@ for arch in "${SUPPORTED_RPM_ARCHITECTURES[@]}"; do fi create_repository "$repo_dir/$arch" "$rpm_path" done - -generate_repository_configuration > "$repo_dir/mullvad.repo" diff --git a/ci/publish-linux-repositories.sh b/ci/publish-linux-repositories.sh index a87c35959aee..e2b0dec2adf7 100755 --- a/ci/publish-linux-repositories.sh +++ b/ci/publish-linux-repositories.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Usage: ./publish-linux-repositories.sh [--production/--staging] +# Usage: ./publish-linux-repositories.sh [--production/--staging/--dev] [--deb ] [--rpm ] # -# Rsyncs a locally prepared and stored apt repository to the dev/staging/production +# Rsyncs a locally prepared and stored apt and/or rpm repository to the dev/staging/production # repository servers. set -eu @@ -15,12 +15,23 @@ while [ "$#" -gt 0 ]; do case "$1" in "--production") repository_servers=("${PRODUCTION_LINUX_REPOSITORY_SERVERS[@]}") + repository_server_url="$PRODUCTION_LINUX_REPOSITORY_PUBLIC_URL" ;; "--staging") repository_servers=("${STAGING_LINUX_REPOSITORY_SERVERS[@]}") + repository_server_url="$STAGING_LINUX_REPOSITORY_PUBLIC_URL" ;; "--dev") repository_servers=("${DEV_LINUX_REPOSITORY_SERVERS[@]}") + repository_server_url="$DEV_LINUX_REPOSITORY_PUBLIC_URL" + ;; + "--deb") + deb_repo_dir=$2 + shift + ;; + "--rpm") + rpm_repo_dir=$2 + shift ;; -*) echo "Unknown option \"$1\"" >&2 @@ -29,8 +40,6 @@ while [ "$#" -gt 0 ]; do *) if [[ -z ${version+x} ]]; then version=$1 - elif [[ -z ${deb_repo_dir+x} ]]; then - deb_repo_dir=$1 else echo "Too many arguments" >&2 exit 1 @@ -44,8 +53,8 @@ if [[ -z ${version+x} ]]; then echo "Please give the release version as an argument to this script" >&2 exit 1 fi -if [[ -z ${deb_repo_dir+x} ]]; then - echo "Please specify the deb repository directory as an argument to this script" >&2 +if [[ -z ${deb_repo_dir+x} && -z ${rpm_repo_dir+x} ]]; then + echo "Please specify at least one of --deb or --rpm" >&2 exit 1 fi if [[ -z ${repository_servers+x} ]]; then @@ -65,12 +74,54 @@ function rsync_repo { done } -if [[ ! -d "$deb_repo_dir" ]]; then - echo "$deb_repo_dir does not exist" >&2 - exit 1 +# Writes the mullvad.repo config file to the repository +# root. This needs to contain the absolute url and path +# to the repository. As such, it depends on what server +# we upload to as well as if it's stable or beta. That's +# why we need to do it just before upload. +function generate_rpm_repository_configuration { + local repository_dir=$1 + local stable_or_beta=$2 + + local repository_name="Mullvad VPN" + if [[ "$stable_or_beta" == "beta" ]]; then + repository_name+=" (beta)" + fi + + echo -e "[mullvad-$stable_or_beta] +name=$repository_name +baseurl=$repository_server_url/rpm/$stable_or_beta/\$basearch +type=rpm +enabled=1 +gpgcheck=1 +gpgkey=$repository_server_url/rpm/mullvad-keyring.asc" > "$repository_dir/mullvad.repo" +} + +if [[ -n ${deb_repo_dir+x} ]]; then + echo "Publishing deb repository from $deb_repo_dir" + if [[ ! -d "$deb_repo_dir" ]]; then + echo "$deb_repo_dir does not exist" >&2 + exit 1 + fi + + rsync_repo "$deb_repo_dir" "deb/beta" + if [[ $version != *"-beta"* ]]; then + rsync_repo "$deb_repo_dir" "deb/stable" + fi fi -rsync_repo "$deb_repo_dir" "deb/beta" -if [[ $version != *"-beta"* ]]; then - rsync_repo "$deb_repo_dir" "deb/stable" +if [[ -n ${rpm_repo_dir+x} ]]; then + echo "Publishing rpm repository from $rpm_repo_dir" + if [[ ! -d "$rpm_repo_dir" ]]; then + echo "$rpm_repo_dir does not exist" >&2 + exit 1 + fi + + generate_rpm_repository_configuration "$rpm_repo_dir" "beta" + rsync_repo "$rpm_repo_dir" "rpm/beta" + if [[ $version != *"-beta"* ]]; then + generate_rpm_repository_configuration "$rpm_repo_dir" "stable" + rsync_repo "$rpm_repo_dir" "rpm/stable" + fi fi +