Skip to content

Commit

Permalink
Merge branch 'master' into umbrella-alliance
Browse files Browse the repository at this point in the history
  • Loading branch information
runcomet authored Nov 29, 2024
2 parents a152e9e + 1e89a31 commit 6a349e9
Show file tree
Hide file tree
Showing 157 changed files with 4,256 additions and 1,269 deletions.
2 changes: 1 addition & 1 deletion .github/env
Original file line number Diff line number Diff line change
@@ -1 +1 @@
IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"
IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-11-19-v202411281558"
33 changes: 31 additions & 2 deletions .github/scripts/common/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -237,15 +237,44 @@ fetch_release_artifacts() {
popd > /dev/null
}

# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
# Fetch deb package from S3. Assumes the ENV are set:
# - RELEASE_ID
# - GITHUB_TOKEN
# - REPO in the form paritytech/polkadot
fetch_release_artifacts_from_s3() {
fetch_debian_package_from_s3() {
BINARY=$1
echo "Version : $VERSION"
echo "Repo : $REPO"
echo "Binary : $BINARY"
echo "Tag : $RELEASE_TAG"
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
echo "OUTPUT_DIR : $OUTPUT_DIR"

URL_BASE=$(get_s3_url_base $BINARY)
echo "URL_BASE=$URL_BASE"

URL=$URL_BASE/$RELEASE_TAG/x86_64-unknown-linux-gnu/${BINARY}_${VERSION}_amd64.deb

mkdir -p "$OUTPUT_DIR"
pushd "$OUTPUT_DIR" > /dev/null

echo "Fetching deb package..."

echo "Fetching %s" "$URL"
curl --progress-bar -LO "$URL" || echo "Missing $URL"

pwd
ls -al --color
popd > /dev/null

}

# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
# - RELEASE_ID
# - GITHUB_TOKEN
# - REPO in the form paritytech/polkadot
fetch_release_artifacts_from_s3() {
BINARY=$1
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
echo "OUTPUT_DIR : $OUTPUT_DIR"

Expand Down
39 changes: 39 additions & 0 deletions .github/scripts/release/distributions
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
Origin: Parity
Label: Parity
Codename: release
Architectures: amd64
Components: main
Description: Apt repository for software made by Parity Technologies Ltd.
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

Origin: Parity
Label: Parity Staging
Codename: staging
Architectures: amd64
Components: main
Description: Staging distribution for Parity Technologies Ltd. packages
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

Origin: Parity
Label: Parity stable2407
Codename: stable2407
Architectures: amd64
Components: main
Description: Apt repository for software made by Parity Technologies Ltd.
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

Origin: Parity
Label: Parity stable2409
Codename: stable2409
Architectures: amd64
Components: main
Description: Apt repository for software made by Parity Technologies Ltd.
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

Origin: Parity
Label: Parity stable2412
Codename: stable2412
Architectures: amd64
Components: main
Description: Apt repository for software made by Parity Technologies Ltd.
SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
4 changes: 2 additions & 2 deletions .github/workflows/build-misc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ jobs:
forklift cargo check -p rococo-runtime
forklift cargo check -p polkadot-test-runtime
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand Down Expand Up @@ -73,7 +73,7 @@ jobs:
cd ./substrate/bin/utils/subkey
forklift cargo build --locked --release
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/check-frame-omni-bencher.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
forklift cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
forklift cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand Down Expand Up @@ -99,7 +99,7 @@ jobs:
echo "Running command: $cmd"
eval "$cmd"
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/check-semver.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ concurrency:
cancel-in-progress: true

env:
TOOLCHAIN: nightly-2024-06-01
TOOLCHAIN: nightly-2024-10-19

jobs:
preflight:
Expand Down Expand Up @@ -74,7 +74,7 @@ jobs:
- name: install parity-publish
# Set the target dir to cache the build.
run: CARGO_TARGET_DIR=./target/ cargo install parity-publish@0.8.0 --locked -q
run: CARGO_TARGET_DIR=./target/ cargo install parity-publish@0.10.1 --locked -q

- name: check semver
run: |
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/checks-quick.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
id: required
run: cargo +nightly fmt --all -- --check
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand Down Expand Up @@ -97,7 +97,6 @@ jobs:
--exclude
"substrate/frame/contracts/fixtures/build"
"substrate/frame/contracts/fixtures/contracts/common"
"substrate/frame/revive/fixtures/build"
"substrate/frame/revive/fixtures/contracts/common"
- name: deny git deps
run: python3 .github/scripts/deny-git-deps.py .
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
cargo clippy --all-targets --locked --workspace --quiet
cargo clippy --all-targets --all-features --locked --workspace --quiet
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand All @@ -62,7 +62,7 @@ jobs:
# experimental code may rely on try-runtime and vice-versa
forklift cargo check --locked --all --features try-runtime,experimental --quiet
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand Down Expand Up @@ -91,7 +91,7 @@ jobs:
./check-features-variants.sh
cd -
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/command-backport.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
uses: korthout/backport-action@v3
id: backport
with:
target_branches: stable2407 stable2409
target_branches: stable2407 stable2409 stable2412
merge_commits: skip
github_token: ${{ steps.generate_token.outputs.token }}
pull_description: |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
env:
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand Down Expand Up @@ -69,7 +69,7 @@ jobs:
retention-days: 1
if-no-files-found: error
- name: Stop all workflows if failed
if: ${{ failure() && steps.required.conclusion == 'failure' }}
if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
uses: ./.github/actions/workflow-stopper
with:
app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
Expand Down
152 changes: 152 additions & 0 deletions .github/workflows/release-40_publish-deb-package.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
name: Release - Publish polakdot deb package

on:
workflow_dispatch:
inputs:
tag:
description: Current final release tag in the format polakdot-stableYYMM or polkadot-stable-YYMM-X
default: polkadot-stable2412
required: true
type: string

distribution:
description: Distribution where to publish deb package (release, staging, stable2407, etc)
default: staging
required: true
type: string

jobs:
check-synchronization:
uses: paritytech-release/sync-workflows/.github/workflows/check-syncronization.yml@main

validate-inputs:
needs: [check-synchronization]
if: ${{ needs.check-synchronization.outputs.checks_passed }} == 'true'
runs-on: ubuntu-latest
outputs:
release_tag: ${{ steps.validate_inputs.outputs.release_tag }}

steps:
- name: Checkout sources
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0

- name: Validate inputs
id: validate_inputs
run: |
. ./.github/scripts/common/lib.sh
RELEASE_TAG=$(validate_stable_tag ${{ inputs.tag }})
echo "release_tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT
fetch-artifacts-from-s3:
runs-on: ubuntu-latest
needs: [validate-inputs]
env:
REPO: ${{ github.repository }}
RELEASE_TAG: ${{ needs.validate-inputs.outputs.release_tag }}
outputs:
VERSION: ${{ steps.fetch_artifacts_from_s3.outputs.VERSION }}

steps:
- name: Checkout sources
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0

- name: Fetch rc artifacts or release artifacts from s3 based on version
id: fetch_artifacts_from_s3
run: |
. ./.github/scripts/common/lib.sh
VERSION="$(get_polkadot_node_version_from_code)"
echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
fetch_debian_package_from_s3 polkadot
- name: Upload artifacts
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: release-artifacts
path: release-artifacts/polkadot/*.deb

publish-deb-package:
runs-on: ubuntu-latest
needs: [fetch-artifacts-from-s3]
environment: release
env:
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_DEB_PATH: "s3://releases-package-repos/deb"
LOCAL_DEB_REPO_PATH: ${{ github.workspace }}/deb
VERSION: ${{ needs.fetch-artifacts-from-s3.outputs.VERSION }}

steps:
- name: Install pgpkkms
run: |
# Install pgpkms that is used to sign built artifacts
python3 -m pip install "pgpkms @ git+https://github.com/paritytech-release/pgpkms.git@1f8555426662ac93a3849480a35449f683b1c89f"
echo "PGPKMS_REPREPRO_PATH=$(which pgpkms-reprepro)" >> $GITHUB_ENV
- name: Install awscli
run: |
python3 -m pip install awscli
which aws
- name: Checkout sources
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0

- name: Import gpg keys
shell: bash
run: |
. ./.github/scripts/common/lib.sh
import_gpg_keys
- name: Download artifacts
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: release-artifacts
path: release-artifacts

- name: Setup local deb repo
run: |
sudo apt-get install -y reprepro
which reprepro
sed -i "s|^SignWith:.*|SignWith: ! ${PGPKMS_REPREPRO_PATH}|" ${{ github.workspace }}/.github/scripts/release/distributions
mkdir -p ${{ github.workspace }}/deb/conf
cp ${{ github.workspace }}/.github/scripts/release/distributions ${{ github.workspace }}/deb/conf/distributions
cat ${{ github.workspace }}/deb/conf/distributions
- name: Sync local deb repo
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
run: |
# Download the current state of the deb repo
aws s3 sync "$AWS_DEB_PATH/db" "$LOCAL_DEB_REPO_PATH/db"
aws s3 sync "$AWS_DEB_PATH/pool" "$LOCAL_DEB_REPO_PATH/pool"
aws s3 sync "$AWS_DEB_PATH/dists" "$LOCAL_DEB_REPO_PATH/dists"
- name: Add deb package to local repo
env:
PGP_KMS_KEY: ${{ secrets.PGP_KMS_KEY }}
PGP_KMS_HASH: ${{ secrets.PGP_KMS_HASH }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
# Add the new deb to the repo
reprepro -b "$LOCAL_DEB_REPO_PATH" includedeb "${{ inputs.distribution }}" "release-artifacts/polkadot_${VERSION}_amd64.deb"
- name: Upload updated deb repo
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
run: |
# Upload the updated repo - dists and pool should be publicly readable
aws s3 sync "$LOCAL_DEB_REPO_PATH/pool" "$AWS_DEB_PATH/pool" --acl public-read
aws s3 sync "$LOCAL_DEB_REPO_PATH/dists" "$AWS_DEB_PATH/dists" --acl public-read
aws s3 sync "$LOCAL_DEB_REPO_PATH/db" "$AWS_DEB_PATH/db"
aws s3 sync "$LOCAL_DEB_REPO_PATH/conf" "$AWS_DEB_PATH/conf"
# Invalidate caches to make sure latest files are served
aws cloudfront create-invalidation --distribution-id E36FKEYWDXAZYJ --paths '/deb/*'
Loading

0 comments on commit 6a349e9

Please sign in to comment.