From 257a41d559c3efbb3946178b95d676a97ba5ebc6 Mon Sep 17 00:00:00 2001 From: bgme Date: Wed, 17 Jan 2024 18:48:24 +0800 Subject: [PATCH] update Dockerfile --- .github/workflows/docker.yml | 37 +- .github/workflows/docs-pr-netlify.yaml | 34 - .github/workflows/docs-pr.yaml | 71 -- .github/workflows/docs.yaml | 117 ---- .github/workflows/fix_lint.yaml | 52 -- .github/workflows/latest_deps.yml | 234 ------- .github/workflows/poetry_lockfile.yaml | 24 - .github/workflows/push_complement_image.yml | 74 -- .github/workflows/release-artifacts.yml | 212 ------ .github/workflows/tests.yml | 720 -------------------- .github/workflows/triage-incoming.yml | 14 - .github/workflows/triage_labelled.yml | 44 -- .github/workflows/twisted_trunk.yml | 215 ------ docker/Dockerfile | 6 +- 14 files changed, 9 insertions(+), 1845 deletions(-) delete mode 100644 .github/workflows/docs-pr-netlify.yaml delete mode 100644 .github/workflows/docs-pr.yaml delete mode 100644 .github/workflows/docs.yaml delete mode 100644 .github/workflows/fix_lint.yaml delete mode 100644 .github/workflows/latest_deps.yml delete mode 100644 .github/workflows/poetry_lockfile.yaml delete mode 100644 .github/workflows/push_complement_image.yml delete mode 100644 .github/workflows/release-artifacts.yml delete mode 100644 .github/workflows/tests.yml delete mode 100644 .github/workflows/triage-incoming.yml delete mode 100644 .github/workflows/triage_labelled.yml delete mode 100644 .github/workflows/twisted_trunk.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 010bce863bcc..e08e09c2b8a0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,7 +5,7 @@ name: Build docker images on: push: tags: ["v*"] - branches: [ master, main, develop ] + branches: [master, main, develop] workflow_dispatch: permissions: @@ -16,12 +16,6 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -42,17 +36,11 @@ jobs: run: | echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - - name: Log in to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Log in to GHCR uses: docker/login-action@v3 with: registry: ghcr.io - username: ${{ github.repository_owner }} + username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Calculate docker image tag @@ -60,8 +48,7 @@ jobs: uses: docker/metadata-action@master with: images: | - docker.io/matrixdotorg/synapse - ghcr.io/element-hq/synapse + ghcr.io/yingziwu/synapse flavor: | latest=false tags: | @@ -80,20 +67,4 @@ jobs: org.opencontainers.image.version=${{ env.SYNAPSE_VERSION }} tags: "${{ steps.set-tag.outputs.tags }}" file: "docker/Dockerfile" - platforms: linux/amd64,linux/arm64 - - # arm64 builds OOM without the git fetch setting. c.f. - # https://github.com/rust-lang/cargo/issues/10583 - build-args: | - CARGO_NET_GIT_FETCH_WITH_CLI=true - - - name: Sign the images with GitHub OIDC Token - env: - DIGEST: ${{ steps.build-and-push.outputs.digest }} - TAGS: ${{ steps.set-tag.outputs.tags }} - run: | - images="" - for tag in ${TAGS}; do - images+="${tag}@${DIGEST} " - done - cosign sign --yes ${images} + platforms: linux/amd64 diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml deleted file mode 100644 index 8b2032230881..000000000000 --- a/.github/workflows/docs-pr-netlify.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Deploy documentation PR preview - -on: - workflow_run: - workflows: [ "Prepare documentation PR preview" ] - types: - - completed - -jobs: - netlify: - if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' - runs-on: ubuntu-latest - steps: - # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action - # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: - - name: 📥 Download artifact - uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e # v2.28.0 - with: - workflow: docs-pr.yaml - run_id: ${{ github.event.workflow_run.id }} - name: book - path: book - - - name: 📤 Deploy to Netlify - uses: matrix-org/netlify-pr-preview@v3 - with: - path: book - owner: ${{ github.event.workflow_run.head_repository.owner.login }} - branch: ${{ github.event.workflow_run.head_branch }} - revision: ${{ github.event.workflow_run.head_sha }} - token: ${{ secrets.NETLIFY_AUTH_TOKEN }} - site_id: ${{ secrets.NETLIFY_SITE_ID }} - desc: Documentation preview - deployment_env: PR Documentation Preview diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml deleted file mode 100644 index 52b0f8802d55..000000000000 --- a/.github/workflows/docs-pr.yaml +++ /dev/null @@ -1,71 +0,0 @@ -name: Prepare documentation PR preview - -on: - pull_request: - paths: - - docs/** - - book.toml - - .github/workflows/docs-pr.yaml - - scripts-dev/schema_versions.py - -jobs: - pages: - name: GitHub Pages - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # Fetch all history so that the schema_versions script works. - fetch-depth: 0 - - - name: Setup mdbook - uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 - with: - mdbook-version: '0.4.17' - - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" - - - name: Build the documentation - # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. - # However, we're using docs/README.md for other purposes and need to pick a new page - # as the default. Let's opt for the welcome page instead. - run: | - mdbook build - cp book/welcome_and_overview.html book/index.html - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: book - path: book - # We'll only use this in a workflow_run, then we're done with it - retention-days: 1 - - link-check: - name: Check links in documentation - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup mdbook - uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 - with: - mdbook-version: '0.4.17' - - - name: Setup htmltest - run: | - wget https://github.com/wjdp/htmltest/releases/download/v0.17.0/htmltest_0.17.0_linux_amd64.tar.gz - echo '775c597ee74899d6002cd2d93076f897f4ba68686bceabe2e5d72e84c57bc0fb htmltest_0.17.0_linux_amd64.tar.gz' | sha256sum -c - tar zxf htmltest_0.17.0_linux_amd64.tar.gz - - - name: Test links with htmltest - # Build the book with `./` as the site URL (to make checks on 404.html possible) - # Then run htmltest (without checking external links since that involves the network and is slow). - run: | - MDBOOK_OUTPUT__HTML__SITE_URL="./" mdbook build - ./htmltest book --skip-external diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml deleted file mode 100644 index d611fdc924f6..000000000000 --- a/.github/workflows/docs.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: Deploy the documentation - -on: - push: - branches: - # For bleeding-edge documentation - - develop - # For documentation specific to a release - - 'release-v*' - # stable docs - - master - - workflow_dispatch: - -jobs: - pre: - name: Calculate variables for GitHub Pages deployment - runs-on: ubuntu-latest - steps: - # Figure out the target directory. - # - # The target directory depends on the name of the branch - # - - name: Get the target directory name - id: vars - run: | - # first strip the 'refs/heads/' prefix with some shell foo - branch="${GITHUB_REF#refs/heads/}" - - case $branch in - release-*) - # strip 'release-' from the name for release branches. - branch="${branch#release-}" - ;; - master) - # deploy to "latest" for the master branch. - branch="latest" - ;; - esac - - # finally, set the 'branch-version' var. - echo "branch-version=$branch" >> "$GITHUB_OUTPUT" - outputs: - branch-version: ${{ steps.vars.outputs.branch-version }} - -################################################################################ - pages-docs: - name: GitHub Pages - runs-on: ubuntu-latest - needs: - - pre - steps: - - uses: actions/checkout@v4 - with: - # Fetch all history so that the schema_versions script works. - fetch-depth: 0 - - - name: Setup mdbook - uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 - with: - mdbook-version: '0.4.17' - - - name: Set version of docs - run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" - - - name: Build the documentation - # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. - # However, we're using docs/README.md for other purposes and need to pick a new page - # as the default. Let's opt for the welcome page instead. - run: | - mdbook build - cp book/welcome_and_overview.html book/index.html - - # Deploy to the target directory. - - name: Deploy to gh pages - uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./book - destination_dir: ./${{ needs.pre.outputs.branch-version }} - -################################################################################ - pages-devdocs: - name: GitHub Pages (developer docs) - runs-on: ubuntu-latest - needs: - - pre - steps: - - uses: actions/checkout@v4 - - - name: "Set up Sphinx" - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: "3.x" - poetry-version: "1.3.2" - groups: "dev-docs" - extras: "" - - - name: Build the documentation - run: | - cd dev-docs - poetry run make html - - # Deploy to the target directory. - - name: Deploy to gh pages - uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./dev-docs/_build/html - destination_dir: ./dev-docs/${{ needs.pre.outputs.branch-version }} diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml deleted file mode 100644 index f1e35fcd99d9..000000000000 --- a/.github/workflows/fix_lint.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# A helper workflow to automatically fixup any linting errors on a PR. Must be -# triggered manually. - -name: Attempt to automatically fix linting errors - -on: - workflow_dispatch: - -jobs: - fixup: - name: Fix up - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - # We use nightly so that `fmt` correctly groups together imports, and - # clippy correctly fixes up the benchmarks. - toolchain: nightly-2022-12-01 - components: rustfmt - - uses: Swatinem/rust-cache@v2 - - - name: Setup Poetry - uses: matrix-org/setup-python-poetry@v1 - with: - install-project: "false" - - - name: Import order (isort) - continue-on-error: true - run: poetry run isort . - - - name: Code style (black) - continue-on-error: true - run: poetry run black . - - - name: Semantic checks (ruff) - continue-on-error: true - run: poetry run ruff --fix . - - - run: cargo clippy --all-features --fix -- -D warnings - continue-on-error: true - - - run: cargo fmt - continue-on-error: true - - - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: "Attempt to fix linting" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml deleted file mode 100644 index f40342f951bd..000000000000 --- a/.github/workflows/latest_deps.yml +++ /dev/null @@ -1,234 +0,0 @@ -# People who are freshly `pip install`ing from PyPI will pull in the latest versions of -# dependencies which match the broad requirements. Since most CI runs are against -# the locked poetry environment, run specifically against the latest dependencies to -# know if there's an upcoming breaking change. -# -# As an overview this workflow: -# - checks out develop, -# - installs from source, pulling in the dependencies like a fresh `pip install` would, and -# - runs mypy and test suites in that checkout. -# -# Based on the twisted trunk CI job. - -name: Latest dependencies - -on: - schedule: - - cron: 0 7 * * * - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - check_repo: - # Prevent this workflow from running on any fork of Synapse other than element-hq/synapse, as it is - # only useful to the Synapse core team. - # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest - # of the workflow will be skipped as well. - runs-on: ubuntu-latest - outputs: - should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }} - steps: - - id: check_condition - run: echo "should_run_workflow=${{ github.repository == 'element-hq/synapse' }}" >> "$GITHUB_OUTPUT" - - mypy: - needs: check_repo - if: needs.check_repo.outputs.should_run_workflow == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - # The dev dependencies aren't exposed in the wheel metadata (at least with current - # poetry-core versions), so we install with poetry. - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: "3.x" - poetry-version: "1.3.2" - extras: "all" - # Dump installed versions for debugging. - - run: poetry run pip list > before.txt - # Upgrade all runtime dependencies only. This is intended to mimic a fresh - # `pip install matrix-synapse[all]` as closely as possible. - - run: poetry update --no-dev - - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) - - name: Remove unhelpful options from mypy config - run: sed -e '/warn_unused_ignores = True/d' -e '/warn_redundant_casts = True/d' -i mypy.ini - - run: poetry run mypy - trial: - needs: check_repo - if: needs.check_repo.outputs.should_run_workflow == 'true' - runs-on: ubuntu-latest - strategy: - matrix: - include: - - database: "sqlite" - - database: "postgres" - postgres-version: "14" - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - run: sudo apt-get -qq install xmlsec1 - - name: Set up PostgreSQL ${{ matrix.postgres-version }} - if: ${{ matrix.postgres-version }} - run: | - docker run -d -p 5432:5432 \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ - postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - run: pip install .[all,test] - - name: Await PostgreSQL - if: ${{ matrix.postgres-version }} - timeout-minutes: 2 - run: until pg_isready -h localhost; do sleep 1; done - - # We nuke the local copy, as we've installed synapse into the virtualenv - # (rather than use an editable install, which we no longer support). If we - # don't do this then python can't find the native lib. - - run: rm -rf synapse/ - - - run: python -m twisted.trial --jobs=2 tests - env: - SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} - SYNAPSE_POSTGRES_HOST: localhost - SYNAPSE_POSTGRES_USER: postgres - SYNAPSE_POSTGRES_PASSWORD: postgres - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - - sytest: - needs: check_repo - if: needs.check_repo.outputs.should_run_workflow == 'true' - runs-on: ubuntu-latest - container: - image: matrixdotorg/sytest-synapse:testing - volumes: - - ${{ github.workspace }}:/src - strategy: - fail-fast: false - matrix: - include: - - sytest-tag: focal - - - sytest-tag: focal - postgres: postgres - workers: workers - redis: redis - env: - POSTGRES: ${{ matrix.postgres && 1}} - WORKERS: ${{ matrix.workers && 1 }} - REDIS: ${{ matrix.redis && 1 }} - BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - name: Ensure sytest runs `pip install` - # Delete the lockfile so sytest will `pip install` rather than `poetry install` - run: rm /src/poetry.lock - working-directory: /src - - name: Prepare test blacklist - run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - - name: Run SyTest - run: /bootstrap.sh synapse - working-directory: /src - - name: Summarise results.tap - if: ${{ always() }} - run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - - name: Upload SyTest logs - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) - path: | - /logs/results.tap - /logs/**/*.log* - - - complement: - needs: check_repo - if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'" - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@v4 - with: - path: synapse - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - uses: actions/setup-go@v5 - with: - cache-dependency-path: complement/go.sum - go-version-file: complement/go.mod - - - run: | - set -o pipefail - TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt - shell: bash - name: Run Complement Tests - - # Open an issue if the build fails, so we know about it. - # Only do this if we're not experimenting with this action in a PR. - open-issue: - if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'" - needs: - # TODO: should mypy be included here? It feels more brittle than the others. - - mypy - - trial - - sytest - - complement - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - update_existing: true - filename: .ci/latest_deps_build_failed_issue_template.md diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml deleted file mode 100644 index 496e536b93bd..000000000000 --- a/.github/workflows/poetry_lockfile.yaml +++ /dev/null @@ -1,24 +0,0 @@ -on: - push: - branches: ["develop", "release-*"] - paths: - - poetry.lock - pull_request: - paths: - - poetry.lock - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - check-sdists: - name: "Check locked dependencies have sdists" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - run: pip install tomli - - run: ./scripts-dev/check_locked_deps_have_sdists.py diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml deleted file mode 100644 index 6fbd2ed0154f..000000000000 --- a/.github/workflows/push_complement_image.yml +++ /dev/null @@ -1,74 +0,0 @@ -# This task does not run complement tests, see tests.yaml instead. -# This task does not build docker images for synapse for use on docker hub, see docker.yaml instead - -name: Store complement-synapse image in ghcr.io -on: - push: - branches: [ "master" ] - schedule: - - cron: '0 5 * * *' - workflow_dispatch: - inputs: - branch: - required: true - default: 'develop' - type: choice - options: - - develop - - master - -# Only run this action once per pull request/branch; restart if a new commit arrives. -# C.f. https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency -# and https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build: - name: Build and push complement image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout specific branch (debug build) - uses: actions/checkout@v4 - if: github.event_name == 'workflow_dispatch' - with: - ref: ${{ inputs.branch }} - - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@v4 - if: github.event_name == 'schedule' - with: - ref: develop - - name: Checkout clean copy of master (on-push) - uses: actions/checkout@v4 - if: github.event_name == 'push' - with: - ref: master - - name: Login to registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Work out labels for complement image - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository }}/complement-synapse - tags: | - type=schedule,pattern=nightly,enable=${{ github.event_name == 'schedule'}} - type=raw,value=develop,enable=${{ github.event_name == 'schedule' || inputs.branch == 'develop' }} - type=raw,value=latest,enable=${{ github.event_name == 'push' || inputs.branch == 'master' }} - type=sha,format=long - - name: Run scripts-dev/complement.sh to generate complement-synapse:latest image. - run: scripts-dev/complement.sh --build-only - - name: Tag and push generated image - run: | - for TAG in ${{ join(fromJson(steps.meta.outputs.json).tags, ' ') }}; do - echo "tag and push $TAG" - docker tag complement-synapse $TAG - docker push $TAG - done diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml deleted file mode 100644 index baf4b62292ff..000000000000 --- a/.github/workflows/release-artifacts.yml +++ /dev/null @@ -1,212 +0,0 @@ -# GitHub actions workflow which builds the release artifacts. - -name: Build release artifacts - -on: - # we build on PRs and develop to (hopefully) get early warning - # of things breaking (but only build one set of debs). PRs skip - # building wheels on macOS & ARM. - pull_request: - push: - branches: ["develop", "release-*"] - - # we do the full build on tags. - tags: ["v*"] - merge_group: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: write - -jobs: - get-distros: - name: "Calculate list of debian distros" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - id: set-distros - run: | - # if we're running from a tag, get the full list of distros; otherwise just use debian:sid - # NOTE: inside the actual Dockerfile-dhvirtualenv, the image name is expanded into its full image path - dists='["debian:sid"]' - if [[ $GITHUB_REF == refs/tags/* ]]; then - dists=$(scripts-dev/build_debian_packages.py --show-dists-json) - fi - echo "distros=$dists" >> "$GITHUB_OUTPUT" - # map the step outputs to job outputs - outputs: - distros: ${{ steps.set-distros.outputs.distros }} - - # now build the packages with a matrix build. - build-debs: - needs: get-distros - name: "Build .deb packages" - runs-on: ubuntu-latest - strategy: - matrix: - distro: ${{ fromJson(needs.get-distros.outputs.distros) }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - path: src - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - with: - install: true - - - name: Set up docker layer caching - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Set up python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Build the packages - # see https://github.com/docker/build-push-action/issues/252 - # for the cache magic here - run: | - ./src/scripts-dev/build_debian_packages.py \ - --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \ - --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \ - --docker-build-arg=--progress=plain \ - --docker-build-arg=--load \ - "${{ matrix.distro }}" - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - - name: Upload debs as artifacts - uses: actions/upload-artifact@v3 - with: - name: debs - path: debs/* - - build-wheels: - name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04, macos-11] - arch: [x86_64, aarch64] - # is_pr is a flag used to exclude certain jobs from the matrix on PRs. - # It is not read by the rest of the workflow. - is_pr: - - ${{ startsWith(github.ref, 'refs/pull/') }} - - exclude: - # Don't build macos wheels on PR CI. - - is_pr: true - os: "macos-11" - # Don't build aarch64 wheels on mac. - - os: "macos-11" - arch: aarch64 - # Don't build aarch64 wheels on PR CI. - - is_pr: true - arch: aarch64 - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - # setup-python@v4 doesn't impose a default python version. Need to use 3.x - # here, because `python` on osx points to Python 2.7. - python-version: "3.x" - - - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.16.2 - - - name: Set up QEMU to emulate aarch64 - if: matrix.arch == 'aarch64' - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - - name: Build aarch64 wheels - if: matrix.arch == 'aarch64' - run: echo 'CIBW_ARCHS_LINUX=aarch64' >> $GITHUB_ENV - - - name: Only build a single wheel on PR - if: startsWith(github.ref, 'refs/pull/') - run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV - - - name: Build wheels - run: python -m cibuildwheel --output-dir wheelhouse - env: - # Skip testing for platforms which various libraries don't have wheels - # for, and so need extra build deps. - CIBW_TEST_SKIP: pp3*-* *i686* *musl* - # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583 - CARGO_NET_GIT_FETCH_WITH_CLI: true - CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI - - - uses: actions/upload-artifact@v3 - with: - name: Wheel - path: ./wheelhouse/*.whl - - build-sdist: - name: Build sdist - runs-on: ubuntu-latest - if: ${{ !startsWith(github.ref, 'refs/pull/') }} - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - run: pip install build - - - name: Build sdist - run: python -m build --sdist - - - uses: actions/upload-artifact@v3 - with: - name: Sdist - path: dist/*.tar.gz - - - # if it's a tag, create a release and attach the artifacts to it - attach-assets: - name: "Attach assets to release" - if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }} - needs: - - build-debs - - build-wheels - - build-sdist - runs-on: ubuntu-latest - steps: - - name: Download all workflow run artifacts - uses: actions/download-artifact@v3 - - name: Build a tarball for the debs - run: tar -cvJf debs.tar.xz debs - - name: Attach to release - uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a # PR#109 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - files: | - Sdist/* - Wheel/* - debs.tar.xz - # if it's not already published, keep the release as a draft. - draft: true - # mark it as a prerelease if the tag contains 'rc'. - prerelease: ${{ contains(github.ref, 'rc') }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index f22ca5f7e60b..000000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,720 +0,0 @@ -name: Tests - -on: - push: - branches: ["develop", "release-*"] - pull_request: - merge_group: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - check-signoff: - if: "github.event_name == 'pull_request'" - uses: "matrix-org/backend-meta/.github/workflows/sign-off.yml@v2" - - # Job to detect what has changed so we don't run e.g. Rust checks on PRs that - # don't modify Rust code. - changes: - runs-on: ubuntu-latest - outputs: - rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }} - trial: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.trial }} - integration: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.integration }} - linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }} - steps: - - uses: dorny/paths-filter@v2 - id: filter - # We only check on PRs - if: startsWith(github.ref, 'refs/pull/') - with: - filters: | - rust: - - 'rust/**' - - 'Cargo.toml' - - 'Cargo.lock' - - '.rustfmt.toml' - - '.github/workflows/tests.yml' - - trial: - - 'synapse/**' - - 'tests/**' - - 'rust/**' - - '.ci/scripts/calculate_jobs.py' - - 'Cargo.toml' - - 'Cargo.lock' - - 'pyproject.toml' - - 'poetry.lock' - - '.github/workflows/tests.yml' - - integration: - - 'synapse/**' - - 'rust/**' - - 'docker/**' - - 'Cargo.toml' - - 'Cargo.lock' - - 'pyproject.toml' - - 'poetry.lock' - - 'docker/**' - - '.ci/**' - - 'scripts-dev/complement.sh' - - '.github/workflows/tests.yml' - - linting: - - 'synapse/**' - - 'docker/**' - - 'tests/**' - - 'scripts-dev/**' - - 'contrib/**' - - 'synmark/**' - - 'stubs/**' - - '.ci/**' - - 'mypy.ini' - - 'pyproject.toml' - - 'poetry.lock' - - '.github/workflows/tests.yml' - - check-sampleconfig: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.linting == 'true' }} - - steps: - - uses: actions/checkout@v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@1.61.0 - - uses: Swatinem/rust-cache@v2 - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: "3.x" - poetry-version: "1.3.2" - extras: "all" - - run: poetry run scripts-dev/generate_sample_config.sh --check - - run: poetry run scripts-dev/config-lint.sh - - check-schema-delta: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.linting == 'true' }} - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" - - run: scripts-dev/check_schema_delta.py --force-colors - - check-lockfile: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - run: .ci/scripts/check_lockfile.py - - lint: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.linting == 'true' }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Poetry - uses: matrix-org/setup-python-poetry@v1 - with: - install-project: "false" - - - name: Import order (isort) - run: poetry run isort --check --diff . - - - name: Code style (black) - run: poetry run black --check --diff . - - - name: Semantic checks (ruff) - # --quiet suppresses the update check. - run: poetry run ruff --quiet . - - lint-mypy: - runs-on: ubuntu-latest - name: Typechecking - needs: changes - if: ${{ needs.changes.outputs.linting == 'true' }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@1.61.0 - - uses: Swatinem/rust-cache@v2 - - - name: Setup Poetry - uses: matrix-org/setup-python-poetry@v1 - with: - # We want to make use of type hints in optional dependencies too. - extras: all - # We have seen odd mypy failures that were resolved when we started - # installing the project again: - # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775 - # To make CI green, err towards caution and install the project. - install-project: "true" - - # Cribbed from - # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - - name: Restore/persist mypy's cache - uses: actions/cache@v3 - with: - path: | - .mypy_cache - key: mypy-cache-${{ github.context.sha }} - restore-keys: mypy-cache- - - - name: Run mypy - run: poetry run mypy - - lint-crlf: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Check line endings - run: scripts-dev/check_line_terminators.sh - - lint-newsfile: - if: ${{ (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - run: "pip install 'towncrier>=18.6.0rc1'" - - run: scripts-dev/check-newsfragment.sh - env: - PULL_REQUEST_NUMBER: ${{ github.event.number }} - - lint-pydantic: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.linting == 'true' }} - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Install Rust - uses: dtolnay/rust-toolchain@1.61.0 - - uses: Swatinem/rust-cache@v2 - - uses: matrix-org/setup-python-poetry@v1 - with: - poetry-version: "1.3.2" - extras: "all" - - run: poetry run scripts-dev/check_pydantic_models.py - - lint-clippy: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.rust == 'true' }} - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@1.61.0 - with: - components: clippy - - uses: Swatinem/rust-cache@v2 - - - run: cargo clippy -- -D warnings - - # We also lint against a nightly rustc so that we can lint the benchmark - # suite, which requires a nightly compiler. - lint-clippy-nightly: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.rust == 'true' }} - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly-2022-12-01 - components: clippy - - uses: Swatinem/rust-cache@v2 - - - run: cargo clippy --all-features -- -D warnings - - lint-rustfmt: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.rust == 'true' }} - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - # We use nightly so that it correctly groups together imports - toolchain: nightly-2022-12-01 - components: rustfmt - - uses: Swatinem/rust-cache@v2 - - - run: cargo fmt --check - - # Dummy step to gate other tests on without repeating the whole list - linting-done: - if: ${{ !cancelled() }} # Run this even if prior jobs were skipped - needs: - - lint - - lint-mypy - - lint-crlf - - lint-newsfile - - lint-pydantic - - check-sampleconfig - - check-schema-delta - - check-lockfile - - lint-clippy - - lint-rustfmt - runs-on: ubuntu-latest - steps: - - run: "true" - - calculate-test-jobs: - if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: linting-done - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - id: get-matrix - run: .ci/scripts/calculate_jobs.py - outputs: - trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }} - sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }} - - trial: - if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail - needs: - - calculate-test-jobs - - changes - runs-on: ubuntu-latest - strategy: - matrix: - job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} - - steps: - - uses: actions/checkout@v4 - - run: sudo apt-get -qq install xmlsec1 - - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} - if: ${{ matrix.job.postgres-version }} - # 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer. - # 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections. - run: | - docker run -d -p 5432:5432 \ - --tmpfs /var/lib/postgres:rw,size=6144m \ - --mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ - postgres:${{ matrix.job.postgres-version }} - - - name: Install Rust - uses: dtolnay/rust-toolchain@1.61.0 - - uses: Swatinem/rust-cache@v2 - - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: ${{ matrix.job.python-version }} - poetry-version: "1.3.2" - extras: ${{ matrix.job.extras }} - - name: Await PostgreSQL - if: ${{ matrix.job.postgres-version }} - timeout-minutes: 2 - run: until pg_isready -h localhost; do sleep 1; done - - run: poetry run trial --jobs=6 tests - env: - SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }} - SYNAPSE_POSTGRES_HOST: /var/run/postgresql - SYNAPSE_POSTGRES_USER: postgres - SYNAPSE_POSTGRES_PASSWORD: postgres - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - trial-olddeps: - # Note: sqlite only; no postgres - if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail - needs: - - linting-done - - changes - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@1.61.0 - - uses: Swatinem/rust-cache@v2 - - # There aren't wheels for some of the older deps, so we need to install - # their build dependencies - - run: | - sudo apt-get -qq update - sudo apt-get -qq install build-essential libffi-dev python-dev \ - libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - - name: Prepare old deps - if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true' - run: .ci/scripts/prepare_old_deps.sh - - # Note: we install using `pip` here, not poetry. `poetry install` ignores the - # build-system section (https://github.com/python-poetry/poetry/issues/6154), but - # we explicitly want to test that you can `pip install` using the oldest version - # of poetry-core and setuptools-rust. - - run: pip install .[all,test] - - # We nuke the local copy, as we've installed synapse into the virtualenv - # (rather than use an editable install, which we no longer support). If we - # don't do this then python can't find the native lib. - - run: rm -rf synapse/ - - # Sanity check we can import/run Synapse - - run: python -m synapse.app.homeserver --help - - - run: python -m twisted.trial -j6 tests - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - trial-pypy: - # Very slow; only run if the branch name includes 'pypy' - # Note: sqlite only; no postgres. Completely untested since poetry move. - if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() && needs.changes.outputs.trial == 'true' }} - needs: - - linting-done - - changes - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["pypy-3.8"] - extras: ["all"] - - steps: - - uses: actions/checkout@v4 - # Install libs necessary for PyPy to build binary wheels for dependencies - - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: ${{ matrix.python-version }} - poetry-version: "1.3.2" - extras: ${{ matrix.extras }} - - run: poetry run trial --jobs=2 tests - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - sytest: - if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }} - needs: - - calculate-test-jobs - - changes - runs-on: ubuntu-latest - container: - image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }} - volumes: - - ${{ github.workspace }}:/src - env: - SYTEST_BRANCH: ${{ github.head_ref }} - POSTGRES: ${{ matrix.job.postgres && 1}} - MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }} - ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }} - WORKERS: ${{ matrix.job.workers && 1 }} - BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }} - TOP: ${{ github.workspace }} - - strategy: - fail-fast: false - matrix: - job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} - - steps: - - uses: actions/checkout@v4 - - name: Prepare test blacklist - run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - - - name: Install Rust - uses: dtolnay/rust-toolchain@1.61.0 - - uses: Swatinem/rust-cache@v2 - - - name: Run SyTest - run: /bootstrap.sh synapse - working-directory: /src - - name: Summarise results.tap - if: ${{ always() }} - run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - - name: Upload SyTest logs - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) - path: | - /logs/results.tap - /logs/**/*.log* - - export-data: - if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true'}} # Allow previous steps to be skipped, but not fail - needs: [linting-done, portdb, changes] - runs-on: ubuntu-latest - env: - TOP: ${{ github.workspace }} - - services: - postgres: - image: postgres - ports: - - 5432:5432 - env: - POSTGRES_PASSWORD: "postgres" - POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v4 - - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 - with: - poetry-version: "1.3.2" - extras: "postgres" - - run: .ci/scripts/test_export_data_command.sh - env: - PGHOST: localhost - PGUSER: postgres - PGPASSWORD: postgres - PGDATABASE: postgres - - - portdb: - if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true'}} # Allow previous steps to be skipped, but not fail - needs: - - linting-done - - changes - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python-version: "3.8" - postgres-version: "11" - - - python-version: "3.11" - postgres-version: "15" - - services: - postgres: - image: postgres:${{ matrix.postgres-version }} - ports: - - 5432:5432 - env: - POSTGRES_PASSWORD: "postgres" - POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v4 - - name: Add PostgreSQL apt repository - # We need a version of pg_dump that can handle the version of - # PostgreSQL being tested against. The Ubuntu package repository lags - # behind new releases, so we have to use the PostreSQL apt repository. - # Steps taken from https://www.postgresql.org/download/linux/ubuntu/ - run: | - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - sudo apt-get update - - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: ${{ matrix.python-version }} - poetry-version: "1.3.2" - extras: "postgres" - - run: .ci/scripts/test_synapse_port_db.sh - id: run_tester_script - env: - PGHOST: localhost - PGUSER: postgres - PGPASSWORD: postgres - PGDATABASE: postgres - - name: "Upload schema differences" - uses: actions/upload-artifact@v3 - if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} - with: - name: Schema dumps - path: | - unported.sql - ported.sql - schema_diff - - complement: - if: "${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}" - needs: - - linting-done - - changes - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@v4 - with: - path: synapse - - - name: Install Rust - uses: dtolnay/rust-toolchain@1.61.0 - - uses: Swatinem/rust-cache@v2 - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - uses: actions/setup-go@v5 - with: - cache-dependency-path: complement/go.sum - go-version-file: complement/go.mod - - # use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once. - - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | synapse/.ci/scripts/gotestfmt - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - name: Run Complement Tests - - cargo-test: - if: ${{ needs.changes.outputs.rust == 'true' }} - runs-on: ubuntu-latest - needs: - - linting-done - - changes - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@1.61.0 - - uses: Swatinem/rust-cache@v2 - - - run: cargo test - - # We want to ensure that the cargo benchmarks still compile, which requires a - # nightly compiler. - cargo-bench: - if: ${{ needs.changes.outputs.rust == 'true' }} - runs-on: ubuntu-latest - needs: - - linting-done - - changes - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly-2022-12-01 - - uses: Swatinem/rust-cache@v2 - - - run: cargo bench --no-run - - # a job which marks all the other jobs as complete, thus allowing PRs to be merged. - tests-done: - if: ${{ always() }} - needs: - - trial - - trial-olddeps - - sytest - - export-data - - portdb - - complement - - cargo-test - - cargo-bench - runs-on: ubuntu-latest - steps: - - uses: matrix-org/done-action@v2 - with: - needs: ${{ toJSON(needs) }} - - # Various bits are skipped if there was no applicable changes. - # The newsfile and signoff lint may be skipped on non PR builds. - skippable: | - trial - trial-olddeps - sytest - portdb - export-data - complement - check-signoff - lint-newsfile - cargo-test - cargo-bench diff --git a/.github/workflows/triage-incoming.yml b/.github/workflows/triage-incoming.yml deleted file mode 100644 index 7a369b77fe46..000000000000 --- a/.github/workflows/triage-incoming.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Move new issues into the issue triage board - -on: - issues: - types: [ opened ] - -jobs: - triage: - uses: matrix-org/backend-meta/.github/workflows/triage-incoming.yml@v2 - with: - project_id: 'PVT_kwDOAIB0Bs4AFDdZ' - content_id: ${{ github.event.issue.node_id }} - secrets: - github_access_token: ${{ secrets.ELEMENT_BOT_TOKEN }} diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml deleted file mode 100644 index d1ac4357b153..000000000000 --- a/.github/workflows/triage_labelled.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Move labelled issues to correct projects - -on: - issues: - types: [ labeled ] - -jobs: - move_needs_info: - name: Move X-Needs-Info on the triage board - runs-on: ubuntu-latest - if: > - contains(github.event.issue.labels.*.name, 'X-Needs-Info') - steps: - - uses: actions/add-to-project@main - id: add_project - with: - project-url: "https://github.com/orgs/matrix-org/projects/67" - github-token: ${{ secrets.ELEMENT_BOT_TOKEN }} - - name: Set status - env: - GITHUB_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }} - run: | - gh api graphql -f query=' - mutation( - $project: ID! - $item: ID! - $fieldid: ID! - $columnid: String! - ) { - updateProjectV2ItemFieldValue( - input: { - projectId: $project - itemId: $item - fieldId: $fieldid - value: { - singleSelectOptionId: $columnid - } - } - ) { - projectV2Item { - id - } - } - }' -f project="PVT_kwDOAIB0Bs4AFDdZ" -f item=${{ steps.add_project.outputs.itemId }} -f fieldid="PVTSSF_lADOAIB0Bs4AFDdZzgC6ZA4" -f columnid=ba22e43c --silent diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml deleted file mode 100644 index 428180fffd6a..000000000000 --- a/.github/workflows/twisted_trunk.yml +++ /dev/null @@ -1,215 +0,0 @@ -name: Twisted Trunk - -on: - schedule: - - cron: 0 8 * * * - - workflow_dispatch: - # NB: inputs are only present when this workflow is dispatched manually. - # (The default below is the default field value in the form to trigger - # a manual dispatch). Otherwise the inputs will evaluate to null. - inputs: - twisted_ref: - description: Commit, branch or tag to checkout from upstream Twisted. - required: false - default: 'trunk' - type: string - - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - check_repo: - # Prevent this workflow from running on any fork of Synapse other than element-hq/synapse, as it is - # only useful to the Synapse core team. - # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest - # of the workflow will be skipped as well. - if: github.repository == 'element-hq/synapse' - runs-on: ubuntu-latest - outputs: - should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }} - steps: - - id: check_condition - run: echo "should_run_workflow=${{ github.repository == 'element-hq/synapse' }}" >> "$GITHUB_OUTPUT" - - mypy: - needs: check_repo - if: needs.check_repo.outputs.should_run_workflow == 'true' - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: "3.x" - extras: "all" - - run: | - poetry remove twisted - poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref || 'trunk' }} - poetry install --no-interaction --extras "all test" - - name: Remove unhelpful options from mypy config - run: sed -e '/warn_unused_ignores = True/d' -e '/warn_redundant_casts = True/d' -i mypy.ini - - run: poetry run mypy - - trial: - needs: check_repo - if: needs.check_repo.outputs.should_run_workflow == 'true' - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - run: sudo apt-get -qq install xmlsec1 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: "3.x" - extras: "all test" - - run: | - poetry remove twisted - poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk - poetry install --no-interaction --extras "all test" - - run: poetry run trial --jobs 2 tests - - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - sytest: - needs: check_repo - if: needs.check_repo.outputs.should_run_workflow == 'true' - runs-on: ubuntu-latest - container: - # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version. - # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if - # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest - # version, assuming that any incompatibilities on newer versions would also be present on the oldest. - image: matrixdotorg/sytest-synapse:focal - volumes: - - ${{ github.workspace }}:/src - - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - name: Patch dependencies - # Note: The poetry commands want to create a virtualenv in /src/.venv/, - # but the sytest-synapse container expects it to be in /venv/. - # We symlink it before running poetry so that poetry actually - # ends up installing to `/venv`. - run: | - ln -s -T /venv /src/.venv - poetry remove twisted - poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk - poetry install --no-interaction --extras "all test" - working-directory: /src - - name: Run SyTest - run: /bootstrap.sh synapse - working-directory: /src - env: - # Use offline mode to avoid reinstalling the pinned version of - # twisted. - OFFLINE: 1 - - name: Summarise results.tap - if: ${{ always() }} - run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - - name: Upload SyTest logs - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) - path: | - /logs/results.tap - /logs/**/*.log* - - complement: - needs: check_repo - if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'" - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@v4 - with: - path: synapse - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - uses: actions/setup-go@v5 - with: - cache-dependency-path: complement/go.sum - go-version-file: complement/go.mod - - # This step is specific to the 'Twisted trunk' test run: - - name: Patch dependencies - run: | - set -x - DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx - pipx install poetry==1.3.2 - - poetry remove -n twisted - poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk - poetry lock --no-update - working-directory: synapse - - - run: | - set -o pipefail - TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt - shell: bash - name: Run Complement Tests - - # open an issue if the build fails, so we know about it. - open-issue: - if: failure() && needs.check_repo.outputs.should_run_workflow == 'true' - needs: - - mypy - - trial - - sytest - - complement - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - update_existing: true - filename: .ci/twisted_trunk_build_failed_issue_template.md diff --git a/docker/Dockerfile b/docker/Dockerfile index d4cb9414ff90..cda9f2c01b7a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,6 +65,10 @@ WORKDIR /synapse # Copy just what we need to run `poetry export`... COPY pyproject.toml poetry.lock /synapse/ +# Install custom extensions +RUN /root/.local/bin/poetry add --lock -- \ + "git+https://github.com/devture/matrix-synapse-shared-secret-auth#master" \ + "git+https://github.com/matrix-org/mjolnir.git#main&subdirectory=synapse_antispam" # If specified, we won't verify the hashes of dependencies. # This is only needed if the hashes of dependencies cannot be checked for some @@ -79,7 +83,7 @@ ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE # Otherwise, just create an empty requirements file so that the Dockerfile can # proceed. RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \ - /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \ + /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt --without-hashes; \ else \ touch /synapse/requirements.txt; \ fi