ci: Basic code coverage setup

Currently only uploads coverage for the unittests run via trial. In
theory we could calculate coverage also from sytest and complement.

The builtin coverage of trial tends to error out when trying to
calculate the coverage of the html files, because it doesn't distinguish
which files got opened. For this reason we are using coverage.py, but
that doesn't work when using multiple trial jobs in parallel (unless
putting in additional work), so this currently uses no concurrency.

I disabled some jobs around the rust parts of synapse for now as those
would take more work to set up.

.codecov.yaml

@@ -0,0 +1,9 @@
+coverage:
+  status:
+    project:
+      default:
+        # Do not pass if PR reduces coverage by more than 5%
+        target: auto
+        threshold: 5
+comment:
+  layout: "header, diff, components, files, footer"

.github/workflows/famedly-tests.yml

@@ -0,0 +1,373 @@
+name: Tests
+
+on:
+  push:
+  pull_request:
+  merge_group:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-sampleconfig:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          poetry-version: "1.3.2"
+          extras: "all"
+      - run: poetry run scripts-dev/generate_sample_config.sh --check
+      - run: poetry run scripts-dev/config-lint.sh
+
+  # We don't update develop currently
+  #check-schema-delta:
+  #  runs-on: ubuntu-latest
+
+  #  steps:
+  #    - uses: actions/checkout@v4
+  #    - uses: actions/setup-python@v5
+  #      with:
+  #        python-version: "3.x"
+  #    - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
+  #    - run: scripts-dev/check_schema_delta.py --force-colors
+
+  check-lockfile:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - run: .ci/scripts/check_lockfile.py
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Poetry
+        uses: matrix-org/setup-python-poetry@v1
+        with:
+          install-project: "false"
+
+      - name: Run ruff check
+        run: poetry run ruff check --output-format=github .
+
+      - name: Run ruff format
+        run: poetry run ruff format --check .
+
+  lint-mypy:
+    runs-on: ubuntu-latest
+    name: Typechecking
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - name: Setup Poetry
+        uses: matrix-org/setup-python-poetry@v1
+        with:
+          # We want to make use of type hints in optional dependencies too.
+          extras: all
+          # We have seen odd mypy failures that were resolved when we started
+          # installing the project again:
+          # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
+          # To make CI green, err towards caution and install the project.
+          install-project: "true"
+
+      # Cribbed from
+      # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
+      - name: Restore/persist mypy's cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            .mypy_cache
+          key: mypy-cache-${{ github.context.sha }}
+          restore-keys: mypy-cache-
+
+      - name: Run mypy
+        run: poetry run mypy
+
+  lint-crlf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check line endings
+        run: scripts-dev/check_line_terminators.sh
+
+  lint-pydantic:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          poetry-version: "1.3.2"
+          extras: "all"
+      - run: poetry run scripts-dev/check_pydantic_models.py
+
+  #lint-clippy:
+  #  runs-on: ubuntu-latest
+
+  #  steps:
+  #    - uses: actions/checkout@v4
+
+  #    - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+  #    - run: cargo clippy -- -D warnings
+
+  #lint-rustfmt:
+  #  runs-on: ubuntu-latest
+
+  #  steps:
+  #    - uses: actions/checkout@v4
+
+  #    - name: Install Rust
+  #      uses: dtolnay/rust-toolchain@master
+  #      with:
+  #        # We use nightly so that it correctly groups together imports
+  #        toolchain: nightly-2022-12-01
+  #        components: rustfmt
+  #    - uses: Swatinem/rust-cache@v2
+
+  #    - run: cargo fmt --check
+
+  # This is to detect issues with the rst file, which can otherwise cause issues
+  # when uploading packages to PyPi.
+  lint-readme:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - run: "pip install rstcheck"
+      - run: "rstcheck --report-level=WARNING README.rst"
+
+  # Dummy step to gate other tests on without repeating the whole list
+  linting-done:
+    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
+    needs:
+      - lint
+      - lint-mypy
+      - lint-crlf
+      - lint-pydantic
+      - check-sampleconfig
+      #- check-schema-delta
+      - check-lockfile
+      #- lint-clippy
+      #- lint-clippy-nightly
+      #- lint-rustfmt
+      - lint-readme
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "done"
+
+  calculate-test-jobs:
+    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - id: get-matrix
+        run: .ci/scripts/calculate_jobs.py
+    outputs:
+      trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
+      sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}
+
+  trial:
+    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
+    needs:
+      - calculate-test-jobs
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}
+    env:
+      TOP: ${{ github.workspace }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
+        if: ${{ matrix.job.postgres-version }}
+        # 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer.
+        # 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections.
+        run: |
+          docker run -d -p 5432:5432 \
+            --tmpfs /var/lib/postgres:rw,size=6144m \
+            --mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.job.postgres-version }}
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: ${{ matrix.job.python-version }}
+          poetry-version: "1.3.2"
+          extras: ${{ matrix.job.extras }}
+      - name: Await PostgreSQL
+        if: ${{ matrix.job.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: python -m pip install --user coverage
+      - run: poetry run coverage run -m twisted.trial tests
+        # can't run parallel jobs as that breaks coverage
+        env:
+          SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: /var/run/postgresql
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Logs are most useful when the command fails, always include them.
+        if: ${{ always() }}
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+      - run: poetry run coverage combine
+      - name: Codecov - Upload coverage
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{secrets.CODECOV_TOKEN}}
+
+  sytest:
+    if: ${{ !failure() && !cancelled() }}
+    needs:
+      - calculate-test-jobs
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }}
+      volumes:
+        - ${{ github.workspace }}:/src
+      env:
+        # If this is a pull request to a release branch, use that branch as default branch for sytest, else use develop
+        # This works because the release script always create a branch on the sytest repo with the same name as the release branch
+        SYTEST_DEFAULT_BRANCH: ${{ startsWith(github.base_ref, 'release-') && github.base_ref || 'develop' }}
+        SYTEST_BRANCH: ${{ github.head_ref }}
+        POSTGRES: ${{ matrix.job.postgres && 1}}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
+        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
+        WORKERS: ${{ matrix.job.workers && 1 }}
+        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
+        TOP: ${{ github.workspace }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+  complement:
+    if: "${{ !failure() && !cancelled() }}"
+    needs:
+      - linting-done
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arrangement: monolith
+            database: SQLite
+
+          - arrangement: monolith
+            database: Postgres
+
+          - arrangement: workers
+            database: Postgres
+
+    steps:
+      - name: Run actions/checkout@v4 for synapse
+        uses: actions/checkout@v4
+        with:
+          path: synapse
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - name: Prepare Complement's Prerequisites
+        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
+
+      - uses: actions/setup-go@v5
+        with:
+          cache-dependency-path: complement/go.sum
+          go-version-file: complement/go.mod
+
+        # use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once.
+      - run: |
+          set -o pipefail
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | synapse/.ci/scripts/gotestfmt
+        shell: bash
+        env:
+          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
+          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
+        name: Run Complement Tests
+
+  cargo-test:
+    runs-on: ubuntu-latest
+    needs:
+      - linting-done
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
+
+      - run: cargo test
+
+  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
+  tests-done:
+    if: ${{ always() }}
+    needs:
+      - trial
+      - sytest
+      - complement
+      - cargo-test
+      - linting-done
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "done"