diff --git a/.github/workflows/capture_new_migrations.yml b/.github/workflows/capture_new_migrations.yml new file mode 100644 index 000000000000..1e0f2ab0157d --- /dev/null +++ b/.github/workflows/capture_new_migrations.yml @@ -0,0 +1,138 @@ +name: Check Django Migrations + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + +jobs: + check_migrations: + name: check migrations + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] + # 'pinned' is used to install the latest patch version of Django + # within the global constraint i.e. Django==3.2.21 in current case + # because we have global constraint of Django<4.2 + django-version: ["pinned", "4.2"] + mongo-version: ["4"] + mysql-version: ["8"] + # excluding mysql5.7 with Django 4.2 since Django 4.2 has + # dropped support for MySQL<8 + exclude: + - django-version: "4.2" + mysql-version: "5.7" + services: + mongo: + image: mongo:${{ matrix.mongo-version }} + ports: + - 27017:27017 + # Note: Calling mongo here only works with mongo 4, in newer versions of mongo + # we'll have to use `mongosh` + options: >- + --health-cmd "mongo --quiet --eval 'db.runCommand(\"ping\")'" + --health-interval 10s + --health-timeout 5s + --health-retries 3 + mysql: + image: mysql:${{ matrix.mysql-version }} + ports: + - 3306:3306 + env: + MYSQL_DATABASE: "edxapp" + MYSQL_USER: "edxapp001" + MYSQL_PASSWORD: "password" + MYSQL_RANDOM_ROOT_PASSWORD: true + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 3 + steps: + - name: Setup mongodb user + run: | + mongosh edxapp --eval ' + db.createUser( + { + user: "edxapp", + pwd: "password", + roles: [ + { role: "readWrite", db: "edxapp" }, + ] + } + ); + ' + + - name: Verify mongo and mysql db credentials + run: | + mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp + mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp + + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system Packages + run: | + sudo apt-get update + make ubuntu-requirements + + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v3 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} + restore-keys: ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: | + make dev-requirements + if [[ "${{ matrix.django-version }}" != "pinned" ]]; then + pip install "django~=${{ matrix.django-version }}.0" + pip check # fail if this test-reqs/Django combination is broken + fi + + - name: list installed package versions + run: | + sudo pip freeze + + - name: Run Tests + env: + LMS_CFG: lms/envs/minimal.yml + # This is from the LMS dir on purpose since we don't need anything different for the CMS yet. + STUDIO_CFG: lms/envs/minimal.yml + run: | + echo "Running the LMS migrations." + ./manage.py lms showmigrations + echo "Running the CMS migrations." + ./manage.py cms showmigrations + + # This job aggregates test results. It's the required check for branch protection. + # https://github.com/marketplace/actions/alls-green#why + # https://github.com/orgs/community/discussions/33579 + success: + name: Migrations checks successful + if: always() + needs: + - check_migrations + runs-on: ubuntu-latest + steps: + - name: Decide whether the needed jobs succeeded or failed + # uses: re-actors/alls-green@v1.2.1 + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c3b1086c20eb..626a2ac54f12 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,156 +1,156 @@ -name: unit-tests - -on: - pull_request: - push: - branches: - - master - -jobs: - run-tests: - name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: [ edx-platform-runner ] - strategy: - matrix: - python-version: - - "3.8" - django-version: - - "pinned" - - "4.2" - # When updating the shards, remember to make the same changes in - # .github/workflows/unit-tests-gh-hosted.yml - shard_name: - - "lms-1" - - "lms-2" - - "lms-3" - - "lms-4" - - "lms-5" - - "lms-6" - - "openedx-1-with-lms" - - "openedx-2-with-lms" - - "openedx-1-with-cms" - - "openedx-2-with-cms" - - "cms-1" - - "cms-2" - - "common-with-lms" - - "common-with-cms" - - "xmodule-with-lms" - - "xmodule-with-cms" - # We expect Django 4.0 to fail, so don't stop when it fails. - continue-on-error: ${{ matrix.django-version == '4.0' }} - - steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - - name: checkout repo - uses: actions/checkout@v3 - - - name: start mongod server for tests - run: | - sudo mkdir -p /data/db - sudo chmod -R a+rw /data/db - mongod & - - - name: install requirements - run: | - sudo make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - sudo pip install "django~=${{ matrix.django-version }}.0" - sudo pip check # fail if this test-reqs/Django combination is broken - fi - - - name: list installed package versions - run: | - sudo pip freeze - - - name: Setup and run tests - uses: ./.github/actions/unit-tests - - - name: Renaming coverage data file - run: | - mv reports/.coverage reports/${{ matrix.shard_name }}.coverage - - - name: Upload coverage - uses: actions/upload-artifact@v3 - with: - name: coverage - path: reports/${{matrix.shard_name}}.coverage - - # This job aggregates test results. It's the required check for branch protection. - # https://github.com/marketplace/actions/alls-green#why - # https://github.com/orgs/community/discussions/33579 - success: - name: Unit tests successful - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - needs: - - run-tests - runs-on: ubuntu-latest - steps: - - name: Decide whether the needed jobs succeeded or failed - # uses: re-actors/alls-green@v1.2.1 - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe - with: - jobs: ${{ toJSON(needs) }} - - compile-warnings-report: - runs-on: [ edx-platform-runner ] - needs: [ run-tests ] - steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - uses: actions/checkout@v3 - - name: collect pytest warnings files - uses: actions/download-artifact@v3 - with: - name: pytest-warnings-json - path: test_root/log - - - name: display structure of downloaded files - run: ls -la test_root/log - - - name: compile warnings report - run: | - python openedx/core/process_warnings.py --dir-path test_root/log --html-path reports/pytest_warnings/warning_report_all.html - - - name: save warning report - if: success() - uses: actions/upload-artifact@v3 - with: - name: pytest-warning-report-html - path: | - reports/pytest_warnings/warning_report_all.html - - # Combine and upload coverage reports. - coverage: - needs: run-tests - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ 3.8 ] - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Download all artifacts - uses: actions/download-artifact@v3 - with: - name: coverage - path: reports - - - name: Install Python dependencies - run: | - pip install -r requirements/edx/coverage.txt - - - name: Run coverage - run: | - coverage combine reports/* - coverage report - coverage xml - - uses: codecov/codecov-action@v3 +#name: unit-tests +# +#on: +# pull_request: +# push: +# branches: +# - master +# +#jobs: +# run-tests: +# name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} +# if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) +# runs-on: [ edx-platform-runner ] +# strategy: +# matrix: +# python-version: +# - "3.8" +# django-version: +# - "pinned" +# - "4.2" +# # When updating the shards, remember to make the same changes in +# # .github/workflows/unit-tests-gh-hosted.yml +# shard_name: +# - "lms-1" +# - "lms-2" +# - "lms-3" +# - "lms-4" +# - "lms-5" +# - "lms-6" +# - "openedx-1-with-lms" +# - "openedx-2-with-lms" +# - "openedx-1-with-cms" +# - "openedx-2-with-cms" +# - "cms-1" +# - "cms-2" +# - "common-with-lms" +# - "common-with-cms" +# - "xmodule-with-lms" +# - "xmodule-with-cms" +# # We expect Django 4.0 to fail, so don't stop when it fails. +# continue-on-error: ${{ matrix.django-version == '4.0' }} +# +# steps: +# - name: sync directory owner +# run: sudo chown runner:runner -R .* +# +# - name: checkout repo +# uses: actions/checkout@v3 +# +# - name: start mongod server for tests +# run: | +# sudo mkdir -p /data/db +# sudo chmod -R a+rw /data/db +# mongod & +# +# - name: install requirements +# run: | +# sudo make test-requirements +# if [[ "${{ matrix.django-version }}" != "pinned" ]]; then +# sudo pip install "django~=${{ matrix.django-version }}.0" +# sudo pip check # fail if this test-reqs/Django combination is broken +# fi +# +# - name: list installed package versions +# run: | +# sudo pip freeze +# +# - name: Setup and run tests +# uses: ./.github/actions/unit-tests +# +# - name: Renaming coverage data file +# run: | +# mv reports/.coverage reports/${{ matrix.shard_name }}.coverage +# +# - name: Upload coverage +# uses: actions/upload-artifact@v3 +# with: +# name: coverage +# path: reports/${{matrix.shard_name}}.coverage +# +# # This job aggregates test results. It's the required check for branch protection. +# # https://github.com/marketplace/actions/alls-green#why +# # https://github.com/orgs/community/discussions/33579 +# success: +# name: Unit tests successful +# if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) +# needs: +# - run-tests +# runs-on: ubuntu-latest +# steps: +# - name: Decide whether the needed jobs succeeded or failed +# # uses: re-actors/alls-green@v1.2.1 +# uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe +# with: +# jobs: ${{ toJSON(needs) }} +# +# compile-warnings-report: +# runs-on: [ edx-platform-runner ] +# needs: [ run-tests ] +# steps: +# - name: sync directory owner +# run: sudo chown runner:runner -R .* +# - uses: actions/checkout@v3 +# - name: collect pytest warnings files +# uses: actions/download-artifact@v3 +# with: +# name: pytest-warnings-json +# path: test_root/log +# +# - name: display structure of downloaded files +# run: ls -la test_root/log +# +# - name: compile warnings report +# run: | +# python openedx/core/process_warnings.py --dir-path test_root/log --html-path reports/pytest_warnings/warning_report_all.html +# +# - name: save warning report +# if: success() +# uses: actions/upload-artifact@v3 +# with: +# name: pytest-warning-report-html +# path: | +# reports/pytest_warnings/warning_report_all.html +# +# # Combine and upload coverage reports. +# coverage: +# needs: run-tests +# runs-on: ubuntu-latest +# strategy: +# matrix: +# python-version: [ 3.8 ] +# steps: +# - name: Checkout repo +# uses: actions/checkout@v3 +# +# - name: Setup Python ${{ matrix.python-version }} +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# +# - name: Download all artifacts +# uses: actions/download-artifact@v3 +# with: +# name: coverage +# path: reports +# +# - name: Install Python dependencies +# run: | +# pip install -r requirements/edx/coverage.txt +# +# - name: Run coverage +# run: | +# coverage combine reports/* +# coverage report +# coverage xml +# - uses: codecov/codecov-action@v3