From d76a9e34eb2cae7bbf26a54b92603518353f9a64 Mon Sep 17 00:00:00 2001 From: AlexWells Date: Fri, 10 Jan 2025 15:18:44 +0000 Subject: [PATCH 1/9] Remove most of the CI to just test the bit we want --- .github/workflows/code.yml | 309 ++++++++++++++++++------------------- 1 file changed, 154 insertions(+), 155 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 461f1d3a..f70f2033 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -13,165 +13,164 @@ on: - cron: '0 8 * * MON' jobs: - lint: - runs-on: "ubuntu-latest" - steps: - - name: Checkout Source - uses: actions/checkout@v4 - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.7" - - - name: Install Python Dependencies - run: pip install flake8 - - - name: Lint - run: flake8 - - sdist: - runs-on: "ubuntu-latest" - steps: - - name: Checkout Source - uses: actions/checkout@v4 - with: - # require history to get back to last tag for version number of branches - fetch-depth: 0 - submodules: true - - - name: Build Sdist - run: pipx run build --sdist . - - - name: Upload Sdist - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist/* - - build: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest, macos-13] - python: [cp37, cp38, cp39, cp310, cp311, cp312] - - - include: - # Put coverage and results files in the project directory for mac - - os: macos-13 - cov_file: "{project}/dist/coverage.xml" - results_file: "{project}/dist/pytest-results.xml" - # And for windows - - os: windows-latest - cov_file: "{project}/dist/coverage.xml" - results_file: "{project}/dist/pytest-results.xml" - # But put coverage and results files in the output dir mounted in docker for linux - - os: ubuntu-latest - cov_file: /output/coverage.xml - results_file: /output/pytest-results.xml - - name: build/${{ matrix.os }}/${{ matrix.python }} - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout Source - uses: actions/checkout@v4 - with: - # require history to get back to last tag for version number of branches - fetch-depth: 0 - submodules: true - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.12" - - - name: Install Python Dependencies - run: pip install build cibuildwheel>=2.16.2 - - - name: Build Wheel - run: cibuildwheel --output-dir dist - env: - CIBW_BUILD: ${{ matrix.python }}*64 - CIBW_TEST_EXTRAS: dev - # Added a sleep command afterwards to let all cleanup finish; sometimes - # cibuildwheel reports it cannot clean up the temp dir used for testing, - # and fails the build. Sleeping seems to give pytest enough time to - # fully clean up. - CIBW_TEST_COMMAND: pytest {project}/tests --cov-report xml:${{ matrix.cov_file }} --junit-xml=${{ matrix.results_file }} && sleep 2 - # Run with faulthandler and -s in the hope we get a stack trace on seg fault on windows... - CIBW_TEST_COMMAND_WINDOWS: python -X faulthandler -m pytest -s {project}/tests --cov-report xml:${{ matrix.cov_file }} --junit-xml=${{ matrix.results_file }} - # Disable auditwheel as it isn't compatible with setuptools_dso approach - # https://github.com/mdavidsaver/setuptools_dso/issues/17 - CIBW_REPAIR_WHEEL_COMMAND: "" - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - CIBW_ENVIRONMENT_LINUX: SETUPTOOLS_DSO_PLAT_NAME=manylinux2014_x86_64 - CIBW_SKIP: "*-musllinux*" # epicscorelibs doesn't build on musllinux platforms - - - name: Upload Wheel - uses: actions/upload-artifact@v4 - with: - name: dist-${{ matrix.os }}-${{ matrix.python }} - path: dist/softioc* - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - name: ${{ matrix.os }}/${{ matrix.python }} - directory: dist - - test-sdist: - needs: [sdist] - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest, macos-13] - python: [cp37, cp38, cp39, cp310, cp311, cp312] - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist - - - name: Install sdist in a venv and check cli works - run: pipx run --spec dist/*.tar.gz pythonSoftIOC --version - shell: bash - - release: - needs: [build, sdist] - runs-on: ubuntu-latest - # upload to PyPI and make a release on every tag - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') - steps: - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist - - - name: Github Release - # We pin to the SHA, not the tag, for security reasons. - # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: softprops/action-gh-release@2d72d869af3bf23602f9593a1e3fd739b80ac1eb # v0.1.12 - with: - files: dist/* - body: See [Changelog](CHANGELOG.rst) for more details - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.pypi_token }} - run: pipx run twine upload dist/* + # lint: + # runs-on: "ubuntu-latest" + # steps: + # - name: Checkout Source + # uses: actions/checkout@v4 + + # - name: Install Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.7" + + # - name: Install Python Dependencies + # run: pip install flake8 + + # - name: Lint + # run: flake8 + + # sdist: + # runs-on: "ubuntu-latest" + # steps: + # - name: Checkout Source + # uses: actions/checkout@v4 + # with: + # # require history to get back to last tag for version number of branches + # fetch-depth: 0 + # submodules: true + + # - name: Build Sdist + # run: pipx run build --sdist . + + # - name: Upload Sdist + # uses: actions/upload-artifact@v4 + # with: + # name: dist + # path: dist/* + + # build: + # strategy: + # fail-fast: false + # matrix: + # os: [ubuntu-latest, windows-latest, macos-13] + # python: [cp37, cp38, cp39, cp310, cp311, cp312] + + + # include: + # # Put coverage and results files in the project directory for mac + # - os: macos-13 + # cov_file: "{project}/dist/coverage.xml" + # results_file: "{project}/dist/pytest-results.xml" + # # And for windows + # - os: windows-latest + # cov_file: "{project}/dist/coverage.xml" + # results_file: "{project}/dist/pytest-results.xml" + # # But put coverage and results files in the output dir mounted in docker for linux + # - os: ubuntu-latest + # cov_file: /output/coverage.xml + # results_file: /output/pytest-results.xml + + # name: build/${{ matrix.os }}/${{ matrix.python }} + # runs-on: ${{ matrix.os }} + + # steps: + # - name: Checkout Source + # uses: actions/checkout@v4 + # with: + # # require history to get back to last tag for version number of branches + # fetch-depth: 0 + # submodules: true + + # - name: Install Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.12" + + # - name: Install Python Dependencies + # run: pip install build cibuildwheel>=2.16.2 + + # - name: Build Wheel + # run: cibuildwheel --output-dir dist + # env: + # CIBW_BUILD: ${{ matrix.python }}*64 + # CIBW_TEST_EXTRAS: dev + # # Added a sleep command afterwards to let all cleanup finish; sometimes + # # cibuildwheel reports it cannot clean up the temp dir used for testing, + # # and fails the build. Sleeping seems to give pytest enough time to + # # fully clean up. + # CIBW_TEST_COMMAND: pytest {project}/tests --cov-report xml:${{ matrix.cov_file }} --junit-xml=${{ matrix.results_file }} && sleep 2 + # # Run with faulthandler and -s in the hope we get a stack trace on seg fault on windows... + # CIBW_TEST_COMMAND_WINDOWS: python -X faulthandler -m pytest -s {project}/tests --cov-report xml:${{ matrix.cov_file }} --junit-xml=${{ matrix.results_file }} + # # Disable auditwheel as it isn't compatible with setuptools_dso approach + # # https://github.com/mdavidsaver/setuptools_dso/issues/17 + # CIBW_REPAIR_WHEEL_COMMAND: "" + # CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + # CIBW_ENVIRONMENT_LINUX: SETUPTOOLS_DSO_PLAT_NAME=manylinux2014_x86_64 + # CIBW_SKIP: "*-musllinux*" # epicscorelibs doesn't build on musllinux platforms + + # - name: Upload Wheel + # uses: actions/upload-artifact@v4 + # with: + # name: dist-${{ matrix.os }}-${{ matrix.python }} + # path: dist/softioc* + + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v4 + # with: + # name: ${{ matrix.os }}/${{ matrix.python }} + # directory: dist + + # test-sdist: + # needs: [sdist] + # strategy: + # fail-fast: false + # matrix: + # os: [ubuntu-latest, windows-latest, macos-13] + # python: [cp37, cp38, cp39, cp310, cp311, cp312] + + # runs-on: ${{ matrix.os }} + + # steps: + # - uses: actions/download-artifact@v4 + # with: + # name: dist + # path: dist + + # - name: Install sdist in a venv and check cli works + # run: pipx run --spec dist/*.tar.gz pythonSoftIOC --version + # shell: bash + + # release: + # needs: [build, sdist] + # runs-on: ubuntu-latest + # # upload to PyPI and make a release on every tag + # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + # steps: + # - uses: actions/download-artifact@v4 + # with: + # name: dist + # path: dist + + # - name: Github Release + # # We pin to the SHA, not the tag, for security reasons. + # # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions + # uses: softprops/action-gh-release@2d72d869af3bf23602f9593a1e3fd739b80ac1eb # v0.1.12 + # with: + # files: dist/* + # body: See [Changelog](CHANGELOG.rst) for more details + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # - name: Publish to PyPI + # env: + # TWINE_USERNAME: __token__ + # TWINE_PASSWORD: ${{ secrets.pypi_token }} + # run: pipx run twine upload dist/* # Check that the master branches of various upstream dependencies are # still compatible when used together master_branch_test: - if: ${{ github.event_name == 'schedule' }} runs-on: "ubuntu-latest" steps: - name: Checkout Source From 2d412e46ab398e51aa5258cbd756a6d270ff7d17 Mon Sep 17 00:00:00 2001 From: AlexWells Date: Fri, 10 Jan 2025 15:23:31 +0000 Subject: [PATCH 2/9] Specify the versions that previously worked --- .github/workflows/code.yml | 1 + .github/workflows/docs.yml | 52 -------------------------------------- 2 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index f70f2033..2a18e437 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -189,6 +189,7 @@ jobs: run: | pip install git+https://github.com/epics-base/p4p.git git+https://github.com/epics-base/pvxs.git git+https://github.com/epics-base/epicscorelibs.git pip install git+https://github.com/DiamondLightSource/aioca git+https://github.com/DiamondLightSource/cothread + pip install Pygments==2.18.0 pytest-asyncio==0.25.0 setuptools==75.6.0 pip install -e .[dev] pip freeze diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index e3866cf4..00000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Docs CI - -on: - push: - branches: - # Add more branches here to publish docs from other branches - - master - - main - tags: - - "*" - pull_request: - -jobs: - build: - name: "Docs CI" - runs-on: ubuntu-latest - - steps: - - name: Checkout Source - uses: actions/checkout@v2 - with: - # require history to get back to last tag for version number of branches - fetch-depth: 0 - submodules: true - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.7" - - - name: Install Python Dependencies - run: | - pip install pipenv - pipenv install --dev --deploy --python $(which python) && pipenv graph - - - name: Build Docs - run: pipenv run docs - - - name: Move to versioned directory - # e.g. master or 0.1.2 - run: mv build/html ".github/pages/${GITHUB_REF##*/}" - - - name: Publish Docs to gh-pages - # Only master and tags are published - if: "${{ github.repository_owner == 'dls-controls' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')) }}" - # We pin to the SHA, not the tag, for security reasons. - # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: peaceiris/actions-gh-pages@bbdfb200618d235585ad98e965f4aafc39b4c501 # v3.7.3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: .github/pages - keep_files: true From 92092733b631a8d7bbbf4e9e0fa7c8bc07e729d6 Mon Sep 17 00:00:00 2001 From: AlexWells Date: Fri, 10 Jan 2025 15:32:40 +0000 Subject: [PATCH 3/9] Downgrade to Ubuntu22 from 24 Specifying these particular modules made no difference --- .github/workflows/code.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 2a18e437..45df4e1b 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -171,7 +171,7 @@ jobs: # Check that the master branches of various upstream dependencies are # still compatible when used together master_branch_test: - runs-on: "ubuntu-latest" + runs-on: "ubuntu-22.04" steps: - name: Checkout Source uses: actions/checkout@v4 @@ -189,7 +189,6 @@ jobs: run: | pip install git+https://github.com/epics-base/p4p.git git+https://github.com/epics-base/pvxs.git git+https://github.com/epics-base/epicscorelibs.git pip install git+https://github.com/DiamondLightSource/aioca git+https://github.com/DiamondLightSource/cothread - pip install Pygments==2.18.0 pytest-asyncio==0.25.0 setuptools==75.6.0 pip install -e .[dev] pip freeze From 4eccded8b2960127a31b108f275f81459f9c46a1 Mon Sep 17 00:00:00 2001 From: AlexWells Date: Fri, 10 Jan 2025 15:58:00 +0000 Subject: [PATCH 4/9] Diagnose what module(s) cause the probem on ubuntu24 --- .github/workflows/code.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 45df4e1b..344a63a9 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -171,7 +171,7 @@ jobs: # Check that the master branches of various upstream dependencies are # still compatible when used together master_branch_test: - runs-on: "ubuntu-22.04" + runs-on: "ubuntu-latest" steps: - name: Checkout Source uses: actions/checkout@v4 @@ -186,9 +186,10 @@ jobs: python-version: "3.12" - name: Install master versions + # + # pip install git+https://github.com/epics-base/p4p.git git+https://github.com/epics-base/pvxs.git git+https://github.com/epics-base/epicscorelibs.git + # pip install git+https://github.com/DiamondLightSource/aioca git+https://github.com/DiamondLightSource/cothread run: | - pip install git+https://github.com/epics-base/p4p.git git+https://github.com/epics-base/pvxs.git git+https://github.com/epics-base/epicscorelibs.git - pip install git+https://github.com/DiamondLightSource/aioca git+https://github.com/DiamondLightSource/cothread pip install -e .[dev] pip freeze From 40438820cea8df4a9ba6be6ca848f89de1d69751 Mon Sep 17 00:00:00 2001 From: AlexWells Date: Fri, 10 Jan 2025 16:00:46 +0000 Subject: [PATCH 5/9] Add epicscorelibs master back in --- .github/workflows/code.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 344a63a9..482d6d3a 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -190,6 +190,7 @@ jobs: # pip install git+https://github.com/epics-base/p4p.git git+https://github.com/epics-base/pvxs.git git+https://github.com/epics-base/epicscorelibs.git # pip install git+https://github.com/DiamondLightSource/aioca git+https://github.com/DiamondLightSource/cothread run: | + pip install git+https://github.com/epics-base/epicscorelibs.git pip install -e .[dev] pip freeze From e16e150c87b4b9825f54649f66d52e1e3330cae9 Mon Sep 17 00:00:00 2001 From: AlexWells Date: Mon, 13 Jan 2025 09:17:59 +0000 Subject: [PATCH 6/9] Try using ci-core-dumper --- .github/workflows/code.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 482d6d3a..9e339d52 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -194,6 +194,11 @@ jobs: pip install -e .[dev] pip freeze - - name: Run tests - run: | + # - name: Run tests + # run: | + # python -m pytest + - name: CI Core DUmper + uses: mdavidsaver/ci-core-dumper@master + runs: | + ulimit -c unlimited python -m pytest From b14511b92383ca3b30e661fb81d10959d747dcee Mon Sep 17 00:00:00 2001 From: AlexWells Date: Mon, 13 Jan 2025 09:21:37 +0000 Subject: [PATCH 7/9] Try ci dumper again --- .github/workflows/code.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 9e339d52..62761c96 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -194,11 +194,20 @@ jobs: pip install -e .[dev] pip freeze - # - name: Run tests - # run: | - # python -m pytest - - name: CI Core DUmper - uses: mdavidsaver/ci-core-dumper@master - runs: | - ulimit -c unlimited + - name: Run tests + run: | python -m pytest + + + test-ci-dumper: + runs-on: "ubuntu-latest" + name: My Job + steps: + - uses: actions/checkout@v4 + - uses: mdavidsaver/ci-core-dumper@master + - run: | + ulimit -c unlimited + pip install git+https://github.com/epics-base/epicscorelibs.git + pip install -e .[dev] + pip freeze + python -m pytest From 8c2cb63e7471eb064f8f81198e49762d626b031b Mon Sep 17 00:00:00 2001 From: AlexWells Date: Mon, 13 Jan 2025 09:25:14 +0000 Subject: [PATCH 8/9] Fix submodule checkout --- .github/workflows/code.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 62761c96..d4cb5ad5 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -204,6 +204,10 @@ jobs: name: My Job steps: - uses: actions/checkout@v4 + with: + # require history to get back to last tag for version number of branches + fetch-depth: 0 + submodules: true - uses: mdavidsaver/ci-core-dumper@master - run: | ulimit -c unlimited From 802cd7e4b60428281d3593f7eeff3a981193d48b Mon Sep 17 00:00:00 2001 From: AlexWells Date: Wed, 15 Jan 2025 09:42:14 +0000 Subject: [PATCH 9/9] Test Michael Davidsaver's possible fix --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index d4cb5ad5..ba007c16 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -190,7 +190,7 @@ jobs: # pip install git+https://github.com/epics-base/p4p.git git+https://github.com/epics-base/pvxs.git git+https://github.com/epics-base/epicscorelibs.git # pip install git+https://github.com/DiamondLightSource/aioca git+https://github.com/DiamondLightSource/cothread run: | - pip install git+https://github.com/epics-base/epicscorelibs.git + pip install git+https://github.com/mdavidsaver/epicscorelibs.git@bypass-fortify pip install -e .[dev] pip freeze