diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d580f54..03b51108 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,116 +1,170 @@ -name: CI +name: Cobaya Tests # for the moment only runs on "action* branches". Coverage/pypi not yet set up. -on: - push: - branches: - - 'action*' - tags: - - '*' - pull_request: - branches: - - '*' +on: [ push, pull_request ] env: - COBAYA_INSTALL_SKIP: polychord,planck_2015,CamSpec2021,2018_highl_CamSpec,unbinned,keck,classy + COBAYA_INSTALL_SKIP_BASE: planck_2015,CamSpec2021,2018_highl_CamSpec,unbinned,keck COBAYA_PACKAGES_PATH: ../packages jobs: - build: + should_run: + # only run on pushes that are not also part of PR + runs-on: ubuntu-latest + outputs: + run_tests: github.event_name == 'push' || ${{ steps.check.outputs.run_tests }} + steps: + - name: Check if tests should run + id: check + if: github.event_name == 'push' + uses: actions/github-script@v7 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const { data: prs } = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: `${context.repo.owner}:${context.ref.replace('refs/heads/', '')}`, + state: 'open' + }); + const shouldRun = prs.length === 0; + core.setOutput('run_tests', shouldRun ? 'true' : 'false'); + + tests: + needs: should_run runs-on: ${{ matrix.os }} + if: github.event_name == 'pull_request' || needs.should_run.outputs.run_tests == 'true' strategy: fail-fast: false matrix: include: - - name: "Anaconda: jammy LTS Python (fast; pip CAMB)" + - name: "Anaconda Python (flake8, no numba)" os: ubuntu-latest pydist: "ANACONDA" - - name: "Latest Python 3.12" + cobaya_skip: classy,polychord + - name: "Latest Python 3.x (with polychord)" os: ubuntu-latest - python-version: 3.12 + python-version: 3.x mpi: openmpi - - name: "OS X Python 3.8" + cobaya_skip: classy,polychord + - name: "OS X Python 3.8 (with classy)" os: macos-latest python-version: 3.8 mpi: openmpi + cobaya_skip: polychord - name: "Windows Python 3.12" os: windows-latest python-version: 3.12 mpi: intelmpi + cobaya_skip: classy,polychord + steps: + - name: Set COBAYA_INSTALL_SKIP + shell: bash + run: echo "COBAYA_INSTALL_SKIP=$COBAYA_INSTALL_SKIP_BASE,${{ matrix.cobaya_skip }}" >> $GITHUB_ENV + + - run: ln -s $(which gfortran-14) /usr/local/bin/gfortran + if: matrix.os == 'macos-latest' + + - run: | + gfortran --version + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python-version }} + if: matrix.pydist != 'ANACONDA' + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up Miniconda + if: matrix.pydist == 'ANACONDA' + uses: conda-incubator/setup-miniconda@v3 + with: + auto-activate-base: false + activate-environment: tests-environment + environment-file: tests-environment.yml + + - name: flake8 Lint + if: matrix.pydist == 'ANACONDA' + shell: bash -el {0} + run: | + flake8 cobaya --max-line-length 90 --select=E713,E704,E703,E714,E10,E11,E20,E22,E23,E25,E27,E301,E302,E304,E9,F405,F406,F5,F6,F7,F8,W1,W2,W3,W6 --show-source --statistics + + - name: Install dependencies (pip) + if: matrix.pydist != 'ANACONDA' + run: | + pip install -r requirements.txt pytest-xdist pytest-cov flaky matplotlib coverage iminuit numba camb + + - name: Install mpi + if: matrix.pydist != 'ANACONDA' && matrix.os != 'windows-latest' + uses: mpi4py/setup-mpi@v1 + with: + mpi: ${{ matrix.mpi }} + + - name: Install mpi4py + if: matrix.pydist != 'ANACONDA' && matrix.os != 'windows-latest' + run: | + pip install mpi4py -i https://pypi.anaconda.org/mpi4py/simple + + # - name: Cache dependencies + # uses: actions/cache@v4 + # with: + # path: | + # ${{ env.COBAYA_PACKAGES_PATH }}/data/ + # key: ${{ env.COBAYA_INSTALL_SKIP }} + # enableCrossOsArchive: true + + + - name: Run cobaya install and tests + shell: bash -el {0} + run: | + coverage run --parallel-mode -m cobaya.install polychord --debug + coverage run --parallel-mode -m pytest tests/ -n auto -k "not cosmo" --skip-not-installed --no-flaky-report + coverage run --parallel-mode -m cobaya.install cosmo-tests --no-progress-bars --debug --skip-global + pytest tests/ --cov -vv -s -k "cosmo" -n 2 --skip-not-installed --no-flaky-report + + - name: Run MPI tests + if: matrix.os != 'windows-latest' + shell: bash -el {0} + run: | + mpiexec -np 2 --oversubscribe coverage run --parallel-mode -m pytest -m mpi tests/ --no-flaky-report + + - name: Run external likelihood tests + shell: bash -el {0} + run: | + git clone --depth=1 https://github.com/CobayaSampler/example_external_likelihood + pip install ./example_external_likelihood --quiet + coverage run --parallel-mode -m unittest test_package.tests.test + + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v4 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + + + deploy: + needs: tests + runs-on: ubuntu-latest + if: github.repository_owner == 'CobayaSampler' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + permissions: + id-token: write + steps: - - run: ln -s $(which gfortran-14) /usr/local/bin/gfortran - if: matrix.os == 'macos-latest' - - - run: gfortran --version - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python ${{ matrix.python-version }} - if: matrix.pydist != 'ANACONDA' - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Set up Miniconda - if: matrix.pydist == 'ANACONDA' - uses: conda-incubator/setup-miniconda@v3 - with: - auto-activate-base: false - activate-environment: tests-environment - environment-file: tests-environment.yml - - - name: Install mpi - if: matrix.pydist != 'ANACONDA' - uses: mpi4py/setup-mpi@v1 - with: - mpi: ${{ matrix.mpi }} - -# - name: Cache dependencies -# uses: actions/cache@v2 -# with: -# path: | -# ${{ github.workspace }}/packages/data/planck_supp_data_and_covmats -# ${{ github.workspace }}/packages/data/bao_data -# ${{ github.workspace }}/packages/data/sn_data -# ${{ github.workspace }}/packages/data/des_data -# ${{ github.workspace }}/packages/data/planck_2018_pliklite_native -# key: ${{ runner.os }}-build-${{ matrix.python-version }}} - - - name: Install dependencies (pip) - if: matrix.pydist != 'ANACONDA' - run: | - pip install mpi4py -i https://pypi.anaconda.org/mpi4py/simple - pip install -r requirements.txt pytest-xdist pytest-cov flaky matplotlib dill coverage flake8 iminuit numba camb - - - name: Run flake8 - shell: bash -el {0} - run: | - flake8 cobaya --select=E713,E704,E703,E714,E741,E10,E11,E20,E22,E23,E25,E27,E301,E302,E304,E9,F405,F406,F5,F6,F7,F8,W1,W2,W3,W6 --show-source --statistics - - - name: Run cobaya install and tests - shell: bash -el {0} - run: | - coverage run --parallel-mode -m cobaya.install polychord --debug - coverage run --parallel-mode -m pytest tests/ -n auto -k "not cosmo" --skip-not-installed --no-flaky-report - coverage run --parallel-mode -m cobaya.install cosmo-tests --no-progress-bars --debug --skip-global - pytest tests/ --cov -vv -s -k "cosmo" -n 2 --skip-not-installed --no-flaky-report - - - name: Run MPI tests - shell: bash -el {0} - run: | - mpiexec -np 2 coverage run --parallel-mode -m pytest -x -m mpi tests/ --no-flaky-report - - - name: Run external likelihood tests - shell: bash -el {0} - run: | - git clone --depth=1 https://github.com/CobayaSampler/example_external_likelihood - pip install ./example_external_likelihood --quiet - coverage run --parallel-mode -m unittest test_package.tests.test - -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v1 + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install -U build twine + - name: Build package + run: python -m build --sdist + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 34442254..00000000 --- a/.travis.yml +++ /dev/null @@ -1,153 +0,0 @@ -language: python -os: linux -git: - depth: false - -#testing -env: - global: - - COBAYA_INSTALL_SKIP=polychord,planck_2015,CamSpec2021,2018_highl_CamSpec,unbinned,keck - -if: (type = pull_request) OR (branch = master) OR (branch =~ ^test.*) OR (branch =~ ^alltest.*) OR (tag IS present) - -#Large CamSpec folders tend to hang, so exclude non-base likelihoods from cache -cache: - directories: - - /home/travis/build/CosmoPars/packages/code - - /home/travis/build/CosmoPars/packages/data/planck_supp_data_and_covmats - - /home/travis/build/CosmoPars/packages/data/planck_2018/baseline - - /home/travis/build/CosmoPars/packages/data/bicep_keck_2018 - - /home/travis/build/CosmoPars/packages/data/bao_data - - /home/travis/build/CosmoPars/packages/data/sn_data - - /home/travis/build/CosmoPars/packages/data/des_data - - /home/travis/build/CosmoPars/packages/data/planck_2018_pliklite_native - -# (Pre)Installation -jobs: - include: - - if: branch !~ ^test.* - dist: focal - name: "Minimum requisites for dependencies: gcc-7, Python 3.8" - addons: - apt: - packages: - - gcc-7 - - g++-7 - - gfortran-7 - env: - - GCC_VERSION="7" - python: "3.8" - - if: branch !~ ^test.* - name: "Typical scenario: latest Ubuntu LTS" - dist: jammy - addons: - apt: - packages: - - gfortran - install: - - pip install -r requirements.txt camb - env: - - GCC_VERSION="ubuntu" - python: "3.10" - - name: "Anaconda: jammy LTS Python (fast; pip CAMB)" - dist: jammy - env: - - GCC_VERSION="ubuntu" - - PYDIST="ANACONDA" - - ANACONDA_CHANNEL="defaults" - - COBAYA_INSTALL_SKIP="$COBAYA_INSTALL_SKIP,classy" - language: minimal - - if: branch !~ ^test.* - name: "Latest jammy Python 3.12" - dist: jammy - addons: - apt: - packages: - - gfortran - env: - - GCC_VERSION="ubuntu" - - COBAYA_INSTALL_SKIP="$COBAYA_INSTALL_SKIP,classy" - python: "3.12" - - -before_install: - # Configure right compiler versions - - if [[ "$GCC_VERSION" != "ubuntu" ]]; then - mkdir -p gcc-symlinks; - ln -s /usr/bin/gfortran-$GCC_VERSION gcc-symlinks/gfortran; - ln -s /usr/bin/gcc-$GCC_VERSION gcc-symlinks/gcc; - ln -s /usr/bin/g++-$GCC_VERSION gcc-symlinks/g++; - export PATH=$PWD/gcc-symlinks:$PATH; - fi - - which gfortran >/dev/null 2>&1 && gfortran --version || echo "gfortran not installed" - # Install rest of system requisites - # - sudo apt install openmpi-bin openmpi-common libopenmpi-dev libopenblas-dev liblapack-dev - # Python requisites - - if [[ "$PYDIST" == "ANACONDA" ]]; then - wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p $HOME/miniconda; - export PATH="$HOME/miniconda/bin:$PATH"; - hash -r; - conda config --set always_yes yes --set changeps1 no; - conda info -a; - conda create -q -n test-environment -c $ANACONDA_CHANNEL scipy matplotlib cython PyYAML dill coverage pytest pandas; - source activate test-environment; - conda install -c conda-forge mpi4py openmpi iminuit; - pip install -r requirements.txt flake8 flaky pytest-xdist pytest-cov camb; - else - python -m pip install --upgrade pip setuptools wheel; - pip install openmpi mpi4py -i https://pypi.anaconda.org/mpi4py/simple; - pip install pytest-xdist pytest-cov flaky matplotlib dill coverage flake8 iminuit numba; - fi - - python --version - -script: - # Find undeclared references and syntax errors, plus standardize whitespace etc; - # See https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes - - flake8 cobaya --select=E713,E704,E703,E714,E741,E10,E11,E20,E22,E23,E25,E27,E301,E302,E304,E9,F405,F406,F5,F6,F7,F8,W1,W2,W3,W6 --show-source --statistics - # General tests: - - export COBAYA_PACKAGES_PATH="../packages" - - coverage run --parallel-mode -m cobaya.install polychord --debug - - coverage run --parallel-mode -m pytest tests/ -n auto -k "not cosmo" --skip-not-installed --no-flaky-report - # Cosmology tests: - - coverage run --parallel-mode -m cobaya.install cosmo-tests --no-progress-bars --debug --skip-global - - if [ -n "${CAMB_BRANCH}" ]; then - rm -rf $COBAYA_PACKAGES_PATH/code/CAMB ; - git clone --recursive --depth 1 -b $CAMB_BRANCH https://github.com/cmbant/CAMB $COBAYA_PACKAGES_PATH/code/CAMB ; - python $COBAYA_PACKAGES_PATH/code/CAMB/setup.py build ; - fi - # mpi tests - - mpiexec -np 2 --mca orte_base_help_aggregate 0 --mca btl ^openib --oversubscribe coverage run --parallel-mode -m pytest -x -m mpi tests/ --no-flaky-report ; - - mkdir covers; mv .coverage.* covers; ls -ltra covers - - pytest tests/ --cov -vv -s -k "cosmo" -n 1 --skip-not-installed --no-flaky-report - - mv .coverage .coverage.pytest; mv covers/.cov* . - # Test external cosmological likelihoods - #- pip install -e $COBAYA_PACKAGES_PATH/code/CAMB --quiet - #- git clone --depth=1 https://github.com/CobayaSampler/planck_lensing_external - #- pip install ./planck_lensing_external --quiet - #- coverage run --parallel-mode -m unittest plancklensing.tests.test_likes - - git clone --depth=1 https://github.com/CobayaSampler/example_external_likelihood - - pip install ./example_external_likelihood --quiet - - coverage run --parallel-mode -m unittest test_package.tests.test - -after_success: - - bash <(curl -s https://codecov.io/bash) - -############################################################################### -# DEPLOYMENT to PyPI (only if it builds and if the commit has been tagged) -deploy: - provider: pypi - on: - tags: true - python: '3.10' - branch: master - repo: CobayaSampler/cobaya -############################################################################### - edge: true - username: "__token__" - password: - secure: rSeqVE21XQnzKbY5bjvDgKofhxdoBOSQ+4h7tU+/HaCPNThTYpXD2vfJgtU+y+9ONoGm0l1665JwcIpajP3O8ruFI79ahD773qisgjukIwvO/pHhLsA8othdNAS4f7nOCBVScVMa92UNYpFFHZ6Kom2eJDLLLfqBJxYNfvAn7sPfetHfuYUe55YMF806xf+2ounXBYjHLZFw9V2iks0OVS3gQOSZYuPUn4oPCV32jojD3ZdYIz6tEDigLG9L+dbrzYfg7tOFkXW4OCpDhgB9XIuIrVcfM2oBByI5AmN0gUwBV2i77T3TjoIL6a+Ea0Y6Gn3lgGtgWVHxC6UGVu8uq6+iUc9+NmkRaJu0+kSLRQNb6AJFx+a+xdI2kMw27hy+S/mOzPowJMLcSjtBEjuuvmSFQzdzwldzFfZL5mYBwKdlLl1O/vca9RTaj+FouQyq6NHngBJ2ifhyuLclgyv35F2Ych13dPSToIIid3oJiDCnUHowcQtaUMqiPJW64tY8hbWMriGd9JgggwKg13Zr3bmsyOI4eh8VQG36OyMNEUYuJVJCuOrFtY8AZrs+/6YLqt2WCs3Mboz84ZgfE4d1cch1HQJzLlUOwW14b05XXfXER/iTylN87BV/q9qieY9q9Zwe3XzbG/TWkyHDXpY+Zm0+4AsZgoOapDSZJKwlUb4= -notifications: - email: - recipients: - - secure: wWS2J0Q413dLa415/iDn/rzwjFjnhCsGogxJeLW64/59NwaXV+spQZvgwfM+dKFt6kEvKZ4yPE+zXOgcpjp5lLSSWoJCaZy3q6rm7qEAEXxJCtH/4yMuLI0nWNxQRqVaOEq7q8e8jppQ/PBIUiqiT3EQlcNqagMReQEhr+M7hKtVnmA4XiFxmrp1H8aWhVvZWPWLPNtafPLX5CA2Ch3CJ09t591N3baZR0yOWwtEDeBZAHjbswTkNCVR6daUmOdbpFFti/Pdj7YmH1WkFTuEqfwuZiRJBrmyOGlARJZ4PybDK6npBXP393W2663TTIx0M4hT98YupWDXhC87WYAICBasySShxt/2lN2wqseHQlnfzPzmurZpim9choX5iDd/TTPyvhBk/xxT5G6RCeq2/vWwwWa4Owe20T2ep+8pYl0x9sNZyS+C67MOWsIs66dWEPxS4fOgsb6lBST9A8pv2Mfr5DHW8OLGAiIXI9mkeOqZkxlTxywQ5kKAO8hPkmzI7TrGH5drMkJDMqdGZSVC0f7y82RfQFLiv5aQp/xFkZKW7FzMi8DYBHJLfelpszfzyCeibQxQR7aPk6r6UbZhMw9Bd9KVpOV9Ag3EZ44MuVIDxA688cmdka4Swjw2VMWhTYQNGF1mB5WfsgtXfzpA8st7YZZJaysy+iXeoPML+is= diff --git a/CHANGELOG.md b/CHANGELOG.md index dc848803..ce216033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -291,9 +291,9 @@ - Added new fast but more realistic running, resuming and post tests with and without mpi - Fixed some randomized test inputs for more reliable running - drag: True running test -- Coverage reporting added to Travis +- Coverage reporting added - More useful traceback and console log when error raised running pytest -- added COBAYA_DEBUG env variable that can be set to force debug output (e.g. set in travis for failed build rerun) +- added COBAYA_DEBUG env variable that can be set to force debug output (e.g. set in tests for failed build rerun) ## 3.0.4 – 2021-03-10 diff --git a/DEVEL.rst b/DEVEL.rst index 6b827e62..4a9de7c2 100644 --- a/DEVEL.rst +++ b/DEVEL.rst @@ -10,7 +10,7 @@ This document gathers some notes about the development flow, release checklist, ``git`` development model ------------------------- -* Non-breaking travis-passing latest changes and fixes are in master +* Non-breaking test-passing latest changes and fixes are in master * Development that may break tests is done in temp branches or forks and merged to master once OK * Breaking changes developed in separate branches, and merged when releases updated * Releases are branched out, and only critical bug fixes are pushed onto them. @@ -45,7 +45,7 @@ can check most formatting and static errors on the command line using:: Release checklist ----------------- -+ Make sure all tests pass in Travis (or the package won't be pushed to PyPI). ++ Make sure all tests pass on GitHub Actions (or the package won't be pushed to PyPI). + Make sure everything relevant has been added to the Changelog. + Delete old deprecation notices (>=2 versions before) + Bump version number in ``__init__.py`` and ``CHANGELOG.md`` (also date) diff --git a/README.rst b/README.rst index 0e9b132b..d67f53ba 100644 --- a/README.rst +++ b/README.rst @@ -15,8 +15,8 @@ :Installation: ``pip install cobaya --upgrade`` (see the `installation instructions `_; in general do *not* clone) -.. image:: https://img.shields.io/travis/com/CobayaSampler/cobaya - :target: https://app.travis-ci.com/CobayaSampler/cobaya +.. image:: https://github.com/CobayaSampler/cobaya/actions/workflows/tests.yml/badge.svg?branch=master + :target: https://github.com/CobayaSampler/cobaya/actions .. image:: https://readthedocs.org/projects/cobaya/badge/?version=latest :target: https://cobaya.readthedocs.org/en/latest .. image:: https://codecov.io/gh/CobayaSampler/cobaya/branch/master/graphs/badge.svg diff --git a/cobaya/likelihoods/base_classes/planck_clik.py b/cobaya/likelihoods/base_classes/planck_clik.py index 7beee7d5..c3e36f63 100644 --- a/cobaya/likelihoods/base_classes/planck_clik.py +++ b/cobaya/likelihoods/base_classes/planck_clik.py @@ -235,7 +235,7 @@ def install(cls, path=None, force=False, code=True, data=True, common_path = "planck" # To see full clik build output even if installs OK (e.g. to check warnings) -_clik_verbose = any((s in os.getenv('TRAVIS_COMMIT_MESSAGE', '')) +_clik_verbose = any((s in os.getenv('COMMIT_MESSAGE', '')) for s in ["clik", "planck"]) # Don't try again to install clik if it failed for a previous likelihood _clik_install_failed = False diff --git a/tests/common.py b/tests/common.py index fc3f8b6b..fd7a1181 100644 --- a/tests/common.py +++ b/tests/common.py @@ -4,8 +4,8 @@ from contextlib import contextmanager -def is_travis(): - return os.environ.get('TRAVIS') == 'true' +def is_ci_test(): + return os.environ.get('GITHUB_ACTIONS') == 'true' def process_packages_path(packages_path) -> str: diff --git a/tests/common_sampler.py b/tests/common_sampler.py index 01af5ad4..036b832e 100644 --- a/tests/common_sampler.py +++ b/tests/common_sampler.py @@ -9,7 +9,7 @@ from cobaya.typing import InputDict, SamplersDict from cobaya.tools import KL_norm from cobaya.run import run -from .common import process_packages_path, is_travis +from .common import process_packages_path, is_ci_test from .conftest import install_test_wrapper from cobaya import mpi @@ -96,7 +96,7 @@ def body_of_sampler_test(info_sampler: SamplersDict, dimension=1, n_modes=1, tmp collection["sample"].to_getdist(label="cluster %d" % (i + 1)) for i, collection in products["clusters"].items()] # Plots! - if do_plots and not is_travis(): + if do_plots and not is_ci_test(): try: import getdist.plots as gdplots from getdist.gaussian_mixtures import MixtureND diff --git a/tests/test_cosmo_planck_2015.py b/tests/test_cosmo_planck_2015.py index 1a521e16..613a24fe 100644 --- a/tests/test_cosmo_planck_2015.py +++ b/tests/test_cosmo_planck_2015.py @@ -4,7 +4,7 @@ from .common_cosmo import body_of_test from cobaya.cosmo_input import planck_precision -# Generating plots in Travis +# Generating plots in tests import matplotlib matplotlib.use('agg') diff --git a/tests/test_cosmo_planck_2018.py b/tests/test_cosmo_planck_2018.py index 09e391c4..57d62ad9 100644 --- a/tests/test_cosmo_planck_2018.py +++ b/tests/test_cosmo_planck_2018.py @@ -4,7 +4,7 @@ from .common_cosmo import body_of_test from cobaya.cosmo_input import planck_precision -# Generating plots in Travis +# Generating plots in tests try: import matplotlib