diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 75a82c6e..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,128 +0,0 @@ -version: 2 -jobs: - checkout_code: - docker: - - image: condaforge/mambaforge - working_directory: ~/repo - steps: - - checkout - - run: git submodule sync - - run: git submodule update --init - - save_cache: - key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} - paths: - - ~/repo - - install_conda_env: - environment: - TIKTORCH_ENV_NAME: tiktorch-server-env - TIKTORCH_ENV_PREFIX: /opt/conda/envs/tiktorch-server-env - docker: - - image: condaforge/mambaforge - working_directory: ~/repo - steps: - - restore_cache: - keys: - - v1-repo-{{ .Environment.CIRCLE_SHA1 }} - - restore_cache: - keys: - - v11-dependencies-{{ checksum "environment.yml" }} - - - run: conda config --set channel_priority strict - - run: mamba update -n base -c conda-forge --update-all - - run: mamba install -c conda-forge conda-build make boa - - run: | - if [ ! -d ${TIKTORCH_ENV_PREFIX} ]; then - echo "Creating new environment ${TIKTORCH_ENV_NAME}" - make devenv - fi - - - save_cache: - paths: - - /opt/conda/envs - key: v11-dependencies-{{ checksum "environment.yml" }} - - pre_commit_check: - docker: - - image: condaforge/mambaforge - working_directory: ~/repo - steps: - - restore_cache: - keys: - - v1-repo-{{ .Environment.CIRCLE_SHA1 }} - - restore_cache: - keys: - - v11-dependencies-{{ checksum "environment.yml" }} - - - run: - name: run pre-commit - command: | - . /opt/conda/etc/profile.d/conda.sh - conda activate tiktorch-server-env - pre-commit run --from-ref origin/${CIRCLE_BRANCH} --to-ref ${CIRCLE_BRANCH} - - tests: - docker: - - image: condaforge/mambaforge - working_directory: ~/repo - steps: - - restore_cache: - keys: - - v1-repo-{{ .Environment.CIRCLE_SHA1 }} - - restore_cache: - keys: - - v11-dependencies-{{ checksum "environment.yml" }} - - - run: - name: run tests - command: | - . /opt/conda/etc/profile.d/conda.sh - conda activate tiktorch-server-env - conda list - python -m pytest -v - - build_conda_packages: - docker: - - image: condaforge/mambaforge - working_directory: ~/repo - steps: - - restore_cache: - keys: - - v1-repo-{{ .Environment.CIRCLE_SHA1 }} - - - run: mamba config --set channel_priority strict - - run: mamba install -c conda-forge conda-build anaconda-client boa - - run: - name: build packages - command: | - . /opt/conda/etc/profile.d/conda.sh - ./scripts/conda_build.sh conda-recipe - - -workflows: - version: 2 - build: - jobs: - - checkout_code: - filters: - tags: - only: /^v.*/ - - install_conda_env: - filters: - tags: - only: /^v.*/ - requires: - - checkout_code - - tests: - requires: - - install_conda_env - - pre_commit_check: - requires: - - install_conda_env - - build_conda_packages: - context: conda-upload - filters: - tags: - only: /^v.*/ - branches: - ignore: /.*/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..ec9ee104 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,168 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + pre_commit_check: + name: Pre-Commit Check + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + auto-activate-base: false + activate-environment: tiktorch-server-env + environment-file: environment.yml + channel-priority: flexible + miniforge-variant: Miniforge3 + - name: Run Pre-Commit + run: | + echo $BRANCH_NAME + echo ${{ github.event.pull_request.base.ref }} + echo ${{ github.event.pull_request.head.sha }} + git fetch origin + pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref ${{ github.event.pull_request.head.sha }} + + test-dev: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + auto-activate-base: false + activate-environment: tiktorch-server-env + environment-file: environment.yml + channel-priority: flexible + miniforge-variant: Miniforge3 + - name: conda diagnostics + run: | + conda info + - name: install submodules + run: | + git submodule init + git submodule update + make install_submodules + - name: Run tests + run: | + pytest -v -s --cov=tiktorch --cov-report=xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + fail_ci_if_error: true + + + conda-noarch-build: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + auto-activate-base: true + channel-priority: flexible + miniforge-variant: Miniforge3 + - name: install common conda dependencies + run: mamba install -n base -c conda-forge conda-build setuptools_scm -y + - name: Cache Conda Packages + uses: actions/cache@v4 + with: + path: | + pkgs/noarch + pkgs/channeldata.json + key: ${{ github.sha }}-packages + - id: version + run: | + vers=$( python setup.py --version ) + echo "version=${vers}" >> $GITHUB_OUTPUT + - name: Linux Conda Build Test + run: | + mkdir -p ./pkgs/noarch + conda build -c conda-forge conda-recipe --no-test --output-folder ./pkgs + + test-build-conda-packages: + needs: [conda-noarch-build] + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} + env: + TIKTORCH_PACKAGE_NAME: tiktorch-${{ needs.conda-noarch-build.outputs.version }}-py_0.tar.bz2 + steps: + # Use GNU tar instead of BSD tar on Windows + - name: "Use GNU tar instead of BSD tar" + if: matrix.os == 'windows-latest' + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + auto-activate-base: true + channel-priority: flexible + miniforge-variant: Miniforge3 + - name: install common conda dependencies + run: mamba install -n base -c conda-forge conda-build setuptools_scm -y + - name: Cache Conda Packages + uses: actions/cache@v4 + with: + path: | + pkgs/noarch + pkgs/channeldata.json + key: ${{ github.sha }}-packages + enableCrossOsArchive: true + - name: Linux Test + if: matrix.os == 'ubuntu-latest' + shell: bash -l {0} + run: | + conda build --test --override-channels \ + -c ./pkgs -c pytorch -c ilastik-forge -c conda-forge \ + ./pkgs/noarch/${TIKTORCH_PACKAGE_NAME} + - name: macOS Test + if: matrix.os == 'macos-latest' + shell: bash -l {0} + run: | + conda build --test --override-channels \ + -c ./pkgs -c pytorch -c ilastik-forge -c conda-forge \ + ./pkgs/noarch/${TIKTORCH_PACKAGE_NAME} + - name: Windows Test + if: matrix.os == 'windows-latest' + # HACK: due to a bug in conda-build need to point to + # libarchive explicitly. + # https://github.com/conda/conda/issues/12563#issuecomment-1494264704 + env: + LIBARCHIVE: C:\Miniconda\Library\bin\archive.dll + shell: cmd /C CALL {0} + run: | + conda build --test --override-channels ^ + -c %CD%\pkgs -c pytorch -c ilastik-forge -c conda-forge ^ + .\pkgs\noarch\%TIKTORCH_PACKAGE_NAME% diff --git a/Makefile b/Makefile index 5e1fee90..262df6ee 100644 --- a/Makefile +++ b/Makefile @@ -27,5 +27,4 @@ install_submodules: remove_devenv: conda env remove --yes --name $(TIKTORCH_ENV_NAME) - .PHONY: * diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index d1d33fc0..f45de0d5 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -15,7 +15,8 @@ source: build: noarch: python number: 0 - script: python -m pip install --no-deps --ignore-installed . + script: + - python -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv entry_points: - tiktorch-server = tiktorch.server.base:main @@ -30,13 +31,11 @@ requirements: - {{ dep.lower() }} {% endfor %} run_constrained: + - mkl <2024.1.0 # [linux] until pytorch is compatible with the current version - cudatoolkit >=10.2 {% for dep in setup_py_data['extras_require']['server-pytorch'] %} - {{ dep.lower() }} {% endfor %} - {% for dep in setup_py_data['extras_require']['server-tensorflow'] %} - - {{ dep.lower() }} - {% endfor %} about: home: https://github.com/ilastik/tiktorch @@ -52,12 +51,10 @@ test: {% for dep in setup_py_data['extras_require']['server-pytorch'] %} - {{ dep.lower() }} {% endfor %} - {% for dep in setup_py_data['extras_require']['server-tensorflow'] %} - - {{ dep.lower() }} - {% endfor %} # this is still necessary, torchvision doesn't work properly with cpuonly mutex - torchvision=*=*cpu - cpuonly + - pytest imports: # client - tiktorch @@ -70,3 +67,12 @@ test: # server - tiktorch.server.base - tiktorch.server.session + + source_files: + - tests + - tiktorch + - pytest.ini + + command: + - pytest -v -s + diff --git a/environment.yml b/environment.yml index 94b04c80..9f493eb3 100644 --- a/environment.yml +++ b/environment.yml @@ -13,8 +13,6 @@ dependencies: - marshmallow=3.12.* - marshmallow-jsonschema - protobuf - - pytest - - pytest-grpc - pyyaml=5.3.* - requests - ruamel.yaml @@ -22,15 +20,16 @@ dependencies: - scipy - typing-extensions - xarray + - setuptools + - pip # pytorch # remove cpuonly, add cudatoolkit and cudnn for gpu support - - pytorch=1.13.* - - inferno=v0.4.* + - pytorch=2.4.* # currently it's necessary to force the cpu version, remove # torchvision pin when going for gpu # - torchvision - #- cpuonly + - cpuonly # - cudatoolkit >=10.2 # - cudnn # - tochvision @@ -46,10 +45,14 @@ dependencies: - typer # dev stuff + - pytest + - pytest-cov + - pytest-grpc - bump2version - mypy - pre_commit - - pip - - mkl <2024.1.0 # [linux] until pytorch is compatible with the current version + + + diff --git a/pytest.ini b/pytest.ini index c2c366b2..f968043d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,6 @@ [pytest] -add_opts = -s --doctest-modules +python_files = test_*.py +addopts = -s --doctest-modules testpaths = tests #log_format = %(asctime)s.%(msecs)03d %(levelname)s %(message)s #log_date_format = %M:%S.%f diff --git a/setup.py b/setup.py index d34ab19e..53caa672 100644 --- a/setup.py +++ b/setup.py @@ -37,14 +37,9 @@ ], extras_require={ "server-pytorch": [ - "inferno", "pytorch>=1.6", "scikit-learn", - ], - "server-tensorflow": [ - "tensorflow>=2.9", - "scikit-learn", - ], + ] }, entry_points={"console_scripts": ["tiktorch=tiktorch.server.base:main"]}, # extras_require={"test": ["pytest"]},