Rework tests

.github/workflows/docker_tests.yaml

@@ -0,0 +1,23 @@
+name: Tests in Docker (3.8)
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  all_tests:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Building docker
+      run: |
+        make docker_build RUNTIME=cpu
+    - name: Running all tests
+      run: |
+        make docker_all_tests RUNTIME=cpu WANDB_API_KEY=${{ secrets.WANDB_API_KEY }} NEPTUNE_API_TOKEN=${{ secrets.NEPTUNE_API_TOKEN }}

.github/workflows/pre-commit-workflow.yaml

@@ -1,6 +1,5 @@
 name: Linters
 
-
 on:
   pull_request:
     branches:
@@ -13,7 +12,6 @@ on:
     paths-ignore:
       - 'README.md'
 
-
 jobs:
   pre_commit:
     runs-on: ubuntu-20.04
@@ -24,10 +22,11 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: "3.8.0"
-    - name: Install dependencies
+    - name: Install test dependencies
       run: |
         apt-get get update && apt-get install cmake
-        python -m pip install --upgrade pre-commit==2.15.0
+        python -m pip install --upgrade pip
+        python -m pip install -r ci/requirements_tests.txt
     - name: Pre-commit tests
       run: |
         make run_precommit

.github/workflows/tests.yaml

@@ -0,0 +1,58 @@
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  short_tests:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Installation
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install -r ci/requirements_tests.txt
+        python3 -m pip install -e .
+    - name: Running short tests
+      run: |
+        make run_short_tests
+
+  all_tests:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Installation
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install -r ci/requirements_tests.txt
+        python3 -m pip install -e .
+    - name: Running all tests
+      run: |
+        if [ "${{ matrix.python-version }}" == "3.8" ]; then
+          make run_all_tests WANDB_API_KEY=${{ secrets.WANDB_API_KEY }} NEPTUNE_API_TOKEN=${{ secrets.NEPTUNE_API_TOKEN }} DOWNLOAD_ZOO_IN_TESTS=yes
+        else
+          make run_all_tests WANDB_API_KEY=${{ secrets.WANDB_API_KEY }} NEPTUNE_API_TOKEN=${{ secrets.NEPTUNE_API_TOKEN }}
+        fi

Makefile

@@ -69,7 +69,6 @@ run_short_tests: download_mock_dataset
 	export PYTORCH_ENABLE_MPS_FALLBACK=1
 	pytest --disable-warnings -sv -m "not long" tests
 	pytest --disable-warnings --doctest-modules --doctest-continue-on-failure -sv oml
-	$(JUPYTER_CMD) --execute pipelines/features_extraction/visualization.ipynb
 
 .PHONY: test_converters
 test_converters:
@@ -83,7 +82,7 @@ test_converters:
 run_precommit:
 	pre-commit install && pre-commit run -a
 
-# ====================================== INFRASTRUCTURE =============================
+# ====================================== DOCKER =============================
 
 .PHONY: docker_build
 docker_build:
@@ -97,6 +96,8 @@ docker_all_tests:
 docker_short_tests:
 	docker run --env WANDB_API_KEY=$(WANDB_API_KEY) --env NEPTUNE_API_TOKEN=$(NEPTUNE_API_TOKEN) -t $(IMAGE_NAME) make run_short_tests
 
+# ====================================== PIP =============================
+
 .PHONY: build_wheel
 build_wheel:
 	python -m pip install --upgrade pip

README.md

@@ -1,15 +1,14 @@
 <div align="center">
 <img src="https://i.ibb.co/wsmD5r4/photo-2022-06-06-17-40-52.jpg" width="400px">
 
-![example workflow](https://github.com/OML-Team/open-metric-learning/actions/workflows/pre-commit-workflow.yaml/badge.svg)
-![example workflow](https://github.com/OML-Team/open-metric-learning/actions/workflows/tests-workflow.yaml/badge.svg?)
+
 [![Documentation Status](https://readthedocs.org/projects/open-metric-learning/badge/?version=latest)](https://open-metric-learning.readthedocs.io/en/latest/?badge=latest)
 [![PyPI Status](https://pepy.tech/badge/open-metric-learning)](https://pepy.tech/project/open-metric-learning)
 [![Pipi version](https://img.shields.io/pypi/v/open-metric-learning.svg)](https://pypi.org/project/open-metric-learning/)
-![example workflow](https://github.com/OML-Team/open-metric-learning/actions/workflows/python-versions.yaml/badge.svg?)
-[![python](https://img.shields.io/badge/python_3.8-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/python-versions.yaml/badge.svg?)
-[![python](https://img.shields.io/badge/python_3.9-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/python-versions.yaml/badge.svg?)
-[![python](https://img.shields.io/badge/python_3.10-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/python-versions.yaml/badge.svg?)
+[![python](https://img.shields.io/badge/python_3.8-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/tests.yaml/badge.svg?)
+[![python](https://img.shields.io/badge/python_3.9-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/tests.yaml/badge.svg?)
+[![python](https://img.shields.io/badge/python_3.10-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/tests.yaml/badge.svg?)
+
 
 OML is a PyTorch-based framework to train and validate the models producing high-quality embeddings.
 

ci/Dockerfile

@@ -28,9 +28,9 @@ RUN pip install Cython==0.29.28
 
 ### Python libs
 COPY ci/requirements.txt requirements.txt
-COPY ci/requirements_dev.txt requirements_dev.txt
+COPY ci/requirements_tests.txt requirements_tests.txt
 RUN pip install --no-cache-dir -r requirements.txt ${EXTRA_INDEX}
-RUN pip install --no-cache-dir -r requirements_dev.txt
+RUN pip install --no-cache-dir -r requirements_tests.txt
 
 WORKDIR /oml-lib
 COPY . /oml-lib

ci/requirements.txt

@@ -1,4 +1,3 @@
-numpy>=1.21
 torch>=2.0
 pytorch-lightning>=2.0
 torchvision
@@ -7,11 +6,15 @@ opencv-python>=4.1.1
 pillow>=8.2
 pandas>=1.0, <=1.5.2
 matplotlib>3.5.2, <3.6.2
-
 omegaconf>=2.0.5, <2.3.0
 hydra-core>=1.2.0, <1.3.0
-neptune-client>=0.14.2, <1.0.0
-wandb>=0.15.4
 python-dotenv>=0.17.0
 validators>=0.18.0
 gdown>=4.0
+
+# todo: make it optional
+grad-cam>=1.4.6
+jupyter>=1.0.0
+tensorboard
+neptune-client>=0.14.2, <1.0.0
+wandb>=0.15.4

ci/requirements_tests.txt

@@ -0,0 +1,2 @@
+pytest>=7.2
+pre-commit==2.15.0

docs/readme/header.md

@@ -1,15 +1,14 @@
 <div align="center">
 <img src="https://i.ibb.co/wsmD5r4/photo-2022-06-06-17-40-52.jpg" width="400px">
 
-![example workflow](https://github.com/OML-Team/open-metric-learning/actions/workflows/pre-commit-workflow.yaml/badge.svg)
-![example workflow](https://github.com/OML-Team/open-metric-learning/actions/workflows/tests-workflow.yaml/badge.svg?)
+
 [![Documentation Status](https://readthedocs.org/projects/open-metric-learning/badge/?version=latest)](https://open-metric-learning.readthedocs.io/en/latest/?badge=latest)
 [![PyPI Status](https://pepy.tech/badge/open-metric-learning)](https://pepy.tech/project/open-metric-learning)
 [![Pipi version](https://img.shields.io/pypi/v/open-metric-learning.svg)](https://pypi.org/project/open-metric-learning/)
-![example workflow](https://github.com/OML-Team/open-metric-learning/actions/workflows/python-versions.yaml/badge.svg?)
-[![python](https://img.shields.io/badge/python_3.8-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/python-versions.yaml/badge.svg?)
-[![python](https://img.shields.io/badge/python_3.9-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/python-versions.yaml/badge.svg?)
-[![python](https://img.shields.io/badge/python_3.10-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/python-versions.yaml/badge.svg?)
+[![python](https://img.shields.io/badge/python_3.8-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/tests.yaml/badge.svg?)
+[![python](https://img.shields.io/badge/python_3.9-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/tests.yaml/badge.svg?)
+[![python](https://img.shields.io/badge/python_3.10-passing-success)](https://github.com/OML-Team/open-metric-learning/actions/workflows/tests.yaml/badge.svg?)
+
 
 OML is a PyTorch-based framework to train and validate the models producing high-quality embeddings.
 

setup.py

@@ -55,7 +55,6 @@ def load_version() -> str:
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: Implementation :: CPython",
     ],
     project_urls={

tests/test_oml/test_functional/test_metrics/test_label_smoothing.py

@@ -4,6 +4,7 @@
 from oml.functional.label_smoothing import label_smoothing
 
 
+@pytest.mark.long
 @pytest.mark.parametrize(
     ["num_classes", "ls", "expected_result"],
     [
@@ -18,6 +19,7 @@ def test_label_smoothing(num_classes: int, ls: float, expected_result: torch.Ten
     assert torch.allclose(expected_result, smoothed, atol=1e-5)
 
 
+@pytest.mark.long
 @pytest.mark.parametrize(
     ["num_classes", "ls", "expected_result", "l2c"],
     [

tests/test_oml/test_losses/test_arcface.py

@@ -106,6 +106,7 @@ def test_arcface_trainable_ls_dataset(dataset2: Tuple[Tuple[torch.Tensor, torch.
     check_arcface_trainable(X, y)
 
 
+@pytest.mark.long
 @pytest.mark.parametrize("label_smoothing", [0.1, 0.2, 0.5])
 def test_label_smoothing_working(
     label_smoothing: float, dataset2: Tuple[Tuple[torch.Tensor, torch.Tensor], Dict[int, int]]
@@ -131,6 +132,7 @@ def test_label_smoothing_working(
     assert l0 > l1
 
 
+@pytest.mark.long
 @pytest.mark.parametrize("label_smoothing", [0.1, 0.5])
 def test_label_smoothing_fuzzy(
     label_smoothing: float, dataset3: Tuple[Tuple[torch.Tensor, torch.Tensor], Dict[int, int]]
@@ -148,12 +150,14 @@ def test_label_smoothing_fuzzy(
     assert loss(X, y).item() > loss_ls(X, y).item()
 
 
+@pytest.mark.long
 @pytest.mark.parametrize("label_smoothing", [-1, 1])
 def test_label_smoothing_raises_bad_ls(label_smoothing: float) -> None:
     with pytest.raises(AssertionError):
         ArcFaceLoss(1, 1, smoothing_epsilon=label_smoothing, label2category={1: 1})
 
 
+@pytest.mark.long
 @pytest.mark.parametrize("seed", [0, 1])
 @pytest.mark.parametrize("m,s", [(0.4, 64), (0.5, 64), (0.5, 48)])
 def test_arface(m: float, s: float, seed: int) -> None:

tests/test_oml/test_miners/test_cross_batch.py

@@ -21,6 +21,7 @@ def get_labels(n_cls: int, sz: int) -> Tensor:
     return tensor(labels)
 
 
+@pytest.mark.long
 @pytest.mark.parametrize("n_cls,cls_sz,n_labels,n_instances", [[50, 10, 10, 4]])
 @pytest.mark.parametrize("bank_sz,bank_k", [[5, 3], [20, 5], [5, 1]])
 def test_mining_with_memory(

tests/test_oml/test_miners/test_inbatch_all_tri.py

@@ -2,6 +2,7 @@
 import sys
 from typing import List, Tuple
 
+import pytest
 import torch
 
 from oml.miners.inbatch_all_tri import AllTripletsMiner, get_available_triplets_naive
@@ -11,6 +12,7 @@
 )
 
 
+@pytest.mark.long
 def test_all_triplets_miner(features_and_labels: List[Tuple[torch.Tensor, List[int]]]) -> None:  # type: ignore
     """
     Args:
@@ -28,6 +30,7 @@ def test_all_triplets_miner(features_and_labels: List[Tuple[torch.Tensor, List[i
         check_triplets_consistency(ids_anchor=ids_a, ids_pos=ids_p, ids_neg=ids_n, labels=labels)
 
 
+@pytest.mark.long
 def test_compare_all_triplets_miner_with_naive_version(
     features_and_labels: List[Tuple[torch.Tensor, List[int]]]
 ) -> None: