From 5b52c162b1396b51d37f0bb48555d1bb0a9a9177 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Dec 2024 00:01:10 +0000 Subject: [PATCH 01/11] Bump numpy from 1.26.4 to 2.2.0 in /requirements Bumps [numpy](https://github.com/numpy/numpy) from 1.26.4 to 2.2.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.26.4...v2.2.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 21cc979b..27c4c9b0 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -16,7 +16,7 @@ matplotlib==3.9.3 mlflow-skinny==2.16.0 nptyping==2.5.0 numba>=0.57.0 # not directly required but pinned to ensure Python 3.11 compatibility. -numpy==1.26.4 +numpy==2.2.0 onnx==1.17.0 onnx2torch==1.5.15 opencv-python-headless==4.10.0.84 From 8a7b1d9487ca1099d52e48c9eea6369d8ee86b20 Mon Sep 17 00:00:00 2001 From: Harry Baker Date: Fri, 20 Dec 2024 16:49:17 +0000 Subject: [PATCH 02/11] Removed references to `nptyping` --- minerva/datasets/utils.py | 4 +- minerva/models/core.py | 4 +- minerva/tasks/core.py | 7 ++-- minerva/utils/utils.py | 53 +++++++++++++------------- minerva/utils/visutils.py | 63 +++++++++++++++---------------- tests/conftest.py | 8 ++-- tests/test_logging.py | 9 ++--- tests/test_utils/test_utils.py | 12 +++--- tests/test_utils/test_visutils.py | 12 +++--- 9 files changed, 86 insertions(+), 86 deletions(-) diff --git a/minerva/datasets/utils.py b/minerva/datasets/utils.py index 25f06cad..3bd54f02 100644 --- a/minerva/datasets/utils.py +++ b/minerva/datasets/utils.py @@ -50,7 +50,7 @@ from typing import Any, Callable, Iterable, Literal, Optional, Sequence, Union import numpy as np -from nptyping import NDArray +from numpy.typing import NDArray from torch.utils.data import ConcatDataset, DataLoader from torchgeo.datasets import ( GeoDataset, @@ -221,7 +221,7 @@ def make_bounding_box(roi: Sequence[float] | bool = False) -> Optional[BoundingB def load_all_samples( dataloader: DataLoader[Iterable[Any]], target_key: Literal["mask", "label"] = "mask", -) -> NDArray[Any, Any]: +) -> NDArray[Any]: """Loads all sample masks from parsed :class:`~torch.utils.data.DataLoader` and computes the modes of their classes. Args: diff --git a/minerva/models/core.py b/minerva/models/core.py index 25ec2116..536bfa97 100644 --- a/minerva/models/core.py +++ b/minerva/models/core.py @@ -62,8 +62,8 @@ from typing import Any, Callable, Iterable, Optional, Sequence, Type, overload import numpy as np +from numpy.typing import NDArray import torch -from nptyping import NDArray from packaging.version import Version from torch import Tensor from torch._dynamo.eval_frame import OptimizedModule @@ -573,7 +573,7 @@ def bilinear_init(in_channels: int, out_channels: int, kernel_size: int) -> Tens og = np.ogrid[:kernel_size, :kernel_size] filt = (1 - abs(og[0] - center) / factor) * (1 - abs(og[1] - center) / factor) - weight: NDArray[Any, Any] = np.zeros( + weight: NDArray[Any] = np.zeros( (in_channels, out_channels, kernel_size, kernel_size), dtype="float32" ) weight[range(in_channels), range(out_channels), :, :] = filt diff --git a/minerva/tasks/core.py b/minerva/tasks/core.py index 7e64cc77..683ed7ff 100644 --- a/minerva/tasks/core.py +++ b/minerva/tasks/core.py @@ -53,7 +53,8 @@ import pandas as pd import torch import torch.distributed as dist -from nptyping import Int, NDArray +import numpy as np +from numpy.typing import NDArray from torch import Tensor from torch._dynamo.eval_frame import OptimizedModule from wandb.sdk.wandb_run import Run @@ -524,8 +525,8 @@ def compute_classification_report( labels (~typing.Sequence[int]): List of corresponding ground truth label masks. """ # Ensures predictions and labels are flattened. - preds: NDArray[Any, Int] = utils.batch_flatten(predictions) - targets: NDArray[Any, Int] = utils.batch_flatten(labels) + preds: NDArray[np.int_] = utils.batch_flatten(predictions) + targets: NDArray[np.int_] = utils.batch_flatten(labels) # Uses utils to create a classification report in a DataFrame. cr_df = utils.make_classification_report(preds, targets, self.params["classes"]) diff --git a/minerva/utils/utils.py b/minerva/utils/utils.py index 4e42662e..c2f3bd65 100644 --- a/minerva/utils/utils.py +++ b/minerva/utils/utils.py @@ -136,8 +136,7 @@ import torch from geopy.exc import GeocoderUnavailable from geopy.geocoders import Photon -from nptyping import Float, Int, NDArray, Shape -from numpy.typing import ArrayLike +from numpy.typing import NDArray, ArrayLike from omegaconf import DictConfig, OmegaConf from pandas import DataFrame from rasterio.crs import CRS @@ -713,7 +712,7 @@ def deg_to_dms(deg: float, axis: str = "lat") -> str: def dec2deg( - dec_co: Sequence[float] | NDArray[Shape["*"], Float], # noqa: F722 + dec_co: Sequence[float] | NDArray[np.float64], # noqa: F722 axis: str = "lat", ) -> list[str]: """Wrapper for :func:`deg_to_dms`. @@ -846,7 +845,7 @@ def find_tensor_mode(mask: LongTensor) -> LongTensor: return mode -def labels_to_ohe(labels: Sequence[int], n_classes: int) -> NDArray[Any, Any]: +def labels_to_ohe(labels: Sequence[int], n_classes: int) -> NDArray[Any]: """Convert an iterable of indices to one-hot encoded (:term:`OHE`) labels. Args: @@ -856,7 +855,7 @@ def labels_to_ohe(labels: Sequence[int], n_classes: int) -> NDArray[Any, Any]: Returns: ~numpy.ndarray[~typing.Any]: Labels in OHE form. """ - targets: NDArray[Any, Any] = np.array(labels).reshape(-1) + targets: NDArray[Any] = np.array(labels).reshape(-1) ohe_labels = np.eye(n_classes)[targets] assert isinstance(ohe_labels, np.ndarray) return ohe_labels @@ -947,7 +946,7 @@ def find_empty_classes( def eliminate_classes( - empty_classes: list[int] | tuple[int, ...] | NDArray[Any, Int], + empty_classes: list[int] | tuple[int, ...] | NDArray[np.int_], old_classes: dict[int, str], old_cmap: Optional[dict[int, str]] = None, ) -> tuple[dict[int, str], dict[int, int], Optional[dict[int, str]]]: @@ -1034,8 +1033,8 @@ def class_transform(label: int, matrix: dict[int, int]) -> int: @overload def mask_transform( # type: ignore[overload-overlap] - array: NDArray[Any, Int], matrix: dict[int, int] -) -> NDArray[Any, Int]: ... # pragma: no cover + array: NDArray[np.int_], matrix: dict[int, int] +) -> NDArray[np.int_]: ... # pragma: no cover @overload @@ -1045,17 +1044,17 @@ def mask_transform( def mask_transform( - array: NDArray[Any, Int] | LongTensor, + array: NDArray[np.int_] | LongTensor, matrix: dict[int, int], -) -> NDArray[Any, Int] | LongTensor: +) -> NDArray[np.int_] | LongTensor: """Transforms all labels of an N-dimensional array from one schema to another mapped by a supplied dictionary. Args: - array (~numpy.ndarray[int]): N-dimensional array containing labels to be transformed. + array (~numpy.ndarray[int] | ~torch.LongTensor): N-dimensional array containing labels to be transformed. matrix (dict[int, int]): Dictionary mapping old labels to new. Returns: - ~numpy.ndarray[int]: Array of transformed labels. + ~numpy.ndarray[int] | ~torch.LongTensor: Array of transformed labels. """ for key in matrix.keys(): array[array == key] = matrix[key] @@ -1064,11 +1063,11 @@ def mask_transform( def check_test_empty( - pred: Sequence[int] | NDArray[Any, Int], - labels: Sequence[int] | NDArray[Any, Int], + pred: Sequence[int] | NDArray[np.int_], + labels: Sequence[int] | NDArray[np.int_], class_labels: Optional[dict[int, str]] = None, p_dist: bool = True, -) -> tuple[NDArray[Any, Int], NDArray[Any, Int], dict[int, str]]: +) -> tuple[NDArray[np.int_], NDArray[np.int_], dict[int, str]]: """Checks if any of the classes in the dataset were not present in both the predictions and ground truth labels. Returns corrected and re-ordered predictions, labels and class labels. @@ -1160,7 +1159,7 @@ def class_frac(patch: pd.Series) -> dict[Any, Any]: return new_columns -def cloud_cover(scene: NDArray[Any, Any]) -> Any: +def cloud_cover(scene: NDArray[Any]) -> float: """Calculates percentage cloud cover for a given scene based on its scene CLD. Args: @@ -1169,7 +1168,9 @@ def cloud_cover(scene: NDArray[Any, Any]) -> Any: Returns: float: Percentage cloud cover of scene. """ - return np.sum(scene) / scene.size + cloud_cover = np.sum(scene) / scene.size + assert isinstance(cloud_cover, float) + return cloud_cover def threshold_scene_select(df: DataFrame, thres: float = 0.3) -> list[str]: @@ -1435,11 +1436,11 @@ def calc_frac(count: float, total: float) -> str: print(tabulate(df, headers="keys", tablefmt="psql")) # type: ignore -def batch_flatten(x: NDArray[Any, Any] | ArrayLike) -> NDArray[Shape["*"], Any]: # noqa: F722 - """Flattens the supplied array with :func:`numpy`. +def batch_flatten(x: ArrayLike) -> NDArray[Any]: # noqa: F722 + """Flattens the supplied array with :func:`numpy.flatten`. Args: - x (~numpy.ndarray[~typing.Any] | ~nptyping.ArrayLike]): Array to be flattened. + x (~numpy.typing.ArrayLike]): Array to be flattened. Returns: ~numpy.ndarray[~typing.Any]: Flattened :class:`~numpy.ndarray`. @@ -1454,8 +1455,8 @@ def batch_flatten(x: NDArray[Any, Any] | ArrayLike) -> NDArray[Shape["*"], Any]: def make_classification_report( - pred: Sequence[int] | NDArray[Any, Int], - labels: Sequence[int] | NDArray[Any, Int], + pred: Sequence[int] | NDArray[np.int_], + labels: Sequence[int] | NDArray[np.int_], class_labels: Optional[dict[int, str]] = None, print_cr: bool = True, p_dist: bool = False, @@ -1625,8 +1626,8 @@ def run_tensorboard( def compute_roc_curves( - probs: NDArray[Any, Float], - labels: Sequence[int] | NDArray[Any, Int], + probs: NDArray[np.float64], + labels: Sequence[int] | NDArray[np.int_], class_labels: list[int], micro: bool = True, macro: bool = True, @@ -1721,7 +1722,7 @@ def compute_roc_curves( if macro: # Aggregate all false positive rates. - all_fpr: NDArray[Any, Any] = np.unique( + all_fpr: NDArray[Any] = np.unique( np.concatenate([fpr[key] for key in populated_classes]) ) @@ -1787,7 +1788,7 @@ def print_config(conf: DictConfig) -> None: def tsne_cluster( - embeddings: NDArray[Any, Any], + embeddings: NDArray[Any], n_dim: int = 2, lr: str = "auto", n_iter: int = 1000, diff --git a/minerva/utils/visutils.py b/minerva/utils/visutils.py index ec515390..59c00b0b 100644 --- a/minerva/utils/visutils.py +++ b/minerva/utils/visutils.py @@ -81,8 +81,7 @@ from matplotlib.image import AxesImage from matplotlib.ticker import MaxNLocator from matplotlib.transforms import Bbox -from nptyping import Float, Int, NDArray, Shape -from numpy.typing import ArrayLike +from numpy.typing import NDArray, ArrayLike from omegaconf import OmegaConf from rasterio.crs import CRS from scipy import stats @@ -117,7 +116,7 @@ # ===================================================================================================================== # METHODS # ===================================================================================================================== -def de_interlace(x: Sequence[Any], f: int) -> NDArray[Any, Any]: +def de_interlace(x: Sequence[Any], f: int) -> NDArray[Any]: """Separates interlaced arrays, ``x`` at a frequency of ``f`` from each other. Args: @@ -127,7 +126,7 @@ def de_interlace(x: Sequence[Any], f: int) -> NDArray[Any, Any]: Returns: ~numpy.ndarray[~typing.Any]: De-interlaced array. Each source array is now sequentially connected. """ - new_x: list[NDArray[Any, Any]] = [] + new_x: list[NDArray[Any]] = [] for i in range(f): x_i = [] for j in np.arange(start=i, stop=len(x), step=f): @@ -143,7 +142,7 @@ def dec_extent_to_deg( src_crs: CRS, new_crs: CRS = WGS84, spacing: int = 32, -) -> tuple[tuple[int, int, int, int], NDArray[Any, Float], NDArray[Any, Float]]: +) -> tuple[tuple[int, int, int, int], NDArray[np.float64], NDArray[np.float64]]: """Gets the extent of the image with ``shape`` and with ``bounds`` in latitude, longitude of system ``new_crs``. Args: @@ -225,7 +224,7 @@ def get_mlp_cmap( def discrete_heatmap( - data: NDArray[Shape["*, *"], Int], # noqa: F722 + data: NDArray[np.int_], # noqa: F722 classes: list[str] | tuple[str, ...], cmap_style: Optional[str | ListedColormap] = None, block_size: int = 32, @@ -268,9 +267,9 @@ def discrete_heatmap( def stack_rgb( - image: NDArray[Shape["3, *, *"], Float], # noqa: F722 + image: NDArray[np.float64], # noqa: F722 max_value: int = 255, -) -> NDArray[Shape["*, *, 3"], Float]: # noqa: F722 +) -> NDArray[np.float64]: # noqa: F722 """Stacks together red, green and blue image bands to create a RGB array. Args: @@ -283,7 +282,7 @@ def stack_rgb( """ # Stack together RGB bands. Assumes RGB bands are in dimensions 0-2. Ignores any other bands. # Note that it has to be order BGR not RGB due to the order numpy stacks arrays. - rgb_image: NDArray[Shape["3, *, *"], Any] = np.dstack( # noqa: F722 + rgb_image: NDArray[Any] = np.dstack( # noqa: F722 (image[2], image[1], image[0]) ) assert isinstance(rgb_image, np.ndarray) @@ -293,7 +292,7 @@ def stack_rgb( def make_rgb_image( - image: NDArray[Shape["3, *, *"], Float], # noqa: F722 + image: NDArray[np.float64], # noqa: F722 block_size: int = 32, max_pixel_value: int = 255, ) -> AxesImage: @@ -326,8 +325,8 @@ def make_rgb_image( def labelled_rgb_image( - image: NDArray[Shape["*, *, 3"], Float], # noqa: F722 - mask: NDArray[Shape["*, *"], Int], # noqa: F722 + image: NDArray[np.float64], # noqa: F722 + mask: NDArray[np.int_], # noqa: F722 bounds: BoundingBox, src_crs: CRS, path: str | Path, @@ -477,8 +476,8 @@ def labelled_rgb_image( def make_gif( dates: Sequence[str], - images: NDArray[Shape["*, *, *, 3"], Any], # noqa: F722 - masks: NDArray[Shape["*, *, *"], Any], # noqa: F722 + images: NDArray[Any], # noqa: F722 + masks: NDArray[Any], # noqa: F722 bounds: BoundingBox, src_crs: CRS, classes: list[str] | tuple[str, ...], @@ -610,7 +609,7 @@ def prediction_plot( gs = GridSpec(nrows=2, ncols=2, figure=fig) - axes: NDArray[Shape["3"], Axes] = np.array( # type: ignore[type-var, assignment] + axes: NDArray[Axes] = np.array( # type: ignore[type-var, assignment] [ fig.add_subplot(gs[0, 0]), fig.add_subplot(gs[0, 1]), @@ -710,16 +709,16 @@ def prediction_plot( def seg_plot( - z: list[int] | NDArray[Any, Any], - y: list[int] | NDArray[Any, Any], + z: list[int] | NDArray[Any], + y: list[int] | NDArray[Any], ids: list[str], - index: Sequence[Any] | NDArray[Any, Any], + index: Sequence[Any] | NDArray[Any], data_dir: Path | str, dataset_params: dict[str, Any], classes: dict[int, str], colours: dict[int, str], fn_prefix: Optional[str | Path], - x: Optional[list[int] | NDArray[Any, Any]] = None, + x: Optional[list[int] | NDArray[Any]] = None, frac: float = 0.05, fig_dim: Optional[tuple[int | float, int | float]] = (9.3, 10.5), model_name: str = "", @@ -761,7 +760,7 @@ def seg_plot( z = np.reshape(z, (z.shape[0] * z.shape[1], z.shape[2], z.shape[3])) y = np.reshape(y, (y.shape[0] * y.shape[1], y.shape[2], y.shape[3])) - flat_ids: NDArray[Any, Any] = np.array(ids).flatten() + flat_ids: NDArray[Any] = np.array(ids).flatten() print("\nRE-CONSTRUCTING DATASET") if cache_dir is not None: @@ -985,8 +984,8 @@ def plot_history( def make_confusion_matrix( - pred: list[int] | NDArray[Any, Int], - labels: list[int] | NDArray[Any, Int], + pred: list[int] | NDArray[np.int_], + labels: list[int] | NDArray[np.int_], classes: dict[int, str], filename: Optional[str | Path] = None, cmap_style: str = "Blues", @@ -1042,8 +1041,8 @@ def make_confusion_matrix( def make_multilabel_confusion_matrix( - preds: list[int] | NDArray[Any, Int], - labels: list[int] | NDArray[Any, Int], + preds: list[int] | NDArray[np.int_], + labels: list[int] | NDArray[np.int_], classes: dict[int, str], filename: Optional[str | Path] = None, cmap_style: str = "Blues", @@ -1118,7 +1117,7 @@ def make_multilabel_confusion_matrix( def make_roc_curves( probs: ArrayLike, - labels: Sequence[int] | NDArray[Any, Int], + labels: Sequence[int] | NDArray[np.int_], class_names: dict[int, str], colours: dict[int, str], micro: bool = True, @@ -1296,7 +1295,7 @@ def plot_embedding( if hasattr(offsetbox, "AnnotationBbox"): # only print thumbnails with matplotlib > 1.0 - shown_images: NDArray[Any, Any] = np.array([[1.0, 1.0]]) # just something big + shown_images: NDArray[Any] = np.array([[1.0, 1.0]]) # just something big for i, image in enumerate(images): dist = np.sum((x[i] - shown_images) ** 2, 1) @@ -1372,14 +1371,14 @@ def standard_format(plot_type: str, *sub_dir) -> str: def plot_results( plots: dict[str, bool], - x: Optional[NDArray[Any, Int]] = None, - y: Optional[list[int] | NDArray[Any, Int]] = None, - z: Optional[list[int] | NDArray[Any, Int]] = None, + x: Optional[NDArray[np.int_]] = None, + y: Optional[list[int] | NDArray[np.int_]] = None, + z: Optional[list[int] | NDArray[np.int_]] = None, metrics: Optional[dict[str, Any]] = None, ids: Optional[list[str]] = None, - index: Optional[NDArray[Any, Any]] = None, - probs: Optional[list[float] | NDArray[Any, Float]] = None, - embeddings: Optional[NDArray[Any, Any]] = None, + index: Optional[NDArray[Any]] = None, + probs: Optional[list[float] | NDArray[np.float64]] = None, + embeddings: Optional[NDArray[Any]] = None, class_names: Optional[dict[int, str]] = None, colours: Optional[dict[int, str]] = None, save: bool = True, diff --git a/tests/conftest.py b/tests/conftest.py index f0f6a287..14baa27b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,7 +48,7 @@ import pytest import torch import torch.nn.modules as nn -from nptyping import Float, Int, NDArray, Shape +from numpy.typing import NDArray from omegaconf import DictConfig, OmegaConf from rasterio.crs import CRS from torch import LongTensor, Tensor @@ -254,7 +254,7 @@ def exp_simconv( @pytest.fixture def random_mask( small_patch_size: tuple[int, int], std_n_classes: int -) -> NDArray[Shape["32, 32"], Int]: +) -> NDArray[np.int_]: mask = np.random.randint(0, std_n_classes - 1, size=small_patch_size) assert isinstance(mask, np.ndarray) return mask @@ -263,14 +263,14 @@ def random_mask( @pytest.fixture def random_image( small_patch_size: tuple[int, int], -) -> NDArray[Shape["32, 32, 3"], Float]: +) -> NDArray[np.float64]: return np.random.rand(*small_patch_size, 3) @pytest.fixture def random_rgbi_image( small_patch_size: tuple[int, int], -) -> NDArray[Shape["32, 32, 4"], Float]: +) -> NDArray[np.float64]: return np.random.rand(*small_patch_size, 4) diff --git a/tests/test_logging.py b/tests/test_logging.py index 9a8d5128..ab3e78a8 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -53,7 +53,7 @@ except (OSError, NewConnectionError, MaxRetryError): NTXentLoss = getattr(importlib.import_module("lightly.loss"), "NTXentLoss") import pytest -from nptyping import NDArray, Shape +from numpy.typing import NDArray from numpy.testing import assert_array_equal from torch import Tensor from torch.nn.modules import Module @@ -102,7 +102,7 @@ def test_SupervisedStepLogger( input_size = (4, *small_patch_size) model = FCN16ResNet18(x_entropy_loss, input_size=input_size).to(default_device) - optimiser = torch.optim.SGD(model.parameters(), lr=1.0e-3) + optimiser = torch.optim.sgd.SGD(model.parameters(), lr=1.0e-3) model.set_optimiser(optimiser) model.determine_output_dim() @@ -191,8 +191,7 @@ def test_SupervisedStepLogger( ) assert np.array(results["ids"]).shape == (std_n_batches, std_batch_size) - shape = f"{std_n_batches}, {std_batch_size}, {small_patch_size[0]}, {small_patch_size[1]}" - y: NDArray[Shape[shape], Any] = np.empty( + y: NDArray[Any] = np.empty( (std_n_batches, std_batch_size, *output_shape), dtype=np.uint8 ) for i in range(std_n_batches): @@ -278,7 +277,7 @@ def test_SSLStepLogger( model: MinervaSiamese = model_cls(criterion, input_size=input_size).to( default_device ) - optimiser = torch.optim.SGD(model.parameters(), lr=1.0e-3) + optimiser = torch.optim.sgd.SGD(model.parameters(), lr=1.0e-3) model.set_optimiser(optimiser) model.determine_output_dim(sample_pairs=True) diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index ad3aaace..1aeafb5f 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -54,7 +54,7 @@ import torch from geopy.exc import GeocoderUnavailable from internet_sabotage import no_connection -from nptyping import Float, NDArray, Shape +from numpy.typing import NDArray from numpy.testing import assert_array_equal from pytest_lazy_fixtures import lf from rasterio.crs import CRS @@ -183,7 +183,7 @@ def test_datetime_reformat() -> None: def test_ohe_labels() -> None: labels = [3, 2, 4, 1, 0] - correct_targets: NDArray[Shape["5, 6"], Float] = np.array( + correct_targets: NDArray[np.float64] = np.array( [ [0.0, 0.0, 0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0, 0.0, 0.0], @@ -606,7 +606,7 @@ def test_compute_roc_curves() -> None: labels = [0, 3, 2, 1, 3, 2, 1, 0] class_labels = [0, 1, 2, 3] - fpr: dict[Any, NDArray[Any, Any]] = { + fpr: dict[Any, NDArray[Any]] = { 0: np.array([0.0, 0.0, 0.0, 0.5, 5.0 / 6.0, 1.0]), 1: np.array([0.0, 0.0, 1.0 / 6.0, 1.0 / 6.0, 1.0]), 2: np.array([0.0, 0.0, 1.0 / 6.0, 0.5, 0.5, 1.0]), @@ -635,7 +635,7 @@ def test_compute_roc_curves() -> None: "macro": np.array([0.0, 1.0 / 6.0, 0.5, 5.0 / 6.0, 1.0]), } - tpr: dict[Any, NDArray[Any, Any]] = { + tpr: dict[Any, NDArray[Any]] = { 0: np.array([0.0, 0.5, 1.0, 1.0, 1.0, 1.0]), 1: np.array([0.0, 0.5, 0.5, 1.0, 1.0]), 2: np.array([0.0, 0.5, 0.5, 0.5, 1.0, 1.0]), @@ -786,7 +786,7 @@ def test_calc_grad(exp_mlp: MinervaModel) -> None: x = torch.rand(batch_size, (64)) y = torch.LongTensor(np.random.randint(0, 8, size=batch_size)) - optimiser = torch.optim.SGD(exp_mlp.parameters(), lr=1.0e-3) + optimiser = torch.optim.sgd.SGD(exp_mlp.parameters(), lr=1.0e-3) exp_mlp.set_optimiser(optimiser) _ = exp_mlp.step(x, y, train=True) @@ -798,7 +798,7 @@ def test_calc_grad(exp_mlp: MinervaModel) -> None: def test_tsne_cluster() -> None: clusters = utils.tsne_cluster(np.random.rand(10, 100)) - assert isinstance(clusters, NDArray) + assert isinstance(clusters, np.ndarray) assert clusters.shape == (10, 2) diff --git a/tests/test_utils/test_visutils.py b/tests/test_utils/test_visutils.py index 251d4105..d32fdec4 100644 --- a/tests/test_utils/test_visutils.py +++ b/tests/test_utils/test_visutils.py @@ -48,7 +48,7 @@ import torch from matplotlib.colors import Colormap, ListedColormap from matplotlib.image import AxesImage -from nptyping import NDArray, Shape +from numpy.typing import NDArray from numpy.testing import assert_array_equal from omegaconf import OmegaConf from rasterio.crs import CRS @@ -68,7 +68,7 @@ def test_de_interlace() -> None: x = [x_1, x_2, x_3, x_1, x_2, x_3] - x2: NDArray[Shape["30"], Any] = np.array([x_1, x_1, x_2, x_2, x_3, x_3]).flatten() + x2: NDArray[Any] = np.array([x_1, x_1, x_2, x_2, x_3, x_3]).flatten() assert_array_equal(visutils.de_interlace(x, 3), x2) @@ -123,19 +123,19 @@ def test_discrete_heatmap( def test_stack_rgb() -> None: - red: NDArray[Shape["3, 3"], Any] = np.array( + red: NDArray[Any] = np.array( [[25.0, 12.0, 11.0], [34.0, 55.0, 89.0], [23.0, 18.0, 76.0]] ) - blue: NDArray[Shape["3, 3"], Any] = np.array( + blue: NDArray[Any] = np.array( [[16.0, 17.0, 18.0], [19.0, 23.0, 24.0], [78.0, 67.0, 54.0]] ) - green: NDArray[Shape["3, 3"], Any] = np.array( + green: NDArray[Any] = np.array( [[3.0, 2.0, 1.0], [9.0, 11.0, 34.0], [23.0, 15.0, 128.0]] ) - image_1: NDArray[Shape["3, 3, 3"], Any] = np.array([red, green, blue]) + image_1: NDArray[Any] = np.array([red, green, blue]) correct = np.dstack((blue, green, red)) / 255.0 result_1 = visutils.stack_rgb(image_1, max_value=255) From 3b14a59c39c79120239a2f1dbfc17a0110af763a Mon Sep 17 00:00:00 2001 From: Harry Baker Date: Fri, 20 Dec 2024 16:49:33 +0000 Subject: [PATCH 03/11] Removed `nptyping` as a dependency --- pyproject.toml | 1 - requirements/requirements.txt | 1 - setup.cfg | 1 - 3 files changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7d671c76..9322ad01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ "matplotlib", "hydra-core", "mlflow-skinny", - "nptyping", "numba>=0.57.0; python_version>='3.11'", "numpy", "overload", diff --git a/requirements/requirements.txt b/requirements/requirements.txt index c2fb115c..22d563f6 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -14,7 +14,6 @@ lightly==1.5.15 lmdb==1.5.1 matplotlib==3.10.0 mlflow-skinny==2.16.0 -nptyping==2.5.0 numba>=0.57.0 # not directly required but pinned to ensure Python 3.11 compatibility. numpy==2.2.0 onnx==1.17.0 diff --git a/setup.cfg b/setup.cfg index 1c9be3bf..6c7c6e27 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,6 @@ install_requires = geopy overload opencv-python - nptyping lightly argcomplete hydra-core From 01a1b57e32930fe7295b0ccc8cf36e24200a5808 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:50:02 +0000 Subject: [PATCH 04/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- minerva/models/core.py | 2 +- minerva/tasks/core.py | 2 +- minerva/utils/utils.py | 2 +- minerva/utils/visutils.py | 2 +- tests/test_logging.py | 2 +- tests/test_utils/test_utils.py | 2 +- tests/test_utils/test_visutils.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/minerva/models/core.py b/minerva/models/core.py index 536bfa97..c6c4817f 100644 --- a/minerva/models/core.py +++ b/minerva/models/core.py @@ -62,8 +62,8 @@ from typing import Any, Callable, Iterable, Optional, Sequence, Type, overload import numpy as np -from numpy.typing import NDArray import torch +from numpy.typing import NDArray from packaging.version import Version from torch import Tensor from torch._dynamo.eval_frame import OptimizedModule diff --git a/minerva/tasks/core.py b/minerva/tasks/core.py index 683ed7ff..385a66dc 100644 --- a/minerva/tasks/core.py +++ b/minerva/tasks/core.py @@ -50,10 +50,10 @@ SummaryWriter = None import hydra +import numpy as np import pandas as pd import torch import torch.distributed as dist -import numpy as np from numpy.typing import NDArray from torch import Tensor from torch._dynamo.eval_frame import OptimizedModule diff --git a/minerva/utils/utils.py b/minerva/utils/utils.py index c2f3bd65..07cbef15 100644 --- a/minerva/utils/utils.py +++ b/minerva/utils/utils.py @@ -136,7 +136,7 @@ import torch from geopy.exc import GeocoderUnavailable from geopy.geocoders import Photon -from numpy.typing import NDArray, ArrayLike +from numpy.typing import ArrayLike, NDArray from omegaconf import DictConfig, OmegaConf from pandas import DataFrame from rasterio.crs import CRS diff --git a/minerva/utils/visutils.py b/minerva/utils/visutils.py index 59c00b0b..04c2d577 100644 --- a/minerva/utils/visutils.py +++ b/minerva/utils/visutils.py @@ -81,7 +81,7 @@ from matplotlib.image import AxesImage from matplotlib.ticker import MaxNLocator from matplotlib.transforms import Bbox -from numpy.typing import NDArray, ArrayLike +from numpy.typing import ArrayLike, NDArray from omegaconf import OmegaConf from rasterio.crs import CRS from scipy import stats diff --git a/tests/test_logging.py b/tests/test_logging.py index ab3e78a8..4f74f495 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -53,8 +53,8 @@ except (OSError, NewConnectionError, MaxRetryError): NTXentLoss = getattr(importlib.import_module("lightly.loss"), "NTXentLoss") import pytest -from numpy.typing import NDArray from numpy.testing import assert_array_equal +from numpy.typing import NDArray from torch import Tensor from torch.nn.modules import Module from torchgeo.datasets.utils import BoundingBox diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 1aeafb5f..35635ab6 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -54,8 +54,8 @@ import torch from geopy.exc import GeocoderUnavailable from internet_sabotage import no_connection -from numpy.typing import NDArray from numpy.testing import assert_array_equal +from numpy.typing import NDArray from pytest_lazy_fixtures import lf from rasterio.crs import CRS from torchgeo.datasets.utils import BoundingBox, stack_samples diff --git a/tests/test_utils/test_visutils.py b/tests/test_utils/test_visutils.py index d32fdec4..4d140026 100644 --- a/tests/test_utils/test_visutils.py +++ b/tests/test_utils/test_visutils.py @@ -48,8 +48,8 @@ import torch from matplotlib.colors import Colormap, ListedColormap from matplotlib.image import AxesImage -from numpy.typing import NDArray from numpy.testing import assert_array_equal +from numpy.typing import NDArray from omegaconf import OmegaConf from rasterio.crs import CRS from torchgeo.datasets import GeoDataset From c1e472b249ca386442244bf6afd31955f151c10c Mon Sep 17 00:00:00 2001 From: Harry Baker Date: Fri, 20 Dec 2024 17:05:14 +0000 Subject: [PATCH 05/11] Bump `tensorflow` --> `2.18.0` and demoted `numpy` --> `2.0.2` --- requirements/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 22d563f6..e065b801 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -15,7 +15,7 @@ lmdb==1.5.1 matplotlib==3.10.0 mlflow-skinny==2.16.0 numba>=0.57.0 # not directly required but pinned to ensure Python 3.11 compatibility. -numpy==2.2.0 +numpy==2.0.2 onnx==1.17.0 onnx2torch==1.5.15 opencv-python-headless==4.10.0.84 @@ -31,7 +31,7 @@ scikit-learn==1.5.2 setuptools==75.6.0 starlette==0.40.0 # not directly required, pinned by Dependabot to avoid a vulnerability. tabulate==0.9.0 -tensorflow==2.17.0 +tensorflow==2.18.0 tifffile==2024.9.20 timm==1.0.11 torch==2.5.1 From 5915ae0836b05677583bd8afc3f0f497a9d16643 Mon Sep 17 00:00:00 2001 From: Harry Baker Date: Fri, 20 Dec 2024 17:33:37 +0000 Subject: [PATCH 06/11] Fixed import error --- tests/test_logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_logging.py b/tests/test_logging.py index 4f74f495..b963afba 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -102,7 +102,7 @@ def test_SupervisedStepLogger( input_size = (4, *small_patch_size) model = FCN16ResNet18(x_entropy_loss, input_size=input_size).to(default_device) - optimiser = torch.optim.sgd.SGD(model.parameters(), lr=1.0e-3) + optimiser = torch.optim.SGD(model.parameters(), lr=1.0e-3) model.set_optimiser(optimiser) model.determine_output_dim() @@ -277,7 +277,7 @@ def test_SSLStepLogger( model: MinervaSiamese = model_cls(criterion, input_size=input_size).to( default_device ) - optimiser = torch.optim.sgd.SGD(model.parameters(), lr=1.0e-3) + optimiser = torch.optim.SGD(model.parameters(), lr=1.0e-3) model.set_optimiser(optimiser) model.determine_output_dim(sample_pairs=True) From 4fac28163daf5f09225db84acca861a591f5f3cb Mon Sep 17 00:00:00 2001 From: Harry Baker Date: Fri, 20 Dec 2024 17:43:07 +0000 Subject: [PATCH 07/11] `numpy>=2.0.0` now minimum `minerva` dependency --- pyproject.toml | 2 +- setup.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9322ad01..a78feea7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "hydra-core", "mlflow-skinny", "numba>=0.57.0; python_version>='3.11'", - "numpy", + "numpy>=2.0.0", "overload", "opencv-python", "pandas", diff --git a/setup.cfg b/setup.cfg index 6c7c6e27..0402af4f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,7 @@ classifiers = packages = minerva, minerva.utils, minerva.models install_requires = torchgeo>=0.6.0 + numpy>=2.0.0 numba>=0.57.0;python_version>='3.11' torch>=2.4.0 pandas From 947023f67a7473172a0555ea9c2dc54474feb6b2 Mon Sep 17 00:00:00 2001 From: Harry Baker Date: Fri, 20 Dec 2024 17:46:32 +0000 Subject: [PATCH 08/11] Update attribute for `numpy>=2.0.0` --- minerva/logger/tasklog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minerva/logger/tasklog.py b/minerva/logger/tasklog.py index 38f6727c..d060817d 100644 --- a/minerva/logger/tasklog.py +++ b/minerva/logger/tasklog.py @@ -270,7 +270,7 @@ def log_null(self) -> None: the length of the logs remains the same as the training logs. """ for metric in self.metrics.keys(): - self.metrics[metric]["y"].append(np.NAN) + self.metrics[metric]["y"].append(np.nan) def get_sub_metrics( self, pattern: tuple[str, ...] = ("train", "val") From 101fc89c626e1654ffe6c66f29418314564fb326 Mon Sep 17 00:00:00 2001 From: Harry Baker Date: Fri, 20 Dec 2024 17:46:53 +0000 Subject: [PATCH 09/11] Fixed `AttributeError` --- tests/test_utils/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 35635ab6..9741a118 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -786,7 +786,7 @@ def test_calc_grad(exp_mlp: MinervaModel) -> None: x = torch.rand(batch_size, (64)) y = torch.LongTensor(np.random.randint(0, 8, size=batch_size)) - optimiser = torch.optim.sgd.SGD(exp_mlp.parameters(), lr=1.0e-3) + optimiser = torch.optim.SGD(exp_mlp.parameters(), lr=1.0e-3) exp_mlp.set_optimiser(optimiser) _ = exp_mlp.step(x, y, train=True) From 178f4e1dd707bc523ba37fdf5da43046c5bca9f5 Mon Sep 17 00:00:00 2001 From: Harry Baker Date: Fri, 20 Dec 2024 17:47:51 +0000 Subject: [PATCH 10/11] Set minimum python version to 3.10 --- pyproject.toml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a78feea7..dba09a46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" authors = [ { name="Harry Baker", email="hjb1d20@soton.ac.uk" }, ] -requires-python = ">=3.9" +requires-python = ">=3.10" license = {file = "LICENSE"} classifiers = [ "Programming Language :: Python :: 3.12", diff --git a/setup.cfg b/setup.cfg index 0402af4f..0ad5e666 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,7 +45,7 @@ install_requires = tifffile kornia torcheval -python_requires = >=3.9 +python_requires = >=3.10 package_dir = scripts = scripts/MinervaExp.py scripts/ManifestMake.py From ab13a3a5d2928de538c8347a56c3a71933a1f81f Mon Sep 17 00:00:00 2001 From: Harry Baker Date: Fri, 20 Dec 2024 18:01:33 +0000 Subject: [PATCH 11/11] `np.Inf` --> `np.inf` --- minerva/pytorchtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minerva/pytorchtools.py b/minerva/pytorchtools.py index 8eb81e56..7b3ba8f3 100644 --- a/minerva/pytorchtools.py +++ b/minerva/pytorchtools.py @@ -91,7 +91,7 @@ def __init__( self.counter: int = 0 self.best_score: Optional[float] = None self.early_stop: bool = False - self.val_loss_min: float = np.Inf + self.val_loss_min: float = np.inf self.delta: float = delta self.path: str | Path = path self.trace_func: Callable[..., None] = trace_func