From 1fc5ff25bff7182e33b79f9921e0c4b5729999d2 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Tue, 5 Sep 2023 17:06:53 +0000 Subject: [PATCH 01/12] Added tt_tensor.py --- .../factorized_tensor/tt_tensor.py | 383 ++++++++++++++++++ 1 file changed, 383 insertions(+) create mode 100644 ivy/data_classes/factorized_tensor/tt_tensor.py diff --git a/ivy/data_classes/factorized_tensor/tt_tensor.py b/ivy/data_classes/factorized_tensor/tt_tensor.py new file mode 100644 index 0000000000000..c3949f5761877 --- /dev/null +++ b/ivy/data_classes/factorized_tensor/tt_tensor.py @@ -0,0 +1,383 @@ +from .base import FactorizedTensor +import ivy + +import warnings + + +class TTTensor(FactorizedTensor): + def __init__(self, factors, inplace=False): + super().__init__() + + shape, rank = ivy.TTTensor.validate_tt_tensor(factors) + + self.shape = tuple(shape) + self.rank = tuple(rank) + self.factors = factors + + # Built-ins # + def __getitem__(self, index): + return self.factors[index] + + def __setitem__(self, index, value): + self.factors[index] = value + + def __iter__(self): + for index in range(len(self)): + yield self[index] + + def __len__(self): + return len(self.factors) + + def __repr__(self): + message = ( + f"factors list : rank-{self.rank} matrix-product-state tensor of shape" + f" {self.shape} " + ) + return message + + # Public Methods # + def to_tensor(self): + return ivy.TTTensor.tt_to_tensor(self) + + def to_unfolding(self, mode): + return ivy.TTTensor.tt_to_unfolded(self, mode) + + def to_vec(self): + return ivy.TTTensor.tt_to_vec(self) + + # Properties # + @property + def n_param(self): + factor_params = [] + for i, s in enumerate(self.shape): + factor_params.append(self.rank[i] * s * self.rank[i + 1]) + return ivy.sum(factor_params) + + # Class Methods # + @staticmethod + def validate_tt_tensor(tt_tensor): + factors = tt_tensor + n_factors = len(factors) + + if isinstance(tt_tensor, TTTensor): + return tt_tensor.shape, tt_tensor.rank + elif isinstance(tt_tensor, (float, int)): + return 0, 0 + + rank = [] + shape = [] + for index, factor in enumerate(factors): + current_rank, current_shape, next_rank = ivy.shape(factor) + + if not len(ivy.shape(factor)) == 3: + raise ValueError( + "TT expresses a tensor as third order factors (tt-cores).\n" + f"However, tl.ndim(factors[{index}]) = {len(ivy.shape(factor))}" + ) + + if index and ivy.shape(factors[index - 1])[2] != current_rank: + raise ValueError( + "Consecutive factors should have matching ranks\n -- e.g." + " tl.shape(factors[0])[2]) == tl.shape(factors[1])[0])\nHowever," + f" tl.shape(factor[{index-1}])[2] ==" + f" {ivy.shape(factors[index - 1])[2]} but" + f" tl.shape(factor[{index}])[0] == {current_rank} " + ) + + if (index == 0) and current_rank != 1: + raise ValueError( + "Boundary conditions dictate factor[0].shape[0] == 1." + f"However, got factor[0].shape[0] = {current_rank}." + ) + + if (index == n_factors - 1) and next_rank != 1: + raise ValueError( + "Boundary conditions dictate factor[-1].shape[2] == 1." + f"However, got factor[{n_factors}].shape[2] = {next_rank}." + ) + + shape.append(current_shape) + rank.append(current_rank) + + rank.append(next_rank) + + return tuple(shape), tuple(rank) + + @staticmethod + def tt_to_tensor(factors): + """ + Return the full tensor whose TT decomposition is given by 'factors'. + + Re-assembles 'factors', which represent a tensor in TT/Matrix-Product-State format # noqa: E501 + into the corresponding full tensor + + Parameters + ---------- + factors + TT factors (TT-cores) + + Returns + ------- + output_tensor + tensor whose TT/MPS decomposition was given by 'factors' + """ + if isinstance(factors, (float, int)): + return factors + + full_shape = [f.shape[1] for f in factors] + full_tensor = ivy.reshape(factors[0], (full_shape[0], -1)) + + for factor in factors[1:]: + rank_prev, _, rank_next = factor.shape + factor = ivy.reshape(factor, (rank_prev, -1)) + full_tensor = ivy.matmul(full_tensor, factor) + full_tensor = ivy.reshape(full_tensor, (-1, rank_next)) + + return ivy.reshape(full_tensor, full_shape) + + @staticmethod + def tt_to_unfolded(factors, mode): + """ + Return the unfolding matrix of a tensor given in TT (or Tensor-Train) format. + + Reassembles a full tensor from 'factors' and returns its unfolding matrix + with mode given by 'mode' + + Parameters + ---------- + factors + TT factors + mode + unfolding matrix to be computed along this mode + + Returns + ------- + 2-D array + unfolding matrix at mode given by 'mode' + """ + return ivy.unfold(ivy.TTTensor.tt_to_tensor(factors), mode) + + @staticmethod + def tt_to_vec(factors): + """ + Return the tensor defined by its TT format ('factors') into its vectorized + format. + + Parameters + ---------- + factors + TT factors + + Returns + ------- + 1-D array + vectorized format of tensor defined by 'factors' + """ + return ivy.reshape(ivy.TTTensor.tt_to_tensor(factors), (-1,)) + + @staticmethod + def _tt_n_param(tensor_shape, rank): + """ + Return the number of parameters of a MPS decomposition for a given `rank` and + full `tensor_shape`. + + Parameters + ---------- + tensor_shape + shape of the full tensor to decompose (or approximate) + rank + rank of the MPS decomposition + + Return + ------- + n_params + Number of parameters of a MPS decomposition of rank `rank` of + a full tensor of shape `tensor_shape` + """ + factor_params = [] + for i, s in enumerate(tensor_shape): + factor_params.append(rank[i] * s * rank[i + 1]) + return ivy.sum(factor_params) + + @staticmethod + def validate_tt_rank( + tensor_shape, + rank="same", + constant_rank=False, + rounding="round", + allow_overparametrization=True, + ): + """ + Return the rank of a TT Decomposition. + + Parameters + ---------- + tensor_shape + shape of the tensor to decompose + rank + way to determine the rank, by default 'same' + if 'same': rank is computed to keep the number of parameters (at most) the same # noqa: E501 + if float, computes a rank so as to keep rank percent of the original number of parameters # noqa: E501 + if int or tuple, just returns rank + constant_rank + if True, the *same* rank will be chosen for each modes + if False (default), the rank of each mode will be + proportional to the corresponding tensor_shape + used only if rank == 'same' or 0 < rank <= 1* + + rounding + Mode for rounding + One of ["round", "floor", "ceil"] + + allow_overparametrization + if False, the rank must be realizable through iterative application of SVD + + Returns + ------- + rank + rank of the decomposition + """ + if rounding == "ceil": + rounding_fn = ivy.ceil + elif rounding == "floor": + rounding_fn = ivy.floor + elif rounding == "round": + rounding_fn = ivy.round + else: + raise ValueError( + f"Rounding should be round, floor or ceil, but got {rounding}" + ) + + if rank == "same": + rank = float(1) + + if isinstance(rank, float) and constant_rank: + n_param_tensor = ivy.prod(tensor_shape) * rank + order = len(tensor_shape) + + if order == 2: + rank = (1, n_param_tensor / (tensor_shape[0] + tensor_shape[1]), 1) + warnings.warn( + "Determining the tt-rank for the trivial case of a matrix (order 2" + f" tensor) of shape {tensor_shape}, not a higher-order tensor." + ) + + a = ivy.sum(tensor_shape[1:-1]) + b = ivy.sum(tensor_shape[0] + tensor_shape[-1]) + c = -n_param_tensor + delta = ivy.sqrt(b**2 - 4 * a * c) + solution = int(rounding_fn((-b + delta) / (2 * a))) + rank = rank = (1,) + (solution,) * (order - 1) + (1,) + + elif isinstance(rank, float): + order = len(tensor_shape) + avg_dim = [ + (tensor_shape[i] + tensor_shape[i + 1]) / 2 for i in range(order - 1) + ] + if len(avg_dim) > 1: + a = sum( + avg_dim[i - 1] * tensor_shape[i] * avg_dim[i] + for i in range(1, order - 1) + ) + else: + warnings.warn( + "Determining the tt-rank for the trivial case of a matrix (order 2" + f" tensor) of shape {tensor_shape}, not a higher-order tensor." + ) + a = avg_dim[0] ** 2 * tensor_shape[0] + b = tensor_shape[0] * avg_dim[0] + tensor_shape[-1] * avg_dim[-1] + c = -ivy.prod(tensor_shape) * rank + delta = ivy.sqrt(b**2 - 4 * a * c) + + fraction_param = (-b + delta) / (2 * a) + rank = tuple( + [max(int(rounding_fn(d * fraction_param)), 1) for d in avg_dim] + ) + rank = (1,) + rank + (1,) + + else: + n_dim = len(tensor_shape) + if isinstance(rank, int): + rank = [1] + [rank] * (n_dim - 1) + [1] + elif n_dim + 1 != len(rank): + message = ( + "Provided incorrect number of ranks. Should verify len(rank) ==" + f" tl.ndim(tensor)+1, but len(rank) = {len(rank)} while" + f" tl.ndim(tensor) + 1 = {n_dim+1}" + ) + raise (ValueError(message)) + + if rank[0] != 1: + message = ( + "Provided rank[0] == {} but boundary conditions dictate rank[0] ==" + " rank[-1] == 1.".format(rank[0]) + ) + raise ValueError(message) + if rank[-1] != 1: + message = ( + "Provided rank[-1] == {} but boundary conditions dictate rank[0] ==" + " rank[-1] == 1.".format(rank[-1]) + ) + raise ValueError(message) + + if allow_overparametrization: + return list(rank) + else: + validated_rank = [1] + for i, s in enumerate(tensor_shape[:-1]): + n_row = int(rank[i] * s) + n_column = ivy.prod(tensor_shape[(i + 1) :]) + validated_rank.append(min(n_row, n_column, rank[i + 1])) + validated_rank.append(1) + + return validated_rank + + @staticmethod + def pad_tt_rank(factor_list, n_padding=1, pad_boundaries=False): + """ + Pad the factors of a Tensor-Train so as to increase its rank without changing + its reconstruction. + + The tensor-train (ring) will be padded with 0s to increase its rank only but + not the underlying tensor it represents. + + Parameters + ---------- + factor_list + tensor list + n_padding + how much to increase the rank (bond dimension) by + pad_boundaries + if True, also pad the boundaries (useful for a tensor-ring) + should be False for a tensor-train to keep the boundary rank to be 1 + + Returns + ------- + padded_factor_list + """ + new_factors = [] + n_factors = len(factor_list) + + for i, factor in enumerate(factor_list): + n_padding_left = n_padding_right = n_padding + if (i == 0) and not pad_boundaries: + n_padding_left = 0 + elif (i == n_factors - 1) and not pad_boundaries: + n_padding_right = 0 + + r1, *s, r2 = ivy.shape(factor) + new_factor = ivy.zeros((r1 + n_padding_left, *s, r2 + n_padding_right)) + new_factors.append( + ivy.TTTensor.index_update( + new_factor, + (slice(None, r1, None), ..., slice(None, r2, None)), + factor, + ) + ) + + return new_factors + + @staticmethod + def index_update(tensor, indices, values): + tensor[indices] = values + return tensor From 6653b7a3ad56d6dc5c487ae862baa33e0f2ce366 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:22:40 +0000 Subject: [PATCH 02/12] Updated ivy/__init__.py, ivy/data_classes/__init__.py --- ivy/__init__.py | 6 +++++- ivy/data_classes/factorized_tensor/__init__.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ivy/__init__.py b/ivy/__init__.py index 0aee1dd56c163..f84ccbaacb1a6 100644 --- a/ivy/__init__.py +++ b/ivy/__init__.py @@ -83,6 +83,10 @@ class CPTensor: pass +class TTTensor: + pass + + class Device(str): def __new__(cls, dev_str): if dev_str != "": @@ -758,7 +762,7 @@ class Node(str): add_ivy_container_instance_methods, ) from .data_classes.nested_array import NestedArray -from .data_classes.factorized_tensor import TuckerTensor, CPTensor +from .data_classes.factorized_tensor import TuckerTensor, CPTensor, TTTensor from ivy.utils.backend import ( current_backend, compiled_backends, diff --git a/ivy/data_classes/factorized_tensor/__init__.py b/ivy/data_classes/factorized_tensor/__init__.py index 89a9c347cfb06..974a3d03ca773 100644 --- a/ivy/data_classes/factorized_tensor/__init__.py +++ b/ivy/data_classes/factorized_tensor/__init__.py @@ -1,2 +1,3 @@ from .tucker_tensor import TuckerTensor from .cp_tensor import CPTensor +from .tt_tensor import TTTensor From f1f770fa2517afb1bb05e2f27d5ea1a78481f32f Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Thu, 7 Sep 2023 08:34:02 +0000 Subject: [PATCH 03/12] Added test_tt_tensor.py, Updated test_creation.py, creation.py, tt_tensor.py --- .../factorized_tensor/tt_tensor.py | 15 ++-- ivy/functional/ivy/experimental/creation.py | 61 ++++++++++++++++ .../test_core/test_creation.py | 70 +++++++++++++++++++ .../test_ivy/test_misc/test_tt_tensor.py | 0 4 files changed, 139 insertions(+), 7 deletions(-) create mode 100644 ivy_tests/test_ivy/test_misc/test_tt_tensor.py diff --git a/ivy/data_classes/factorized_tensor/tt_tensor.py b/ivy/data_classes/factorized_tensor/tt_tensor.py index c3949f5761877..03f2bf7602971 100644 --- a/ivy/data_classes/factorized_tensor/tt_tensor.py +++ b/ivy/data_classes/factorized_tensor/tt_tensor.py @@ -71,17 +71,18 @@ def validate_tt_tensor(tt_tensor): if not len(ivy.shape(factor)) == 3: raise ValueError( - "TT expresses a tensor as third order factors (tt-cores).\n" - f"However, tl.ndim(factors[{index}]) = {len(ivy.shape(factor))}" + "TT expresses a tensor as third order factors" + f" (tt-cores).\nHowever, len(ivy.shape(factors[{index}])) =" + f" {len(ivy.shape(factor))}" ) if index and ivy.shape(factors[index - 1])[2] != current_rank: raise ValueError( "Consecutive factors should have matching ranks\n -- e.g." - " tl.shape(factors[0])[2]) == tl.shape(factors[1])[0])\nHowever," - f" tl.shape(factor[{index-1}])[2] ==" + " ivy.shape(factors[0])[2]) == ivy.shape(factors[1])[0])\nHowever," + f" ivy.shape(factor[{index-1}])[2] ==" f" {ivy.shape(factors[index - 1])[2]} but" - f" tl.shape(factor[{index}])[0] == {current_rank} " + f" ivy.shape(factor[{index}])[0] == {current_rank} " ) if (index == 0) and current_rank != 1: @@ -302,8 +303,8 @@ def validate_tt_rank( elif n_dim + 1 != len(rank): message = ( "Provided incorrect number of ranks. Should verify len(rank) ==" - f" tl.ndim(tensor)+1, but len(rank) = {len(rank)} while" - f" tl.ndim(tensor) + 1 = {n_dim+1}" + f" len(ivy.shape(tensor)) + 1, but len(rank) = {len(rank)} while" + f" len(ivy.shape(tensor)) + 1 = {n_dim+1}" ) raise (ValueError(message)) diff --git a/ivy/functional/ivy/experimental/creation.py b/ivy/functional/ivy/experimental/creation.py index fc57537624d08..ecc9eb8ed0d65 100644 --- a/ivy/functional/ivy/experimental/creation.py +++ b/ivy/functional/ivy/experimental/creation.py @@ -861,6 +861,67 @@ def random_cp( return ivy.CPTensor((weights, factors)) +@handle_exceptions +@handle_nestable +@infer_dtype +def random_tt( + shape: Sequence[int], + rank: Union[Sequence[int], int], + /, + *, + full: Optional[bool] = False, + dtype: Optional[Union[ivy.Dtype, ivy.NativeDtype]] = None, + seed: Optional[int] = None, +) -> Union[ivy.TTTensor, ivy.Array]: + """ + Generate a random TT/MPS tensor. + + Parameters + ---------- + shape + shape of the tensor to generate + rank + rank of the TT decomposition + must verify rank[0] == rank[-1] ==1 (boundary conditions) + and len(rank) == len(shape)+1 + full + if True, a full tensor is returned + otherwise, the decomposed tensor is returned + seed + seed for generating random numbers + + Returns + ------- + ivy.TTTensor + """ + len(shape) + rank = ivy.TTTensor.validate_tt_rank(shape, rank) + + rank = list(rank) + if rank[0] != 1: + message = ( + "Provided rank[0] == {} but boundaring conditions dictatate rank[0] ==" + " rank[-1] == 1.".format(rank[0]) + ) + raise ValueError(message) + if rank[-1] != 1: + message = ( + "Provided rank[-1] == {} but boundaring conditions dictatate rank[0] ==" + " rank[-1] == 1.".format(rank[-1]) + ) + raise ValueError(message) + + factors = [ + (ivy.random_uniform(shape=(rank[i], s, rank[i + 1]), dtype=dtype, seed=seed)) + for i, s in enumerate(shape) + ] + + if full: + return ivy.TTTensor.tt_to_tensor(factors) + else: + return ivy.TTTensor(factors) + + @handle_nestable @handle_array_like_without_promotion @handle_out_argument diff --git a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py index eabf25027caee..5cbe086c6b724 100644 --- a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py +++ b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py @@ -27,6 +27,18 @@ def _random_cp_data(draw): return shape, rank, dtype[0], full, orthogonal, seed, normalise_factors +@st.composite +def _random_tt_data(draw): + shape = draw( + st.lists(helpers.ints(min_value=1, max_value=5), min_size=2, max_size=4) + ) + rank = draw(helpers.ints(min_value=1, max_value=10)) + dtype = draw(helpers.get_dtypes("float", full=False)) + full = draw(st.booleans()) + seed = draw(st.one_of((st.just(None), helpers.ints(min_value=0, max_value=2000)))) + return shape, rank, dtype[0], full, seed + + @st.composite def _random_tucker_data(draw): shape = draw( @@ -383,6 +395,64 @@ def test_random_cp( assert np.prod(f.shape) == np.prod(f_gt.shape) +@handle_test( + fn_tree="functional.ivy.experimental.random_tt", + data=_random_tt_data(), + test_with_out=st.just(False), + test_instance_method=st.just(False), +) +def test_random_tt( + *, + data, + test_flags, + backend_fw, + fn_name, + on_device, +): + shape, rank, dtype, full, seed = data + results = helpers.test_function( + input_dtypes=[], + backend_to_test=backend_fw, + test_flags=test_flags, + on_device=on_device, + fn_name=fn_name, + shape=shape, + rank=rank, + dtype=dtype, + full=full, + seed=seed, + test_values=False, + ) + + ret_np, ret_from_gt_np = results + + if full: + reconstructed_tensor = helpers.flatten_and_to_np(ret=ret_np, backend=backend_fw) + reconstructed_tensor_gt = helpers.flatten_and_to_np( + ret=ret_from_gt_np, backend=test_flags.ground_truth_backend + ) + for x, x_gt in zip(reconstructed_tensor, reconstructed_tensor_gt): + assert np.prod(shape) == np.prod(x.shape) + assert np.prod(shape) == np.prod(x_gt.shape) + + else: + weights = helpers.flatten_and_to_np(ret=ret_np[0], backend=backend_fw) + factors = helpers.flatten_and_to_np(ret=ret_np[1], backend=backend_fw) + weights_gt = helpers.flatten_and_to_np( + ret=ret_from_gt_np[0], backend=test_flags.ground_truth_backend + ) + factors_gt = helpers.flatten_and_to_np( + ret=ret_from_gt_np[1], backend=test_flags.ground_truth_backend + ) + + for w, w_gt in zip(weights, weights_gt): + assert len(w) == rank + assert len(w_gt) == rank + + for f, f_gt in zip(factors, factors_gt): + assert np.prod(f.shape) == np.prod(f_gt.shape) + + @handle_test( fn_tree="functional.ivy.experimental.random_tucker", data=_random_tucker_data(), diff --git a/ivy_tests/test_ivy/test_misc/test_tt_tensor.py b/ivy_tests/test_ivy/test_misc/test_tt_tensor.py new file mode 100644 index 0000000000000..e69de29bb2d1d From 31984977ad78df2a81a7ea4ab89cb2d3372e7975 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Mon, 11 Sep 2023 08:25:41 +0000 Subject: [PATCH 04/12] Updated testing strategy - test_creation.py --- docs/demos | 1 + .../test_experimental/test_core/test_creation.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 160000 docs/demos diff --git a/docs/demos b/docs/demos new file mode 160000 index 0000000000000..0491d7886cb34 --- /dev/null +++ b/docs/demos @@ -0,0 +1 @@ +Subproject commit 0491d7886cb34dedc0416f0acc09891ca4cee70e diff --git a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py index 5cbe086c6b724..37d49d815599b 100644 --- a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py +++ b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py @@ -32,8 +32,12 @@ def _random_tt_data(draw): shape = draw( st.lists(helpers.ints(min_value=1, max_value=5), min_size=2, max_size=4) ) - rank = draw(helpers.ints(min_value=1, max_value=10)) - dtype = draw(helpers.get_dtypes("float", full=False)) + rank = len(shape) + dtype = draw( + helpers.get_dtypes("float", full=False).filter( + lambda x: x not in ["bfloat16", "float16"] + ) + ) full = draw(st.booleans()) seed = draw(st.one_of((st.just(None), helpers.ints(min_value=0, max_value=2000)))) return shape, rank, dtype[0], full, seed @@ -446,8 +450,8 @@ def test_random_tt( ) for w, w_gt in zip(weights, weights_gt): - assert len(w) == rank - assert len(w_gt) == rank + assert w.shape[-1] == rank + assert w_gt.shape[-1] == rank for f, f_gt in zip(factors, factors_gt): assert np.prod(f.shape) == np.prod(f_gt.shape) From 939899d11adf80a827fc3007b0be9b152bd91576 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Mon, 11 Sep 2023 08:44:19 +0000 Subject: [PATCH 05/12] Updated testing strategy - test_creation.py --- docs/demos | 1 + .../test_experimental/test_core/test_creation.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 160000 docs/demos diff --git a/docs/demos b/docs/demos new file mode 160000 index 0000000000000..0491d7886cb34 --- /dev/null +++ b/docs/demos @@ -0,0 +1 @@ +Subproject commit 0491d7886cb34dedc0416f0acc09891ca4cee70e diff --git a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py index 5cbe086c6b724..37d49d815599b 100644 --- a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py +++ b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py @@ -32,8 +32,12 @@ def _random_tt_data(draw): shape = draw( st.lists(helpers.ints(min_value=1, max_value=5), min_size=2, max_size=4) ) - rank = draw(helpers.ints(min_value=1, max_value=10)) - dtype = draw(helpers.get_dtypes("float", full=False)) + rank = len(shape) + dtype = draw( + helpers.get_dtypes("float", full=False).filter( + lambda x: x not in ["bfloat16", "float16"] + ) + ) full = draw(st.booleans()) seed = draw(st.one_of((st.just(None), helpers.ints(min_value=0, max_value=2000)))) return shape, rank, dtype[0], full, seed @@ -446,8 +450,8 @@ def test_random_tt( ) for w, w_gt in zip(weights, weights_gt): - assert len(w) == rank - assert len(w_gt) == rank + assert w.shape[-1] == rank + assert w_gt.shape[-1] == rank for f, f_gt in zip(factors, factors_gt): assert np.prod(f.shape) == np.prod(f_gt.shape) From 180507ca4010d31a566d51e2e9c825489f844889 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Mon, 11 Sep 2023 08:49:16 +0000 Subject: [PATCH 06/12] Updated test_creation.py --- docs/demos | 1 + .../test_experimental/test_core/test_creation.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 160000 docs/demos diff --git a/docs/demos b/docs/demos new file mode 160000 index 0000000000000..0491d7886cb34 --- /dev/null +++ b/docs/demos @@ -0,0 +1 @@ +Subproject commit 0491d7886cb34dedc0416f0acc09891ca4cee70e diff --git a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py index 5cbe086c6b724..4b8ef256f2fdf 100644 --- a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py +++ b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py @@ -32,7 +32,12 @@ def _random_tt_data(draw): shape = draw( st.lists(helpers.ints(min_value=1, max_value=5), min_size=2, max_size=4) ) - rank = draw(helpers.ints(min_value=1, max_value=10)) + rank = len(shape) + dtype = draw( + helpers.get_dtypes("float", full=False).filter( + lambda x: x not in ["bfloat16", "float16"] + ) + ) dtype = draw(helpers.get_dtypes("float", full=False)) full = draw(st.booleans()) seed = draw(st.one_of((st.just(None), helpers.ints(min_value=0, max_value=2000)))) @@ -446,8 +451,8 @@ def test_random_tt( ) for w, w_gt in zip(weights, weights_gt): - assert len(w) == rank - assert len(w_gt) == rank + assert w.shape[-1] == rank + assert w_gt.shape[-1] == rank for f, f_gt in zip(factors, factors_gt): assert np.prod(f.shape) == np.prod(f_gt.shape) From 005e8799f2f6f12929f9600c7bec40273ccf0734 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Mon, 11 Sep 2023 09:26:13 +0000 Subject: [PATCH 07/12] Updated test_creation.py --- .../test_experimental/test_core/test_creation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py index 37d49d815599b..c3d5686656b19 100644 --- a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py +++ b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py @@ -32,7 +32,7 @@ def _random_tt_data(draw): shape = draw( st.lists(helpers.ints(min_value=1, max_value=5), min_size=2, max_size=4) ) - rank = len(shape) + rank = draw(helpers.ints(min_value=1, max_value=len(shape))) dtype = draw( helpers.get_dtypes("float", full=False).filter( lambda x: x not in ["bfloat16", "float16"] From 3cc6f6d57ab9e2e18d3e9dd2c3f3c108e0b83f96 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Mon, 11 Sep 2023 10:22:01 +0000 Subject: [PATCH 08/12] Updated creation.py, test_tt_tensor.py --- ivy/functional/ivy/experimental/creation.py | 1 - .../test_ivy/test_misc/test_tt_tensor.py | 76 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/ivy/functional/ivy/experimental/creation.py b/ivy/functional/ivy/experimental/creation.py index ecc9eb8ed0d65..e4527da1b6e58 100644 --- a/ivy/functional/ivy/experimental/creation.py +++ b/ivy/functional/ivy/experimental/creation.py @@ -894,7 +894,6 @@ def random_tt( ------- ivy.TTTensor """ - len(shape) rank = ivy.TTTensor.validate_tt_rank(shape, rank) rank = list(rank) diff --git a/ivy_tests/test_ivy/test_misc/test_tt_tensor.py b/ivy_tests/test_ivy/test_misc/test_tt_tensor.py index e69de29bb2d1d..804a42cf6648a 100644 --- a/ivy_tests/test_ivy/test_misc/test_tt_tensor.py +++ b/ivy_tests/test_ivy/test_misc/test_tt_tensor.py @@ -0,0 +1,76 @@ +import ivy + +import numpy as np +import pytest + + +@pytest.mark.parametrize( + "n1, n2, n3, shape1, shape2, shape3", + [(3, 4, 2, (1, 3, 2), (2, 4, 2), (2, 2, 1))], +) +def test_tt_tensor(n1, n2, n3, shape1, shape2, shape3): + tensor = ivy.zeros((n1, n2, n3)) + + for i in range(n1): + for j in range(n2): + for k in range(n3): + tensor[i][j][k] = (i + 1) + (j + 1) + (k + 1) + + tensor = ivy.array(tensor) + + factors = [None] * 3 + + factors[0] = ivy.zeros(shape1) + factors[1] = ivy.zeros(shape2) + factors[2] = ivy.zeros(shape3) + + for i in range(3): + for j in range(4): + for k in range(2): + factors[0][0][i][0] = i + 1 + factors[0][0][i][1] = 1 + + factors[1][0][j][0] = 1 + factors[1][0][j][1] = 0 + factors[1][1][j][0] = j + 1 + factors[1][1][j][1] = 1 + + factors[2][0][k][0] = 1 + factors[2][1][k][0] = k + 1 + + factors = [ivy.array(f) for f in factors] + + np.testing.assert_array_almost_equal(tensor, ivy.TTTensor.tt_to_tensor(factors)) + + +@pytest.mark.parametrize( + "true_shape, true_rank", + [ + ( + (3, 4, 5), + (1, 3, 2, 1), + ) + ], +) +def test_validate_tt_tensor(true_shape, true_rank): + factors = ivy.random_tt(true_shape, true_rank).factors + shape, rank = ivy.TTTensor.validate_tt_tensor(factors) + + np.testing.assert_equal( + shape, + true_shape, + err_msg=f"Returned incorrect shape (got {shape}, expected {true_shape})", + ) + np.testing.assert_equal( + rank, + true_rank, + err_msg=f"Returned incorrect rank (got {rank}, expected {true_rank})", + ) + + factors[0] = ivy.random_uniform(shape=(4, 4)) + with np.testing.assert_raises(ValueError): + ivy.TTTensor.validate_tt_tensor(factors) + + factors[0] = ivy.random_uniform(shape=(3, 3, 2)) + with np.testing.assert_raises(ValueError): + ivy.TTTensor.validate_tt_tensor(factors) From 7020c1684b4f729d42ee333445f8b7fb176bd8f4 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Mon, 11 Sep 2023 14:13:30 +0000 Subject: [PATCH 09/12] Updated test_tt_tensor.py --- .../test_ivy/test_misc/test_tt_tensor.py | 68 ++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/ivy_tests/test_ivy/test_misc/test_tt_tensor.py b/ivy_tests/test_ivy/test_misc/test_tt_tensor.py index 804a42cf6648a..664159e64584c 100644 --- a/ivy_tests/test_ivy/test_misc/test_tt_tensor.py +++ b/ivy_tests/test_ivy/test_misc/test_tt_tensor.py @@ -3,12 +3,60 @@ import numpy as np import pytest +# These tests have been adapetd from Tensorly +# https://github.com/tensorly/tensorly/blob/main/tensorly/tests/test_tt_tensor.py + + +@pytest.mark.parametrize("n_pad", [1, 2]) +def test_pad_tt_rank(n_pad): + rank = (1, 2, 2, 2, 1) + tt = ivy.random_tt((4, 3, 5, 2), rank) + padded_tt = ivy.TTTensor( + ivy.TTTensor.pad_tt_rank(tt, n_padding=n_pad, pad_boundaries=False) + ) + rec = tt.to_tensor() + rec_padded = padded_tt.to_tensor() + + np.testing.assert_array_almost_equal(rec, rec_padded, decimal=4) + np.testing.assert_(padded_tt.rank == (1, *[i + n_pad for i in rank[1:-1]], 1)) + + +# TODO: Uncomment once ivy.tensor_train is implemented +# @pytest.mark.parametrize( +# "shape, rank", +# [((3, 4, 5, 6, 2, 10), 10)], +# ) +# def test_tt_to_tensor_random(shape, rank): +# tensor = ivy.random_uniform(shape) +# tensor_shape = tensor.shape + +# factors = ivy.tensor_train(tensor, rank) +# reconstructed_tensor = ivy.TTTensor.tt_to_tensor(factors) +# np.testing.assert_(ivy.shape(reconstructed_tensor) == tensor_shape) + +# D = len(factors) +# for k in range(D): +# (r_prev, _, r_k) = factors[k].shape +# assert r_prev <= rank, "TT rank with index " + str(k) + "exceeds rank" +# assert r_k <= rank, "TT rank with index " + str(k + 1) + "exceeds rank" + + +@pytest.mark.parametrize( + "shape, rank", + [((4, 5, 4, 8, 5), (1, 3, 2, 2, 4, 1))], +) +def test_tt_n_param(shape, rank): + factors = ivy.random_tt(shape, rank) + true_n_param = ivy.sum([ivy.prod(f.shape) for f in factors]) + n_param = ivy.TTTensor._tt_n_param(shape, rank) + np.testing.assert_equal(n_param, true_n_param) + @pytest.mark.parametrize( "n1, n2, n3, shape1, shape2, shape3", [(3, 4, 2, (1, 3, 2), (2, 4, 2), (2, 2, 1))], ) -def test_tt_tensor(n1, n2, n3, shape1, shape2, shape3): +def test_tt_to_tensor(n1, n2, n3, shape1, shape2, shape3): tensor = ivy.zeros((n1, n2, n3)) for i in range(n1): @@ -43,6 +91,24 @@ def test_tt_tensor(n1, n2, n3, shape1, shape2, shape3): np.testing.assert_array_almost_equal(tensor, ivy.TTTensor.tt_to_tensor(factors)) +@pytest.mark.parametrize( + "coef", + [((0.2))], +) +def test_validate_tt_rank(coef): + tensor_shape = tuple(ivy.random.randint(5, 10, shape=(4,))) + n_param_tensor = ivy.prod(tensor_shape) + + # TODO: This test fails even for the native implementation + # rank = ivy.TTTensor.validate_tt_rank(tensor_shape, coef, rounding="floor") + # n_param = ivy.TTTensor._tt_n_param(tensor_shape, rank) + # np.testing.assert_(n_param >= n_param_tensor * coef) + + rank = ivy.TTTensor.validate_tt_rank(tensor_shape, coef, rounding="ceil") + n_param = ivy.TTTensor._tt_n_param(tensor_shape, rank) + np.testing.assert_(n_param >= n_param_tensor * coef) + + @pytest.mark.parametrize( "true_shape, true_rank", [ From b1f33727c7c7b4a51fb97824af7261f1dc4951d0 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Tue, 12 Sep 2023 06:40:43 +0000 Subject: [PATCH 10/12] Updated demos --- docs/demos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/demos b/docs/demos index 0491d7886cb34..1ba2b6ee7fe97 160000 --- a/docs/demos +++ b/docs/demos @@ -1 +1 @@ -Subproject commit 0491d7886cb34dedc0416f0acc09891ca4cee70e +Subproject commit 1ba2b6ee7fe97fff704f103c90b2ec8ca19a703e From 0adbe9061426444d6193b15d9690bc017f485986 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Thu, 14 Sep 2023 13:53:59 +0000 Subject: [PATCH 11/12] Updated test_creation.py --- .../test_experimental/test_core/test_creation.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py index a982a667e5c85..7a8a4ec277976 100644 --- a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py +++ b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py @@ -480,19 +480,10 @@ def test_random_tt( assert np.prod(shape) == np.prod(x_gt.shape) else: - weights = helpers.flatten_and_to_np(ret=ret_np[0], backend=backend_fw) - factors = helpers.flatten_and_to_np(ret=ret_np[1], backend=backend_fw) - weights_gt = helpers.flatten_and_to_np( - ret=ret_from_gt_np[0], backend=test_flags.ground_truth_backend - ) + factors = helpers.flatten_and_to_np(ret=ret_np, backend=backend_fw) factors_gt = helpers.flatten_and_to_np( - ret=ret_from_gt_np[1], backend=test_flags.ground_truth_backend + ret=ret_from_gt_np, backend=test_flags.ground_truth_backend ) - - for w, w_gt in zip(weights, weights_gt): - assert w.shape[-1] == rank - assert w_gt.shape[-1] == rank - for f, f_gt in zip(factors, factors_gt): assert np.prod(f.shape) == np.prod(f_gt.shape) From 605ede3a151eae205138890b138015a9b3632488 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Sun, 17 Sep 2023 07:56:31 +0000 Subject: [PATCH 12/12] Updated test_creation.py, test_tt_tensor.py --- .../test_experimental/test_core/test_creation.py | 6 +----- ivy_tests/test_ivy/test_misc/test_tt_tensor.py | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py index 7a8a4ec277976..314631d99e9ed 100644 --- a/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py +++ b/ivy_tests/test_ivy/test_functional/test_experimental/test_core/test_creation.py @@ -33,11 +33,7 @@ def _random_tt_data(draw): st.lists(helpers.ints(min_value=1, max_value=5), min_size=2, max_size=4) ) rank = draw(helpers.ints(min_value=1, max_value=len(shape))) - dtype = draw( - helpers.get_dtypes("float", full=False).filter( - lambda x: x not in ["bfloat16", "float16"] - ) - ) + dtype = draw(helpers.get_dtypes("float", full=False)) full = draw(st.booleans()) seed = draw(st.one_of((st.just(None), helpers.ints(min_value=0, max_value=2000)))) return shape, rank, dtype[0], full, seed diff --git a/ivy_tests/test_ivy/test_misc/test_tt_tensor.py b/ivy_tests/test_ivy/test_misc/test_tt_tensor.py index 664159e64584c..0382dc7c55398 100644 --- a/ivy_tests/test_ivy/test_misc/test_tt_tensor.py +++ b/ivy_tests/test_ivy/test_misc/test_tt_tensor.py @@ -100,6 +100,7 @@ def test_validate_tt_rank(coef): n_param_tensor = ivy.prod(tensor_shape) # TODO: This test fails even for the native implementation + # https://github.com/tensorly/tensorly/issues/529 # rank = ivy.TTTensor.validate_tt_rank(tensor_shape, coef, rounding="floor") # n_param = ivy.TTTensor._tt_n_param(tensor_shape, rank) # np.testing.assert_(n_param >= n_param_tensor * coef)