From f53ce018f28f9ac9c0cb79315deb980c5e69a969 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Fri, 30 Aug 2024 14:21:50 -0700 Subject: [PATCH 1/5] Add test case to check if all non-method ops are being verified --- .../test_dtype_constraints.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tripy/tests/spec_verification/test_dtype_constraints.py b/tripy/tests/spec_verification/test_dtype_constraints.py index 0f5c41429..1268f27d4 100755 --- a/tripy/tests/spec_verification/test_dtype_constraints.py +++ b/tripy/tests/spec_verification/test_dtype_constraints.py @@ -22,7 +22,7 @@ import itertools import pytest from tests.spec_verification.object_builders import create_obj -from tripy.constraints import TYPE_VERIFICATION, RETURN_VALUE +from tripy.constraints import TYPE_VERIFICATION, RETURN_VALUE, FUNC_W_DOC_VERIF import tripy as tp from contextlib import ExitStack @@ -181,3 +181,21 @@ def test_neg_dtype_constraints(test_data): api_call_locals, namespace = _run_dtype_constraints_subtest(test_data) if isinstance(api_call_locals[RETURN_VALUE], tp.Tensor): assert api_call_locals[RETURN_VALUE].dtype == namespace[return_dtype] + + +operations = [obj.__qualname__ for _, obj in inspect.getmembers(tp) if inspect.isfunction(obj)] +# add any function that you do not want to be verified: +func_exceptions = ["plugin", "dequantize"] + + +# Check if there are any operations that are not included (Currently does not test any ____ functions) +@pytest.mark.parametrize("func_qualname", operations, ids=lambda val: f"is_{val}_verified") +def test_all_ops_verified(func_qualname): + if func_qualname in FUNC_W_DOC_VERIF: + return + elif func_qualname.startswith("_") or func_qualname in func_exceptions: + return + else: + pytest.fail( + f"function {func_qualname}'s data types have not been verified. Please add data type verification by following the guide within tripy/tests/spec_verification or exclude it from this test." + ) From 864a830d117ca0938d6370c6f998c7f4b2e90f63 Mon Sep 17 00:00:00 2001 From: Mgluhovskoi Date: Fri, 30 Aug 2024 18:18:32 -0400 Subject: [PATCH 2/5] Update tripy/tests/spec_verification/test_dtype_constraints.py Co-authored-by: pranavm-nvidia <49246958+pranavm-nvidia@users.noreply.github.com> Signed-off-by: Mgluhovskoi --- .../tests/spec_verification/test_dtype_constraints.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tripy/tests/spec_verification/test_dtype_constraints.py b/tripy/tests/spec_verification/test_dtype_constraints.py index 1268f27d4..9860b0479 100755 --- a/tripy/tests/spec_verification/test_dtype_constraints.py +++ b/tripy/tests/spec_verification/test_dtype_constraints.py @@ -191,11 +191,7 @@ def test_neg_dtype_constraints(test_data): # Check if there are any operations that are not included (Currently does not test any ____ functions) @pytest.mark.parametrize("func_qualname", operations, ids=lambda val: f"is_{val}_verified") def test_all_ops_verified(func_qualname): - if func_qualname in FUNC_W_DOC_VERIF: - return - elif func_qualname.startswith("_") or func_qualname in func_exceptions: - return + if not func_qualname.startswith("_") and func_qualname not in func_exceptions: + assert func_qualname in FUNC_W_DOC_VERIF, f"function {func_qualname}'s data types have not been verified. Please add data type verification by following the guide within tripy/tests/spec_verification or exclude it from this test." else: - pytest.fail( - f"function {func_qualname}'s data types have not been verified. Please add data type verification by following the guide within tripy/tests/spec_verification or exclude it from this test." - ) + pytest.skip("Data type constraints are not required for this API") From 35fafd26fc32c67d3613b5a4f39a1694eb7bcc08 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Fri, 30 Aug 2024 16:55:39 -0700 Subject: [PATCH 3/5] Add checking for all other ops --- .../test_dtype_constraints.py | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/tripy/tests/spec_verification/test_dtype_constraints.py b/tripy/tests/spec_verification/test_dtype_constraints.py index 9860b0479..0d4593e56 100755 --- a/tripy/tests/spec_verification/test_dtype_constraints.py +++ b/tripy/tests/spec_verification/test_dtype_constraints.py @@ -24,7 +24,7 @@ from tests.spec_verification.object_builders import create_obj from tripy.constraints import TYPE_VERIFICATION, RETURN_VALUE, FUNC_W_DOC_VERIF import tripy as tp -from contextlib import ExitStack +import tests.helper def _method_handler(func_name, kwargs, func_obj, api_call_locals): @@ -183,15 +183,61 @@ def test_neg_dtype_constraints(test_data): assert api_call_locals[RETURN_VALUE].dtype == namespace[return_dtype] -operations = [obj.__qualname__ for _, obj in inspect.getmembers(tp) if inspect.isfunction(obj)] +def get_all_possible_verif_ops(): + qualnames = set() + tripy_interfaces = tests.helper.get_all_tripy_interfaces() + for obj in tripy_interfaces: + if not obj.__doc__: + continue + + blocks = [ + (block.code()) + for block in tests.helper.consolidate_code_blocks(obj.__doc__) + if isinstance(block, tests.helper.DocstringCodeBlock) + ] + if blocks is None: + continue + + if ( + isinstance(obj, property) + or "." in obj.__qualname__ + or obj.__qualname__[0].isupper() + or obj in DATA_TYPES.values() + or (not obj.__qualname__.startswith("__") and "_" in obj.__qualname__) + ): + continue + + qualnames.add(obj.__qualname__) + + return qualnames + + +operations = get_all_possible_verif_ops() # add any function that you do not want to be verified: -func_exceptions = ["plugin", "dequantize"] +func_exceptions = [ + "plugin", + "dequantize", + "default", + "dtype", + "function", + "type", + "tolist", + "md5", + "integer", + "volume", + "save", + "floating", + "load", + "device", +] # Check if there are any operations that are not included (Currently does not test any ____ functions) @pytest.mark.parametrize("func_qualname", operations, ids=lambda val: f"is_{val}_verified") def test_all_ops_verified(func_qualname): if not func_qualname.startswith("_") and func_qualname not in func_exceptions: - assert func_qualname in FUNC_W_DOC_VERIF, f"function {func_qualname}'s data types have not been verified. Please add data type verification by following the guide within tripy/tests/spec_verification or exclude it from this test." + assert ( + func_qualname in FUNC_W_DOC_VERIF + ), f"function {func_qualname}'s data types have not been verified. Please add data type verification by following the guide within tripy/tests/spec_verification or exclude it from this test." else: pytest.skip("Data type constraints are not required for this API") From 8bf5f76d95f02e352411434403d01adc46fb9fe2 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Fri, 30 Aug 2024 16:58:11 -0700 Subject: [PATCH 4/5] Amendment --- tripy/tests/spec_verification/test_dtype_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tripy/tests/spec_verification/test_dtype_constraints.py b/tripy/tests/spec_verification/test_dtype_constraints.py index 0d4593e56..049bfcb1b 100755 --- a/tripy/tests/spec_verification/test_dtype_constraints.py +++ b/tripy/tests/spec_verification/test_dtype_constraints.py @@ -235,7 +235,7 @@ def get_all_possible_verif_ops(): # Check if there are any operations that are not included (Currently does not test any ____ functions) @pytest.mark.parametrize("func_qualname", operations, ids=lambda val: f"is_{val}_verified") def test_all_ops_verified(func_qualname): - if not func_qualname.startswith("_") and func_qualname not in func_exceptions: + if func_qualname not in func_exceptions: assert ( func_qualname in FUNC_W_DOC_VERIF ), f"function {func_qualname}'s data types have not been verified. Please add data type verification by following the guide within tripy/tests/spec_verification or exclude it from this test." From 91a98bbec0d111d0a9d9376afa864b8a70821efd Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Fri, 30 Aug 2024 17:32:45 -0700 Subject: [PATCH 5/5] Pushed what I finished, might have to revert to previous commit since this commit has lots of issue and it may be easier to just to add tensor checking from scratch --- .../test_dtype_constraints.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tripy/tests/spec_verification/test_dtype_constraints.py b/tripy/tests/spec_verification/test_dtype_constraints.py index 049bfcb1b..1bef0b863 100755 --- a/tripy/tests/spec_verification/test_dtype_constraints.py +++ b/tripy/tests/spec_verification/test_dtype_constraints.py @@ -17,7 +17,7 @@ import inspect -from typing import List +from typing import List, Union, Optional, get_origin, get_args, ForwardRef, get_type_hints from tripy.common.datatype import DATA_TYPES import itertools import pytest @@ -198,13 +198,24 @@ def get_all_possible_verif_ops(): if blocks is None: continue - if ( - isinstance(obj, property) - or "." in obj.__qualname__ - or obj.__qualname__[0].isupper() - or obj in DATA_TYPES.values() - or (not obj.__qualname__.startswith("__") and "_" in obj.__qualname__) - ): + if isinstance(obj, property): + continue + + func_sig = inspect.signature(func_obj) + param_dict = func_sig.parameters + contains_tensor_input = False + for type_hint in param_dict.values(): + type_hint = type_hint.annotation + while get_origin(type_hint) in [Union, Optional, list] and not contains_tensor_input: + type_hint = get_args(type_hint)[0] + # ForwardRef refers to any case where type hint is a string. + if isinstance(type_hint, ForwardRef): + type_hint = type_hint.__forward_arg__ + if type_hint == "tripy.Tensor": + print(type_hint) + contains_tensor_input = True + + if not contains_tensor_input: continue qualnames.add(obj.__qualname__) @@ -212,6 +223,9 @@ def get_all_possible_verif_ops(): return qualnames +print(get_all_possible_verif_ops()) +print(len(get_all_possible_verif_ops())) + operations = get_all_possible_verif_ops() # add any function that you do not want to be verified: func_exceptions = [