From 90f8878cf616d590852650b5f223d407b79e9ad1 Mon Sep 17 00:00:00 2001 From: Mostafa Hani <71686115+CatB1t@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:44:41 +0000 Subject: [PATCH 1/4] Add dynamic flag --- .../test_ivy/helpers/test_parameter_flags.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/ivy_tests/test_ivy/helpers/test_parameter_flags.py b/ivy_tests/test_ivy/helpers/test_parameter_flags.py index be0395e883262..a011249ed4699 100644 --- a/ivy_tests/test_ivy/helpers/test_parameter_flags.py +++ b/ivy_tests/test_ivy/helpers/test_parameter_flags.py @@ -3,6 +3,14 @@ from . import globals as test_globals from .pipeline_helper import BackendHandler +from dataclasses import dataclass +from hypothesis.strategies import SearchStrategy + + +@dataclass +class DynamicFlag: + strategy: SearchStrategy + @st.composite def _gradient_strategy(draw): @@ -27,16 +35,16 @@ def _as_varaible_strategy(draw): return draw(st.lists(st.booleans(), min_size=1, max_size=1)) -BuiltNativeArrayStrategy = st.lists(st.booleans(), min_size=1, max_size=1) -BuiltAsVariableStrategy = _as_varaible_strategy() -BuiltContainerStrategy = st.lists(st.booleans(), min_size=1, max_size=1) -BuiltInstanceStrategy = st.booleans() -BuiltInplaceStrategy = st.just(False) -BuiltGradientStrategy = _gradient_strategy() -BuiltWithOutStrategy = st.booleans() -BuiltCompileStrategy = st.just(False) -BuiltFrontendArrayStrategy = st.booleans() -BuiltPrecisionModeStrategy = st.booleans() +BuiltNativeArrayStrategy = DynamicFlag(st.lists(st.booleans(), min_size=1, max_size=1)) +BuiltAsVariableStrategy = DynamicFlag(_as_varaible_strategy()) +BuiltContainerStrategy = DynamicFlag(st.lists(st.booleans(), min_size=1, max_size=1)) +BuiltInstanceStrategy = DynamicFlag(st.booleans()) +BuiltInplaceStrategy = DynamicFlag(st.just(False)) +BuiltGradientStrategy = DynamicFlag(_gradient_strategy()) +BuiltWithOutStrategy = DynamicFlag(st.booleans()) +BuiltCompileStrategy = DynamicFlag(st.booleans()) +BuiltFrontendArrayStrategy = DynamicFlag(st.booleans()) +BuiltPrecisionModeStrategy = DynamicFlag(st.booleans()) flags_mapping = { @@ -59,7 +67,7 @@ def build_flag(key: str, value: bool): assert ( flags_mapping[key] in globals().keys() ), f"{flags_mapping[key]} is not a valid flag variable." - globals()[flags_mapping[key]] = value + globals()[flags_mapping[key]].strategy = value # Strategy Helpers # From 7209912cb9ad7d6cab38096f99a4f9eeb2fb6948 Mon Sep 17 00:00:00 2001 From: Mostafa Hani <71686115+CatB1t@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:45:18 +0000 Subject: [PATCH 2/4] Add Helper function for retrieving value of flag at runtime. --- ivy_tests/test_ivy/helpers/testing_helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ivy_tests/test_ivy/helpers/testing_helpers.py b/ivy_tests/test_ivy/helpers/testing_helpers.py index 670f6106ac880..4da860aa4f181 100644 --- a/ivy_tests/test_ivy/helpers/testing_helpers.py +++ b/ivy_tests/test_ivy/helpers/testing_helpers.py @@ -15,6 +15,7 @@ from . import test_globals as t_globals from .pipeline_helper import BackendHandler from ivy_tests.test_ivy.helpers.test_parameter_flags import ( + DynamicFlag, BuiltInstanceStrategy, BuiltAsVariableStrategy, BuiltNativeArrayStrategy, @@ -48,6 +49,10 @@ ) +def _get_runtime_flag_value(flag): + return flag.strategy if isinstance(flag, DynamicFlag) else flag + + @st.composite def num_positional_args_method(draw, *, method): """ From 168fe5547a8a05e710ec9fd6c6d43862aaacbd25 Mon Sep 17 00:00:00 2001 From: Mostafa Hani <71686115+CatB1t@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:07:37 +0000 Subject: [PATCH 3/4] Update handle_test. --- ivy_tests/test_ivy/helpers/testing_helpers.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ivy_tests/test_ivy/helpers/testing_helpers.py b/ivy_tests/test_ivy/helpers/testing_helpers.py index 4da860aa4f181..e8f37387e5ab8 100644 --- a/ivy_tests/test_ivy/helpers/testing_helpers.py +++ b/ivy_tests/test_ivy/helpers/testing_helpers.py @@ -361,14 +361,14 @@ def handle_test( possible_arguments["test_flags"] = pf.function_flags( ground_truth_backend=st.just(ground_truth_backend), num_positional_args=number_positional_args, - instance_method=test_instance_method, - with_out=test_with_out, - test_gradients=test_gradients, - test_compile=test_compile, - as_variable=as_variable_flags, - native_arrays=native_array_flags, - container_flags=container_flags, - precision_mode=precision_mode, + instance_method=_get_runtime_flag_value(test_instance_method), + with_out=_get_runtime_flag_value(test_with_out), + test_gradients=_get_runtime_flag_value(test_gradients), + test_compile=_get_runtime_flag_value(test_compile), + as_variable=_get_runtime_flag_value(as_variable_flags), + native_arrays=_get_runtime_flag_value(native_array_flags), + container_flags=_get_runtime_flag_value(container_flags), + precision_mode=_get_runtime_flag_value(precision_mode), ) def test_wrapper(test_fn): From f286b1508ddc9e120858ea085524134cf09c5082 Mon Sep 17 00:00:00 2001 From: Mostafa Hani <71686115+CatB1t@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:12:03 +0000 Subject: [PATCH 4/4] Update handle_method, handle_frontend_method, handle_frontend_test --- ivy_tests/test_ivy/helpers/testing_helpers.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/ivy_tests/test_ivy/helpers/testing_helpers.py b/ivy_tests/test_ivy/helpers/testing_helpers.py index e8f37387e5ab8..f9fb2ec29c8a7 100644 --- a/ivy_tests/test_ivy/helpers/testing_helpers.py +++ b/ivy_tests/test_ivy/helpers/testing_helpers.py @@ -491,13 +491,13 @@ def handle_frontend_test( # Generate the test flags strategy test_flags = pf.frontend_function_flags( num_positional_args=number_positional_args, - with_out=test_with_out, - inplace=test_inplace, - as_variable=as_variable_flags, - native_arrays=native_array_flags, - test_compile=test_compile, - generate_frontend_arrays=generate_frontend_arrays, - precision_mode=precision_mode, + with_out=_get_runtime_flag_value(test_with_out), + inplace=_get_runtime_flag_value(test_inplace), + as_variable=_get_runtime_flag_value(as_variable_flags), + native_arrays=_get_runtime_flag_value(native_array_flags), + test_compile=_get_runtime_flag_value(test_compile), + generate_frontend_arrays=_get_runtime_flag_value(generate_frontend_arrays), + precision_mode=_get_runtime_flag_value(precision_mode), ) def test_wrapper(test_fn): @@ -599,9 +599,9 @@ def handle_method( is_hypothesis_test = len(_given_kwargs) != 0 possible_arguments = { "ground_truth_backend": st.just(ground_truth_backend), - "test_gradients": test_gradients, - "test_compile": test_compile, - "precision_mode": precision_mode, + "test_gradients": _get_runtime_flag_value(test_gradients), + "test_compile": _get_runtime_flag_value(test_compile), + "precision_mode": _get_runtime_flag_value(precision_mode), } if is_hypothesis_test and is_method_tree_provided: @@ -614,9 +614,9 @@ def handle_method( possible_arguments["init_flags"] = pf.init_method_flags( num_positional_args=init_num_positional_args, - as_variable=init_as_variable_flags, - native_arrays=init_native_arrays, - precision_mode=precision_mode, + as_variable=_get_runtime_flag_value(init_as_variable_flags), + native_arrays=_get_runtime_flag_value(init_native_arrays), + precision_mode=_get_runtime_flag_value(precision_mode), ) if method_num_positional_args is None: @@ -626,10 +626,10 @@ def handle_method( possible_arguments["method_flags"] = pf.method_flags( num_positional_args=method_num_positional_args, - as_variable=method_as_variable_flags, - native_arrays=method_native_arrays, - container_flags=method_container_flags, - precision_mode=precision_mode, + as_variable=_get_runtime_flag_value(method_as_variable_flags), + native_arrays=_get_runtime_flag_value(method_native_arrays), + container_flags=_get_runtime_flag_value(method_container_flags), + precision_mode=_get_runtime_flag_value(precision_mode), ) def test_wrapper(test_fn): @@ -747,18 +747,18 @@ def test_wrapper(test_fn): param_names = inspect.signature(test_fn).parameters.keys() init_flags = pf.frontend_method_flags( num_positional_args=init_num_positional_args, - as_variable=init_as_variable_flags, - native_arrays=init_native_arrays, - test_compile=test_compile, - precision_mode=precision_mode, + as_variable=_get_runtime_flag_value(init_as_variable_flags), + native_arrays=_get_runtime_flag_value(init_native_arrays), + test_compile=_get_runtime_flag_value(test_compile), + precision_mode=_get_runtime_flag_value(precision_mode), ) method_flags = pf.frontend_method_flags( num_positional_args=method_num_positional_args, - as_variable=method_as_variable_flags, - native_arrays=method_native_arrays, - test_compile=test_compile, - precision_mode=precision_mode, + as_variable=_get_runtime_flag_value(method_as_variable_flags), + native_arrays=_get_runtime_flag_value(method_native_arrays), + test_compile=_get_runtime_flag_value(test_compile), + precision_mode=_get_runtime_flag_value(precision_mode), ) ivy_init_modules = str(ivy_init_module) framework_init_modules = str(framework_init_module)