diff --git a/backends/test/suite/__init__.py b/backends/test/suite/__init__.py index 7190da4e0fd..43d4e16818f 100644 --- a/backends/test/suite/__init__.py +++ b/backends/test/suite/__init__.py @@ -9,18 +9,11 @@ import logging import os -import unittest - -from enum import Enum -from typing import Callable import executorch.backends.test.suite.flow -import torch -from executorch.backends.test.suite.context import get_active_test_context, TestContext from executorch.backends.test.suite.flow import TestFlow -from executorch.backends.test.suite.reporting import log_test_summary -from executorch.backends.test.suite.runner import run_test, runner_main +from executorch.backends.test.suite.runner import runner_main logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -62,109 +55,6 @@ def get_test_flows() -> dict[str, TestFlow]: return _ALL_TEST_FLOWS -DTYPES = [ - # torch.int8, - # torch.uint8, - # torch.int16, - # torch.uint16, - # torch.int32, - # torch.uint32, - # torch.int64, - # torch.uint64, - # torch.float16, - torch.float32, - # torch.float64, -] - -FLOAT_DTYPES = [ - torch.float16, - torch.float32, - torch.float64, -] - - -# The type of test function. This controls the test generation and expected signature. -# Standard tests are run, as is. Dtype tests get a variant generated for each dtype and -# take an additional dtype parameter. -class TestType(Enum): - STANDARD = 1 - DTYPE = 2 - - -# Function annotation for dtype tests. This instructs the test framework to run the test -# for each supported dtype and to pass dtype as a test parameter. -def dtype_test(func): - func.test_type = TestType.DTYPE - return func - - -# Class annotation for operator tests. This triggers the test framework to register -# the tests. -def operator_test(cls): - _create_tests(cls) - return cls - - -# Generate test cases for each backend flow. -def _create_tests(cls): - for key in dir(cls): - if key.startswith("test_"): - _expand_test(cls, key) - - -# Expand a test into variants for each registered flow. -def _expand_test(cls, test_name: str): - test_func = getattr(cls, test_name) - for flow in get_test_flows().values(): - _create_test_for_backend(cls, test_func, flow) - delattr(cls, test_name) - - -def _make_wrapped_test( - test_func: Callable, - test_name: str, - flow: TestFlow, - params: dict | None = None, -): - def wrapped_test(self): - with TestContext(test_name, flow.name, params): - test_kwargs = params or {} - test_kwargs["flow"] = flow - - test_func(self, **test_kwargs) - - wrapped_test._name = test_name - wrapped_test._flow = flow - - return wrapped_test - - -def _create_test_for_backend( - cls, - test_func: Callable, - flow: TestFlow, -): - test_type = getattr(test_func, "test_type", TestType.STANDARD) - - if test_type == TestType.STANDARD: - wrapped_test = _make_wrapped_test(test_func, test_func.__name__, flow) - test_name = f"{test_func.__name__}_{flow.name}" - setattr(cls, test_name, wrapped_test) - elif test_type == TestType.DTYPE: - for dtype in DTYPES: - wrapped_test = _make_wrapped_test( - test_func, - test_func.__name__, - flow, - {"dtype": dtype}, - ) - dtype_name = str(dtype)[6:] # strip "torch." - test_name = f"{test_func.__name__}_{dtype_name}_{flow.name}" - setattr(cls, test_name, wrapped_test) - else: - raise NotImplementedError(f"Unknown test type {test_type}.") - - def load_tests(loader, suite, pattern): package_dir = os.path.dirname(__file__) discovered_suite = loader.discover( @@ -174,32 +64,5 @@ def load_tests(loader, suite, pattern): return suite -class OperatorTest(unittest.TestCase): - def _test_op(self, model, inputs, flow: TestFlow): - context = get_active_test_context() - - # This should be set in the wrapped test. See _make_wrapped_test above. - assert context is not None, "Missing test context." - - run_summary = run_test( - model, - inputs, - flow, - context.test_name, - context.params, - ) - - log_test_summary(run_summary) - - if not run_summary.result.is_success(): - if run_summary.result.is_backend_failure(): - raise RuntimeError("Test failure.") from run_summary.error - else: - # Non-backend failure indicates a bad test. Mark as skipped. - raise unittest.SkipTest( - f"Test failed for reasons other than backend failure. Error: {run_summary.error}" - ) - - if __name__ == "__main__": runner_main() diff --git a/backends/test/suite/discovery.py b/backends/test/suite/discovery.py index 7ccc52ba4e7..92de356f550 100644 --- a/backends/test/suite/discovery.py +++ b/backends/test/suite/discovery.py @@ -69,8 +69,17 @@ def _filter_tests( def _is_test_enabled(test_case: unittest.TestCase, test_filter: TestFilter) -> bool: test_method = getattr(test_case, test_case._testMethodName) + # Handle import / discovery failures - leave them enabled to report nicely at the + # top level. There might be a better way to do this. Internally, unittest seems to + # replace it with a stub method to report the failure. + if "testFailure" in str(test_method): + print(f"Warning: Test {test_case._testMethodName} failed to import.") + return True + if not hasattr(test_method, "_flow"): - print(f"Test missing flow: {test_method}") + raise RuntimeError( + f"Test missing flow: {test_case._testMethodName} {test_method}" + ) flow: TestFlow = test_method._flow diff --git a/backends/test/suite/operators/__init__.py b/backends/test/suite/operators/__init__.py index 0fb9ecd1dff..79684fd43ec 100644 --- a/backends/test/suite/operators/__init__.py +++ b/backends/test/suite/operators/__init__.py @@ -7,6 +7,17 @@ # pyre-unsafe import os +import unittest + +from enum import Enum +from typing import Callable + +import torch +from executorch.backends.test.suite import get_test_flows +from executorch.backends.test.suite.context import get_active_test_context, TestContext +from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.reporting import log_test_summary +from executorch.backends.test.suite.runner import run_test def load_tests(loader, suite, pattern): @@ -16,3 +27,133 @@ def load_tests(loader, suite, pattern): ) suite.addTests(discovered_suite) return suite + + +DTYPES = [ + # torch.int8, + # torch.uint8, + # torch.int16, + # torch.uint16, + # torch.int32, + # torch.uint32, + # torch.int64, + # torch.uint64, + # torch.float16, + torch.float32, + # torch.float64, +] + +FLOAT_DTYPES = [ + torch.float16, + torch.float32, + torch.float64, +] + + +# The type of test function. This controls the test generation and expected signature. +# Standard tests are run, as is. Dtype tests get a variant generated for each dtype and +# take an additional dtype parameter. +class TestType(Enum): + STANDARD = 1 + DTYPE = 2 + + +# Function annotation for dtype tests. This instructs the test framework to run the test +# for each supported dtype and to pass dtype as a test parameter. +def dtype_test(func): + func.test_type = TestType.DTYPE + return func + + +# Class annotation for operator tests. This triggers the test framework to register +# the tests. +def operator_test(cls): + _create_tests(cls) + return cls + + +# Generate test cases for each backend flow. +def _create_tests(cls): + for key in dir(cls): + if key.startswith("test_"): + _expand_test(cls, key) + + +# Expand a test into variants for each registered flow. +def _expand_test(cls, test_name: str): + test_func = getattr(cls, test_name) + for flow in get_test_flows().values(): + _create_test_for_backend(cls, test_func, flow) + delattr(cls, test_name) + + +def _make_wrapped_test( + test_func: Callable, + test_name: str, + flow: TestFlow, + params: dict | None = None, +): + def wrapped_test(self): + with TestContext(test_name, flow.name, params): + test_kwargs = params or {} + test_kwargs["flow"] = flow + + test_func(self, **test_kwargs) + + wrapped_test._name = test_name + wrapped_test._flow = flow + + return wrapped_test + + +def _create_test_for_backend( + cls, + test_func: Callable, + flow: TestFlow, +): + test_type = getattr(test_func, "test_type", TestType.STANDARD) + + if test_type == TestType.STANDARD: + wrapped_test = _make_wrapped_test(test_func, test_func.__name__, flow) + test_name = f"{test_func.__name__}_{flow.name}" + setattr(cls, test_name, wrapped_test) + elif test_type == TestType.DTYPE: + for dtype in DTYPES: + wrapped_test = _make_wrapped_test( + test_func, + test_func.__name__, + flow, + {"dtype": dtype}, + ) + dtype_name = str(dtype)[6:] # strip "torch." + test_name = f"{test_func.__name__}_{dtype_name}_{flow.name}" + setattr(cls, test_name, wrapped_test) + else: + raise NotImplementedError(f"Unknown test type {test_type}.") + + +class OperatorTest(unittest.TestCase): + def _test_op(self, model, inputs, flow: TestFlow): + context = get_active_test_context() + + # This should be set in the wrapped test. See _make_wrapped_test above. + assert context is not None, "Missing test context." + + run_summary = run_test( + model, + inputs, + flow, + context.test_name, + context.params, + ) + + log_test_summary(run_summary) + + if not run_summary.result.is_success(): + if run_summary.result.is_backend_failure(): + raise RuntimeError("Test failure.") from run_summary.error + else: + # Non-backend failure indicates a bad test. Mark as skipped. + raise unittest.SkipTest( + f"Test failed for reasons other than backend failure. Error: {run_summary.error}" + ) diff --git a/backends/test/suite/operators/test_add.py b/backends/test/suite/operators/test_add.py index 2ff1644d672..6b21c3bf985 100644 --- a/backends/test/suite/operators/test_add.py +++ b/backends/test/suite/operators/test_add.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def forward(self, x, y): diff --git a/backends/test/suite/operators/test_div.py b/backends/test/suite/operators/test_div.py index 1367a4bc8f7..656d350585d 100644 --- a/backends/test/suite/operators/test_div.py +++ b/backends/test/suite/operators/test_div.py @@ -10,10 +10,14 @@ from typing import Optional import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def forward(self, x, y): diff --git a/backends/test/suite/operators/test_elu.py b/backends/test/suite/operators/test_elu.py index be4bb99bba0..f768a426954 100644 --- a/backends/test/suite/operators/test_elu.py +++ b/backends/test/suite/operators/test_elu.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, alpha=1.0, inplace=False): diff --git a/backends/test/suite/operators/test_gelu.py b/backends/test/suite/operators/test_gelu.py index 948947907d9..5c6a9f8f415 100644 --- a/backends/test/suite/operators/test_gelu.py +++ b/backends/test/suite/operators/test_gelu.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, approximate="none"): diff --git a/backends/test/suite/operators/test_glu.py b/backends/test/suite/operators/test_glu.py index b7126d5fdf5..cd19377c36b 100644 --- a/backends/test/suite/operators/test_glu.py +++ b/backends/test/suite/operators/test_glu.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, dim=-1): diff --git a/backends/test/suite/operators/test_hardsigmoid.py b/backends/test/suite/operators/test_hardsigmoid.py index 7ad92819506..238b18b1e0d 100644 --- a/backends/test/suite/operators/test_hardsigmoid.py +++ b/backends/test/suite/operators/test_hardsigmoid.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, inplace=False): diff --git a/backends/test/suite/operators/test_hardswish.py b/backends/test/suite/operators/test_hardswish.py index e8d25266af5..66902791c33 100644 --- a/backends/test/suite/operators/test_hardswish.py +++ b/backends/test/suite/operators/test_hardswish.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, inplace=False): diff --git a/backends/test/suite/operators/test_hardtanh.py b/backends/test/suite/operators/test_hardtanh.py index ffef9977e01..2fcd1dbf563 100644 --- a/backends/test/suite/operators/test_hardtanh.py +++ b/backends/test/suite/operators/test_hardtanh.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, min_val=-1.0, max_val=1.0, inplace=False): diff --git a/backends/test/suite/operators/test_leaky_relu.py b/backends/test/suite/operators/test_leaky_relu.py index e753abf8bb6..983da47bba3 100644 --- a/backends/test/suite/operators/test_leaky_relu.py +++ b/backends/test/suite/operators/test_leaky_relu.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, negative_slope=0.01, inplace=False): diff --git a/backends/test/suite/operators/test_logsigmoid.py b/backends/test/suite/operators/test_logsigmoid.py index ff62358a98e..1df1d11546f 100644 --- a/backends/test/suite/operators/test_logsigmoid.py +++ b/backends/test/suite/operators/test_logsigmoid.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def forward(self, x): diff --git a/backends/test/suite/operators/test_mul.py b/backends/test/suite/operators/test_mul.py index 5914b455762..ceadc1edf7a 100644 --- a/backends/test/suite/operators/test_mul.py +++ b/backends/test/suite/operators/test_mul.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def forward(self, x, y): diff --git a/backends/test/suite/operators/test_prelu.py b/backends/test/suite/operators/test_prelu.py index 5987f6bd75b..c02fc5692a5 100644 --- a/backends/test/suite/operators/test_prelu.py +++ b/backends/test/suite/operators/test_prelu.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, num_parameters=1, init=0.25): diff --git a/backends/test/suite/operators/test_relu.py b/backends/test/suite/operators/test_relu.py index d90a7c6f04e..c9f416f090f 100644 --- a/backends/test/suite/operators/test_relu.py +++ b/backends/test/suite/operators/test_relu.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, inplace=False): diff --git a/backends/test/suite/operators/test_sigmoid.py b/backends/test/suite/operators/test_sigmoid.py index 2a2c8c0539e..df083218884 100644 --- a/backends/test/suite/operators/test_sigmoid.py +++ b/backends/test/suite/operators/test_sigmoid.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def forward(self, x): diff --git a/backends/test/suite/operators/test_silu.py b/backends/test/suite/operators/test_silu.py index 9d8afbaa716..69b6576734f 100644 --- a/backends/test/suite/operators/test_silu.py +++ b/backends/test/suite/operators/test_silu.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, inplace=False): diff --git a/backends/test/suite/operators/test_sub.py b/backends/test/suite/operators/test_sub.py index 30c0db5878c..be7b871fdad 100644 --- a/backends/test/suite/operators/test_sub.py +++ b/backends/test/suite/operators/test_sub.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def forward(self, x, y): diff --git a/backends/test/suite/operators/test_tanh.py b/backends/test/suite/operators/test_tanh.py index b7e4ce7166b..7f961493ce9 100644 --- a/backends/test/suite/operators/test_tanh.py +++ b/backends/test/suite/operators/test_tanh.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def forward(self, x): diff --git a/backends/test/suite/operators/test_threshold.py b/backends/test/suite/operators/test_threshold.py index 1dfac7dd007..42b6fb801e5 100644 --- a/backends/test/suite/operators/test_threshold.py +++ b/backends/test/suite/operators/test_threshold.py @@ -8,10 +8,14 @@ import torch - -from executorch.backends.test.suite import dtype_test, operator_test, OperatorTest from executorch.backends.test.suite.flow import TestFlow +from executorch.backends.test.suite.operators import ( + dtype_test, + operator_test, + OperatorTest, +) + class Model(torch.nn.Module): def __init__(self, threshold=0.0, value=0.0, inplace=False):