From f120e70e935996ce8a0b274f74625fbe0f1252e8 Mon Sep 17 00:00:00 2001
From: Gregory James Comer <gjcomer@meta.com>
Date: Thu, 17 Jul 2025 17:26:58 -0700
Subject: [PATCH 1/6] Update

[ghstack-poisoned]
---
 backends/test/suite/__init__.py           | 56 ++++++++++----------
 backends/test/suite/discovery.py          | 63 +++++++++++++++++++++++
 backends/test/suite/flow.py               | 63 +++++++++++++++++++++++
 backends/test/suite/operators/__init__.py | 11 ++++
 backends/test/suite/reporting.py          |  6 +--
 backends/test/suite/runner.py             | 40 ++++++++++++--
 6 files changed, 200 insertions(+), 39 deletions(-)
 create mode 100644 backends/test/suite/discovery.py
 create mode 100644 backends/test/suite/flow.py

diff --git a/backends/test/suite/__init__.py b/backends/test/suite/__init__.py
index bce62ce1d63..cf73a7bdd0c 100644
--- a/backends/test/suite/__init__.py
+++ b/backends/test/suite/__init__.py
@@ -12,11 +12,13 @@
 import unittest
 
 from enum import Enum
-from typing import Any, Callable, Tuple
+from typing import Callable, Sequence, Sequence
+
+import executorch.backends.test.suite.flow
 
 import torch
-from executorch.backends.test.harness import Tester
 from executorch.backends.test.suite.context import get_active_test_context, TestContext
+from executorch.backends.test.suite.flow import TestFlow
 from executorch.backends.test.suite.reporting import log_test_summary
 from executorch.backends.test.suite.runner import run_test, runner_main
 
@@ -44,22 +46,20 @@ def is_backend_enabled(backend):
         return backend in _ENABLED_BACKENDS
 
 
-ALL_TEST_FLOWS = []
+_ALL_TEST_FLOWS: Sequence[TestFlow] | None = None
 
-if is_backend_enabled("xnnpack"):
-    from executorch.backends.xnnpack.test.tester import Tester as XnnpackTester
 
-    XNNPACK_TEST_FLOW = ("xnnpack", XnnpackTester)
-    ALL_TEST_FLOWS.append(XNNPACK_TEST_FLOW)
+def get_test_flows() -> Sequence[TestFlow]:
+    global _ALL_TEST_FLOWS
 
-if is_backend_enabled("coreml"):
-    try:
-        from executorch.backends.apple.coreml.test.tester import CoreMLTester
+    if _ALL_TEST_FLOWS is None:
+        _ALL_TEST_FLOWS = [
+            f
+            for f in executorch.backends.test.suite.flow.all_flows()
+            if is_backend_enabled(f.backend)
+        ]
 
-        COREML_TEST_FLOW = ("coreml", CoreMLTester)
-        ALL_TEST_FLOWS.append(COREML_TEST_FLOW)
-    except Exception:
-        print("Core ML AOT is not available.")
+    return _ALL_TEST_FLOWS
 
 
 DTYPES = [
@@ -115,53 +115,51 @@ def _create_tests(cls):
 # Expand a test into variants for each registered flow.
 def _expand_test(cls, test_name: str):
     test_func = getattr(cls, test_name)
-    for flow_name, tester_factory in ALL_TEST_FLOWS:
-        _create_test_for_backend(cls, test_func, flow_name, tester_factory)
+    for flow in get_test_flows():
+        _create_test_for_backend(cls, test_func, flow)
     delattr(cls, test_name)
 
 
 def _make_wrapped_test(
     test_func: Callable,
     test_name: str,
-    test_flow: str,
-    tester_factory: Callable,
+    flow: TestFlow,
     params: dict | None = None,
 ):
     def wrapped_test(self):
-        with TestContext(test_name, test_flow, params):
+        with TestContext(test_name, flow.name, params):
             test_kwargs = params or {}
-            test_kwargs["tester_factory"] = tester_factory
+            test_kwargs["tester_factory"] = flow.tester_factory
 
             test_func(self, **test_kwargs)
 
+    setattr(wrapped_test, "_name", test_name)
+    setattr(wrapped_test, "_flow", flow)
+
     return wrapped_test
 
 
 def _create_test_for_backend(
     cls,
     test_func: Callable,
-    flow_name: str,
-    tester_factory: Callable[[torch.nn.Module, Tuple[Any]], Tester],
+    flow: TestFlow,
 ):
     test_type = getattr(test_func, "test_type", TestType.STANDARD)
 
     if test_type == TestType.STANDARD:
-        wrapped_test = _make_wrapped_test(
-            test_func, test_func.__name__, flow_name, tester_factory
-        )
-        test_name = f"{test_func.__name__}_{flow_name}"
+        wrapped_test = _make_wrapped_test(test_func, test_func.__name__, flow)
+        test_name = f"{test_func.__name__}_{flow.name}"
         setattr(cls, test_name, wrapped_test)
     elif test_type == TestType.DTYPE:
         for dtype in DTYPES:
             wrapped_test = _make_wrapped_test(
                 test_func,
                 test_func.__name__,
-                flow_name,
-                tester_factory,
+                flow,
                 {"dtype": dtype},
             )
             dtype_name = str(dtype)[6:]  # strip "torch."
-            test_name = f"{test_func.__name__}_{dtype_name}_{flow_name}"
+            test_name = f"{test_func.__name__}_{dtype_name}_{flow.name}"
             setattr(cls, test_name, wrapped_test)
     else:
         raise NotImplementedError(f"Unknown test type {test_type}.")
diff --git a/backends/test/suite/discovery.py b/backends/test/suite/discovery.py
new file mode 100644
index 00000000000..5abd194cbcd
--- /dev/null
+++ b/backends/test/suite/discovery.py
@@ -0,0 +1,63 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# pyre-unsafe
+
+import os
+import unittest
+
+from types import ModuleType
+
+from executorch.backends.test.suite.flow import TestFlow
+
+#
+# This file contains logic related to test discovery and filtering.
+#
+
+
+def discover_tests(
+    root_module: ModuleType, backends: set[str] | None
+) -> unittest.TestSuite:
+    # Collect all tests using the unittest discovery mechanism then filter down.
+
+    # Find the file system path corresponding to the root module.
+    module_file = root_module.__file__
+    if module_file is None:
+        raise RuntimeError(f"Module {root_module} has no __file__ attribute")
+
+    loader = unittest.TestLoader()
+    module_dir = os.path.dirname(module_file)
+    suite = loader.discover(module_dir)
+
+    return _filter_tests(suite, backends)
+
+
+def _filter_tests(
+    suite: unittest.TestSuite, backends: set[str] | None
+) -> unittest.TestSuite:
+    # Recursively traverse the test suite and add them to the filtered set.
+    filtered_suite = unittest.TestSuite()
+
+    for child in suite:
+        if isinstance(child, unittest.TestSuite):
+            filtered_suite.addTest(_filter_tests(child, backends))
+        elif isinstance(child, unittest.TestCase):
+            if _is_test_enabled(child, backends):
+                filtered_suite.addTest(child)
+        else:
+            raise RuntimeError(f"Unexpected test type: {type(child)}")
+
+    return filtered_suite
+
+
+def _is_test_enabled(test_case: unittest.TestCase, backends: set[str] | None) -> bool:
+    test_method = getattr(test_case, test_case._testMethodName)
+
+    if backends is not None:
+        flow: TestFlow = getattr(test_method, "_flow")
+        return flow.backend in backends
+    else:
+        return True
diff --git a/backends/test/suite/flow.py b/backends/test/suite/flow.py
new file mode 100644
index 00000000000..4410d382401
--- /dev/null
+++ b/backends/test/suite/flow.py
@@ -0,0 +1,63 @@
+import logging
+
+from dataclasses import dataclass
+from math import log
+from typing import Callable, Sequence
+
+from executorch.backends.test.harness import Tester
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+@dataclass
+class TestFlow:
+    """
+    A lowering flow to test. This typically corresponds to a combination of a backend and
+    a lowering recipe.
+    """
+
+    name: str
+    """ The name of the lowering flow. """
+
+    backend: str
+    """ The name of the target backend. """
+
+    tester_factory: Callable[[], Tester]
+    """ A factory function that returns a Tester instance for this lowering flow. """
+
+
+def create_xnnpack_flow() -> TestFlow | None:
+    try:
+        from executorch.backends.xnnpack.test.tester import Tester as XnnpackTester
+
+        return TestFlow(
+            name="xnnpack",
+            backend="xnnpack",
+            tester_factory=XnnpackTester,
+        )
+    except Exception:
+        logger.info("Skipping XNNPACK flow registration due to import failure.")
+        return None
+
+
+def create_coreml_flow() -> TestFlow | None:
+    try:
+        from executorch.backends.apple.coreml.test.tester import CoreMLTester
+
+        return TestFlow(
+            name="coreml",
+            backend="coreml",
+            tester_factory=CoreMLTester,
+        )
+    except Exception:
+        logger.info("Skipping Core ML flow registration due to import failure.")
+        return None
+
+
+def all_flows() -> Sequence[TestFlow]:
+    flows = [
+        create_xnnpack_flow(),
+        create_coreml_flow(),
+    ]
+    return [f for f in flows if f is not None]
diff --git a/backends/test/suite/operators/__init__.py b/backends/test/suite/operators/__init__.py
index 6ac1a72bde6..0fb9ecd1dff 100644
--- a/backends/test/suite/operators/__init__.py
+++ b/backends/test/suite/operators/__init__.py
@@ -5,3 +5,14 @@
 # LICENSE file in the root directory of this source tree.
 
 # pyre-unsafe
+
+import os
+
+
+def load_tests(loader, suite, pattern):
+    package_dir = os.path.dirname(__file__)
+    discovered_suite = loader.discover(
+        start_dir=package_dir, pattern=pattern or "test_*.py"
+    )
+    suite.addTests(discovered_suite)
+    return suite
diff --git a/backends/test/suite/reporting.py b/backends/test/suite/reporting.py
index 948a6187b41..d7181300873 100644
--- a/backends/test/suite/reporting.py
+++ b/backends/test/suite/reporting.py
@@ -1,6 +1,6 @@
 from collections import Counter
 from dataclasses import dataclass
-from enum import IntEnum, nonmember
+from enum import IntEnum
 
 
 class TestResult(IntEnum):
@@ -33,19 +33,15 @@ class TestResult(IntEnum):
     UNKNOWN_FAIL = 8
     """ The test failed in an unknown or unexpected manner. """
 
-    @nonmember
     def is_success(self):
         return self in {TestResult.SUCCESS, TestResult.SUCCESS_UNDELEGATED}
 
-    @nonmember
     def is_non_backend_failure(self):
         return self in {TestResult.EAGER_FAIL, TestResult.EAGER_FAIL}
 
-    @nonmember
     def is_backend_failure(self):
         return not self.is_success() and not self.is_non_backend_failure()
 
-    @nonmember
     def display_name(self):
         if self == TestResult.SUCCESS:
             return "Success (Delegated)"
diff --git a/backends/test/suite/runner.py b/backends/test/suite/runner.py
index 2a626a5e35f..34a860e8f0b 100644
--- a/backends/test/suite/runner.py
+++ b/backends/test/suite/runner.py
@@ -1,4 +1,5 @@
 import argparse
+import importlib
 import unittest
 
 from typing import Callable
@@ -6,6 +7,7 @@
 import torch
 
 from executorch.backends.test.harness import Tester
+from executorch.backends.test.suite.discovery import discover_tests
 from executorch.backends.test.suite.reporting import (
     begin_test_session,
     complete_test_session,
@@ -15,6 +17,12 @@
 )
 
 
+# A list of all runnable test suites and the corresponding python package.
+NAMED_SUITES = {
+    "operators": "executorch.backends.test.suite.operators",
+}
+
+
 def run_test(  # noqa: C901
     model: torch.nn.Module,
     inputs: any,
@@ -130,20 +138,42 @@ def parse_args():
         prog="ExecuTorch Backend Test Suite",
         description="Run ExecuTorch backend tests.",
     )
-    parser.add_argument("test_path", nargs="?", help="Prefix filter for tests to run.")
+    parser.add_argument(
+        "suite",
+        nargs="*",
+        help="The test suite to run.",
+        choices=NAMED_SUITES.keys(),
+        default=["operators"],
+    )
+    parser.add_argument(
+        "-b", "--backend", nargs="*", help="The backend or backends to test."
+    )
     return parser.parse_args()
 
 
+def test(suite):
+    if isinstance(suite, unittest.TestSuite):
+        print(f"Suite: {suite}")
+        for t in suite:
+            test(t)
+    else:
+        print(f"Leaf: {type(suite)} {suite}")
+        print(f" {suite.__name__}")
+        print(f" {callable(suite)}")
+
+
 def runner_main():
     args = parse_args()
 
     begin_test_session()
 
-    test_path = args.test_path or "executorch.backends.test.suite.operators"
+    if len(args.suite) > 1:
+        raise NotImplementedError("TODO Support multiple suites.")
 
-    loader = unittest.TestLoader()
-    suite = loader.loadTestsFromName(test_path)
-    unittest.TextTestRunner().run(suite)
+    test_path = NAMED_SUITES[args.suite[0]]
+    test_root = importlib.import_module(test_path)
+    suite = discover_tests(test_root, args.backend)
+    unittest.TextTestRunner(verbosity=2).run(suite)
 
     summary = complete_test_session()
     print_summary(summary)

From 0fb85e693bfebb44c217d84df9eb9087066330d0 Mon Sep 17 00:00:00 2001
From: Gregory James Comer <gjcomer@meta.com>
Date: Thu, 17 Jul 2025 17:40:29 -0700
Subject: [PATCH 2/6] Update

[ghstack-poisoned]
---
 backends/test/suite/discovery.py | 40 +++++++++++++++++++++++---------
 backends/test/suite/runner.py    | 24 ++++++++++---------
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/backends/test/suite/discovery.py b/backends/test/suite/discovery.py
index 5abd194cbcd..929a426d430 100644
--- a/backends/test/suite/discovery.py
+++ b/backends/test/suite/discovery.py
@@ -9,7 +9,9 @@
 import os
 import unittest
 
+from dataclasses import dataclass
 from types import ModuleType
+from typing import Pattern
 
 from executorch.backends.test.suite.flow import TestFlow
 
@@ -18,8 +20,19 @@
 #
 
 
+@dataclass
+class TestFilter:
+    """A set of filters for test discovery."""
+
+    backends: set[str] | None
+    """ The set of backends to include. If None, all backends are included. """
+
+    name_regex: Pattern[str] | None
+    """ A regular expression to filter test names. If None, all tests are included. """
+
+
 def discover_tests(
-    root_module: ModuleType, backends: set[str] | None
+    root_module: ModuleType, test_filter: TestFilter
 ) -> unittest.TestSuite:
     # Collect all tests using the unittest discovery mechanism then filter down.
 
@@ -32,20 +45,20 @@ def discover_tests(
     module_dir = os.path.dirname(module_file)
     suite = loader.discover(module_dir)
 
-    return _filter_tests(suite, backends)
+    return _filter_tests(suite, test_filter)
 
 
 def _filter_tests(
-    suite: unittest.TestSuite, backends: set[str] | None
+    suite: unittest.TestSuite, test_filter: TestFilter
 ) -> unittest.TestSuite:
     # Recursively traverse the test suite and add them to the filtered set.
     filtered_suite = unittest.TestSuite()
 
     for child in suite:
         if isinstance(child, unittest.TestSuite):
-            filtered_suite.addTest(_filter_tests(child, backends))
+            filtered_suite.addTest(_filter_tests(child, test_filter))
         elif isinstance(child, unittest.TestCase):
-            if _is_test_enabled(child, backends):
+            if _is_test_enabled(child, test_filter):
                 filtered_suite.addTest(child)
         else:
             raise RuntimeError(f"Unexpected test type: {type(child)}")
@@ -53,11 +66,16 @@ def _filter_tests(
     return filtered_suite
 
 
-def _is_test_enabled(test_case: unittest.TestCase, backends: set[str] | None) -> bool:
+def _is_test_enabled(test_case: unittest.TestCase, test_filter: TestFilter) -> bool:
     test_method = getattr(test_case, test_case._testMethodName)
+    flow: TestFlow = getattr(test_method, "_flow")
+
+    if test_filter.backends is not None and flow.backend not in test_filter.backends:
+        return False
+
+    if test_filter.name_regex is not None and not test_filter.name_regex.search(
+        test_case.id()
+    ):
+        return False
 
-    if backends is not None:
-        flow: TestFlow = getattr(test_method, "_flow")
-        return flow.backend in backends
-    else:
-        return True
+    return True
diff --git a/backends/test/suite/runner.py b/backends/test/suite/runner.py
index 34a860e8f0b..36905d0dabc 100644
--- a/backends/test/suite/runner.py
+++ b/backends/test/suite/runner.py
@@ -1,5 +1,6 @@
 import argparse
 import importlib
+import re
 import unittest
 
 from typing import Callable
@@ -7,7 +8,7 @@
 import torch
 
 from executorch.backends.test.harness import Tester
-from executorch.backends.test.suite.discovery import discover_tests
+from executorch.backends.test.suite.discovery import discover_tests, TestFilter
 from executorch.backends.test.suite.reporting import (
     begin_test_session,
     complete_test_session,
@@ -148,18 +149,17 @@ def parse_args():
     parser.add_argument(
         "-b", "--backend", nargs="*", help="The backend or backends to test."
     )
+    parser.add_argument(
+        "-f", "--filter", nargs="?", help="A regular expression filter for test names."
+    )
     return parser.parse_args()
 
 
-def test(suite):
-    if isinstance(suite, unittest.TestSuite):
-        print(f"Suite: {suite}")
-        for t in suite:
-            test(t)
-    else:
-        print(f"Leaf: {type(suite)} {suite}")
-        print(f" {suite.__name__}")
-        print(f" {callable(suite)}")
+def build_test_filter(args: argparse.Namespace) -> TestFilter:
+    return TestFilter(
+        backends=set(args.backend) if args.backend is not None else None,
+        name_regex=re.compile(args.filter) if args.filter is not None else None,
+    )
 
 
 def runner_main():
@@ -172,7 +172,9 @@ def runner_main():
 
     test_path = NAMED_SUITES[args.suite[0]]
     test_root = importlib.import_module(test_path)
-    suite = discover_tests(test_root, args.backend)
+    test_filter = build_test_filter(args)
+
+    suite = discover_tests(test_root, test_filter)
     unittest.TextTestRunner(verbosity=2).run(suite)
 
     summary = complete_test_session()

From 4d8d844ebf040e5aff9b8d4a60cd5948a3b38157 Mon Sep 17 00:00:00 2001
From: Gregory James Comer <gjcomer@meta.com>
Date: Fri, 18 Jul 2025 19:52:50 -0700
Subject: [PATCH 3/6] Update

[ghstack-poisoned]
---
 backends/test/suite/discovery.py              |   4 +
 backends/test/suite/models/__init__.py        | 124 +++++++++++++++
 .../test/suite/models/test_torchvision.py     | 145 ++++++++++++++++++
 backends/test/suite/runner.py                 |  12 +-
 4 files changed, 282 insertions(+), 3 deletions(-)
 create mode 100644 backends/test/suite/models/__init__.py
 create mode 100644 backends/test/suite/models/test_torchvision.py

diff --git a/backends/test/suite/discovery.py b/backends/test/suite/discovery.py
index 929a426d430..ec77f5a90cd 100644
--- a/backends/test/suite/discovery.py
+++ b/backends/test/suite/discovery.py
@@ -68,6 +68,10 @@ def _filter_tests(
 
 def _is_test_enabled(test_case: unittest.TestCase, test_filter: TestFilter) -> bool:
     test_method = getattr(test_case, test_case._testMethodName)
+
+    if not hasattr(test_method, "_flow"):
+        print(f"Test missing flow: {test_method}")
+
     flow: TestFlow = getattr(test_method, "_flow")
 
     if test_filter.backends is not None and flow.backend not in test_filter.backends:
diff --git a/backends/test/suite/models/__init__.py b/backends/test/suite/models/__init__.py
new file mode 100644
index 00000000000..496bcb6f194
--- /dev/null
+++ b/backends/test/suite/models/__init__.py
@@ -0,0 +1,124 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# pyre-unsafe
+
+from executorch.backends.test.harness import Tester
+from executorch.backends.test.suite import get_test_flows
+from executorch.backends.test.suite.context import get_active_test_context, TestContext
+from executorch.backends.test.suite.flow import TestFlow
+from executorch.backends.test.suite.reporting import log_test_summary
+from executorch.backends.test.suite.runner import run_test 
+from typing import Any, Callable
+
+import itertools
+import os
+import torch
+import unittest
+
+
+DTYPES = [
+    torch.float16,
+    torch.float32,
+    torch.float64,
+]
+
+
+def load_tests(loader, suite, pattern):
+    package_dir = os.path.dirname(__file__)
+    discovered_suite = loader.discover(
+        start_dir=package_dir, pattern=pattern or "test_*.py"
+    )
+    suite.addTests(discovered_suite)
+    return suite
+
+
+def _create_test(
+    cls, 
+    test_func: Callable, 
+    flow: TestFlow, 
+    dtype: torch.dtype, 
+    use_dynamic_shapes: bool,
+):
+    def wrapped_test(self):
+        params = {
+            "dtype": dtype,
+            "use_dynamic_shapes": use_dynamic_shapes,
+        }
+        with TestContext(test_name, flow.name, params):
+            test_func(self, dtype, use_dynamic_shapes, flow.tester_factory)
+    
+    dtype_name = str(dtype)[6:]  # strip "torch."
+    test_name = f"{test_func.__name__}_{flow.name}_{dtype_name}"
+    if use_dynamic_shapes:
+        test_name += "_dynamic_shape"
+
+    setattr(wrapped_test, "_name", test_func.__name__)
+    setattr(wrapped_test, "_flow", flow)
+    
+    setattr(cls, test_name, wrapped_test)
+
+
+# Expand a test into variants for each registered flow.
+def _expand_test(cls, test_name: str) -> None:
+    test_func = getattr(cls, test_name)
+    supports_dynamic_shapes = getattr(test_func, "supports_dynamic_shapes", True)
+    dynamic_shape_values = [True, False] if supports_dynamic_shapes else [False]
+
+    for flow, dtype, use_dynamic_shapes in itertools.product(get_test_flows(), DTYPES, dynamic_shape_values):
+        _create_test(cls, test_func, flow, dtype, use_dynamic_shapes)
+    delattr(cls, test_name)
+
+
+def model_test_cls(cls) -> Callable | None:
+    """ Decorator for model tests. Handles generating test variants for each test flow and configuration. """
+    for key in dir(cls):
+        if key.startswith("test_"):
+            _expand_test(cls, key)
+    return cls
+
+
+def model_test_params(supports_dynamic_shapes: bool) -> Callable:
+    """ Optional parameter decorator for model tests. Specifies test pararameters. Only valid with a class decorated by model_test_cls. """
+    def inner_decorator(func: Callable) -> Callable:
+        setattr(func, "supports_dynamic_shapes", supports_dynamic_shapes)
+        return func
+    return inner_decorator
+
+
+def run_model_test(
+    model: torch.nn.Module,
+    inputs: tuple[Any],
+    dtype: torch.dtype,
+    dynamic_shapes: Any | None,
+    tester_factory: Callable[[], Tester],    
+):
+    model = model.to(dtype)
+    context = get_active_test_context()
+    
+    # This should be set in the wrapped test. See _create_test above.
+    assert context is not None, "Missing test context."
+
+    run_summary = run_test(
+        model,
+        inputs,
+        tester_factory,
+        context.test_name,
+        context.flow_name,
+        context.params,
+        dynamic_shapes=dynamic_shapes,
+    )
+    
+    log_test_summary(run_summary)
+
+    if not run_summary.result.is_success():
+        if run_summary.result.is_backend_failure():
+            raise RuntimeError("Test failure.") from run_summary.error
+        else:
+            # Non-backend failure indicates a bad test. Mark as skipped.
+            raise unittest.SkipTest(
+                f"Test failed for reasons other than backend failure. Error: {run_summary.error}"
+            )
diff --git a/backends/test/suite/models/test_torchvision.py b/backends/test/suite/models/test_torchvision.py
new file mode 100644
index 00000000000..6e6a8f6b36e
--- /dev/null
+++ b/backends/test/suite/models/test_torchvision.py
@@ -0,0 +1,145 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# pyre-unsafe
+
+import torch
+import torchvision
+import unittest
+
+from executorch.backends.test.suite.models import model_test_params, model_test_cls, run_model_test
+from torch.export import Dim
+from typing import Callable
+
+#
+# This file contains model integration tests for supported torchvision models.
+# 
+
+@model_test_cls
+class TorchVision(unittest.TestCase):
+    def _test_cv_model(
+        self,
+        model: torch.nn.Module,
+        dtype: torch.dtype,
+        use_dynamic_shapes: bool,
+        tester_factory: Callable,
+    ):
+        # Test a CV model that follows the standard conventions.
+        inputs = (
+            torch.randn(1, 3, 224, 224, dtype=dtype),
+        )
+            
+        dynamic_shapes = (
+            {
+                2: Dim("height", min=1, max=16)*16,
+                3: Dim("width", min=1, max=16)*16,
+            },
+        ) if use_dynamic_shapes else None
+        
+        run_model_test(model, inputs, dtype, dynamic_shapes, tester_factory)
+
+
+    def test_alexnet(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.alexnet()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    def test_convnext_small(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.convnext_small()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    def test_densenet161(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.densenet161()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    def test_efficientnet_b4(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.efficientnet_b4()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+        
+
+    def test_efficientnet_v2_s(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.efficientnet_v2_s()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    def test_googlenet(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.googlenet()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    def test_inception_v3(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.inception_v3()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    @model_test_params(supports_dynamic_shapes=False)
+    def test_maxvit_t(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.maxvit_t()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    def test_mnasnet1_0(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.mnasnet1_0()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    def test_mobilenet_v2(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.mobilenet_v2()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    def test_mobilenet_v3_small(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.mobilenet_v3_small()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    def test_regnet_y_1_6gf(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.regnet_y_1_6gf()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+    
+
+    def test_resnet50(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.resnet50()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+    
+        
+    def test_resnext50_32x4d(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.resnext50_32x4d()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+    
+        
+    def test_shufflenet_v2_x1_0(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.shufflenet_v2_x1_0()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+    
+        
+    def test_squeezenet1_1(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.squeezenet1_1()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+    
+        
+    def test_swin_v2_t(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.swin_v2_t()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+    
+        
+    def test_vgg11(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.vgg11()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+
+
+    @model_test_params(supports_dynamic_shapes=False)
+    def test_vit_b_16(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.vit_b_16()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+        
+
+    def test_wide_resnet50_2(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchvision.models.wide_resnet50_2()
+        self._test_cv_model(model, dtype, use_dynamic_shapes, tester_factory)
+    
\ No newline at end of file
diff --git a/backends/test/suite/runner.py b/backends/test/suite/runner.py
index 36905d0dabc..09554521d41 100644
--- a/backends/test/suite/runner.py
+++ b/backends/test/suite/runner.py
@@ -3,11 +3,12 @@
 import re
 import unittest
 
-from typing import Callable
+from typing import Any, Callable
 
 import torch
 
 from executorch.backends.test.harness import Tester
+from executorch.backends.test.harness.stages import StageType
 from executorch.backends.test.suite.discovery import discover_tests, TestFilter
 from executorch.backends.test.suite.reporting import (
     begin_test_session,
@@ -20,17 +21,19 @@
 
 # A list of all runnable test suites and the corresponding python package.
 NAMED_SUITES = {
+    "models": "executorch.backends.test.suite.models",
     "operators": "executorch.backends.test.suite.operators",
 }
 
 
 def run_test(  # noqa: C901
     model: torch.nn.Module,
-    inputs: any,
+    inputs: Any,
     tester_factory: Callable[[], Tester],
     test_name: str,
     flow_name: str,
     params: dict | None,
+    dynamic_shapes: Any | None = None,
 ) -> TestCaseSummary:
     """
     Top-level test run function for a model, input set, and tester. Handles test execution
@@ -61,7 +64,10 @@ def build_result(
         return build_result(TestResult.UNKNOWN_FAIL, e)
 
     try:
-        tester.export()
+        # TODO Use Tester dynamic_shapes parameter once input generation can properly handle derived dims.
+        tester.export(
+            tester._get_default_stage(StageType.EXPORT, dynamic_shapes=dynamic_shapes),
+        )
     except Exception as e:
         return build_result(TestResult.EXPORT_FAIL, e)
 

From dc12b40463afd520e2a9f5edc027b29c139d9a60 Mon Sep 17 00:00:00 2001
From: Gregory James Comer <gjcomer@meta.com>
Date: Sun, 20 Jul 2025 18:36:51 -0700
Subject: [PATCH 4/6] Update

[ghstack-poisoned]
---
 backends/test/suite/models/__init__.py        | 12 ++-
 backends/test/suite/models/test_torchaudio.py | 81 +++++++++++++++++++
 backends/test/suite/runner.py                 |  2 +
 3 files changed, 93 insertions(+), 2 deletions(-)
 create mode 100644 backends/test/suite/models/test_torchaudio.py

diff --git a/backends/test/suite/models/__init__.py b/backends/test/suite/models/__init__.py
index 496bcb6f194..278423353ea 100644
--- a/backends/test/suite/models/__init__.py
+++ b/backends/test/suite/models/__init__.py
@@ -67,8 +67,9 @@ def _expand_test(cls, test_name: str) -> None:
     test_func = getattr(cls, test_name)
     supports_dynamic_shapes = getattr(test_func, "supports_dynamic_shapes", True)
     dynamic_shape_values = [True, False] if supports_dynamic_shapes else [False]
+    dtypes = getattr(test_func, "dtypes", DTYPES)
 
-    for flow, dtype, use_dynamic_shapes in itertools.product(get_test_flows(), DTYPES, dynamic_shape_values):
+    for flow, dtype, use_dynamic_shapes in itertools.product(get_test_flows(), dtypes, dynamic_shape_values):
         _create_test(cls, test_func, flow, dtype, use_dynamic_shapes)
     delattr(cls, test_name)
 
@@ -81,10 +82,17 @@ def model_test_cls(cls) -> Callable | None:
     return cls
 
 
-def model_test_params(supports_dynamic_shapes: bool) -> Callable:
+def model_test_params(
+    supports_dynamic_shapes: bool = True,
+    dtypes: list[torch.dtype] | None = None,
+) -> Callable:
     """ Optional parameter decorator for model tests. Specifies test pararameters. Only valid with a class decorated by model_test_cls. """
     def inner_decorator(func: Callable) -> Callable:
         setattr(func, "supports_dynamic_shapes", supports_dynamic_shapes)
+        
+        if dtypes is not None:
+            setattr(func, "dtypes", dtypes)
+
         return func
     return inner_decorator
 
diff --git a/backends/test/suite/models/test_torchaudio.py b/backends/test/suite/models/test_torchaudio.py
new file mode 100644
index 00000000000..620dbae07f0
--- /dev/null
+++ b/backends/test/suite/models/test_torchaudio.py
@@ -0,0 +1,81 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# pyre-unsafe
+
+import torch
+import torchaudio
+import unittest
+
+from executorch.backends.test.suite.models import model_test_params, model_test_cls, run_model_test
+from torch.export import Dim
+from typing import Callable, Tuple
+
+#
+# This file contains model integration tests for supported torchaudio models.
+# 
+
+class PatchedConformer(torch.nn.Module):
+    """
+    A lightly modified version of the top-level Conformer module, such that it can be exported.
+    Instead of taking lengths and computing the padding mask, it takes the padding mask directly.
+    See https://github.com/pytorch/audio/blob/main/src/torchaudio/models/conformer.py#L215
+    """
+
+    def __init__(self, conformer):
+        super().__init__()
+        self.conformer = conformer
+
+    def forward(self, input: torch.Tensor, encoder_padding_mask: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        x = input.transpose(0, 1)
+        for layer in self.conformer.conformer_layers:
+            x = layer(x, encoder_padding_mask)
+        return x.transpose(0, 1)
+
+@model_test_cls
+class TorchAudio(unittest.TestCase):
+    @model_test_params(dtypes=[torch.float32], supports_dynamic_shapes=False)
+    def test_conformer(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        inner_model = torchaudio.models.Conformer(
+            input_dim=80,
+            num_heads=4,
+            ffn_dim=128,
+            num_layers=4,
+            depthwise_conv_kernel_size=31,
+        )
+        model = PatchedConformer(inner_model)
+        lengths = torch.randint(1, 400, (10,))
+        
+        encoder_padding_mask = torchaudio.models.conformer._lengths_to_padding_mask(lengths)
+        inputs = (
+            torch.rand(10, int(lengths.max()), 80),
+            encoder_padding_mask,
+        )
+        
+        run_model_test(model, inputs, dtype, None, tester_factory)
+
+    @model_test_params(dtypes=[torch.float32])
+    def test_wav2letter(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchaudio.models.Wav2Letter()
+        inputs = (torch.randn(1, 1, 1024, dtype=dtype),)
+        dynamic_shapes = {
+            "x": {
+                2: Dim("d", min=900, max=1024),
+            }
+        } if use_dynamic_shapes else None
+        run_model_test(model, inputs, dtype, dynamic_shapes, tester_factory)
+        
+    @unittest.skip("This model times out on all backends.")
+    def test_wavernn(self, dtype: torch.dtype, use_dynamic_shapes: bool, tester_factory: Callable):
+        model = torchaudio.models.WaveRNN(upsample_scales=[5,5,8], n_classes=512, hop_length=200).eval()
+
+        # See https://docs.pytorch.org/audio/stable/generated/torchaudio.models.WaveRNN.html#forward
+        inputs = (
+            torch.randn(1, 1, (64 - 5 + 1) * 200), # waveform
+            torch.randn(1, 1, 128, 64), # specgram
+        )
+
+        run_model_test(model, inputs, dtype, None, tester_factory)
diff --git a/backends/test/suite/runner.py b/backends/test/suite/runner.py
index 09554521d41..064ead2a9ba 100644
--- a/backends/test/suite/runner.py
+++ b/backends/test/suite/runner.py
@@ -51,6 +51,8 @@ def build_result(
             result=result,
             error=error,
         )
+    
+    model.eval()
 
     # Ensure the model can run in eager mode.
     try:

From ead0616cc34a5db81f1c8bc7e998cfad8c0b00dd Mon Sep 17 00:00:00 2001
From: Gregory James Comer <gjcomer@meta.com>
Date: Mon, 21 Jul 2025 17:43:47 -0700
Subject: [PATCH 5/6] Update

[ghstack-poisoned]
---
 backends/test/suite/__init__.py  | 22 +++++++++++-----------
 backends/test/suite/discovery.py |  2 +-
 backends/test/suite/flow.py      |  7 +++----
 3 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/backends/test/suite/__init__.py b/backends/test/suite/__init__.py
index cf73a7bdd0c..86cb5a5716f 100644
--- a/backends/test/suite/__init__.py
+++ b/backends/test/suite/__init__.py
@@ -12,7 +12,7 @@
 import unittest
 
 from enum import Enum
-from typing import Callable, Sequence, Sequence
+from typing import Callable
 
 import executorch.backends.test.suite.flow
 
@@ -46,18 +46,18 @@ def is_backend_enabled(backend):
         return backend in _ENABLED_BACKENDS
 
 
-_ALL_TEST_FLOWS: Sequence[TestFlow] | None = None
+_ALL_TEST_FLOWS: dict[str, TestFlow] = {}
 
 
-def get_test_flows() -> Sequence[TestFlow]:
+def get_test_flows() -> dict[str, TestFlow]:
     global _ALL_TEST_FLOWS
 
-    if _ALL_TEST_FLOWS is None:
-        _ALL_TEST_FLOWS = [
-            f
-            for f in executorch.backends.test.suite.flow.all_flows()
+    if not _ALL_TEST_FLOWS:
+        _ALL_TEST_FLOWS = {
+            name: f
+            for name, f in executorch.backends.test.suite.flow.all_flows().items()
             if is_backend_enabled(f.backend)
-        ]
+        }
 
     return _ALL_TEST_FLOWS
 
@@ -115,7 +115,7 @@ def _create_tests(cls):
 # Expand a test into variants for each registered flow.
 def _expand_test(cls, test_name: str):
     test_func = getattr(cls, test_name)
-    for flow in get_test_flows():
+    for flow in get_test_flows().values():
         _create_test_for_backend(cls, test_func, flow)
     delattr(cls, test_name)
 
@@ -133,8 +133,8 @@ def wrapped_test(self):
 
             test_func(self, **test_kwargs)
 
-    setattr(wrapped_test, "_name", test_name)
-    setattr(wrapped_test, "_flow", flow)
+    wrapped_test._name = test_name
+    wrapped_test._flow = flow
 
     return wrapped_test
 
diff --git a/backends/test/suite/discovery.py b/backends/test/suite/discovery.py
index 5abd194cbcd..e7af0d0923d 100644
--- a/backends/test/suite/discovery.py
+++ b/backends/test/suite/discovery.py
@@ -57,7 +57,7 @@ def _is_test_enabled(test_case: unittest.TestCase, backends: set[str] | None) ->
     test_method = getattr(test_case, test_case._testMethodName)
 
     if backends is not None:
-        flow: TestFlow = getattr(test_method, "_flow")
+        flow: TestFlow = test_method._flow
         return flow.backend in backends
     else:
         return True
diff --git a/backends/test/suite/flow.py b/backends/test/suite/flow.py
index 4410d382401..bda85a76ffa 100644
--- a/backends/test/suite/flow.py
+++ b/backends/test/suite/flow.py
@@ -1,8 +1,7 @@
 import logging
 
 from dataclasses import dataclass
-from math import log
-from typing import Callable, Sequence
+from typing import Callable
 
 from executorch.backends.test.harness import Tester
 
@@ -55,9 +54,9 @@ def create_coreml_flow() -> TestFlow | None:
         return None
 
 
-def all_flows() -> Sequence[TestFlow]:
+def all_flows() -> dict[str, TestFlow]:
     flows = [
         create_xnnpack_flow(),
         create_coreml_flow(),
     ]
-    return [f for f in flows if f is not None]
+    return {f.name: f for f in flows if f is not None}

From 81dfb07dda684312018b2d2562ff911d9e74c3d1 Mon Sep 17 00:00:00 2001
From: Gregory James Comer <gjcomer@meta.com>
Date: Wed, 23 Jul 2025 13:07:15 -0700
Subject: [PATCH 6/6] Update

[ghstack-poisoned]
---
 backends/test/suite/models/test_torchaudio.py  | 4 +++-
 backends/test/suite/models/test_torchvision.py | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/backends/test/suite/models/test_torchaudio.py b/backends/test/suite/models/test_torchaudio.py
index ac1bc21a526..5d526fe708e 100644
--- a/backends/test/suite/models/test_torchaudio.py
+++ b/backends/test/suite/models/test_torchaudio.py
@@ -20,7 +20,9 @@
 from torch.export import Dim
 
 #
-# This file contains model integration tests for supported torchaudio models.
+# This file contains model integration tests for supported torchaudio models. As many torchaudio
+# models are not export-compatible, this suite contains a subset of the available models and may
+# grow over time.
 #
 
 
diff --git a/backends/test/suite/models/test_torchvision.py b/backends/test/suite/models/test_torchvision.py
index faa4212e1c4..2ef864ef42c 100644
--- a/backends/test/suite/models/test_torchvision.py
+++ b/backends/test/suite/models/test_torchvision.py
@@ -20,7 +20,9 @@
 from torch.export import Dim
 
 #
-# This file contains model integration tests for supported torchvision models.
+# This file contains model integration tests for supported torchvision models. This
+# suite intends to include all export-compatible torchvision models. For models with
+# multiple size variants, one small or medium variant is used.
 #