[Backend Tester] Clean up a few test issues

backends/test/harness/stages/to_edge_transform_and_lower.py

@@ -23,7 +23,9 @@ def __init__(
             if default_partitioner_cls is not None
             else []
         )
-        self.edge_compile_conf = edge_compile_config or EdgeCompileConfig()
+        self.edge_compile_conf = edge_compile_config or EdgeCompileConfig(
+            _check_ir_validity=False
+        )
         self.edge_dialect_program = None
 
     def stage_type(self) -> StageType:

backends/test/suite/operators/test_amax.py

@@ -207,39 +207,39 @@ def test_amax_edge_cases(self, flow: TestFlow) -> None:
             AmaxModel(),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             AmaxModel(dim=0),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             AmaxModel(dim=1),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
 
         x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]])
         self._test_op(
             AmaxModel(),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             AmaxModel(dim=0),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             AmaxModel(dim=1),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
 
     def test_amax_scalar(self, flow: TestFlow) -> None:

backends/test/suite/operators/test_amin.py

@@ -209,39 +209,39 @@ def test_amin_edge_cases(self, flow: TestFlow) -> None:
             AminModel(),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             AminModel(dim=0),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             AminModel(dim=1),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
 
         x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]])
         self._test_op(
             AminModel(),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             AminModel(dim=0),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             AminModel(dim=1),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
 
     def test_amin_scalar(self, flow: TestFlow) -> None:

backends/test/suite/operators/test_argmax.py

@@ -149,39 +149,39 @@ def test_argmax_edge_cases(self, flow: TestFlow) -> None:
             ArgmaxModel(),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             ArgmaxModel(dim=0),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             ArgmaxModel(dim=1),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
 
         x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]])
         self._test_op(
             ArgmaxModel(),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             ArgmaxModel(dim=0),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             ArgmaxModel(dim=1),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
 
         x = torch.tensor([5.0])

backends/test/suite/operators/test_argmin.py

@@ -149,39 +149,39 @@ def test_argmin_edge_cases(self, flow: TestFlow) -> None:
             ArgminModel(),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             ArgminModel(dim=0),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             ArgminModel(dim=1),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
 
         x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]])
         self._test_op(
             ArgminModel(),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             ArgminModel(dim=0),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
         self._test_op(
             ArgminModel(dim=1),
             (x,),
             flow,
-            use_random_test_inputs=False,
+            generate_random_test_inputs=False,
         )
 
         x = torch.tensor([5.0])

backends/test/suite/operators/test_floor.py

@@ -18,8 +18,8 @@
 
 
 class FloorModel(torch.nn.Module):
-    def __init__(self):
-        super().__init__()
+    def forward(self, x):
+        return torch.floor(x)
 
 
 @operator_test

backends/test/suite/reporting.py

@@ -28,35 +28,32 @@ class TestResult(IntEnum):
     SUCCESS_UNDELEGATED = 1
     """ The test succeeded without the backend delegating anything. """
 
-    EAGER_FAIL = 2
-    """ The test failed due to the model failing to run in eager mode. """
+    SKIPPED = 2
+    """ The test was skipped due to a non-backend failure. """
 
     QUANTIZE_FAIL = 3
     """ The test failed due to the quantization stage failing. """
 
-    EXPORT_FAIL = 4
-    """ The test failed due to the model failing to export. """
-
-    LOWER_FAIL = 5
+    LOWER_FAIL = 4
     """ The test failed due to a failure in partitioning or lowering. """
 
-    PTE_LOAD_FAIL = 6
+    PTE_LOAD_FAIL = 5
     """ The test failed due to the resulting PTE failing to load. """
 
-    PTE_RUN_FAIL = 7
+    PTE_RUN_FAIL = 6
     """ The test failed due to the resulting PTE failing to run. """
 
-    OUTPUT_MISMATCH_FAIL = 8
+    OUTPUT_MISMATCH_FAIL = 7
     """ The test failed due to a mismatch between runtime and reference outputs. """
 
-    UNKNOWN_FAIL = 9
+    UNKNOWN_FAIL = 8
     """ The test failed in an unknown or unexpected manner. """
 
     def is_success(self):
         return self in {TestResult.SUCCESS, TestResult.SUCCESS_UNDELEGATED}
 
     def is_non_backend_failure(self):
-        return self in {TestResult.EAGER_FAIL, TestResult.EAGER_FAIL}
+        return self in {TestResult.SKIPPED}
 
     def is_backend_failure(self):
         return not self.is_success() and not self.is_non_backend_failure()
@@ -66,12 +63,10 @@ def display_name(self):
             return "Success (Delegated)"
         elif self == TestResult.SUCCESS_UNDELEGATED:
             return "Success (Undelegated)"
-        elif self == TestResult.EAGER_FAIL:
-            return "Fail (Eager)"
+        elif self == TestResult.SKIPPED:
+            return "Skipped"
         elif self == TestResult.QUANTIZE_FAIL:
             return "Fail (Quantize)"
-        elif self == TestResult.EXPORT_FAIL:
-            return "Fail (Export)"
         elif self == TestResult.LOWER_FAIL:
             return "Fail (Lowering)"
         elif self == TestResult.PTE_LOAD_FAIL:

backends/test/suite/runner.py

@@ -9,6 +9,14 @@
 
 import torch
 
+# Set of unsupported ops that should cause tests to be skipped
+UNSUPPORTED_PORTABLE_OPS = {
+    "aten::_embedding_bag",
+    "aten::median",
+    "aten::median.dim",
+    "aten::round.decimals",
+}
+
 from executorch.backends.test.harness.error_statistics import ErrorStatistics
 from executorch.backends.test.harness.stages import StageType
 from executorch.backends.test.suite.discovery import discover_tests, TestFilter
@@ -70,7 +78,7 @@ def build_result(
     try:
         model(*inputs)
     except Exception as e:
-        return build_result(TestResult.EAGER_FAIL, e)
+        return build_result(TestResult.SKIPPED, e)
 
     try:
         tester = flow.tester_factory(model, inputs)
@@ -96,7 +104,7 @@ def build_result(
             tester._get_default_stage(StageType.EXPORT, dynamic_shapes=dynamic_shapes),
         )
     except Exception as e:
-        return build_result(TestResult.EXPORT_FAIL, e)
+        return build_result(TestResult.SKIPPED, e)
 
     lower_start_time = time.perf_counter()
     try:
@@ -125,7 +133,16 @@ def build_result(
         if n.op == "call_function"
     )
 
-    # Only run the runtime portion if something was delegated (or the flow doesn't delegate).
+    # Check if any undelegated ops are in the unsupported ops set.
+    has_unsupported_ops = any(
+        op in UNSUPPORTED_PORTABLE_OPS for op in undelegated_op_counts.keys()
+    )
+
+    # Skip the test if there are unsupported portable ops remaining.
+    if has_unsupported_ops:
+        return build_result(TestResult.SKIPPED)
+
+    # Only run the runtime portion if something was delegated (or the flow doesn't delegate)
     if is_delegated or not flow.is_delegated:
         try:
             tester.to_executorch().serialize()
@@ -142,12 +159,15 @@ def build_result(
             tester.run_method_and_compare_outputs(
                 inputs=None if generate_random_test_inputs else inputs,
                 statistics_callback=lambda stats: error_statistics.append(stats),
+                atol=1e-1,
+                rtol=4e-2,
             )
         except AssertionError as e:
             return build_result(TestResult.OUTPUT_MISMATCH_FAIL, e)
         except Exception as e:
             return build_result(TestResult.PTE_RUN_FAIL, e)
     else:
+        # Skip the test if nothing is delegated
         return build_result(TestResult.SUCCESS_UNDELEGATED)
 
     return build_result(TestResult.SUCCESS)

backends/test/suite/tests/test_reporting.py

@@ -54,7 +54,7 @@
         flow="flow1",
         name="test2_backend2_flow1",
         params={"use_dynamic_shapes": True},
-        result=TestResult.EXPORT_FAIL,
+        result=TestResult.SKIPPED,
         error=None,
         tensor_error_statistics=[],
     ),
@@ -108,7 +108,7 @@ def test_csv_report_simple(self):
         self.assertEqual(records[3]["Test Case"], "test2")
         self.assertEqual(records[3]["Backend"], "backend2")
         self.assertEqual(records[3]["Flow"], "flow1")
-        self.assertEqual(records[3]["Result"], "Fail (Export)")
+        self.assertEqual(records[3]["Result"], "Skipped")
         self.assertEqual(records[3]["Dtype"], "")
         self.assertEqual(records[3]["Use_dynamic_shapes"], "True")