Frame comparison pixel threshold (#3053)

* implemented threshold for number of pixel value errors

* fix implementation, throw proper warning

* added test for pixel value error threshold

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* remove forgotten debug print

* introduce constants for frame_comparison tolerance

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* rename constant

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: behackl

manim/utils/testing/_frames_testers.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import contextlib
+import warnings
 from pathlib import Path
 
 import numpy as np
@@ -9,6 +10,9 @@
 
 from ._show_diff import show_diff_helper
 
+FRAME_ABSOLUTE_TOLERANCE = 1.01
+FRAME_MISMATCH_RATIO_TOLERANCE = 1e-5
+
 
 class _FramesTester:
     def __init__(self, file_path: Path, show_diff=False) -> None:
@@ -42,12 +46,27 @@ def check_frame(self, frame_number: int, frame: np.ndarray):
             np.testing.assert_allclose(
                 frame,
                 self._frames[frame_number],
-                atol=1.01,
+                atol=FRAME_ABSOLUTE_TOLERANCE,
                 err_msg=f"Frame no {frame_number}. You can use --show_diff to visually show the difference.",
                 verbose=False,
             )
             self._frames_compared += 1
         except AssertionError as e:
+            number_of_matches = np.isclose(
+                frame, self._frames[frame_number], atol=FRAME_ABSOLUTE_TOLERANCE
+            ).sum()
+            number_of_mismatches = frame.size - number_of_matches
+            if number_of_mismatches / frame.size < FRAME_MISMATCH_RATIO_TOLERANCE:
+                # we tolerate a small (< 0.001%) amount of pixel value errors
+                # in the tests, this accounts for minor OS dependent inconsistencies
+                self._frames_compared += 1
+                warnings.warn(
+                    f"Mismatch of {number_of_mismatches} pixel values in frame {frame_number} "
+                    f"against control data in {self._file_path}. Below error threshold, "
+                    "continuing..."
+                )
+                return
+
             if self._show_diff:
                 show_diff_helper(
                     frame_number,

tests/test_graphical_units/control_data/utils/pixel_error_threshold.npz

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from manim.utils.testing.frames_comparison import frames_comparison
+
+__module_test__ = "utils"
+
+
+@frames_comparison
+def test_pixel_error_threshold(scene):
+    """Scene produces black frame, control data has 11 modified pixel values."""
+    pass