Implement specified timeout for slow doctests

Author: user202729

src/sage/doctest/forker.py

@@ -517,6 +517,52 @@ def getvalue(self):
 TestResults = namedtuple('TestResults', 'failed attempted')
 
 
+def _parse_example_timeout(source: str, default_timeout: float) -> float:
+    """
+    Parse the timeout value from a doctest example's source.
+
+    INPUT:
+
+    - ``source`` -- the source code of a ``doctest.Example``
+    - ``default_timeout`` -- the default timeout value to use
+
+    OUTPUT:
+
+    - a float, the timeout value to use for the example
+
+    TESTS::
+
+        sage: from sage.doctest.forker import _parse_example_timeout
+        sage: _parse_example_timeout("sleep(10)  # long time (10s)", 5.0r)
+        10.0
+        sage: _parse_example_timeout("sleep(10)  # long time", 5.0r)
+        5.0
+        sage: _parse_example_timeout("sleep(10)  # long time (1a2s)", 5.0r)
+        Traceback (most recent call last):
+        ...
+        ValueError: malformed optional tag '# long time (1a2s)', should be <number>s
+        sage: _parse_example_timeout("sleep(10)  # long time (:issue:`12345`)", 5.0r)
+        5.0
+    """
+    # TODO this double-parsing is inefficient, should make :meth:`SageDocTestParser.parse`
+    # return subclass of doctest.Example that already include the timeout value
+    from sage.doctest.parsing import parse_optional_tags
+    value = parse_optional_tags(source).get("long time", None)
+    if value is None:
+        # either has the "long time" tag without any value in parentheses,
+        # or tag not present
+        return default_timeout
+    assert isinstance(value, str)
+    match = re.fullmatch(r'(\S*\d)\s*s', value.strip())
+    if match:
+        try:
+            return float(match[1])
+        except ValueError:
+            raise ValueError(f"malformed optional tag '# long time ({value})', should be <number>s")
+    else:
+        return default_timeout
+
+
 class SageDocTestRunner(doctest.DocTestRunner):
     def __init__(self, *args, **kwds):
         """
@@ -582,6 +628,8 @@ def _run(self, test, compileflags, out):
         Since it needs to be able to read stdout, it should be called
         while spoofing using :class:`SageSpoofInOut`.
 
+        INPUT: see :meth:`run`.
+
         EXAMPLES::
 
             sage: from sage.doctest.parsing import SageOutputChecker
@@ -627,6 +675,7 @@ def _run(self, test, compileflags, out):
         check = self._checker.check_output
 
         # Process each example.
+        example: doctest.Example
         for examplenum, example in enumerate(test.examples):
             if failures:
                 # If exitfirst is set, abort immediately after a
@@ -816,8 +865,10 @@ def compiler(example):
             if example.warnings:
                 for warning in example.warnings:
                     out(self._failure_header(test, example, f'Warning: {warning}'))
+
             if outcome is SUCCESS:
-                if self.options.warn_long > 0 and example.cputime + check_timer.cputime > self.options.warn_long:
+                if self.options.warn_long > 0 and example.cputime + check_timer.cputime > _parse_example_timeout(
+                        example.source, self.options.warn_long):
                     self.report_overtime(out, test, example, got,
                                          check_timer=check_timer)
                 elif example.warnings:

src/sage/doctest/parsing.py

@@ -829,7 +829,7 @@ def __ne__(self, other):
         """
         return not (self == other)
 
-    def parse(self, string, *args):
+    def parse(self, string, *args) -> list[doctest.Example | str]:
         r"""
         A Sage specialization of :class:`doctest.DocTestParser`.
 
@@ -1015,8 +1015,8 @@ def parse(self, string, *args):
             string = find_python_continuation.sub(r"\1" + ellipsis_tag + r"\2", string)
         string = find_sage_prompt.sub(r"\1>>> sage: ", string)
         string = find_sage_continuation.sub(r"\1...", string)
-        res = doctest.DocTestParser.parse(self, string, *args)
-        filtered = []
+        res: list[doctest.Example | str] = doctest.DocTestParser.parse(self, string, *args)
+        filtered: list[doctest.Example | str] = []
         persistent_optional_tags = self.file_optional_tags
         persistent_optional_tag_setter = None
         persistent_optional_tag_setter_index = None

src/sage/doctest/sources.py

@@ -193,7 +193,7 @@ def __ne__(self, other):
         """
         return not (self == other)
 
-    def _process_doc(self, doctests, doc, namespace, start):
+    def _process_doc(self, doctests: list[doctest.DocTest], doc, namespace, start):
         """
         Appends doctests defined in ``doc`` to the list ``doctests``.
 
@@ -266,7 +266,7 @@ def file_optional_tags(self):
         """
         return set()
 
-    def _create_doctests(self, namespace, tab_okay=None):
+    def _create_doctests(self, namespace, tab_okay=None) -> tuple[list[doctest.DocTest], dict]:
         """
         Create a list of doctests defined in this source.
 
@@ -314,7 +314,7 @@ def _create_doctests(self, namespace, tab_okay=None):
                                         probed_tags=self.options.probe,
                                         file_optional_tags=self.file_optional_tags)
         self.linking = False
-        doctests = []
+        doctests: list[doctest.DocTest] = []
         in_docstring = False
         unparsed_doc = False
         doc = []
@@ -480,7 +480,7 @@ def __iter__(self):
         for lineno, line in enumerate(self.source.split('\n')):
             yield lineno + self.lineno_shift, line + '\n'
 
-    def create_doctests(self, namespace):
+    def create_doctests(self, namespace) -> tuple[list[doctest.DocTest], dict]:
         r"""
         Create doctests from this string.
 
@@ -492,8 +492,8 @@ def create_doctests(self, namespace):
 
         - ``doctests`` -- list of doctests defined by this string
 
-        - ``tab_locations`` -- either ``False`` or a list of linenumbers
-          on which tabs appear
+        - ``extras`` -- dictionary with ``extras['tab']`` either
+          ``False`` or a list of linenumbers on which tabs appear
 
         EXAMPLES::
 
@@ -503,10 +503,12 @@ def create_doctests(self, namespace):
             sage: s = "'''\n    sage: 2 + 2\n    4\n'''"
             sage: PythonStringSource = dynamic_class('PythonStringSource',(StringDocTestSource, PythonSource))
             sage: PSS = PythonStringSource('<runtime>', s, DocTestDefaults(), 'runtime')
-            sage: dt, tabs = PSS.create_doctests({})
+            sage: dt, extras = PSS.create_doctests({})
             sage: for t in dt:
             ....:     print("{} {}".format(t.name, t.examples[0].sage_source))
             <runtime> 2 + 2
+            sage: extras
+            {...'tab': []...}
         """
         return self._create_doctests(namespace)
 
@@ -736,7 +738,7 @@ def file_optional_tags(self):
         from .parsing import parse_file_optional_tags
         return parse_file_optional_tags(self)
 
-    def create_doctests(self, namespace):
+    def create_doctests(self, namespace) -> tuple[list[doctest.DocTest], dict]:
         r"""
         Return a list of doctests for this file.
 
@@ -910,7 +912,7 @@ class SourceLanguage:
 
     Currently supported languages include Python, ReST and LaTeX.
     """
-    def parse_docstring(self, docstring, namespace, start):
+    def parse_docstring(self, docstring, namespace, start) -> list[doctest.DocTest]:
         """
         Return a list of doctest defined in this docstring.