pythongh-108851: Fix tomllib recursion tests

* sys.setrecursionlimit() now allows setting the limit to the current
  recursion depth.
* Add get_recursion_available() and get_recursion_depth() functions
  to the test.support module.
* Fix test_tomllib recursion tests: reduce the limit to fix tests on
  WASI buildbots and compute the maximum nested array/dict depending
  on the current available recursion limit.

Author: vstinner

Lib/test/support/__init__.py

@@ -2242,6 +2242,34 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds):
     msg = f"cannot create '{re.escape(qualname)}' instances"
     testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds)
 
+def get_recursion_depth():
+    try:
+        import _testinternalcapi
+    except ImportError:
+        _testinternalcapi = None
+
+    if _testinternalcapi is not None:
+        depth = _testinternalcapi.get_recursion_depth()
+    else:
+        # Implementation using sys._getframe() and frame.f_back
+        try:
+            depth = 0
+            frame = sys._getframe()
+            while frame is not None:
+                depth += 1
+                frame = frame.f_back
+        finally:
+            # Break reference cycle
+            frame = None
+
+    # Ignore get_recursion_depth() frame
+    return max(depth - 1, 1)
+
+def get_recursion_available():
+    limit = sys.getrecursionlimit()
+    depth = get_recursion_depth()
+    return limit - depth
+
 @contextlib.contextmanager
 def set_recursion_limit(limit):
     """Temporarily change the recursion limit."""
@@ -2252,14 +2280,16 @@ def set_recursion_limit(limit):
     finally:
         sys.setrecursionlimit(original_limit)
 
-def infinite_recursion(max_depth=75):
+def infinite_recursion(max_depth=100):
     """Set a lower limit for tests that interact with infinite recursions
     (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some
     debug windows builds, due to not enough functions being inlined the
     stack size might not handle the default recursion limit (1000). See
     bpo-11105 for details."""
-    return set_recursion_limit(max_depth)
-
+    if max_depth < 1:
+        raise ValueError("max_depth must be at least 1")
+    limit = get_recursion_depth() + max_depth
+    return set_recursion_limit(limit)
 
 def ignore_deprecations_from(module: str, *, like: str) -> object:
     token = object()

Lib/test/test_support.py

@@ -685,6 +685,64 @@ def test_has_strftime_extensions(self):
         else:
             self.assertTrue(support.has_strftime_extensions)
 
+    def test_get_recursion_depth(self):
+        # test support.get_recursion_depth()
+        code = textwrap.dedent("""
+            from test import support
+            import sys
+
+            def check(cond):
+                if not cond:
+                    raise AssertionError("test failed")
+
+            check(support.get_recursion_depth() == 1)
+
+            def test_func():
+                check(support.get_recursion_depth() == 2)
+            test_func()
+
+            def test_recursive(depth, limit):
+                if depth >= limit:
+                    return
+                get_depth = support.get_recursion_depth()
+                print(f"test_recursive: {depth}/{limit}: get_recursion_depth() says {get_depth}")
+                check(get_depth == depth)
+                test_recursive(depth + 1, limit)
+
+            with support.infinite_recursion():
+                limit = sys.getrecursionlimit()
+                test_recursive(2, limit)
+        """)
+        script_helper.assert_python_ok("-c", code)
+
+    def test_recursion(self):
+        # test infinite_recursion() and get_recursion_available() functions
+        def recursive_function(depth):
+            if depth:
+                recursive_function(depth - 1)
+
+        # test also the bare minimum, max_depth=1: only allow one frame, one
+        # function call!
+        for max_depth in (1, 5, 25, 250):
+            with support.infinite_recursion(max_depth):
+                limit = sys.getrecursionlimit()
+                available = support.get_recursion_available()
+                try:
+                    recursive_function(available)
+                    # avoid self.assertRaises(RecursionError) which consumes
+                    # more than one frame and so raises RecursionError
+                    try:
+                        recursive_function(available + 1)
+                    except RecursionError:
+                        pass
+                    else:
+                        self.fail("RecursionError was not raised")
+                except Exception as exc:
+                    # avoid self.subTest() since it consumes more
+                    # than one frame and so raises RecursionError!
+                    raise AssertionError(f"test failed with {max_depth=}, "
+                                         f"{limit=}, {available=}") from exc
+
     # XXX -follows a list of untested API
     # make_legacy_pyc
     # is_resource_enabled

Lib/test/test_sys.py

@@ -279,20 +279,29 @@ def test_switchinterval(self):
         finally:
             sys.setswitchinterval(orig)
 
-    def test_recursionlimit(self):
+    def test_getrecursionlimit(self):
+        limit = sys.getrecursionlimit()
+        self.assertIsInstance(limit, int)
+        self.assertGreater(limit, 1)
+
         self.assertRaises(TypeError, sys.getrecursionlimit, 42)
-        oldlimit = sys.getrecursionlimit()
-        self.assertRaises(TypeError, sys.setrecursionlimit)
-        self.assertRaises(ValueError, sys.setrecursionlimit, -42)
-        sys.setrecursionlimit(10000)
-        self.assertEqual(sys.getrecursionlimit(), 10000)
-        sys.setrecursionlimit(oldlimit)
+
+    def test_setrecursionlimit(self):
+        old_limit = sys.getrecursionlimit()
+        try:
+            sys.setrecursionlimit(10000)
+            self.assertEqual(sys.getrecursionlimit(), 10000)
+
+            self.assertRaises(TypeError, sys.setrecursionlimit)
+            self.assertRaises(ValueError, sys.setrecursionlimit, -42)
+        finally:
+            sys.setrecursionlimit(old_limit)
 
     def test_recursionlimit_recovery(self):
         if hasattr(sys, 'gettrace') and sys.gettrace():
             self.skipTest('fatal error if run with a trace function')
 
-        oldlimit = sys.getrecursionlimit()
+        old_limit = sys.getrecursionlimit()
         def f():
             f()
         try:
@@ -311,35 +320,63 @@ def f():
                 with self.assertRaises(RecursionError):
                     f()
         finally:
-            sys.setrecursionlimit(oldlimit)
+            sys.setrecursionlimit(old_limit)
 
     @test.support.cpython_only
-    def test_setrecursionlimit_recursion_depth(self):
+    def test_setrecursionlimit_recursionerror(self):
         # Issue #25274: Setting a low recursion limit must be blocked if the
         # current recursion depth is already higher than limit.
 
-        from _testinternalcapi import get_recursion_depth
-
-        def set_recursion_limit_at_depth(depth, limit):
-            recursion_depth = get_recursion_depth()
-            if recursion_depth >= depth:
+        def set_recursion_limit_at_depth(desired_depth, limit):
+            recursion_depth = support.get_recursion_depth()
+            if recursion_depth < desired_depth:
+                set_recursion_limit_at_depth(desired_depth, limit)
+            else:
                 with self.assertRaises(RecursionError) as cm:
                     sys.setrecursionlimit(limit)
                 self.assertRegex(str(cm.exception),
                                  "cannot set the recursion limit to [0-9]+ "
                                  "at the recursion depth [0-9]+: "
                                  "the limit is too low")
-            else:
-                set_recursion_limit_at_depth(depth, limit)
 
-        oldlimit = sys.getrecursionlimit()
+        old_limit = sys.getrecursionlimit()
+        try:
+            with support.infinite_recursion(max_depth=500):
+                for set_limit in (10, 25, 100, 250):
+                    # At depth limit+1, sys.setrecursionlimit(limit) must
+                    # raises RecursionError
+                    depth = set_limit + 1
+
+                    # Add details if the test fails
+                    with self.subTest(limit=sys.getrecursionlimit(),
+                                      available=support.get_recursion_depth(),
+                                      set_limit=set_limit):
+                        set_recursion_limit_at_depth(depth, set_limit)
+        finally:
+            sys.setrecursionlimit(old_limit)
+
+    @test.support.cpython_only
+    def test_setrecursionlimit_to_depth(self):
+        def func():
+            pass
+
+        old_limit = sys.getrecursionlimit()
         try:
-            sys.setrecursionlimit(1000)
+            depth = support.get_recursion_depth()
 
-            for limit in (10, 25, 50, 75, 100, 150, 200):
-                set_recursion_limit_at_depth(limit, limit)
+            # gh-108851: Corner case: set the limit to current recursion depth
+            # should be permitted. Calling any Python function raises
+            # RecursionError!  Calling C functions is still ok, like the
+            # sys.setrecursionlimit() call below.
+            sys.setrecursionlimit(depth)
+            try:
+                func()
+            except RecursionError:
+                pass
+            else:
+                self.fail("RecursionError was not raised")
         finally:
-            sys.setrecursionlimit(oldlimit)
+            sys.setrecursionlimit(old_limit)
 
     def test_getwindowsversion(self):
         # Raise SkipTest if sys doesn't have getwindowsversion attribute

Lib/test/test_tomllib/test_misc.py

@@ -9,6 +9,7 @@
 import sys
 import tempfile
 import unittest
+from test import support
 
 from . import tomllib
 
@@ -92,13 +93,23 @@ def test_deepcopy(self):
         self.assertEqual(obj_copy, expected_obj)
 
     def test_inline_array_recursion_limit(self):
-        # 465 with default recursion limit
-        nest_count = int(sys.getrecursionlimit() * 0.465)
-        recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]"
-        tomllib.loads(recursive_array_toml)
+        with support.infinite_recursion(max_depth=100):
+            available = support.get_recursion_available()
+            nest_count = (available // 2) - 2
+            # Add details if the test fails
+            with self.subTest(limit=sys.getrecursionlimit(),
+                              available=available,
+                              nest_count=nest_count):
+                recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]"
+                tomllib.loads(recursive_array_toml)
 
     def test_inline_table_recursion_limit(self):
-        # 310 with default recursion limit
-        nest_count = int(sys.getrecursionlimit() * 0.31)
-        recursive_table_toml = nest_count * "key = {" + nest_count * "}"
-        tomllib.loads(recursive_table_toml)
+        with support.infinite_recursion(max_depth=100):
+            available = support.get_recursion_available()
+            nest_count = (available // 3) - 1
+            # Add details if the test fails
+            with self.subTest(limit=sys.getrecursionlimit(),
+                              available=available,
+                              nest_count=nest_count):
+                recursive_table_toml = nest_count * "key = {" + nest_count * "}"
+                tomllib.loads(recursive_table_toml)

Misc/NEWS.d/next/Library/2023-09-03-21-17-01.gh-issue-108851.XABIWw.rst

@@ -0,0 +1,2 @@
+:func:`sys.setrecursionlimit` now allows setting the limit to the current
+recursion depth. Patch by Victor Stinner.

Misc/NEWS.d/next/Tests/2023-09-03-21-18-35.gh-issue-108851.CCuHyI.rst

@@ -0,0 +1,2 @@
+Add ``get_recursion_available()`` and ``get_recursion_depth()`` functions to
+the :mod:`test.support` module. Patch by Victor Stinner.

Misc/NEWS.d/next/Tests/2023-09-03-21-41-10.gh-issue-108851.xFTYOE.rst

@@ -0,0 +1,3 @@
+Fix ``test_tomllib`` recursion tests: reduce the limit to fix tests on WASI
+buildbots and compute the maximum nested array/dict depending on the current
+available recursion limit. Patch by Victor Stinner.

Python/sysmodule.c

@@ -1271,7 +1271,7 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit)
     /* Reject too low new limit if the current recursion depth is higher than
        the new low-water mark. */
     int depth = tstate->py_recursion_limit - tstate->py_recursion_remaining;
-    if (depth >= new_limit) {
+    if (depth > new_limit) {
         _PyErr_Format(tstate, PyExc_RecursionError,
                       "cannot set the recursion limit to %i at "
                       "the recursion depth %i: the limit is too low",