From 7c6cdbe86970c45f1b1013bc951db5852678c78f Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 13 Jul 2025 19:10:10 +0900 Subject: [PATCH 01/11] NEWS --- .../next/Library/2025-07-13-01-02-31.gh-issue-136421.5HKeoS.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-07-13-01-02-31.gh-issue-136421.5HKeoS.rst diff --git a/Misc/NEWS.d/next/Library/2025-07-13-01-02-31.gh-issue-136421.5HKeoS.rst b/Misc/NEWS.d/next/Library/2025-07-13-01-02-31.gh-issue-136421.5HKeoS.rst new file mode 100644 index 00000000000000..e61114f825a446 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-13-01-02-31.gh-issue-136421.5HKeoS.rst @@ -0,0 +1,2 @@ +Fix crash in :mod:`datetime` when its static types are initialized or +finalized by multiple interpreters concurrently. From c9ad7faf250b6faf9bfc704a3ace927d8fa265ff Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 13 Jul 2025 19:14:30 +0900 Subject: [PATCH 02/11] Add a test --- Lib/test/datetimetester.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 93b3382b9c654e..c4bccb64403984 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7295,6 +7295,27 @@ def test_update_type_cache(self): """) script_helper.assert_python_ok('-c', script) + @unittest.skipIf(_interpreters is None, "missing _interpreters module") + def test_static_type_concurrent_init_fini(self): + # gh-136421 + script = textwrap.dedent(""" + import threading + import _interpreters + + def run(id): + _interpreters.exec(id, "import _datetime; print('a', end='')") + _interpreters.destroy(id) + + ids = [_interpreters.create() for i in range(5)] + ts = [threading.Thread(target=run, args=(id,)) for id in ids] + for t in ts: + t.start() + for t in ts: + t.join() + """) + res = script_helper.assert_python_ok('-c', script) + self.assertEqual(res.out, b'a' * 5) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) From 61f1faf9550e8f43571206677ac0fd709b86fc84 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 13 Jul 2025 19:48:37 +0900 Subject: [PATCH 03/11] Tweak test --- Lib/test/datetimetester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index c4bccb64403984..9246ad2d9220b0 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7306,7 +7306,7 @@ def run(id): _interpreters.exec(id, "import _datetime; print('a', end='')") _interpreters.destroy(id) - ids = [_interpreters.create() for i in range(5)] + ids = [_interpreters.create() for i in range(10)] ts = [threading.Thread(target=run, args=(id,)) for id in ids] for t in ts: t.start() @@ -7314,7 +7314,7 @@ def run(id): t.join() """) res = script_helper.assert_python_ok('-c', script) - self.assertEqual(res.out, b'a' * 5) + self.assertEqual(res.out, b'a' * 10) def load_tests(loader, standard_tests, pattern): From df7ae54c8c776262916b68a198b8be5262acda58 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 13 Jul 2025 20:18:34 +0900 Subject: [PATCH 04/11] ditto --- Lib/test/datetimetester.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9246ad2d9220b0..ca6f9ddf32e880 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7295,26 +7295,21 @@ def test_update_type_cache(self): """) script_helper.assert_python_ok('-c', script) + @support.skip_if_pgo_task @unittest.skipIf(_interpreters is None, "missing _interpreters module") def test_static_type_concurrent_init_fini(self): - # gh-136421 script = textwrap.dedent(""" - import threading - import _interpreters - - def run(id): - _interpreters.exec(id, "import _datetime; print('a', end='')") - _interpreters.destroy(id) - - ids = [_interpreters.create() for i in range(10)] - ts = [threading.Thread(target=run, args=(id,)) for id in ids] - for t in ts: - t.start() - for t in ts: - t.join() - """) - res = script_helper.assert_python_ok('-c', script) - self.assertEqual(res.out, b'a' * 10) + from concurrent.futures import InterpreterPoolExecutor + def func(): + import _datetime + print('a', end='') + with InterpreterPoolExecutor() as executor: + for _ in range(8): + executor.submit(func) + """) + _, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(out, b'a' * 8) + self.assertEqual(err, b'') def load_tests(loader, standard_tests, pattern): From 69762527fdfe5a298c837050b34d0564937cb900 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 13 Jul 2025 20:45:55 +0900 Subject: [PATCH 05/11] ditto --- Lib/test/datetimetester.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index ca6f9ddf32e880..9cf7310ac401f0 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7298,17 +7298,24 @@ def test_update_type_cache(self): @support.skip_if_pgo_task @unittest.skipIf(_interpreters is None, "missing _interpreters module") def test_static_type_concurrent_init_fini(self): + # gh-136421 script = textwrap.dedent(""" - from concurrent.futures import InterpreterPoolExecutor - def func(): - import _datetime - print('a', end='') - with InterpreterPoolExecutor() as executor: - for _ in range(8): - executor.submit(func) - """) - _, out, err = script_helper.assert_python_ok("-c", script) - self.assertEqual(out, b'a' * 8) + import threading + import _interpreters + + def run(id): + _interpreters.exec(id, "import _datetime; print('a', end='')") + _interpreters.destroy(id) + + ids = [_interpreters.create() for i in range(20)] + ts = [threading.Thread(target=run, args=(id,)) for id in ids] + for t in ts: + t.start() + for t in ts: + t.join() + """) + _, out, err = script_helper.assert_python_ok('-c', script) + self.assertEqual(out, b'a' * 20) self.assertEqual(err, b'') From 7d85f55d0a2db84fc64d6e40cc051438854e629f Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 13 Jul 2025 21:09:10 +0900 Subject: [PATCH 06/11] ditto --- Lib/test/datetimetester.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9cf7310ac401f0..e6693edc47e119 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7300,23 +7300,17 @@ def test_update_type_cache(self): def test_static_type_concurrent_init_fini(self): # gh-136421 script = textwrap.dedent(""" - import threading - import _interpreters - - def run(id): - _interpreters.exec(id, "import _datetime; print('a', end='')") - _interpreters.destroy(id) - - ids = [_interpreters.create() for i in range(20)] - ts = [threading.Thread(target=run, args=(id,)) for id in ids] - for t in ts: - t.start() - for t in ts: - t.join() - """) - _, out, err = script_helper.assert_python_ok('-c', script) - self.assertEqual(out, b'a' * 20) - self.assertEqual(err, b'') + from concurrent.futures import InterpreterPoolExecutor + def func(): + import _datetime + print('a', end='') + with InterpreterPoolExecutor(max_workers=10) as executor: + for _ in range(10): + executor.submit(func) + """) + res = script_helper.assert_python_ok("-c", script) + self.assertEqual(res.out, b'a' * 10) + self.assertEqual(res.err, b'') def load_tests(loader, standard_tests, pattern): From 3a6f8449ff8efc24fa2d8a88c1cbf1df3ad5ac75 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 13 Jul 2025 21:38:03 +0900 Subject: [PATCH 07/11] Fix crash --- Modules/_datetimemodule.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 7a6426593d021f..7f76df169f6f13 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7335,6 +7335,11 @@ init_static_types(PyInterpreterState *interp, int reloading) if (reloading) { return 0; } + if (_Py_IsMainInterpreter(interp) + && PyType_HasFeature(&PyDateTime_DateType, Py_TPFLAGS_READY)) { + // This function was already called from PyInit__datetime() + return 0; + } // `&...` is not a constant expression according to a strict reading // of C standards. Fill tp_base at run-time rather than statically. @@ -7564,6 +7569,11 @@ static PyModuleDef datetimemodule = { PyMODINIT_FUNC PyInit__datetime(void) { + PyInterpreterState *interp = PyInterpreterState_Get(); + // gh-136421: Ensure static types are fully finalized at the shutdown of + // the main interpreter rather than subinterpreters for concurrency. + assert(_Py_IsMainInterpreter(interp)); + init_static_types(interp, 0); return PyModuleDef_Init(&datetimemodule); } From c3b3a35f28a59647a299668a350e6756ad34dd05 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 13 Jul 2025 22:04:14 +0900 Subject: [PATCH 08/11] Make test less expensive --- Lib/test/datetimetester.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index e6693edc47e119..10a9a152d59bdb 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7295,22 +7295,27 @@ def test_update_type_cache(self): """) script_helper.assert_python_ok('-c', script) - @support.skip_if_pgo_task @unittest.skipIf(_interpreters is None, "missing _interpreters module") def test_static_type_concurrent_init_fini(self): # gh-136421 script = textwrap.dedent(""" - from concurrent.futures import InterpreterPoolExecutor - def func(): - import _datetime - print('a', end='') - with InterpreterPoolExecutor(max_workers=10) as executor: - for _ in range(10): - executor.submit(func) - """) - res = script_helper.assert_python_ok("-c", script) - self.assertEqual(res.out, b'a' * 10) - self.assertEqual(res.err, b'') + import threading + import _interpreters + + def run(id): + _interpreters.exec(id, "import _datetime; print('a', end='')") + _interpreters.destroy(id) + + ids = [_interpreters.create() for i in range(10)] + ts = [threading.Thread(target=run, args=(id,)) for id in ids] + for t in ts: + t.start() + for t in ts: + t.join() + """) + _, out, err = script_helper.assert_python_ok('-c', script) + self.assertEqual(out, b'a' * 10) + self.assertEqual(err, b'') def load_tests(loader, standard_tests, pattern): From 9cae616e0a3af16d316457b6db3eb943d3a3c1e1 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 14 Jul 2025 09:23:43 +0900 Subject: [PATCH 09/11] Apply suggestions --- Lib/test/datetimetester.py | 4 +++- Modules/_datetimemodule.c | 28 +++++++++++++++------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 10a9a152d59bdb..819163e081dc6d 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7297,7 +7297,9 @@ def test_update_type_cache(self): @unittest.skipIf(_interpreters is None, "missing _interpreters module") def test_static_type_concurrent_init_fini(self): - # gh-136421 + # gh-136421: When a managed static extension type is concurrently + # used by multiple interpreters, there was a crash due to misjudging + # its first initialization stage or last finalization one. script = textwrap.dedent(""" import threading import _interpreters diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 7f76df169f6f13..b02dd089619385 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7335,18 +7335,18 @@ init_static_types(PyInterpreterState *interp, int reloading) if (reloading) { return 0; } - if (_Py_IsMainInterpreter(interp) - && PyType_HasFeature(&PyDateTime_DateType, Py_TPFLAGS_READY)) { - // This function was already called from PyInit__datetime() - return 0; + if (_Py_IsMainInterpreter(interp)) { + if (PyType_HasFeature(&PyDateTime_DateType, Py_TPFLAGS_READY)) { + // This function was already called from PyInit__datetime() + return 0; + } + // `&...` is not a constant expression according to a strict reading + // of C standards. Fill tp_base at run-time rather than statically. + // See https://bugs.python.org/issue40777 + PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; + PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; } - // `&...` is not a constant expression according to a strict reading - // of C standards. Fill tp_base at run-time rather than statically. - // See https://bugs.python.org/issue40777 - PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; - PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - /* Bases classes must be initialized before subclasses, * so capi_types must have the types in the appropriate order. */ for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { @@ -7569,11 +7569,13 @@ static PyModuleDef datetimemodule = { PyMODINIT_FUNC PyInit__datetime(void) { - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); // gh-136421: Ensure static types are fully finalized at the shutdown of // the main interpreter rather than subinterpreters for concurrency. - assert(_Py_IsMainInterpreter(interp)); - init_static_types(interp, 0); + assert(interp != NULL && _Py_IsMainInterpreter(interp)); + if (init_static_types(interp, 0) < 0) { + return NULL; + } return PyModuleDef_Init(&datetimemodule); } From 478a75e525da032ed69bb07511daed17b869e0c2 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 14 Jul 2025 15:10:28 +0900 Subject: [PATCH 10/11] Clarify comments --- Lib/test/datetimetester.py | 7 ++++--- Modules/_datetimemodule.c | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 819163e081dc6d..c446cced99a020 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7297,9 +7297,10 @@ def test_update_type_cache(self): @unittest.skipIf(_interpreters is None, "missing _interpreters module") def test_static_type_concurrent_init_fini(self): - # gh-136421: When a managed static extension type is concurrently - # used by multiple interpreters, there was a crash due to misjudging - # its first initialization stage or last finalization one. + # gh-136421: When a managed static extension type is concurrently used + # by multiple interpreters, there was a crash due to the runtime state + # rather than an interpreter state being updated wrongly by misjudging + # the type's first initialization stage or last finalization one. script = textwrap.dedent(""" import threading import _interpreters diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index b02dd089619385..07196ef0667b70 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7570,8 +7570,8 @@ PyMODINIT_FUNC PyInit__datetime(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - // gh-136421: Ensure static types are fully finalized at the shutdown of - // the main interpreter rather than subinterpreters for concurrency. + // gh-136421: Ensure static types' runtime state gets cleared in shutting + // down the main interpreter rather than subinterpreters for concurrency. assert(interp != NULL && _Py_IsMainInterpreter(interp)); if (init_static_types(interp, 0) < 0) { return NULL; From 790a034a61d36a0a5acd442ada37b8f8fafd3ffc Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 16 Jul 2025 01:19:19 +0900 Subject: [PATCH 11/11] reword --- Lib/test/datetimetester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index c446cced99a020..856d47be479444 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7298,8 +7298,8 @@ def test_update_type_cache(self): @unittest.skipIf(_interpreters is None, "missing _interpreters module") def test_static_type_concurrent_init_fini(self): # gh-136421: When a managed static extension type is concurrently used - # by multiple interpreters, there was a crash due to the runtime state - # rather than an interpreter state being updated wrongly by misjudging + # by only subinterpreters, there was a crash due to the runtime state + # rather than an interpreter state being updated wrongly by mistaking # the type's first initialization stage or last finalization one. script = textwrap.dedent(""" import threading