From 33fb4a6ee84a3b6637fc117e1474aced8db79ea6 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Thu, 22 Jan 2026 21:26:02 +0800 Subject: [PATCH 01/39] Bump internals version --- include/pybind11/detail/internals.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index d66cf72cc7..e2dee77ec9 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -39,7 +39,7 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 11 +# define PYBIND11_INTERNALS_VERSION 12 #endif #if PYBIND11_INTERNALS_VERSION < 11 From a4a6a1e7e4b31d2e0bfdd2469ff77e5eec9ac5b7 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 23 Jan 2026 01:19:47 +0800 Subject: [PATCH 02/39] Prevent internals destruction before all pybind11 types are destroyed --- include/pybind11/detail/class.h | 22 ++++++++++++++++ include/pybind11/detail/internals.h | 39 ++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 480c369aa6..5d2432d5b8 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -251,6 +251,15 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { } }); + // Release the references to the internals capsules that were acquired in make_new_python_type. + // See the comment there for details on preventing use-after-free during interpreter shutdown. + if (PyObject *capsule = get_internals_capsule()) { + Py_DECREF(capsule); + } + if (PyObject *capsule = get_local_internals_capsule()) { + Py_DECREF(capsule); + } + PyType_Type.tp_dealloc(obj); } @@ -829,6 +838,19 @@ inline PyObject *make_new_python_type(const type_record &rec) { PYBIND11_SET_OLDPY_QUALNAME(type, qualname); + // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so the + // internals capsule may be destroyed (resetting internals via internals_shutdown) before all + // pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls py::cast, it + // would recreate an empty internals and fail because the type registry is gone. By holding + // references to the capsules, we ensure they outlive all pybind11 types. The decref happens + // in pybind11_meta_dealloc. + if (PyObject *capsule = get_internals_capsule()) { + Py_INCREF(capsule); + } + if (PyObject *capsule = get_local_internals_capsule()) { + Py_INCREF(capsule); + } + return reinterpret_cast(type); } diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index e2dee77ec9..619481a5b9 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -777,7 +777,7 @@ class internals_pp_manager { on_fetch_function *on_fetch_ = nullptr; // Pointer-to-pointer to the singleton internals for the first seen interpreter (may not be the // main interpreter) - std::unique_ptr *internals_singleton_pp_; + std::unique_ptr *internals_singleton_pp_ = nullptr; }; // If We loaded the internals through `state_dict`, our `error_already_set` @@ -829,6 +829,35 @@ PYBIND11_NOINLINE internals &get_internals() { return *internals_ptr; } +/// Return the PyObject* for the internals capsule (borrowed reference). +/// Returns nullptr if the capsule doesn't exist yet. +/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types +/// to hold a reference to the capsule (see make_new_python_type in class.h). +inline PyObject *get_internals_capsule() { + auto state_dict = reinterpret_borrow(get_python_state_dict()); + return dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID); +} + +/// Return the key used for local_internals in the state dict. +/// This function ensures a consistent key is used across all call sites within the same +/// compilation unit. The key includes the address of a static variable to make it unique per +/// module (DSO), matching the behavior of get_local_internals_pp_manager(). +inline const std::string &get_local_internals_key() { + static const std::string key + = PYBIND11_MODULE_LOCAL_ID + std::to_string(reinterpret_cast(&key)); + return key; +} + +/// Return the PyObject* for the local_internals capsule (borrowed reference). +/// Returns nullptr if the capsule doesn't exist yet. +/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types +/// to hold a reference to the capsule (see make_new_python_type in class.h). +inline PyObject *get_local_internals_capsule() { + const auto &key = get_local_internals_key(); + auto state_dict = reinterpret_borrow(get_python_state_dict()); + return dict_getitemstring(state_dict.ptr(), key.c_str()); +} + inline void ensure_internals() { pybind11::detail::get_internals_pp_manager().unref(); #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT @@ -840,12 +869,10 @@ inline void ensure_internals() { } inline internals_pp_manager &get_local_internals_pp_manager() { - // Use the address of this static itself as part of the key, so that the value is uniquely tied + // Use the address of a static variable as part of the key, so that the value is uniquely tied // to where the module is loaded in memory - static const std::string this_module_idstr - = PYBIND11_MODULE_LOCAL_ID - + std::to_string(reinterpret_cast(&this_module_idstr)); - return internals_pp_manager::get_instance(this_module_idstr.c_str(), nullptr); + return internals_pp_manager::get_instance(get_local_internals_key().c_str(), + nullptr); } /// Works like `get_internals`, but for things which are locally registered. From 1d490067284c5f5ce2d629ebe5ed6c659fdcc2f3 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 23 Jan 2026 01:36:59 +0800 Subject: [PATCH 03/39] Use Py_XINCREF and Py_XDECREF --- include/pybind11/detail/class.h | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 5d2432d5b8..8d3a49750a 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -251,16 +251,12 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { } }); + PyType_Type.tp_dealloc(obj); + // Release the references to the internals capsules that were acquired in make_new_python_type. // See the comment there for details on preventing use-after-free during interpreter shutdown. - if (PyObject *capsule = get_internals_capsule()) { - Py_DECREF(capsule); - } - if (PyObject *capsule = get_local_internals_capsule()) { - Py_DECREF(capsule); - } - - PyType_Type.tp_dealloc(obj); + Py_XDECREF(get_internals_capsule()); + Py_XDECREF(get_local_internals_capsule()); } /** This metaclass is assigned by default to all pybind11 types and is required in order @@ -844,12 +840,8 @@ inline PyObject *make_new_python_type(const type_record &rec) { // would recreate an empty internals and fail because the type registry is gone. By holding // references to the capsules, we ensure they outlive all pybind11 types. The decref happens // in pybind11_meta_dealloc. - if (PyObject *capsule = get_internals_capsule()) { - Py_INCREF(capsule); - } - if (PyObject *capsule = get_local_internals_capsule()) { - Py_INCREF(capsule); - } + Py_XINCREF(get_internals_capsule()); + Py_XINCREF(get_local_internals_capsule()); return reinterpret_cast(type); } From b147430e467ce8f7b43d3d4cac044f70bca56191 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 23 Jan 2026 02:08:45 +0800 Subject: [PATCH 04/39] Hold GIL before decref --- include/pybind11/detail/internals.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 0f35a9611e..25a5cbe2bf 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -345,6 +345,7 @@ struct internals { // original owning interpreter is active. auto *cur_istate = get_interpreter_state_unchecked(); if (cur_istate && cur_istate == istate) { + gil_scoped_acquire_simple gil; Py_CLEAR(instance_base); Py_CLEAR(default_metaclass); Py_CLEAR(static_property_type); @@ -378,6 +379,7 @@ struct local_internals { // original owning interpreter is active. auto *cur_istate = get_interpreter_state_unchecked(); if (cur_istate && cur_istate == istate) { + gil_scoped_acquire_simple gil; Py_CLEAR(function_record_py_type); } } @@ -877,6 +879,7 @@ inline local_internals &get_local_internals() { auto &ppmgr = get_local_internals_pp_manager(); auto &internals_ptr = *ppmgr.get_pp(); if (!internals_ptr) { + gil_scoped_acquire_simple gil; internals_ptr.reset(new local_internals()); } return *internals_ptr; From 05576f11a7cdffa3c028d7c3a4138da13e29011f Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 23 Jan 2026 03:00:40 +0800 Subject: [PATCH 05/39] Use weakrefs --- include/pybind11/detail/class.h | 20 ++++++-------------- include/pybind11/pybind11.h | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 8d3a49750a..62bb513ba3 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -11,10 +11,16 @@ #include #include +#include +#include #include "exception_translation.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +class weakref; +class cpp_function; + PYBIND11_NAMESPACE_BEGIN(detail) #if !defined(PYPY_VERSION) @@ -252,11 +258,6 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { }); PyType_Type.tp_dealloc(obj); - - // Release the references to the internals capsules that were acquired in make_new_python_type. - // See the comment there for details on preventing use-after-free during interpreter shutdown. - Py_XDECREF(get_internals_capsule()); - Py_XDECREF(get_local_internals_capsule()); } /** This metaclass is assigned by default to all pybind11 types and is required in order @@ -834,15 +835,6 @@ inline PyObject *make_new_python_type(const type_record &rec) { PYBIND11_SET_OLDPY_QUALNAME(type, qualname); - // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so the - // internals capsule may be destroyed (resetting internals via internals_shutdown) before all - // pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls py::cast, it - // would recreate an empty internals and fail because the type registry is gone. By holding - // references to the capsules, we ensure they outlive all pybind11 types. The decref happens - // in pybind11_meta_dealloc. - Py_XINCREF(get_internals_capsule()); - Py_XINCREF(get_local_internals_capsule()); - return reinterpret_cast(type); } diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 02d2e72c2c..2e25c011f3 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1692,6 +1692,30 @@ class generic_type : public object { m_ptr = make_new_python_type(rec); + // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so the + // internals capsule may be destroyed (resetting internals via internals_shutdown) before + // all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls py::cast, + // it would recreate an empty internals and fail because the type registry is gone. By + // holding references to the capsules, we ensure they outlive all pybind11 types. + // We use weakrefs on the type with a cpp_function callback. When the type is destroyed, + // Python will call the callback which releases the capsule reference and the weakref. + if (PyObject *capsule = get_internals_capsule()) { + Py_INCREF(capsule); + (void) weakref(handle(m_ptr), cpp_function([](handle prevent_release) -> void { + Py_XDECREF(get_internals_capsule()); + prevent_release.dec_ref(); + })) + .release(); + } + if (PyObject *capsule = get_local_internals_capsule()) { + Py_INCREF(capsule); + (void) weakref(handle(m_ptr), cpp_function([](handle prevent_release) -> void { + Py_XDECREF(get_local_internals_capsule()); + prevent_release.dec_ref(); + })) + .release(); + } + /* Register supplemental type information in C++ dict */ auto *tinfo = new detail::type_info(); tinfo->type = reinterpret_cast(m_ptr); From 740f69343c44ef2dd8070218e3ba657149b269cb Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 23 Jan 2026 03:01:39 +0800 Subject: [PATCH 06/39] Remove unused code --- include/pybind11/detail/class.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 62bb513ba3..480c369aa6 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -11,16 +11,10 @@ #include #include -#include -#include #include "exception_translation.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) - -class weakref; -class cpp_function; - PYBIND11_NAMESPACE_BEGIN(detail) #if !defined(PYPY_VERSION) From d9227ce1c341d05061c75a374e2da458a0e838ef Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 23 Jan 2026 03:04:08 +0800 Subject: [PATCH 07/39] Move code location --- include/pybind11/pybind11.h | 49 +++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 2e25c011f3..5bba01668f 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1692,30 +1692,6 @@ class generic_type : public object { m_ptr = make_new_python_type(rec); - // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so the - // internals capsule may be destroyed (resetting internals via internals_shutdown) before - // all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls py::cast, - // it would recreate an empty internals and fail because the type registry is gone. By - // holding references to the capsules, we ensure they outlive all pybind11 types. - // We use weakrefs on the type with a cpp_function callback. When the type is destroyed, - // Python will call the callback which releases the capsule reference and the weakref. - if (PyObject *capsule = get_internals_capsule()) { - Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle prevent_release) -> void { - Py_XDECREF(get_internals_capsule()); - prevent_release.dec_ref(); - })) - .release(); - } - if (PyObject *capsule = get_local_internals_capsule()) { - Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle prevent_release) -> void { - Py_XDECREF(get_local_internals_capsule()); - prevent_release.dec_ref(); - })) - .release(); - } - /* Register supplemental type information in C++ dict */ auto *tinfo = new detail::type_info(); tinfo->type = reinterpret_cast(m_ptr); @@ -1745,6 +1721,31 @@ class generic_type : public object { #endif } + // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so + // the internals capsule may be destroyed (resetting internals via internals_shutdown) + // before all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls + // py::cast, it would recreate an empty internals and fail because the type registry is + // gone. By holding references to the capsules, we ensure they outlive all pybind11 + // types. We use weakrefs on the type with a cpp_function callback. When the type is + // destroyed, Python will call the callback which releases the capsule reference and + // the weakref. + if (PyObject *capsule = get_internals_capsule()) { + Py_INCREF(capsule); + (void) weakref(handle(m_ptr), cpp_function([](handle prevent_release) -> void { + Py_XDECREF(get_internals_capsule()); + prevent_release.dec_ref(); + })) + .release(); + } + if (PyObject *capsule = get_local_internals_capsule()) { + Py_INCREF(capsule); + (void) weakref(handle(m_ptr), cpp_function([](handle prevent_release) -> void { + Py_XDECREF(get_local_internals_capsule()); + prevent_release.dec_ref(); + })) + .release(); + } + PYBIND11_WARNING_PUSH #if defined(__GNUC__) && __GNUC__ == 12 // When using GCC 12 these warnings are disabled as they trigger From 7c5d505b736f34043bb276c8640587da61f35fb9 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 23 Jan 2026 03:07:27 +0800 Subject: [PATCH 08/39] Move code location --- include/pybind11/pybind11.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 5bba01668f..67718ea745 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1725,23 +1725,24 @@ class generic_type : public object { // the internals capsule may be destroyed (resetting internals via internals_shutdown) // before all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls // py::cast, it would recreate an empty internals and fail because the type registry is - // gone. By holding references to the capsules, we ensure they outlive all pybind11 - // types. We use weakrefs on the type with a cpp_function callback. When the type is - // destroyed, Python will call the callback which releases the capsule reference and - // the weakref. + // gone. + // + // By holding references to the capsules, we ensure they outlive all pybind11 types. We + // use weakrefs on the type with a cpp_function callback. When the type is destroyed, + // Python will call the callback which releases the capsule reference and the weakref. if (PyObject *capsule = get_internals_capsule()) { Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle prevent_release) -> void { + (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { Py_XDECREF(get_internals_capsule()); - prevent_release.dec_ref(); + wr.dec_ref(); })) .release(); } if (PyObject *capsule = get_local_internals_capsule()) { Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle prevent_release) -> void { + (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { Py_XDECREF(get_local_internals_capsule()); - prevent_release.dec_ref(); + wr.dec_ref(); })) .release(); } From 436d812f76300eb693923816f9bed046e16e915b Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 23 Jan 2026 03:10:24 +0800 Subject: [PATCH 09/39] Move code location --- include/pybind11/detail/internals.h | 4 +-- include/pybind11/pybind11.h | 51 ++++++++++++++--------------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 25a5cbe2bf..db6044aecc 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -831,7 +831,7 @@ PYBIND11_NOINLINE internals &get_internals() { /// Return the PyObject* for the internals capsule (borrowed reference). /// Returns nullptr if the capsule doesn't exist yet. /// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types -/// to hold a reference to the capsule (see make_new_python_type in class.h). +/// to hold a reference to the capsule (see comments in generic_type::initialize). inline PyObject *get_internals_capsule() { auto state_dict = reinterpret_borrow(get_python_state_dict()); return dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID); @@ -850,7 +850,7 @@ inline const std::string &get_local_internals_key() { /// Return the PyObject* for the local_internals capsule (borrowed reference). /// Returns nullptr if the capsule doesn't exist yet. /// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types -/// to hold a reference to the capsule (see make_new_python_type in class.h). +/// to hold a reference to the capsule (see comments in generic_type::initialize). inline PyObject *get_local_internals_capsule() { const auto &key = get_local_internals_key(); auto state_dict = reinterpret_borrow(get_python_state_dict()); diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 67718ea745..4ee4f977d9 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1721,32 +1721,6 @@ class generic_type : public object { #endif } - // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so - // the internals capsule may be destroyed (resetting internals via internals_shutdown) - // before all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls - // py::cast, it would recreate an empty internals and fail because the type registry is - // gone. - // - // By holding references to the capsules, we ensure they outlive all pybind11 types. We - // use weakrefs on the type with a cpp_function callback. When the type is destroyed, - // Python will call the callback which releases the capsule reference and the weakref. - if (PyObject *capsule = get_internals_capsule()) { - Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { - Py_XDECREF(get_internals_capsule()); - wr.dec_ref(); - })) - .release(); - } - if (PyObject *capsule = get_local_internals_capsule()) { - Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { - Py_XDECREF(get_local_internals_capsule()); - wr.dec_ref(); - })) - .release(); - } - PYBIND11_WARNING_PUSH #if defined(__GNUC__) && __GNUC__ == 12 // When using GCC 12 these warnings are disabled as they trigger @@ -1759,6 +1733,31 @@ class generic_type : public object { PYBIND11_WARNING_POP }); + // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so the + // internals capsule may be destroyed (resetting internals via internals_shutdown) before + // all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls py::cast, + // it would recreate an empty internals and fail because the type registry is gone. + // + // By holding references to the capsules, we ensure they outlive all pybind11 types. We use + // weakrefs on the type with a cpp_function callback. When the type is destroyed, Python + // will call the callback which releases the capsule reference and the weakref. + if (PyObject *capsule = get_internals_capsule()) { + Py_INCREF(capsule); + (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { + Py_XDECREF(get_internals_capsule()); + wr.dec_ref(); + })) + .release(); + } + if (PyObject *capsule = get_local_internals_capsule()) { + Py_INCREF(capsule); + (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { + Py_XDECREF(get_local_internals_capsule()); + wr.dec_ref(); + })) + .release(); + } + if (rec.bases.size() > 1 || rec.multiple_inheritance) { mark_parents_nonsimple(tinfo->type); tinfo->simple_ancestors = false; From ce9ca7f3aaaa9e37f7b1566ee54ac8399350fa72 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 24 Jan 2026 12:59:38 +0800 Subject: [PATCH 10/39] Try add tests --- docs/advanced/classes.rst | 37 +++++++++++---- tests/env.py | 28 ++++++++++++ tests/test_custom_type_setup.cpp | 70 +++++++++++++++++++++++++---- tests/test_custom_type_setup.py | 24 +++++++--- tests/test_multiple_interpreters.py | 36 +++------------ 5 files changed, 141 insertions(+), 54 deletions(-) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index faaba38b8d..bfbaea60ab 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -1381,11 +1381,22 @@ You can do that using ``py::custom_type_setup``: .. code-block:: cpp - struct OwnsPythonObjects { - py::object value = py::none(); + struct ContainerOwnsPythonObjects { + std::vector list; + + void append(const py::object &obj) { list.emplace_back(obj); } + py::object at(py::ssize_t index) const { + if (index >= size() || index < 0) { + throw py::index_error("Index out of range"); + } + return list.at(py::size_t(index)); + } + py::ssize_t size() const { return py::ssize_t_cast(list.size()); } + void clear() { list.clear(); } }; - py::class_ cls( - m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { + + py::class_ cls( + m, "ContainerOwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) { @@ -1394,20 +1405,28 @@ You can do that using ``py::custom_type_setup``: Py_VISIT(Py_TYPE(self_base)); #endif if (py::detail::is_holder_constructed(self_base)) { - auto &self = py::cast(py::handle(self_base)); - Py_VISIT(self.value.ptr()); + auto &self = py::cast(py::handle(self_base)); + for (auto &item : self.list) { + Py_VISIT(item.ptr()); + } } return 0; }; type->tp_clear = [](PyObject *self_base) { if (py::detail::is_holder_constructed(self_base)) { - auto &self = py::cast(py::handle(self_base)); - self.value = py::none(); + auto &self = py::cast(py::handle(self_base)); + for (auto &item : self.list) { + Py_CLEAR(item.ptr()); + } + self.list.clear(); } return 0; }; })); cls.def(py::init<>()); - cls.def_readwrite("value", &OwnsPythonObjects::value); + cls.def("append", &ContainerOwnsPythonObjects::append); + cls.def("at", &ContainerOwnsPythonObjects::at); + cls.def("size", &ContainerOwnsPythonObjects::size); + cls.def("clear", &ContainerOwnsPythonObjects::clear); .. versionadded:: 2.8 diff --git a/tests/env.py b/tests/env.py index ee932ad77a..8c06178307 100644 --- a/tests/env.py +++ b/tests/env.py @@ -29,3 +29,31 @@ or GRAALPY or (CPYTHON and PY_GIL_DISABLED and (3, 13) <= sys.version_info < (3, 14)) ) + + +def check_script_success_in_subprocess(code: str, *, rerun: int = 8) -> None: + """Runs the given code in a subprocess.""" + import os + import subprocess + import sys + import textwrap + + code = textwrap.dedent(code).strip() + try: + for _ in range(rerun): # run flakily failing test multiple times + subprocess.check_output( + [sys.executable, "-c", code], + cwd=os.getcwd(), + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError as ex: + raise RuntimeError( + f"Subprocess failed with exit code {ex.returncode}.\n\n" + f"Code:\n" + f"```python\n" + f"{code}\n" + f"```\n\n" + f"Output:\n" + f"{ex.output}" + ) from None diff --git a/tests/test_custom_type_setup.cpp b/tests/test_custom_type_setup.cpp index 35d30abcf7..d1eba912d2 100644 --- a/tests/test_custom_type_setup.cpp +++ b/tests/test_custom_type_setup.cpp @@ -7,22 +7,64 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "pybind11_tests.h" +#include + namespace py = pybind11; namespace { +struct ContainerOwnsPythonObjects { + std::vector list; -struct OwnsPythonObjects { - py::object value = py::none(); + void append(const py::object &obj) { list.emplace_back(obj); } + py::object at(py::ssize_t index) const { + if (index >= size() || index < 0) { + throw py::index_error("Index out of range"); + } + return list.at(py::size_t(index)); + } + py::ssize_t size() const { return py::ssize_t_cast(list.size()); } + void clear() { list.clear(); } }; + +void add_gc_checkers_with_weakrefs(const py::object &obj) { + py::handle global_capsule = py::detail::get_internals_capsule(); + if (!global_capsule) { + throw std::runtime_error("No global internals capsule found"); + } + (void) py::weakref(obj, py::cpp_function([global_capsule, obj](py::handle weakref) -> void { + py::handle new_global_capsule = py::detail::get_internals_capsule(); + if (!new_global_capsule.is(global_capsule)) { + throw std::runtime_error( + "Global internals capsule was destroyed prematurely"); + } + weakref.dec_ref(); + })) + .release(); + + py::handle local_capsule = py::detail::get_local_internals_capsule(); + if (!local_capsule) { + throw std::runtime_error("No local internals capsule found"); + } + (void) py::weakref( + obj, py::cpp_function([local_capsule, obj](py::handle weakref) -> void { + py::handle new_local_capsule = py::detail::get_local_internals_capsule(); + if (!new_local_capsule.is(local_capsule)) { + throw std::runtime_error("Local internals capsule was destroyed prematurely"); + } + weakref.dec_ref(); + })) + .release(); +} } // namespace TEST_SUBMODULE(custom_type_setup, m) { - py::class_ cls( - m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { + py::class_ cls( + m, "ContainerOwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) { @@ -31,19 +73,29 @@ TEST_SUBMODULE(custom_type_setup, m) { Py_VISIT(Py_TYPE(self_base)); #endif if (py::detail::is_holder_constructed(self_base)) { - auto &self = py::cast(py::handle(self_base)); - Py_VISIT(self.value.ptr()); + auto &self = py::cast(py::handle(self_base)); + for (auto &item : self.list) { + Py_VISIT(item.ptr()); + } } return 0; }; type->tp_clear = [](PyObject *self_base) { if (py::detail::is_holder_constructed(self_base)) { - auto &self = py::cast(py::handle(self_base)); - self.value = py::none(); + auto &self = py::cast(py::handle(self_base)); + for (auto &item : self.list) { + Py_CLEAR(item.ptr()); + } + self.list.clear(); } return 0; }; })); cls.def(py::init<>()); - cls.def_readwrite("value", &OwnsPythonObjects::value); + cls.def("append", &ContainerOwnsPythonObjects::append); + cls.def("at", &ContainerOwnsPythonObjects::at); + cls.def("size", &ContainerOwnsPythonObjects::size); + cls.def("clear", &ContainerOwnsPythonObjects::clear); + + m.def("add_gc_checkers_with_weakrefs", &add_gc_checkers_with_weakrefs); } diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index bb2865cade..d488fde0b0 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -5,7 +5,7 @@ import pytest -import env # noqa: F401 +import env from pybind11_tests import custom_type_setup as m @@ -36,15 +36,27 @@ def add_ref(obj): # PyPy does not seem to reliably garbage collect. @pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_self_cycle(gc_tester): - obj = m.OwnsPythonObjects() - obj.value = obj + obj = m.ContainerOwnsPythonObjects() + obj.append(obj) gc_tester(obj) # PyPy does not seem to reliably garbage collect. @pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_indirect_cycle(gc_tester): - obj = m.OwnsPythonObjects() - obj_list = [obj] - obj.value = obj_list + obj = m.ContainerOwnsPythonObjects() + obj.append([obj]) gc_tester(obj) + + +@pytest.mark.skipif("env.PYPY or env.GRAALPY") +def test_py_cast_useable_on_shutdown(): + env.check_script_success_in_subprocess( + """ + from pybind11_tests import custom_type_setup as m + + obj = m.ContainerOwnsPythonObjects() + obj.append(obj) + m.add_gc_checkers_with_weakrefs(obj) + """ + ) diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index 44877e772a..c52c7bf722 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -3,7 +3,6 @@ import contextlib import os import pickle -import subprocess import sys import textwrap @@ -269,36 +268,13 @@ def test_import_module_with_singleton_per_interpreter(): interp.exec(code) -def check_script_success_in_subprocess(code: str, *, rerun: int = 8) -> None: - """Runs the given code in a subprocess.""" - code = textwrap.dedent(code).strip() - try: - for _ in range(rerun): # run flakily failing test multiple times - subprocess.check_output( - [sys.executable, "-c", code], - cwd=os.getcwd(), - stderr=subprocess.STDOUT, - text=True, - ) - except subprocess.CalledProcessError as ex: - raise RuntimeError( - f"Subprocess failed with exit code {ex.returncode}.\n\n" - f"Code:\n" - f"```python\n" - f"{code}\n" - f"```\n\n" - f"Output:\n" - f"{ex.output}" - ) from None - - @pytest.mark.skipif( sys.platform.startswith("emscripten"), reason="Requires loadable modules" ) @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") def test_import_in_subinterpreter_after_main(): """Tests that importing a module in a subinterpreter after the main interpreter works correctly""" - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -319,7 +295,7 @@ def test_import_in_subinterpreter_after_main(): ) ) - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -354,7 +330,7 @@ def test_import_in_subinterpreter_after_main(): @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") def test_import_in_subinterpreter_before_main(): """Tests that importing a module in a subinterpreter before the main interpreter works correctly""" - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -375,7 +351,7 @@ def test_import_in_subinterpreter_before_main(): ) ) - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -401,7 +377,7 @@ def test_import_in_subinterpreter_before_main(): ) ) - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -434,7 +410,7 @@ def test_import_in_subinterpreter_before_main(): @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") def test_import_in_subinterpreter_concurrently(): """Tests that importing a module in multiple subinterpreters concurrently works correctly""" - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ From fed174912bb93f1b4d770e32e71a0ada1e9b914d Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 24 Jan 2026 16:28:43 +0800 Subject: [PATCH 11/39] Fix PYTHONPATH --- tests/test_custom_type_setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index d488fde0b0..cf56d4a1aa 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -1,11 +1,13 @@ from __future__ import annotations import gc +import os import weakref import pytest import env +import pybind11_tests from pybind11_tests import custom_type_setup as m @@ -52,7 +54,11 @@ def test_indirect_cycle(gc_tester): @pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_py_cast_useable_on_shutdown(): env.check_script_success_in_subprocess( - """ + f""" + import sys + + sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r}) + from pybind11_tests import custom_type_setup as m obj = m.ContainerOwnsPythonObjects() From a407438fde4444519e28415ea62cf4b069a0dda4 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 24 Jan 2026 16:36:50 +0800 Subject: [PATCH 12/39] Fix PYTHONPATH --- tests/test_custom_type_setup.py | 1 + tests/test_multiple_interpreters.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index cf56d4a1aa..979ba6259f 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -57,6 +57,7 @@ def test_py_cast_useable_on_shutdown(): f""" import sys + sys.path.insert(0, {os.path.dirname(env.__file__)!r}) sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r}) from pybind11_tests import custom_type_setup as m diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index c52c7bf722..56d303a36e 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -218,6 +218,7 @@ def test_dependent_subinterpreters(): def test(): import sys + sys.path.insert(0, {os.path.dirname(env.__file__)!r}) sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r}) import collections From 3df427ce50ba9b2ba0b2e408842e2ce2710b71b2 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 24 Jan 2026 17:10:51 +0800 Subject: [PATCH 13/39] Skip tests for subprocess --- tests/test_custom_type_setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index 979ba6259f..83a10ecac5 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -2,6 +2,7 @@ import gc import os +import sys import weakref import pytest @@ -51,6 +52,10 @@ def test_indirect_cycle(gc_tester): gc_tester(obj) +@pytest.mark.skipif( + env.IOS or sys.platform.startswith("emscripten"), + reason="Requires subprocess support", +) @pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_py_cast_useable_on_shutdown(): env.check_script_success_in_subprocess( From 72c2e0aa9b481debd94557b629e86b954543d64b Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 24 Jan 2026 21:14:26 +0800 Subject: [PATCH 14/39] Revert to leak internals --- include/pybind11/detail/internals.h | 70 ++++++++++------------------- include/pybind11/pybind11.h | 25 ----------- 2 files changed, 23 insertions(+), 72 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index db6044aecc..eaf6c89e30 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -337,20 +337,7 @@ struct internals { internals(internals &&other) = delete; internals &operator=(const internals &other) = delete; internals &operator=(internals &&other) = delete; - ~internals() { - // Normally this destructor runs during interpreter finalization and it may DECREF things. - // In odd finalization scenarios it might end up running after the interpreter has - // completely shut down, In that case, we should not decref these objects because pymalloc - // is gone. This also applies across sub-interpreters, we should only DECREF when the - // original owning interpreter is active. - auto *cur_istate = get_interpreter_state_unchecked(); - if (cur_istate && cur_istate == istate) { - gil_scoped_acquire_simple gil; - Py_CLEAR(instance_base); - Py_CLEAR(default_metaclass); - Py_CLEAR(static_property_type); - } - } + ~internals() = default; }; // the internals struct (above) is shared between all the modules. local_internals are only @@ -371,18 +358,7 @@ struct local_internals { PyTypeObject *function_record_py_type = nullptr; PyInterpreterState *istate = nullptr; - ~local_internals() { - // Normally this destructor runs during interpreter finalization and it may DECREF things. - // In odd finalization scenarios it might end up running after the interpreter has - // completely shut down, In that case, we should not decref these objects because pymalloc - // is gone. This also applies across sub-interpreters, we should only DECREF when the - // original owning interpreter is active. - auto *cur_istate = get_interpreter_state_unchecked(); - if (cur_istate && cur_istate == istate) { - gil_scoped_acquire_simple gil; - Py_CLEAR(function_record_py_type); - } - } + ~local_internals() = default; }; enum class holder_enum_t : uint8_t { @@ -578,6 +554,10 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif +// Sentinel value for the `dtor` parameter of `atomic_get_or_create_in_state_dict`. +// Indicates no destructor was explicitly provided (distinct from nullptr, which means "leak"). +#define PYBIND11_DTOR_UNSPECIFIED (reinterpret_cast(1)) + // Get or create per-storage capsule in the current interpreter's state dict. // - The storage is interpreter-dependent: different interpreters will have different storage. // This is important when using multiple-interpreters, to avoid sharing unshareable objects @@ -594,9 +574,14 @@ inline void translate_local_exception(std::exception_ptr p) { // // Returns: pair of (pointer to storage, bool indicating if newly created). // The bool follows std::map::insert convention: true = created, false = existed. +// `dtor`: optional destructor called when the interpreter shuts down. +// - If not provided: the storage will be deleted using `delete`. +// - If nullptr: the storage will be leaked (useful for singletons that outlive the interpreter). +// - If a function: that function will be called with the capsule object. template std::pair atomic_get_or_create_in_state_dict(const char *key, - void (*dtor)(PyObject *) = nullptr) { + void (*dtor)(PyObject *) + = PYBIND11_DTOR_UNSPECIFIED) { error_scope err_scope; // preserve any existing Python error states auto state_dict = reinterpret_borrow(get_python_state_dict()); @@ -642,7 +627,7 @@ std::pair atomic_get_or_create_in_state_dict(const char *key, // - Otherwise, our `new_capsule` is now in the dict, and it owns the storage and the state // dict will incref it. We need to set the caller's destructor on it, which will be // called when the interpreter shuts down. - if (created && dtor) { + if (created && dtor != PYBIND11_DTOR_UNSPECIFIED) { if (PyCapsule_SetDestructor(capsule_obj, dtor) < 0) { throw error_already_set(); } @@ -659,6 +644,8 @@ std::pair atomic_get_or_create_in_state_dict(const char *key, return std::pair(static_cast(raw_ptr), created); } +#undef PYBIND11_DTOR_UNSPECIFIED + template class internals_pp_manager { public: @@ -730,27 +717,16 @@ class internals_pp_manager { internals_pp_manager(char const *id, on_fetch_function *on_fetch) : holder_id_(id), on_fetch_(on_fetch) {} - static void internals_shutdown(PyObject *capsule) { - auto *pp = static_cast *>( - PyCapsule_GetPointer(capsule, nullptr)); - if (pp) { - pp->reset(); - } - // We reset the unique_ptr's contents but cannot delete the unique_ptr itself here. - // The pp_manager in this module (and possibly other modules sharing internals) holds - // a raw pointer to this unique_ptr, and that pointer would dangle if we deleted it now. - // - // For pybind11-owned interpreters (via embed.h or subinterpreter.h), destroy() is - // called after Py_Finalize/Py_EndInterpreter completes, which safely deletes the - // unique_ptr. For interpreters not owned by pybind11 (e.g., a pybind11 extension - // loaded into an external interpreter), destroy() is never called and the unique_ptr - // shell (8 bytes, not its contents) is leaked. - // (See PR #5958 for ideas to eliminate this leak.) - } - std::unique_ptr *get_or_create_pp_in_state_dict() { + // The `unique_ptr` is intentionally leaked on interpreter shutdown. + // Once an instance is created, it will never be deleted until the process exits (compare + // to interpreter shutdown in multiple-interpreter scenarios). + // We cannot guarantee the destruction order of capsules in the interpreter state dict on + // interpreter shutdown, so deleting internals too early could cause undefined behavior + // when other pybind11 objects access `get_internals()` during finalization (which would + // recreate empty internals). auto result = atomic_get_or_create_in_state_dict>( - holder_id_, &internals_shutdown); + holder_id_, /*dtor=*/nullptr /* leak the capsule content */); auto *pp = result.first; bool created = result.second; // Only call on_fetch_ when fetching existing internals, not when creating new ones. diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 4ee4f977d9..02d2e72c2c 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1733,31 +1733,6 @@ class generic_type : public object { PYBIND11_WARNING_POP }); - // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so the - // internals capsule may be destroyed (resetting internals via internals_shutdown) before - // all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls py::cast, - // it would recreate an empty internals and fail because the type registry is gone. - // - // By holding references to the capsules, we ensure they outlive all pybind11 types. We use - // weakrefs on the type with a cpp_function callback. When the type is destroyed, Python - // will call the callback which releases the capsule reference and the weakref. - if (PyObject *capsule = get_internals_capsule()) { - Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { - Py_XDECREF(get_internals_capsule()); - wr.dec_ref(); - })) - .release(); - } - if (PyObject *capsule = get_local_internals_capsule()) { - Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { - Py_XDECREF(get_local_internals_capsule()); - wr.dec_ref(); - })) - .release(); - } - if (rec.bases.size() > 1 || rec.multiple_inheritance) { mark_parents_nonsimple(tinfo->type); tinfo->simple_ancestors = false; From c5ec1cf8862b43c125bdaf582445f3cbb1347017 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 24 Jan 2026 21:53:40 +0800 Subject: [PATCH 15/39] Revert to leak internals --- include/pybind11/detail/internals.h | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index eaf6c89e30..29f801fe6c 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -39,7 +39,7 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 12 +# define PYBIND11_INTERNALS_VERSION 11 #endif #if PYBIND11_INTERNALS_VERSION < 11 @@ -347,8 +347,6 @@ struct internals { // impact any other modules, because the only things accessing the local internals is the // module that contains them. struct local_internals { - local_internals() : istate(get_interpreter_state_unchecked()) {} - // It should be safe to use fast_type_map here because this entire // data structure is scoped to our single module, and thus a single // DSO and single instance of type_info for any particular type. @@ -356,9 +354,6 @@ struct local_internals { std::forward_list registered_exception_translators; PyTypeObject *function_record_py_type = nullptr; - PyInterpreterState *istate = nullptr; - - ~local_internals() = default; }; enum class holder_enum_t : uint8_t { @@ -806,8 +801,6 @@ PYBIND11_NOINLINE internals &get_internals() { /// Return the PyObject* for the internals capsule (borrowed reference). /// Returns nullptr if the capsule doesn't exist yet. -/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types -/// to hold a reference to the capsule (see comments in generic_type::initialize). inline PyObject *get_internals_capsule() { auto state_dict = reinterpret_borrow(get_python_state_dict()); return dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID); @@ -825,8 +818,6 @@ inline const std::string &get_local_internals_key() { /// Return the PyObject* for the local_internals capsule (borrowed reference). /// Returns nullptr if the capsule doesn't exist yet. -/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types -/// to hold a reference to the capsule (see comments in generic_type::initialize). inline PyObject *get_local_internals_capsule() { const auto &key = get_local_internals_key(); auto state_dict = reinterpret_borrow(get_python_state_dict()); From 8f25a254e893817c2ed70d97cde7f62778a14de2 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sun, 25 Jan 2026 01:23:43 +0800 Subject: [PATCH 16/39] Revert "Revert to leak internals" This reverts commit c5ec1cf8862b43c125bdaf582445f3cbb1347017. This reverts commit 72c2e0aa9b481debd94557b629e86b954543d64b. --- include/pybind11/detail/internals.h | 64 ++++++++++++++++++++++++----- include/pybind11/pybind11.h | 25 +++++++++++ 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 29f801fe6c..d24a773289 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -39,7 +39,7 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 11 +# define PYBIND11_INTERNALS_VERSION 12 #endif #if PYBIND11_INTERNALS_VERSION < 11 @@ -337,7 +337,20 @@ struct internals { internals(internals &&other) = delete; internals &operator=(const internals &other) = delete; internals &operator=(internals &&other) = delete; - ~internals() = default; + ~internals() { + // Normally this destructor runs during interpreter finalization and it may DECREF things. + // In odd finalization scenarios it might end up running after the interpreter has + // completely shut down, In that case, we should not decref these objects because pymalloc + // is gone. This also applies across sub-interpreters, we should only DECREF when the + // original owning interpreter is active. + auto *cur_istate = get_interpreter_state_unchecked(); + if (cur_istate && cur_istate == istate) { + gil_scoped_acquire_simple gil; + Py_CLEAR(instance_base); + Py_CLEAR(default_metaclass); + Py_CLEAR(static_property_type); + } + } }; // the internals struct (above) is shared between all the modules. local_internals are only @@ -347,6 +360,8 @@ struct internals { // impact any other modules, because the only things accessing the local internals is the // module that contains them. struct local_internals { + local_internals() : istate(get_interpreter_state_unchecked()) {} + // It should be safe to use fast_type_map here because this entire // data structure is scoped to our single module, and thus a single // DSO and single instance of type_info for any particular type. @@ -354,6 +369,20 @@ struct local_internals { std::forward_list registered_exception_translators; PyTypeObject *function_record_py_type = nullptr; + PyInterpreterState *istate = nullptr; + + ~local_internals() { + // Normally this destructor runs during interpreter finalization and it may DECREF things. + // In odd finalization scenarios it might end up running after the interpreter has + // completely shut down, In that case, we should not decref these objects because pymalloc + // is gone. This also applies across sub-interpreters, we should only DECREF when the + // original owning interpreter is active. + auto *cur_istate = get_interpreter_state_unchecked(); + if (cur_istate && cur_istate == istate) { + gil_scoped_acquire_simple gil; + Py_CLEAR(function_record_py_type); + } + } }; enum class holder_enum_t : uint8_t { @@ -712,16 +741,27 @@ class internals_pp_manager { internals_pp_manager(char const *id, on_fetch_function *on_fetch) : holder_id_(id), on_fetch_(on_fetch) {} + static void internals_shutdown(PyObject *capsule) { + auto *pp = static_cast *>( + PyCapsule_GetPointer(capsule, nullptr)); + if (pp) { + pp->reset(); + } + // We reset the unique_ptr's contents but cannot delete the unique_ptr itself here. + // The pp_manager in this module (and possibly other modules sharing internals) holds + // a raw pointer to this unique_ptr, and that pointer would dangle if we deleted it now. + // + // For pybind11-owned interpreters (via embed.h or subinterpreter.h), destroy() is + // called after Py_Finalize/Py_EndInterpreter completes, which safely deletes the + // unique_ptr. For interpreters not owned by pybind11 (e.g., a pybind11 extension + // loaded into an external interpreter), destroy() is never called and the unique_ptr + // shell (8 bytes, not its contents) is leaked. + // (See PR #5958 for ideas to eliminate this leak.) + } + std::unique_ptr *get_or_create_pp_in_state_dict() { - // The `unique_ptr` is intentionally leaked on interpreter shutdown. - // Once an instance is created, it will never be deleted until the process exits (compare - // to interpreter shutdown in multiple-interpreter scenarios). - // We cannot guarantee the destruction order of capsules in the interpreter state dict on - // interpreter shutdown, so deleting internals too early could cause undefined behavior - // when other pybind11 objects access `get_internals()` during finalization (which would - // recreate empty internals). auto result = atomic_get_or_create_in_state_dict>( - holder_id_, /*dtor=*/nullptr /* leak the capsule content */); + holder_id_, &internals_shutdown); auto *pp = result.first; bool created = result.second; // Only call on_fetch_ when fetching existing internals, not when creating new ones. @@ -801,6 +841,8 @@ PYBIND11_NOINLINE internals &get_internals() { /// Return the PyObject* for the internals capsule (borrowed reference). /// Returns nullptr if the capsule doesn't exist yet. +/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types +/// to hold a reference to the capsule (see comments in generic_type::initialize). inline PyObject *get_internals_capsule() { auto state_dict = reinterpret_borrow(get_python_state_dict()); return dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID); @@ -818,6 +860,8 @@ inline const std::string &get_local_internals_key() { /// Return the PyObject* for the local_internals capsule (borrowed reference). /// Returns nullptr if the capsule doesn't exist yet. +/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types +/// to hold a reference to the capsule (see comments in generic_type::initialize). inline PyObject *get_local_internals_capsule() { const auto &key = get_local_internals_key(); auto state_dict = reinterpret_borrow(get_python_state_dict()); diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 02d2e72c2c..4ee4f977d9 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1733,6 +1733,31 @@ class generic_type : public object { PYBIND11_WARNING_POP }); + // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so the + // internals capsule may be destroyed (resetting internals via internals_shutdown) before + // all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls py::cast, + // it would recreate an empty internals and fail because the type registry is gone. + // + // By holding references to the capsules, we ensure they outlive all pybind11 types. We use + // weakrefs on the type with a cpp_function callback. When the type is destroyed, Python + // will call the callback which releases the capsule reference and the weakref. + if (PyObject *capsule = get_internals_capsule()) { + Py_INCREF(capsule); + (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { + Py_XDECREF(get_internals_capsule()); + wr.dec_ref(); + })) + .release(); + } + if (PyObject *capsule = get_local_internals_capsule()) { + Py_INCREF(capsule); + (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { + Py_XDECREF(get_local_internals_capsule()); + wr.dec_ref(); + })) + .release(); + } + if (rec.bases.size() > 1 || rec.multiple_inheritance) { mark_parents_nonsimple(tinfo->type); tinfo->simple_ancestors = false; From 97e12d2c5773ffa196e385aec1a93d490c40072f Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sun, 25 Jan 2026 20:39:44 +0800 Subject: [PATCH 17/39] Revert internals version bump --- include/pybind11/detail/internals.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index d24a773289..9dbafeea9e 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -39,7 +39,7 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 12 +# define PYBIND11_INTERNALS_VERSION 11 #endif #if PYBIND11_INTERNALS_VERSION < 11 From cdefbf3547afa4da9f92ae8a4e661707e5f4ee04 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sun, 25 Jan 2026 20:50:44 +0800 Subject: [PATCH 18/39] Reapply to leak internals This reverts commit 8f25a254e893817c2ed70d97cde7f62778a14de2. --- include/pybind11/detail/internals.h | 70 ++++++----------------------- include/pybind11/pybind11.h | 25 ----------- tests/test_custom_type_setup.cpp | 12 ++--- tests/test_custom_type_setup.py | 7 +++ 4 files changed, 26 insertions(+), 88 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 9dbafeea9e..fa7ad5d5b0 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -337,20 +337,7 @@ struct internals { internals(internals &&other) = delete; internals &operator=(const internals &other) = delete; internals &operator=(internals &&other) = delete; - ~internals() { - // Normally this destructor runs during interpreter finalization and it may DECREF things. - // In odd finalization scenarios it might end up running after the interpreter has - // completely shut down, In that case, we should not decref these objects because pymalloc - // is gone. This also applies across sub-interpreters, we should only DECREF when the - // original owning interpreter is active. - auto *cur_istate = get_interpreter_state_unchecked(); - if (cur_istate && cur_istate == istate) { - gil_scoped_acquire_simple gil; - Py_CLEAR(instance_base); - Py_CLEAR(default_metaclass); - Py_CLEAR(static_property_type); - } - } + ~internals() = default; }; // the internals struct (above) is shared between all the modules. local_internals are only @@ -360,8 +347,6 @@ struct internals { // impact any other modules, because the only things accessing the local internals is the // module that contains them. struct local_internals { - local_internals() : istate(get_interpreter_state_unchecked()) {} - // It should be safe to use fast_type_map here because this entire // data structure is scoped to our single module, and thus a single // DSO and single instance of type_info for any particular type. @@ -369,20 +354,6 @@ struct local_internals { std::forward_list registered_exception_translators; PyTypeObject *function_record_py_type = nullptr; - PyInterpreterState *istate = nullptr; - - ~local_internals() { - // Normally this destructor runs during interpreter finalization and it may DECREF things. - // In odd finalization scenarios it might end up running after the interpreter has - // completely shut down, In that case, we should not decref these objects because pymalloc - // is gone. This also applies across sub-interpreters, we should only DECREF when the - // original owning interpreter is active. - auto *cur_istate = get_interpreter_state_unchecked(); - if (cur_istate && cur_istate == istate) { - gil_scoped_acquire_simple gil; - Py_CLEAR(function_record_py_type); - } - } }; enum class holder_enum_t : uint8_t { @@ -580,7 +551,7 @@ inline void translate_local_exception(std::exception_ptr p) { // Sentinel value for the `dtor` parameter of `atomic_get_or_create_in_state_dict`. // Indicates no destructor was explicitly provided (distinct from nullptr, which means "leak"). -#define PYBIND11_DTOR_UNSPECIFIED (reinterpret_cast(1)) +#define PYBIND11_DTOR_USE_DELETE (reinterpret_cast(1)) // Get or create per-storage capsule in the current interpreter's state dict. // - The storage is interpreter-dependent: different interpreters will have different storage. @@ -605,7 +576,7 @@ inline void translate_local_exception(std::exception_ptr p) { template std::pair atomic_get_or_create_in_state_dict(const char *key, void (*dtor)(PyObject *) - = PYBIND11_DTOR_UNSPECIFIED) { + = PYBIND11_DTOR_USE_DELETE) { error_scope err_scope; // preserve any existing Python error states auto state_dict = reinterpret_borrow(get_python_state_dict()); @@ -651,7 +622,7 @@ std::pair atomic_get_or_create_in_state_dict(const char *key, // - Otherwise, our `new_capsule` is now in the dict, and it owns the storage and the state // dict will incref it. We need to set the caller's destructor on it, which will be // called when the interpreter shuts down. - if (created && dtor != PYBIND11_DTOR_UNSPECIFIED) { + if (created && dtor != PYBIND11_DTOR_USE_DELETE) { if (PyCapsule_SetDestructor(capsule_obj, dtor) < 0) { throw error_already_set(); } @@ -668,7 +639,7 @@ std::pair atomic_get_or_create_in_state_dict(const char *key, return std::pair(static_cast(raw_ptr), created); } -#undef PYBIND11_DTOR_UNSPECIFIED +#undef PYBIND11_DTOR_USE_DELETE template class internals_pp_manager { @@ -741,27 +712,16 @@ class internals_pp_manager { internals_pp_manager(char const *id, on_fetch_function *on_fetch) : holder_id_(id), on_fetch_(on_fetch) {} - static void internals_shutdown(PyObject *capsule) { - auto *pp = static_cast *>( - PyCapsule_GetPointer(capsule, nullptr)); - if (pp) { - pp->reset(); - } - // We reset the unique_ptr's contents but cannot delete the unique_ptr itself here. - // The pp_manager in this module (and possibly other modules sharing internals) holds - // a raw pointer to this unique_ptr, and that pointer would dangle if we deleted it now. - // - // For pybind11-owned interpreters (via embed.h or subinterpreter.h), destroy() is - // called after Py_Finalize/Py_EndInterpreter completes, which safely deletes the - // unique_ptr. For interpreters not owned by pybind11 (e.g., a pybind11 extension - // loaded into an external interpreter), destroy() is never called and the unique_ptr - // shell (8 bytes, not its contents) is leaked. - // (See PR #5958 for ideas to eliminate this leak.) - } - std::unique_ptr *get_or_create_pp_in_state_dict() { + // The `unique_ptr` is intentionally leaked on interpreter shutdown. + // Once an instance is created, it will never be deleted until the process exits (compare + // to interpreter shutdown in multiple-interpreter scenarios). + // We cannot guarantee the destruction order of capsules in the interpreter state dict on + // interpreter shutdown, so deleting internals too early could cause undefined behavior + // when other pybind11 objects access `get_internals()` during finalization (which would + // recreate empty internals). auto result = atomic_get_or_create_in_state_dict>( - holder_id_, &internals_shutdown); + holder_id_, /*dtor=*/nullptr /* leak the capsule content */); auto *pp = result.first; bool created = result.second; // Only call on_fetch_ when fetching existing internals, not when creating new ones. @@ -841,8 +801,6 @@ PYBIND11_NOINLINE internals &get_internals() { /// Return the PyObject* for the internals capsule (borrowed reference). /// Returns nullptr if the capsule doesn't exist yet. -/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types -/// to hold a reference to the capsule (see comments in generic_type::initialize). inline PyObject *get_internals_capsule() { auto state_dict = reinterpret_borrow(get_python_state_dict()); return dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID); @@ -860,8 +818,6 @@ inline const std::string &get_local_internals_key() { /// Return the PyObject* for the local_internals capsule (borrowed reference). /// Returns nullptr if the capsule doesn't exist yet. -/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types -/// to hold a reference to the capsule (see comments in generic_type::initialize). inline PyObject *get_local_internals_capsule() { const auto &key = get_local_internals_key(); auto state_dict = reinterpret_borrow(get_python_state_dict()); diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 4ee4f977d9..02d2e72c2c 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1733,31 +1733,6 @@ class generic_type : public object { PYBIND11_WARNING_POP }); - // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so the - // internals capsule may be destroyed (resetting internals via internals_shutdown) before - // all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls py::cast, - // it would recreate an empty internals and fail because the type registry is gone. - // - // By holding references to the capsules, we ensure they outlive all pybind11 types. We use - // weakrefs on the type with a cpp_function callback. When the type is destroyed, Python - // will call the callback which releases the capsule reference and the weakref. - if (PyObject *capsule = get_internals_capsule()) { - Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { - Py_XDECREF(get_internals_capsule()); - wr.dec_ref(); - })) - .release(); - } - if (PyObject *capsule = get_local_internals_capsule()) { - Py_INCREF(capsule); - (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void { - Py_XDECREF(get_local_internals_capsule()); - wr.dec_ref(); - })) - .release(); - } - if (rec.bases.size() > 1 || rec.multiple_inheritance) { mark_parents_nonsimple(tinfo->type); tinfo->simple_ancestors = false; diff --git a/tests/test_custom_type_setup.cpp b/tests/test_custom_type_setup.cpp index d1eba912d2..a25e4c109a 100644 --- a/tests/test_custom_type_setup.cpp +++ b/tests/test_custom_type_setup.cpp @@ -36,9 +36,9 @@ void add_gc_checkers_with_weakrefs(const py::object &obj) { if (!global_capsule) { throw std::runtime_error("No global internals capsule found"); } - (void) py::weakref(obj, py::cpp_function([global_capsule, obj](py::handle weakref) -> void { - py::handle new_global_capsule = py::detail::get_internals_capsule(); - if (!new_global_capsule.is(global_capsule)) { + (void) py::weakref(obj, py::cpp_function([global_capsule](py::handle weakref) -> void { + py::handle current_global_capsule = py::detail::get_internals_capsule(); + if (!current_global_capsule.is(global_capsule)) { throw std::runtime_error( "Global internals capsule was destroyed prematurely"); } @@ -51,9 +51,9 @@ void add_gc_checkers_with_weakrefs(const py::object &obj) { throw std::runtime_error("No local internals capsule found"); } (void) py::weakref( - obj, py::cpp_function([local_capsule, obj](py::handle weakref) -> void { - py::handle new_local_capsule = py::detail::get_local_internals_capsule(); - if (!new_local_capsule.is(local_capsule)) { + obj, py::cpp_function([local_capsule](py::handle weakref) -> void { + py::handle current_local_capsule = py::detail::get_local_internals_capsule(); + if (!current_local_capsule.is(local_capsule)) { throw std::runtime_error("Local internals capsule was destroyed prematurely"); } weakref.dec_ref(); diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index 83a10ecac5..42cd42eb7e 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -58,6 +58,7 @@ def test_indirect_cycle(gc_tester): ) @pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_py_cast_useable_on_shutdown(): + """Test that py::cast works during interpreter shutdown.""" env.check_script_success_in_subprocess( f""" import sys @@ -67,8 +68,14 @@ def test_py_cast_useable_on_shutdown(): from pybind11_tests import custom_type_setup as m + # Create a self-referential cycle that will be collected during shutdown. + # The tp_traverse and tp_clear callbacks call py::cast, which requires + # internals to still be valid. obj = m.ContainerOwnsPythonObjects() obj.append(obj) + + # Add weakref callbacks that verify the capsule is still alive when the + # pybind11 object is garbage collected during shutdown. m.add_gc_checkers_with_weakrefs(obj) """ ) From 6ed28308c9033bf289e244b84ee3069ab3ea5c4f Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sun, 25 Jan 2026 21:52:03 +0800 Subject: [PATCH 19/39] Add re-entrancy detection for internals creation Prevent re-creation of internals after destruction during interpreter shutdown. If pybind11 code runs after internals have been destroyed, fail early with a clear error message instead of silently creating new empty internals that would cause type lookup failures. Co-Authored-By: Claude Opus 4.5 --- include/pybind11/detail/internals.h | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index fa7ad5d5b0..cf3fa9c1c3 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -697,17 +697,29 @@ class internals_pp_manager { // this could be called without an active interpreter, just use what was cached if (!tstate || tstate->interp == last_istate_tls()) { auto tpp = internals_p_tls(); - + pps_have_created_content_.erase(tpp); delete tpp; } unref(); return; } #endif + pps_have_created_content_.erase(internals_singleton_pp_); delete internals_singleton_pp_; unref(); } + static void fail_if_internals_recreated(std::unique_ptr *pp) { + // Prevent re-creation of internals after destruction during interpreter shutdown. + // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals + // have been destroyed, a new empty internals would be created, causing type lookup + // failures. See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. + if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { + pybind11_fail("Reentrant call detected while fetching pybind11 internals!"); + } + pps_have_created_content_.insert(pp); + } + private: internals_pp_manager(char const *id, on_fetch_function *on_fetch) : holder_id_(id), on_fetch_(on_fetch) {} @@ -748,6 +760,9 @@ class internals_pp_manager { // Pointer-to-pointer to the singleton internals for the first seen interpreter (may not be the // main interpreter) std::unique_ptr *internals_singleton_pp_ = nullptr; + + // Tracks pointer-to-pointers whose internals have been created, to detect re-entrancy. + inline static std::unordered_set pps_have_created_content_{}; }; // If We loaded the internals through `state_dict`, our `error_already_set` @@ -788,6 +803,7 @@ PYBIND11_NOINLINE internals &get_internals() { // Slow path, something needs fetched from the state dict or created gil_scoped_acquire_simple gil; error_scope err_scope; + internals_pp_manager::fail_if_internals_recreated(&internals_ptr); internals_ptr.reset(new internals()); if (!internals_ptr->instance_base) { @@ -847,6 +863,7 @@ inline local_internals &get_local_internals() { auto &internals_ptr = *ppmgr.get_pp(); if (!internals_ptr) { gil_scoped_acquire_simple gil; + internals_pp_manager::fail_if_internals_recreated(&internals_ptr); internals_ptr.reset(new local_internals()); } return *internals_ptr; From 2c8346280ec9108343b3f05dcd910ab24276683e Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sun, 25 Jan 2026 23:02:56 +0800 Subject: [PATCH 20/39] Fix C++11/C++14 support --- include/pybind11/detail/internals.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index cf3fa9c1c3..66ef0fb851 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -691,6 +691,8 @@ class internals_pp_manager { } void destroy() { + auto &pps_have_created_content_ = pps_have_created_content(); + #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT if (has_seen_non_main_interpreter()) { auto *tstate = get_thread_state_unchecked(); @@ -710,6 +712,8 @@ class internals_pp_manager { } static void fail_if_internals_recreated(std::unique_ptr *pp) { + auto &pps_have_created_content_ = pps_have_created_content(); + // Prevent re-creation of internals after destruction during interpreter shutdown. // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals // have been destroyed, a new empty internals would be created, causing type lookup @@ -717,6 +721,7 @@ class internals_pp_manager { if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { pybind11_fail("Reentrant call detected while fetching pybind11 internals!"); } + // Each pp can only create its internals once. Mark this pp as having created its content. pps_have_created_content_.insert(pp); } @@ -762,7 +767,10 @@ class internals_pp_manager { std::unique_ptr *internals_singleton_pp_ = nullptr; // Tracks pointer-to-pointers whose internals have been created, to detect re-entrancy. - inline static std::unordered_set pps_have_created_content_{}; + static std::unordered_set &pps_have_created_content() { + static std::unordered_set value{}; + return value; + } }; // If We loaded the internals through `state_dict`, our `error_already_set` From 6922d7dfec137c8d29e6fad90a6db5fa9a72f324 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sun, 25 Jan 2026 23:46:22 +0800 Subject: [PATCH 21/39] Add lock under multiple interpreters --- include/pybind11/detail/internals.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 66ef0fb851..c3fda7aee1 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -712,6 +712,11 @@ class internals_pp_manager { } static void fail_if_internals_recreated(std::unique_ptr *pp) { +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT + static std::mutex mtx; + std::lock_guard lock(mtx); +#endif + auto &pps_have_created_content_ = pps_have_created_content(); // Prevent re-creation of internals after destruction during interpreter shutdown. From d61f17cb978e2376cc70ba7d9c73890db7346862 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 00:04:12 +0800 Subject: [PATCH 22/39] Try fix tests --- include/pybind11/detail/internals.h | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index c3fda7aee1..1246212cd5 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -691,8 +691,6 @@ class internals_pp_manager { } void destroy() { - auto &pps_have_created_content_ = pps_have_created_content(); - #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT if (has_seen_non_main_interpreter()) { auto *tstate = get_thread_state_unchecked(); @@ -711,13 +709,13 @@ class internals_pp_manager { unref(); } - static void fail_if_internals_recreated(std::unique_ptr *pp) { + void create_pp_content_once(std::unique_ptr *pp) { #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT static std::mutex mtx; std::lock_guard lock(mtx); #endif - auto &pps_have_created_content_ = pps_have_created_content(); + assert(*pp == nullptr); // Prevent re-creation of internals after destruction during interpreter shutdown. // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals @@ -726,8 +724,9 @@ class internals_pp_manager { if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { pybind11_fail("Reentrant call detected while fetching pybind11 internals!"); } - // Each pp can only create its internals once. Mark this pp as having created its content. + // Each pp can only create its internals once. pps_have_created_content_.insert(pp); + pp->reset(new InternalsType()); } private: @@ -772,10 +771,8 @@ class internals_pp_manager { std::unique_ptr *internals_singleton_pp_ = nullptr; // Tracks pointer-to-pointers whose internals have been created, to detect re-entrancy. - static std::unordered_set &pps_have_created_content() { - static std::unordered_set value{}; - return value; - } + // Use instance member over static due to singleton pattern of this class. + std::unordered_set pps_have_created_content_; }; // If We loaded the internals through `state_dict`, our `error_already_set` @@ -816,8 +813,8 @@ PYBIND11_NOINLINE internals &get_internals() { // Slow path, something needs fetched from the state dict or created gil_scoped_acquire_simple gil; error_scope err_scope; - internals_pp_manager::fail_if_internals_recreated(&internals_ptr); - internals_ptr.reset(new internals()); + + ppmgr.create_pp_content_once(&internals_ptr); if (!internals_ptr->instance_base) { // This calls get_internals, so cannot be called from within the internals constructor @@ -876,8 +873,9 @@ inline local_internals &get_local_internals() { auto &internals_ptr = *ppmgr.get_pp(); if (!internals_ptr) { gil_scoped_acquire_simple gil; - internals_pp_manager::fail_if_internals_recreated(&internals_ptr); - internals_ptr.reset(new local_internals()); + error_scope err_scope; + + ppmgr.create_pp_content_once(&internals_ptr); } return *internals_ptr; } From 26e7509e4b552498618fb5cd357cb263f2668b59 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 00:16:20 +0800 Subject: [PATCH 23/39] Try fix tests --- include/pybind11/detail/internals.h | 32 ++++++++++++++++------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 1246212cd5..46cc452edf 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -710,22 +710,26 @@ class internals_pp_manager { } void create_pp_content_once(std::unique_ptr *pp) { -#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT - static std::mutex mtx; - std::lock_guard lock(mtx); + // Assume the GIL is held + { +#ifndef Py_GIL_DISABLED + const gil_scoped_release_simple gil_release{}; #endif - - assert(*pp == nullptr); - - // Prevent re-creation of internals after destruction during interpreter shutdown. - // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals - // have been destroyed, a new empty internals would be created, causing type lookup - // failures. See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. - if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { - pybind11_fail("Reentrant call detected while fetching pybind11 internals!"); + static std::mutex mtx; + std::lock_guard lock(mtx); + + // Prevent re-creation of internals after destruction during interpreter shutdown. + // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals + // have been destroyed, a new empty internals would be created, causing type lookup + // failures. See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. + if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { + pybind11_fail("Reentrant call detected while fetching pybind11 internals!"); + } + // Each pp can only create its internals once. + pps_have_created_content_.insert(pp); } - // Each pp can only create its internals once. - pps_have_created_content_.insert(pp); + + // Create the internals content. May call back into Python. pp->reset(new InternalsType()); } From 6820eadb2d83a369dcdf06fd3c3f426839b7bc71 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 00:35:43 +0800 Subject: [PATCH 24/39] Try fix tests --- include/pybind11/detail/internals.h | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 46cc452edf..98093fe2fe 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -697,33 +697,37 @@ class internals_pp_manager { // this could be called without an active interpreter, just use what was cached if (!tstate || tstate->interp == last_istate_tls()) { auto tpp = internals_p_tls(); - pps_have_created_content_.erase(tpp); - delete tpp; + { + std::lock_guard lock(pps_creation_mutex_); + pps_have_created_content_.erase(tpp); + delete tpp; + } } unref(); return; } #endif - pps_have_created_content_.erase(internals_singleton_pp_); - delete internals_singleton_pp_; + { + std::lock_guard lock(pps_creation_mutex_); + pps_have_created_content_.erase(internals_singleton_pp_); + delete internals_singleton_pp_; + } unref(); } void create_pp_content_once(std::unique_ptr *pp) { // Assume the GIL is held { -#ifndef Py_GIL_DISABLED - const gil_scoped_release_simple gil_release{}; -#endif - static std::mutex mtx; - std::lock_guard lock(mtx); + std::lock_guard lock(pps_creation_mutex_); // Prevent re-creation of internals after destruction during interpreter shutdown. // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals // have been destroyed, a new empty internals would be created, causing type lookup // failures. See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { - pybind11_fail("Reentrant call detected while fetching pybind11 internals!"); + pybind11_fail( + "pybind11::detail::internals_pp_manager::create_pp_content_once() FAILED: " + "reentrant call detected while fetching pybind11 internals!"); } // Each pp can only create its internals once. pps_have_created_content_.insert(pp); @@ -776,7 +780,8 @@ class internals_pp_manager { // Tracks pointer-to-pointers whose internals have been created, to detect re-entrancy. // Use instance member over static due to singleton pattern of this class. - std::unordered_set pps_have_created_content_; + std::unordered_set *> pps_have_created_content_; + std::mutex pps_creation_mutex_; }; // If We loaded the internals through `state_dict`, our `error_already_set` From 15bcbf89dd6e7f34970dfebd0cb78ebfafd0d769 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 01:49:38 +0800 Subject: [PATCH 25/39] Update comments and assertion messages --- include/pybind11/detail/internals.h | 35 +++++++++++++---------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 98093fe2fe..ed98c33d41 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -698,7 +698,7 @@ class internals_pp_manager { if (!tstate || tstate->interp == last_istate_tls()) { auto tpp = internals_p_tls(); { - std::lock_guard lock(pps_creation_mutex_); + std::lock_guard lock(pp_set_mutex_); pps_have_created_content_.erase(tpp); delete tpp; } @@ -708,7 +708,7 @@ class internals_pp_manager { } #endif { - std::lock_guard lock(pps_creation_mutex_); + std::lock_guard lock(pp_set_mutex_); pps_have_created_content_.erase(internals_singleton_pp_); delete internals_singleton_pp_; } @@ -716,24 +716,21 @@ class internals_pp_manager { } void create_pp_content_once(std::unique_ptr *pp) { - // Assume the GIL is held - { - std::lock_guard lock(pps_creation_mutex_); - - // Prevent re-creation of internals after destruction during interpreter shutdown. - // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals - // have been destroyed, a new empty internals would be created, causing type lookup - // failures. See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. - if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { - pybind11_fail( - "pybind11::detail::internals_pp_manager::create_pp_content_once() FAILED: " - "reentrant call detected while fetching pybind11 internals!"); - } - // Each pp can only create its internals once. - pps_have_created_content_.insert(pp); + std::lock_guard lock(pp_set_mutex_); + + // Prevent re-creation of internals after destruction during interpreter shutdown. + // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals + // have been destroyed, a new empty internals would be created, causing type lookup + // failures. See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. + if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { + pybind11_fail("pybind11::detail::internals_pp_manager::create_pp_content_once() " + "FAILED: reentrant call detected while fetching pybind11 internals!"); } + // Each pp can only create its internals once. + pps_have_created_content_.insert(pp); - // Create the internals content. May call back into Python. + // Assume the GIL is held here. May call back into Python. + // Create the internals content. pp->reset(new InternalsType()); } @@ -781,7 +778,7 @@ class internals_pp_manager { // Tracks pointer-to-pointers whose internals have been created, to detect re-entrancy. // Use instance member over static due to singleton pattern of this class. std::unordered_set *> pps_have_created_content_; - std::mutex pps_creation_mutex_; + std::mutex pp_set_mutex_; }; // If We loaded the internals through `state_dict`, our `error_already_set` From b0d350e6c28b9203246150f61fe639c61d065ec4 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 01:55:35 +0800 Subject: [PATCH 26/39] Update comments and assertion messages --- include/pybind11/detail/internals.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index ed98c33d41..b30b4c2d88 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -699,7 +699,7 @@ class internals_pp_manager { auto tpp = internals_p_tls(); { std::lock_guard lock(pp_set_mutex_); - pps_have_created_content_.erase(tpp); + pps_have_created_content_.erase(tpp); // untrack deleted pp delete tpp; } } @@ -709,24 +709,24 @@ class internals_pp_manager { #endif { std::lock_guard lock(pp_set_mutex_); - pps_have_created_content_.erase(internals_singleton_pp_); + pps_have_created_content_.erase(internals_singleton_pp_); // untrack deleted pp delete internals_singleton_pp_; } unref(); } - void create_pp_content_once(std::unique_ptr *pp) { + void create_pp_content_once(std::unique_ptr *const pp) { std::lock_guard lock(pp_set_mutex_); - // Prevent re-creation of internals after destruction during interpreter shutdown. - // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals - // have been destroyed, a new empty internals would be created, causing type lookup - // failures. See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. + // Detect re-creation of internals after destruction during interpreter shutdown. + // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals have + // been destroyed, a new empty internals would be created, causing type lookup failures. + // See also get_or_create_pp_in_state_dict() comments. if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { pybind11_fail("pybind11::detail::internals_pp_manager::create_pp_content_once() " "FAILED: reentrant call detected while fetching pybind11 internals!"); } - // Each pp can only create its internals once. + // Each interpreter can only create its internals once. pps_have_created_content_.insert(pp); // Assume the GIL is held here. May call back into Python. @@ -745,7 +745,8 @@ class internals_pp_manager { // We cannot guarantee the destruction order of capsules in the interpreter state dict on // interpreter shutdown, so deleting internals too early could cause undefined behavior // when other pybind11 objects access `get_internals()` during finalization (which would - // recreate empty internals). + // recreate empty internals). See also create_pp_content_once() above. + // See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. auto result = atomic_get_or_create_in_state_dict>( holder_id_, /*dtor=*/nullptr /* leak the capsule content */); auto *pp = result.first; @@ -775,9 +776,9 @@ class internals_pp_manager { // main interpreter) std::unique_ptr *internals_singleton_pp_ = nullptr; - // Tracks pointer-to-pointers whose internals have been created, to detect re-entrancy. + // Track pointer-to-pointers whose internals have been created, to detect re-entrancy. // Use instance member over static due to singleton pattern of this class. - std::unordered_set *> pps_have_created_content_; + std::unordered_set pps_have_created_content_; std::mutex pp_set_mutex_; }; From 33ffa8ec1e45368ec43ca753aadbd9a5cec353f4 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 11:57:17 +0800 Subject: [PATCH 27/39] Update comments --- tests/test_custom_type_setup.cpp | 5 ++++- tests/test_custom_type_setup.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_custom_type_setup.cpp b/tests/test_custom_type_setup.cpp index a25e4c109a..15516b21b0 100644 --- a/tests/test_custom_type_setup.cpp +++ b/tests/test_custom_type_setup.cpp @@ -64,7 +64,10 @@ void add_gc_checkers_with_weakrefs(const py::object &obj) { TEST_SUBMODULE(custom_type_setup, m) { py::class_ cls( - m, "ContainerOwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { + m, + "ContainerOwnsPythonObjects", + // Please review/update docs/advanced/classes.rst after making changes here. + py::custom_type_setup([](PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) { diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index 42cd42eb7e..4c6b9510ae 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -58,7 +58,10 @@ def test_indirect_cycle(gc_tester): ) @pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_py_cast_useable_on_shutdown(): - """Test that py::cast works during interpreter shutdown.""" + """Test that py::cast works during interpreter shutdown. + + See PR #5972 and https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. + """ env.check_script_success_in_subprocess( f""" import sys From 708ca55d4c2567d7f0aebaa2e12d4b2cb703263b Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 12:32:15 +0800 Subject: [PATCH 28/39] Update lock scope --- include/pybind11/detail/internals.h | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index b30b4c2d88..5f64bd0dc7 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -716,18 +716,21 @@ class internals_pp_manager { } void create_pp_content_once(std::unique_ptr *const pp) { - std::lock_guard lock(pp_set_mutex_); - - // Detect re-creation of internals after destruction during interpreter shutdown. - // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals have - // been destroyed, a new empty internals would be created, causing type lookup failures. - // See also get_or_create_pp_in_state_dict() comments. - if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { - pybind11_fail("pybind11::detail::internals_pp_manager::create_pp_content_once() " - "FAILED: reentrant call detected while fetching pybind11 internals!"); + { + std::lock_guard lock(pp_set_mutex_); + + // Detect re-creation of internals after destruction during interpreter shutdown. + // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals + // have been destroyed, a new empty internals would be created, causing type lookup + // failures. See also get_or_create_pp_in_state_dict() comments. + if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { + pybind11_fail( + "pybind11::detail::internals_pp_manager::create_pp_content_once() " + "FAILED: reentrant call detected while fetching pybind11 internals!"); + } + // Each interpreter can only create its internals once. + pps_have_created_content_.insert(pp); } - // Each interpreter can only create its internals once. - pps_have_created_content_.insert(pp); // Assume the GIL is held here. May call back into Python. // Create the internals content. From 85dc7e6e177c62b419850bec34aa36c81d560ef2 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 13:03:50 +0800 Subject: [PATCH 29/39] Use original pointer type for Windows --- include/pybind11/detail/internals.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 5f64bd0dc7..5e3bc51fec 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -781,7 +781,7 @@ class internals_pp_manager { // Track pointer-to-pointers whose internals have been created, to detect re-entrancy. // Use instance member over static due to singleton pattern of this class. - std::unordered_set pps_have_created_content_; + std::unordered_set *> pps_have_created_content_; std::mutex pp_set_mutex_; }; From 404457ef3d52bd05761df123989f5498f18be876 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 15:14:52 +0800 Subject: [PATCH 30/39] Change hard error to warning --- include/pybind11/detail/internals.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 5e3bc51fec..1434d7a514 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -724,9 +724,10 @@ class internals_pp_manager { // have been destroyed, a new empty internals would be created, causing type lookup // failures. See also get_or_create_pp_in_state_dict() comments. if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { - pybind11_fail( - "pybind11::detail::internals_pp_manager::create_pp_content_once() " - "FAILED: reentrant call detected while fetching pybind11 internals!"); + PyErr_WarnEx(PyExc_RuntimeWarning, + "pybind11::detail::internals_pp_manager::create_pp_content_once() " + "FAILED: reentrant call detected while fetching pybind11 internals!", + /*stack_level=*/2); } // Each interpreter can only create its internals once. pps_have_created_content_.insert(pp); From 51a70ab602c04a3f6712d33cf6861809254c541b Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 26 Jan 2026 17:40:15 +0800 Subject: [PATCH 31/39] Update lock scope --- include/pybind11/detail/internals.h | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 1434d7a514..4d42c77e9f 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -716,22 +716,23 @@ class internals_pp_manager { } void create_pp_content_once(std::unique_ptr *const pp) { - { - std::lock_guard lock(pp_set_mutex_); + std::lock_guard lock(pp_set_mutex_); - // Detect re-creation of internals after destruction during interpreter shutdown. - // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals - // have been destroyed, a new empty internals would be created, causing type lookup - // failures. See also get_or_create_pp_in_state_dict() comments. - if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { - PyErr_WarnEx(PyExc_RuntimeWarning, - "pybind11::detail::internals_pp_manager::create_pp_content_once() " - "FAILED: reentrant call detected while fetching pybind11 internals!", - /*stack_level=*/2); - } - // Each interpreter can only create its internals once. - pps_have_created_content_.insert(pp); + if (*pp) { + // Already created in another thread. + return; + } + + // Detect re-creation of internals after destruction during interpreter shutdown. + // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals have + // been destroyed, a new empty internals would be created, causing type lookup failures. + // See also get_or_create_pp_in_state_dict() comments. + if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { + pybind11_fail("pybind11::detail::internals_pp_manager::create_pp_content_once() " + "FAILED: reentrant call detected while fetching pybind11 internals!"); } + // Each interpreter can only create its internals once. + pps_have_created_content_.insert(pp); // Assume the GIL is held here. May call back into Python. // Create the internals content. From 5f96327ee70f835c73e129cf19d1a3bd3c8101b4 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 27 Jan 2026 01:05:58 +0800 Subject: [PATCH 32/39] Update lock scope to resolve deadlock --- include/pybind11/detail/internals.h | 34 +++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 4d42c77e9f..165d21a9e3 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -716,23 +716,29 @@ class internals_pp_manager { } void create_pp_content_once(std::unique_ptr *const pp) { - std::lock_guard lock(pp_set_mutex_); + { +#ifndef Py_GIL_DISABLED + gil_scoped_release_simple gil_release{}; +#endif + std::lock_guard lock(pp_set_mutex_); - if (*pp) { - // Already created in another thread. - return; - } + if (*pp) { + // Already created in another thread. + return; + } - // Detect re-creation of internals after destruction during interpreter shutdown. - // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals have - // been destroyed, a new empty internals would be created, causing type lookup failures. - // See also get_or_create_pp_in_state_dict() comments. - if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { - pybind11_fail("pybind11::detail::internals_pp_manager::create_pp_content_once() " - "FAILED: reentrant call detected while fetching pybind11 internals!"); + // Detect re-creation of internals after destruction during interpreter shutdown. + // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals + // have been destroyed, a new empty internals would be created, causing type lookup + // failures. See also get_or_create_pp_in_state_dict() comments. + if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { + pybind11_fail( + "pybind11::detail::internals_pp_manager::create_pp_content_once() " + "FAILED: reentrant call detected while fetching pybind11 internals!"); + } + // Each interpreter can only create its internals once. + pps_have_created_content_.insert(pp); } - // Each interpreter can only create its internals once. - pps_have_created_content_.insert(pp); // Assume the GIL is held here. May call back into Python. // Create the internals content. From 40731b790737aa5bf605daa9fb69fa878f19ab65 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 27 Jan 2026 01:21:03 +0800 Subject: [PATCH 33/39] Remove scope release of GIL --- include/pybind11/detail/internals.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 165d21a9e3..1a14d2f759 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -717,9 +717,6 @@ class internals_pp_manager { void create_pp_content_once(std::unique_ptr *const pp) { { -#ifndef Py_GIL_DISABLED - gil_scoped_release_simple gil_release{}; -#endif std::lock_guard lock(pp_set_mutex_); if (*pp) { From aa1767c5d245c74234bffd9e126d2d8a19dab5a1 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 27 Jan 2026 01:37:28 +0800 Subject: [PATCH 34/39] Update comments --- include/pybind11/detail/internals.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 1a14d2f759..a0f646034c 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -717,6 +717,8 @@ class internals_pp_manager { void create_pp_content_once(std::unique_ptr *const pp) { { + // Lock scope must not include Python calls, which may require the GIL and cause + // deadlocks. std::lock_guard lock(pp_set_mutex_); if (*pp) { @@ -724,6 +726,9 @@ class internals_pp_manager { return; } + // At this point, pp->get() is nullptr. + // The content is either not yet created, or was previously destroyed via pp->reset(). + // Detect re-creation of internals after destruction during interpreter shutdown. // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals // have been destroyed, a new empty internals would be created, causing type lookup From dea56609fbc5c8c561ba5a8e337d267238822395 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 27 Jan 2026 02:37:38 +0800 Subject: [PATCH 35/39] Lock pp on reset --- include/pybind11/detail/internals.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index a0f646034c..766c475523 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -744,7 +744,20 @@ class internals_pp_manager { // Assume the GIL is held here. May call back into Python. // Create the internals content. - pp->reset(new InternalsType()); + auto tmp = std::unique_ptr(new InternalsType()); + + { + // Lock scope must not include Python calls, which may require the GIL and cause + // deadlocks. + std::lock_guard lock(pp_set_mutex_); + + // Double-check that another thread didn't create the content while we were creating + // it above without holding the lock. + if (!*pp) { + // Install the created content. + pp->swap(tmp); + } + } } private: From 691241a0e27665b0145aea0901c3a2ab03ac8839 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 27 Jan 2026 03:13:00 +0800 Subject: [PATCH 36/39] Mark content created after assignment --- include/pybind11/detail/internals.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 766c475523..ae799e0890 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -738,8 +738,6 @@ class internals_pp_manager { "pybind11::detail::internals_pp_manager::create_pp_content_once() " "FAILED: reentrant call detected while fetching pybind11 internals!"); } - // Each interpreter can only create its internals once. - pps_have_created_content_.insert(pp); } // Assume the GIL is held here. May call back into Python. @@ -757,6 +755,9 @@ class internals_pp_manager { // Install the created content. pp->swap(tmp); } + + // Each interpreter can only create its internals once. + pps_have_created_content_.insert(pp); } } From 552f8b0eb53c84fec646daa26ccb0686381b3517 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 27 Jan 2026 03:36:51 +0800 Subject: [PATCH 37/39] Update comments --- include/pybind11/detail/internals.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index ae799e0890..f8a304eb8a 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -740,8 +740,9 @@ class internals_pp_manager { } } - // Assume the GIL is held here. May call back into Python. - // Create the internals content. + // Assume the GIL is held here. May call back into Python. We cannot hold the lock with our + // mutex here. So there may be multiple threads creating the content at the same time. Only + // one will install its content to pp below. Others will be freed when going out of scope. auto tmp = std::unique_ptr(new InternalsType()); { From 79a80d209530e5cd792f1e9d17339131bd98d06b Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 27 Jan 2026 14:30:07 +0800 Subject: [PATCH 38/39] Simplify implementation --- include/pybind11/detail/internals.h | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index f8a304eb8a..a6e4c6fbf6 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -716,6 +716,11 @@ class internals_pp_manager { } void create_pp_content_once(std::unique_ptr *const pp) { + // Assume the GIL is held here. May call back into Python. We cannot hold the lock with our + // mutex here. So there may be multiple threads creating the content at the same time. Only + // one will install its content to pp below. Others will be freed when going out of scope. + auto tmp = std::unique_ptr(new InternalsType()); + { // Lock scope must not include Python calls, which may require the GIL and cause // deadlocks. @@ -738,27 +743,11 @@ class internals_pp_manager { "pybind11::detail::internals_pp_manager::create_pp_content_once() " "FAILED: reentrant call detected while fetching pybind11 internals!"); } - } - - // Assume the GIL is held here. May call back into Python. We cannot hold the lock with our - // mutex here. So there may be multiple threads creating the content at the same time. Only - // one will install its content to pp below. Others will be freed when going out of scope. - auto tmp = std::unique_ptr(new InternalsType()); - - { - // Lock scope must not include Python calls, which may require the GIL and cause - // deadlocks. - std::lock_guard lock(pp_set_mutex_); - - // Double-check that another thread didn't create the content while we were creating - // it above without holding the lock. - if (!*pp) { - // Install the created content. - pp->swap(tmp); - } // Each interpreter can only create its internals once. pps_have_created_content_.insert(pp); + // Install the created content. + pp->swap(tmp); } } From 56926f67bfd1e95175f3558d60dbade82b6ef192 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 27 Jan 2026 18:44:09 +0800 Subject: [PATCH 39/39] Update lock scope when delete unique_ptr --- include/pybind11/detail/internals.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index a6e4c6fbf6..7cfa5da921 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -700,8 +700,8 @@ class internals_pp_manager { { std::lock_guard lock(pp_set_mutex_); pps_have_created_content_.erase(tpp); // untrack deleted pp - delete tpp; } + delete tpp; // may call back into Python } unref(); return; @@ -710,8 +710,8 @@ class internals_pp_manager { { std::lock_guard lock(pp_set_mutex_); pps_have_created_content_.erase(internals_singleton_pp_); // untrack deleted pp - delete internals_singleton_pp_; } + delete internals_singleton_pp_; // may call back into Python unref(); }