diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 2954411d7b..a5de5b1efa 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -875,6 +875,12 @@ The ``__setstate__`` part of the ``py::pickle()`` definition follows the same rules as the single-argument version of ``py::init()``. The return type can be a value, pointer or holder type. See :ref:`custom_constructors` for details. +Calling ``__new__`` directly creates the Python wrapper without constructing its C++ value. +Outside deprecated placement-new constructor dispatch, passing such an uninitialized wrapper to +bound C++ code raises ``ValueError``. Calling its ``__init__`` or a pickle-generated +``__setstate__`` can still finish construction normally. See :ref:`old_style_placement_new` for +the compatibility behavior and safety limitations of deprecated placement-new callbacks. + An instance can now be pickled as follows: .. code-block:: python @@ -1427,4 +1433,12 @@ You can do that using ``py::custom_type_setup``: cls.def("size", &ContainerOwnsPythonObjects::size); cls.def("clear", &ContainerOwnsPythonObjects::clear); +.. note:: + + The ``py::detail::is_holder_constructed()`` guards above are required. During garbage + collection, ``tp_traverse`` and ``tp_clear`` may be handed an instance whose C++ value has + not been constructed yet -- for example one created with ``__new__`` before ``__init__`` + has run. Casting such an instance raises ``ValueError``, and an exception must not be + allowed to escape either of these slots. + .. versionadded:: 2.8 diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 966a319a80..236494a5cd 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -381,6 +381,8 @@ The old macro emits a compile-time deprecation warning. } +.. _old_style_placement_new: + New API for defining custom constructors and pickling functions --------------------------------------------------------------- @@ -408,6 +410,25 @@ constructors prevent such mistakes. See :ref:`custom_constructors` for details. // or: return Foo(...); // return by value (move constructor) })); +.. warning:: + + Deprecated placement-new ``__init__`` and ``__setstate__`` callbacks receive access to raw + storage before the C++ object's lifetime begins. For compatibility, pybind11 retains this + behavior for the complete constructor overload chain whenever the chain contains such a + callback. This compatibility feature has important caveats: other loads triggered during + argument conversion or callback execution may receive a C++ pointer to the storage even + though no C++ object has been constructed there yet. Accessing the storage through such + a pointer as though it contained a live C++ object results in undefined behavior. + + Consequently, until placement-new completes, the binding must not otherwise load or inspect + the instance as a C++ object. Unsafe access can occur through reentrant argument conversion + or callback code, nested initialization, another C++ base in a Python multiple-inheritance + instance, or concurrent access. Mixing old- and new-style constructor overloads does not + narrow the window. Such access may treat unconstructed storage as a live object and result + in undefined behavior. To avoid these hazards, use ``py::init()`` factories and + ``py::pickle()`` for new bindings, and migrate existing placement-new callbacks wherever + practical. + Mirroring the custom constructor changes, ``py::pickle()`` is now the preferred way to get and set object state. See :ref:`pickling` for details. diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 9f8b3b3266..7738b3f39b 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -676,6 +676,14 @@ struct instance { bool has_patients : 1; /// If true, this Python object needs to be kept alive for the lifetime of the C++ value. bool is_alias : 1; + /// If true, this instance is being dispatched through a constructor chain containing a + /// deprecated old-style placement-new `__init__`/`__setstate__`. Such chains retain the + /// historical ability to lazily allocate C++ value storage. This is an instance-wide + /// compatibility marker, not per-value construction state or a synchronization mechanism. + /// Its intentionally retained safety limitations are documented under + /// `old_style_placement_new` in `docs/upgrade.rst` and referenced from + /// `docs/advanced/classes.rst`. + bool old_style_init_active : 1; /// Initializes all of the above type/values/holders data (but not the instance values /// themselves) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 82bfa0b27c..65b6bd07df 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -525,6 +525,7 @@ PYBIND11_NOINLINE void instance::allocate_layout() { = reinterpret_cast(&nonsimple.values_and_holders[flags_at]); } owned = true; + old_style_init_active = false; } // NOLINTNEXTLINE(readability-make-member-function-const) @@ -534,6 +535,45 @@ PYBIND11_NOINLINE void instance::deallocate_layout() { } } +/// RAII helper preserving lazy value allocation for a constructor chain containing a deprecated +/// old-style placement-new `__init__`/`__setstate__`. Passing `nullptr` makes this a no-op. The +/// compatibility window covers the whole chain and all value slots in the Python instance; it does +/// not attempt to distinguish the old-style `self` load from reentrant, later-argument, +/// cross-base, nested, or concurrent loads. Nesting restores the previous state but is not made +/// safe by this scope. +/// Before narrowing this window, review `old_style_placement_new` in `docs/upgrade.rst` and its +/// reference from `docs/advanced/classes.rst`: the broad scope preserves historical behavior, +/// with documented reentrancy, multiple-inheritance, nesting, and concurrency limitations. +/// +/// If construction fails (the holder was never constructed) after storage was lazily allocated +/// inside this scope, the destructor frees that storage and resets the value pointer, so that the +/// uninitialized-value guard in `load_value()` stays effective for later uses of the instance. +class old_style_init_scope { +public: + explicit old_style_init_scope(value_and_holder *v_h) : v_h_{v_h} { + if (v_h_ != nullptr) { + was_active_ = v_h_->inst->old_style_init_active; + value_was_null_ = v_h_->value_ptr() == nullptr; + v_h_->inst->old_style_init_active = true; + } + } + ~old_style_init_scope() { + if (v_h_ != nullptr) { + v_h_->inst->old_style_init_active = was_active_; + if (value_was_null_ && !v_h_->holder_constructed() && v_h_->value_ptr() != nullptr) { + v_h_->type->dealloc(*v_h_); // Frees the storage and nulls the value pointer. + } + } + } + old_style_init_scope(const old_style_init_scope &) = delete; + old_style_init_scope &operator=(const old_style_init_scope &) = delete; + +private: + value_and_holder *v_h_; + bool was_active_ = false; + bool value_was_null_ = false; +}; + PYBIND11_NOINLINE bool isinstance_generic(handle obj, const std::type_info &tp) { handle type = detail::get_type_handle(tp, false); if (!type) { @@ -1140,6 +1180,20 @@ class type_caster_generic { auto *&vptr = v_h.value_ptr(); // Lazy allocation for unallocated values: if (vptr == nullptr) { + // Lazy allocation exists only to support the deprecated old-style placement-new + // `__init__`/`__setstate__` idiom, which is handed a reference to uninitialized + // storage and constructs the C++ value into it. In any other context a null value + // pointer means the C++ object was never constructed -- e.g. the instance was created + // with `__new__()`, bypassing `__init__()` -- and handing out a pointer to + // uninitialized memory from here is undefined behavior (typically a segfault on the + // first virtual call). Fail loudly instead. + if (!v_h.inst->old_style_init_active) { + throw value_error("Missing value for wrapped C++ type `" + + clean_type_id(cpptype->name()) + + "`: Python instance is uninitialized: the C++ object was " + "never constructed (`__init__()` was bypassed, e.g. by " + "calling `__new__()` directly)."); + } const auto *type = v_h.type ? v_h.type : typeinfo; if (type->operator_new) { vptr = type->operator_new(type->type_size); diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 3687983460..953e7c8844 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1001,6 +1001,25 @@ class cpp_function : public function { } } + // While a constructor chain containing an old-style placement-new + // `__init__`/`__setstate__` runs, `type_caster_generic::load_value()` is permitted to + // lazily allocate storage for the C++ value that the constructor is about to construct + // into. New-style constructors never load `self` through a type caster (it is injected + // directly below), so the scope stays disarmed for chains that contain only new-style + // constructors and loading a not-yet-constructed instance remains an error even while they + // run. The scope also frees storage that was lazily allocated by a constructor call that + // then failed. + detail::value_and_holder *lazily_allocatable_v_h = nullptr; + if (overloads->is_constructor) { + for (const function_record *fr = overloads; fr != nullptr; fr = fr->next) { + if (!fr->is_new_style_constructor) { + lazily_allocatable_v_h = &self_value_and_holder; + break; + } + } + } + detail::old_style_init_scope old_style_init_guard(lazily_allocatable_v_h); + try { // We do this in two passes: in the first pass, we load arguments with `convert=false`; // in the second, we allow conversion (except for arguments with an explicit diff --git a/tests/test_class.cpp b/tests/test_class.cpp index e520f29ec5..21ff617365 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -77,6 +77,27 @@ static_assert(!py::detail::is_same_or_base_of< test_class::pr5396_forward_declared_class::ForwardClass>::value, ""); +// test_new_bypasses_init +struct NewNoInit { + int m_data; + explicit NewNoInit(int data) : m_data(data) {} + NewNoInit(const NewNoInit &) = default; + virtual ~NewNoInit() = default; + int data() const { return m_data; } + // Virtual on purpose: using a not-yet-constructed instance reads the vtable pointer out of + // uninitialized storage, which segfaults rather than merely returning a garbage value. + virtual int v_data() const { return m_data; } +}; + +// test_failed_old_style_init_does_not_leave_lazy_storage +struct OldStyleInit { + int m_data; + explicit OldStyleInit(int data) : m_data(data) {} + virtual ~OldStyleInit() = default; + int data() const { return m_data; } + virtual int v_data() const { return m_data; } +}; + TEST_SUBMODULE(class_, m) { m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); @@ -597,6 +618,38 @@ TEST_SUBMODULE(class_, m) { m.def("return_universal_recipient", []() -> test_class::ConvertibleFromAnything { return test_class::ConvertibleFromAnything{}; }); + + py::class_(m, "NewNoInit") + .def(py::init()) + .def("data", &NewNoInit::data) + .def("v_data", &NewNoInit::v_data) + .def(py::pickle([](const NewNoInit &p) { return py::make_tuple(p.m_data); }, + [](const py::tuple &t) { + if (t.size() != 1) { + throw std::runtime_error("Invalid state!"); + } + return NewNoInit(t[0].cast()); + })); + + py::class_ old_style_init(m, "OldStyleInit"); + ignoreOldStyleInitWarnings([&old_style_init]() { + old_style_init + .def("__init__", + [](OldStyleInit &self, int x) { + if (x < 0) { + throw std::runtime_error("negative data"); + } + new (&self) OldStyleInit(x); + }) + .def("__setstate__", [](const py::object &self, int x) { + auto &typed_self = self.cast(); + new (&typed_self) OldStyleInit(x); + }); + }); + old_style_init.def("data", &OldStyleInit::data).def("v_data", &OldStyleInit::v_data); + // This probe intentionally does not dereference the pointer. It documents the narrow scope of + // this fix without itself reading storage before an OldStyleInit lifetime has begun. + m.def("expose_old_style_init_pointer", [](OldStyleInit *value) { return value != nullptr; }); } template diff --git a/tests/test_class.py b/tests/test_class.py index 201c7e339e..645b799c9f 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -1,6 +1,7 @@ from __future__ import annotations import gc +import pickle import sys from unittest import mock @@ -251,6 +252,115 @@ def __init__(self): assert msg(exc_info.value) == expected +def test_new_bypasses_init(): + """`__new__` allocates the Python object but not the C++ one; using the instance before + `__init__` has run must raise instead of segfaulting.""" + + class PythonDerived(m.NewNoInit): + pass + + for cls in (m.NewNoInit, PythonDerived): + obj = cls.__new__(cls) + for use in (obj.data, obj.v_data, obj.__getstate__): + with pytest.raises(ValueError) as exc_info: + use() + assert "Python instance is uninitialized" in str(exc_info.value) + assert "NewNoInit" in str(exc_info.value) + + # Calling `__init__()` is the sanctioned way to finish an object made with `__new__()`. + obj.__init__(42) + assert obj.data() == 42 + assert obj.v_data() == 42 + + +def test_new_then_setstate(): + """`__new__` must not be blocked: pickle relies on it, and `__setstate__` finishes the + object off. This walks the protocol by hand, then checks the real thing.""" + real_obj = m.NewNoInit(42) + assert real_obj.data() == 42 + state = real_obj.__getstate__() + + obj = m.NewNoInit.__new__(m.NewNoInit) # NEWOBJ + obj.__setstate__(state) # BUILD + assert obj.data() == 42 + assert obj.v_data() == 42 + + for protocol in range(2, pickle.HIGHEST_PROTOCOL + 1): + assert pickle.loads(pickle.dumps(m.NewNoInit(7), protocol)).v_data() == 7 + + +def test_failed_old_style_init_does_not_leave_lazy_storage(): + """If an old-style placement-new `__init__` throws before constructing the value, the + lazily allocated storage must not linger: later use must still raise, not segfault.""" + obj = m.OldStyleInit.__new__(m.OldStyleInit) + with pytest.raises(RuntimeError, match="negative data"): + obj.__init__(-1) + + # The failed __init__ already lazily allocated storage for `self`, so without cleanup the + # uninitialized-instance guard never fires again and this reads a garbage vtable pointer. + with pytest.raises(ValueError, match="uninitialized"): + obj.v_data() + + # A successful retry is still allowed. + obj.__init__(42) + assert obj.v_data() == 42 + + +def test_old_style_setstate_remains_supported(): + """Deprecated placement-new `__setstate__` may still obtain storage inside its callback.""" + obj = m.OldStyleInit.__new__(m.OldStyleInit) + obj.__setstate__(43) + assert obj.data() == 43 + + +def test_old_style_init_reentrant_load_current_limitation(): + """The historical broad lazy-allocation window retained during an old-style constructor + chain can expose a pointer to unconstructed storage through a reentrant load.""" + obj = m.OldStyleInit.__new__(m.OldStyleInit) + seen = {} + + class LoadOnIndex: + def __index__(self): + # This pointer-only probe deliberately does not inspect or dereference the storage. + seen["exposed"] = m.expose_old_style_init_pointer(obj) + raise TypeError("stop the constructor") + + with pytest.raises(TypeError): + obj.__init__(LoadOnIndex()) + + assert seen == {"exposed": True} + + # Failure cleanup removes the raw storage, so subsequent ordinary loads are rejected and a + # normal initialization retry remains possible. + with pytest.raises(ValueError, match="uninitialized"): + m.expose_old_style_init_pointer(obj) + obj.__init__(44) + assert obj.data() == 44 + + +def test_reentrant_load_during_new_style_init(): + """New-style constructors never need lazy allocation, so passing the half-built instance + to another bound function while `__init__` runs must raise, not hand out garbage.""" + obj = m.NewNoInit.__new__(m.NewNoInit) + seen = {} + + class Evil: + def __index__(self): + # Runs during int conversion of a pure new-style constructor. Its chain has no + # compatibility window for lazy allocation. + try: + seen["data"] = obj.data() + except ValueError as exc: + seen["error"] = exc + raise TypeError("stop the constructor") + + with pytest.raises(TypeError): + obj.__init__(Evil()) + + assert "data" not in seen, f"handed out uninitialized storage: {seen['data']!r}" + assert "error" in seen + + @pytest.mark.parametrize( "mock_return_value", [None, (1, 2, 3), m.Pet("Polly", "parrot"), m.Dog("Molly")] )