Merge pull request #437 from dean0x7d/dynamic-attrs

Add dynamic attribute support

docs/classes.rst

@@ -165,6 +165,66 @@ the setter and getter functions:
     static variables and properties. Please also see the section on
     static variables and properties. Please also see the section on
     :ref:`static_properties` in the advanced part of the documentation.
     :ref:`static_properties` in the advanced part of the documentation.
 
 
+Dynamic attributes
+==================
+
+Native Python classes can pick up new attributes dynamically:
+
+.. code-block:: pycon
+
+    >>> class Pet:
+    ...     name = 'Molly'
+    ...
+    >>> p = Pet()
+    >>> p.name = 'Charly'  # overwrite existing
+    >>> p.age = 2  # dynamically add a new attribute
+
+By default, classes exported from C++ do not support this and the only writable
+attributes are the ones explicitly defined using :func:`class_::def_readwrite`
+or :func:`class_::def_property`.
+
+.. code-block:: cpp
+
+    py::class_<Pet>(m, "Pet")
+        .def(py::init<>())
+        .def_readwrite("name", &Pet::name);
+
+Trying to set any other attribute results in an error:
+
+.. code-block:: pycon
+
+    >>> p = example.Pet()
+    >>> p.name = 'Charly'  # OK, attribute defined in C++
+    >>> p.age = 2  # fail
+    AttributeError: 'Pet' object has no attribute 'age'
+
+To enable dynamic attributes for C++ classes, the :class:`py::dynamic_attr` tag
+must be added to the :class:`py::class_` constructor:
+
+.. code-block:: cpp
+
+    py::class_<Pet>(m, "Pet", py::dynamic_attr())
+        .def(py::init<>())
+        .def_readwrite("name", &Pet::name);
+
+Now everything works as expected:
+
+.. code-block:: pycon
+
+    >>> p = example.Pet()
+    >>> p.name = 'Charly'  # OK, overwrite value in C++
+    >>> p.age = 2  # OK, dynamically add a new attribute
+    >>> p.__dict__  # just like a native Python class
+    {'age': 2}
+
+Note that there is a small runtime cost for a class with dynamic attributes.
+Not only because of the addition of a ``__dict__``, but also because of more
+expensive garbage collection tracking which must be activated to resolve
+possible circular references. Native Python classes incur this same cost by
+default, so this is not anything to worry about. By default, pybind11 classes
+are more efficient than native Python classes. Enabling dynamic attributes
+just brings them on par.
+
 .. _inheritance:
 .. _inheritance:
 
 
 Inheritance
 Inheritance

include/pybind11/attr.h

@@ -44,6 +44,9 @@ template <int Nurse, int Patient> struct keep_alive { };
 /// Annotation indicating that a class is involved in a multiple inheritance relationship
 /// Annotation indicating that a class is involved in a multiple inheritance relationship
 struct multiple_inheritance { };
 struct multiple_inheritance { };
 
 
+/// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class
+struct dynamic_attr { };
+
 NAMESPACE_BEGIN(detail)
 NAMESPACE_BEGIN(detail)
 /* Forward declarations */
 /* Forward declarations */
 enum op_id : int;
 enum op_id : int;
@@ -162,6 +165,9 @@ struct type_record {
     /// Multiple inheritance marker
     /// Multiple inheritance marker
     bool multiple_inheritance = false;
     bool multiple_inheritance = false;
 
 
+    /// Does the class manage a __dict__?
+    bool dynamic_attr = false;
+
     PYBIND11_NOINLINE void add_base(const std::type_info *base, void *(*caster)(void *)) {
     PYBIND11_NOINLINE void add_base(const std::type_info *base, void *(*caster)(void *)) {
         auto base_info = detail::get_type_info(*base, false);
         auto base_info = detail::get_type_info(*base, false);
         if (!base_info) {
         if (!base_info) {
@@ -292,6 +298,11 @@ struct process_attribute<multiple_inheritance> : process_attribute_default<multi
     static void init(const multiple_inheritance &, type_record *r) { r->multiple_inheritance = true; }
     static void init(const multiple_inheritance &, type_record *r) { r->multiple_inheritance = true; }
 };
 };
 
 
+template <>
+struct process_attribute<dynamic_attr> : process_attribute_default<dynamic_attr> {
+    static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; }
+};
+
 /***
 /***
  * Process a keep_alive call policy -- invokes keep_alive_impl during the
  * Process a keep_alive call policy -- invokes keep_alive_impl during the
  * pre-call handler if both Nurse, Patient != 0 and use the post-call handler
  * pre-call handler if both Nurse, Patient != 0 and use the post-call handler

include/pybind11/pybind11.h

@@ -573,6 +573,33 @@ public:
 };
 };
 
 
 NAMESPACE_BEGIN(detail)
 NAMESPACE_BEGIN(detail)
+extern "C" inline PyObject *get_dict(PyObject *op, void *) {
+    PyObject *&dict = *_PyObject_GetDictPtr(op);
+    if (!dict) {
+        dict = PyDict_New();
+    }
+    Py_XINCREF(dict);
+    return dict;
+}
+
+extern "C" inline int set_dict(PyObject *op, PyObject *new_dict, void *) {
+    if (!PyDict_Check(new_dict)) {
+        PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, not a '%.200s'",
+                     Py_TYPE(new_dict)->tp_name);
+        return -1;
+    }
+    PyObject *&dict = *_PyObject_GetDictPtr(op);
+    Py_INCREF(new_dict);
+    Py_CLEAR(dict);
+    dict = new_dict;
+    return 0;
+}
+
+static PyGetSetDef generic_getset[] = {
+    {const_cast<char*>("__dict__"), get_dict, set_dict, nullptr, nullptr},
+    {nullptr, nullptr, nullptr, nullptr, nullptr}
+};
+
 /// Generic support for creating new Python heap types
 /// Generic support for creating new Python heap types
 class generic_type : public object {
 class generic_type : public object {
     template <typename...> friend class class_;
     template <typename...> friend class class_;
@@ -684,6 +711,16 @@ protected:
 #endif
 #endif
         type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC;
         type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC;
 
 
+        /* Support dynamic attributes */
+        if (rec->dynamic_attr) {
+            type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_GC;
+            type->ht_type.tp_dictoffset = type->ht_type.tp_basicsize; // place the dict at the end
+            type->ht_type.tp_basicsize += sizeof(PyObject *); // and allocate enough space for it
+            type->ht_type.tp_getset = generic_getset;
+            type->ht_type.tp_traverse = traverse;
+            type->ht_type.tp_clear = clear;
+        }
+
         type->ht_type.tp_doc = tp_doc;
         type->ht_type.tp_doc = tp_doc;
 
 
         if (PyType_Ready(&type->ht_type) < 0)
         if (PyType_Ready(&type->ht_type) < 0)
@@ -785,10 +822,27 @@ protected:
 
 
             if (self->weakrefs)
             if (self->weakrefs)
                 PyObject_ClearWeakRefs((PyObject *) self);
                 PyObject_ClearWeakRefs((PyObject *) self);
+
+            PyObject **dict_ptr = _PyObject_GetDictPtr((PyObject *) self);
+            if (dict_ptr) {
+                Py_CLEAR(*dict_ptr);
+            }
         }
         }
         Py_TYPE(self)->tp_free((PyObject*) self);
         Py_TYPE(self)->tp_free((PyObject*) self);
     }
     }
 
 
+    static int traverse(PyObject *op, visitproc visit, void *arg) {
+        PyObject *&dict = *_PyObject_GetDictPtr(op);
+        Py_VISIT(dict);
+        return 0;
+    }
+
+    static int clear(PyObject *op) {
+        PyObject *&dict = *_PyObject_GetDictPtr(op);
+        Py_CLEAR(dict);
+        return 0;
+    }
+
     void install_buffer_funcs(
     void install_buffer_funcs(
             buffer_info *(*get_buffer)(PyObject *, void *),
             buffer_info *(*get_buffer)(PyObject *, void *),
             void *get_buffer_data) {
             void *get_buffer_data) {

tests/test_methods_and_attributes.cpp

@@ -53,6 +53,12 @@ public:
     int value = 0;
     int value = 0;
 };
 };
 
 
+class DynamicClass {
+public:
+    DynamicClass() { print_default_created(this); }
+    ~DynamicClass() { print_destroyed(this); }
+};
+
 test_initializer methods_and_attributes([](py::module &m) {
 test_initializer methods_and_attributes([](py::module &m) {
     py::class_<ExampleMandA>(m, "ExampleMandA")
     py::class_<ExampleMandA>(m, "ExampleMandA")
         .def(py::init<>())
         .def(py::init<>())
@@ -81,4 +87,7 @@ test_initializer methods_and_attributes([](py::module &m) {
         .def("__str__", &ExampleMandA::toString)
         .def("__str__", &ExampleMandA::toString)
         .def_readwrite("value", &ExampleMandA::value)
         .def_readwrite("value", &ExampleMandA::value)
         ;
         ;
+
+    py::class_<DynamicClass>(m, "DynamicClass", py::dynamic_attr())
+        .def(py::init());
 });
 });

tests/test_methods_and_attributes.py

+import pytest
 from pybind11_tests import ExampleMandA, ConstructorStats
 from pybind11_tests import ExampleMandA, ConstructorStats
 
 
 
 
@@ -44,3 +45,67 @@ def test_methods_and_attributes():
     assert cstats.move_constructions >= 1
     assert cstats.move_constructions >= 1
     assert cstats.copy_assignments == 0
     assert cstats.copy_assignments == 0
     assert cstats.move_assignments == 0
     assert cstats.move_assignments == 0
+
+
+def test_dynamic_attributes():
+    from pybind11_tests import DynamicClass
+
+    instance = DynamicClass()
+    assert not hasattr(instance, "foo")
+    assert "foo" not in dir(instance)
+
+    # Dynamically add attribute
+    instance.foo = 42
+    assert hasattr(instance, "foo")
+    assert instance.foo == 42
+    assert "foo" in dir(instance)
+
+    # __dict__ should be accessible and replaceable
+    assert "foo" in instance.__dict__
+    instance.__dict__ = {"bar": True}
+    assert not hasattr(instance, "foo")
+    assert hasattr(instance, "bar")
+
+    with pytest.raises(TypeError) as excinfo:
+        instance.__dict__ = []
+    assert str(excinfo.value) == "__dict__ must be set to a dictionary, not a 'list'"
+
+    cstats = ConstructorStats.get(DynamicClass)
+    assert cstats.alive() == 1
+    del instance
+    assert cstats.alive() == 0
+
+    # Derived classes should work as well
+    class Derived(DynamicClass):
+        pass
+
+    derived = Derived()
+    derived.foobar = 100
+    assert derived.foobar == 100
+
+    assert cstats.alive() == 1
+    del derived
+    assert cstats.alive() == 0
+
+
+def test_cyclic_gc():
+    from pybind11_tests import DynamicClass
+
+    # One object references itself
+    instance = DynamicClass()
+    instance.circular_reference = instance
+
+    cstats = ConstructorStats.get(DynamicClass)
+    assert cstats.alive() == 1
+    del instance
+    assert cstats.alive() == 0
+
+    # Two object reference each other
+    i1 = DynamicClass()
+    i2 = DynamicClass()
+    i1.cycle = i2
+    i2.cycle = i1
+
+    assert cstats.alive() == 2
+    del i1, i2
+    assert cstats.alive() == 0

tests/test_pickling.cpp

@@ -24,6 +24,14 @@ private:
     int m_extra2 = 0;
     int m_extra2 = 0;
 };
 };
 
 
+class PickleableWithDict {
+public:
+    PickleableWithDict(const std::string &value) : value(value) { }
+
+    std::string value;
+    int extra;
+};
+
 test_initializer pickling([](py::module &m) {
 test_initializer pickling([](py::module &m) {
     py::class_<Pickleable>(m, "Pickleable")
     py::class_<Pickleable>(m, "Pickleable")
         .def(py::init<std::string>())
         .def(py::init<std::string>())
@@ -48,4 +56,26 @@ test_initializer pickling([](py::module &m) {
             p.setExtra1(t[1].cast<int>());
             p.setExtra1(t[1].cast<int>());
             p.setExtra2(t[2].cast<int>());
             p.setExtra2(t[2].cast<int>());
         });
         });
+
+    py::class_<PickleableWithDict>(m, "PickleableWithDict", py::dynamic_attr())
+        .def(py::init<std::string>())
+        .def_readwrite("value", &PickleableWithDict::value)
+        .def_readwrite("extra", &PickleableWithDict::extra)
+        .def("__getstate__", [](py::object self) {
+            /* Also include __dict__ in state */
+            return py::make_tuple(self.attr("value"), self.attr("extra"), self.attr("__dict__"));
+        })
+        .def("__setstate__", [](py::object self, py::tuple t) {
+            if (t.size() != 3)
+                throw std::runtime_error("Invalid state!");
+            /* Cast and construct */
+            auto& p = self.cast<PickleableWithDict&>();
+            new (&p) Pickleable(t[0].cast<std::string>());
+
+            /* Assign C++ state */
+            p.extra = t[1].cast<int>();
+
+            /* Assign Python state */
+            self.attr("__dict__") = t[2];
+        });
 });
 });

tests/test_pickling.py

@@ -3,10 +3,10 @@ try:
 except ImportError:
 except ImportError:
     import pickle
     import pickle
 
 
-from pybind11_tests import Pickleable
-
 
 
 def test_roundtrip():
 def test_roundtrip():
+    from pybind11_tests import Pickleable
+
     p = Pickleable("test_value")
     p = Pickleable("test_value")
     p.setExtra1(15)
     p.setExtra1(15)
     p.setExtra2(48)
     p.setExtra2(48)
@@ -16,3 +16,17 @@ def test_roundtrip():
     assert p2.value() == p.value()
     assert p2.value() == p.value()
     assert p2.extra1() == p.extra1()
     assert p2.extra1() == p.extra1()
     assert p2.extra2() == p.extra2()
     assert p2.extra2() == p.extra2()
+
+
+def test_roundtrip_with_dict():
+    from pybind11_tests import PickleableWithDict
+
+    p = PickleableWithDict("test_value")
+    p.extra = 15
+    p.dynamic = "Attribute"
+
+    data = pickle.dumps(p, pickle.HIGHEST_PROTOCOL)
+    p2 = pickle.loads(data)
+    assert p2.value == p.value
+    assert p2.extra == p.extra
+    assert p2.dynamic == p.dynamic