Add dynamic attribute support

include/pybind11/attr.h

@@ -44,6 +44,9 @@ template <int Nurse, int Patient> struct keep_alive { };
 /// Annotation indicating that a class is involved in a multiple inheritance relationship
 struct multiple_inheritance { };
 
+/// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class
+struct dynamic_attr { };
+
 NAMESPACE_BEGIN(detail)
 /* Forward declarations */
 enum op_id : int;
@@ -162,6 +165,9 @@ struct type_record {
     /// Multiple inheritance marker
     bool multiple_inheritance = false;
 
+    /// Does the class manage a __dict__?
+    bool dynamic_attr = false;
+
     PYBIND11_NOINLINE void add_base(const std::type_info *base, void *(*caster)(void *)) {
         auto base_info = detail::get_type_info(*base, false);
         if (!base_info) {
@@ -292,6 +298,11 @@ struct process_attribute<multiple_inheritance> : process_attribute_default<multi
     static void init(const multiple_inheritance &, type_record *r) { r->multiple_inheritance = true; }
 };
 
+template <>
+struct process_attribute<dynamic_attr> : process_attribute_default<dynamic_attr> {
+    static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; }
+};
+
 /***
  * Process a keep_alive call policy -- invokes keep_alive_impl during the
  * pre-call handler if both Nurse, Patient != 0 and use the post-call handler

include/pybind11/common.h

@@ -297,6 +297,7 @@ inline std::string error_string();
 template <typename type> struct instance_essentials {
     PyObject_HEAD
     type *value;
+    PyObject *dict;
     PyObject *weakrefs;
     bool owned : 1;
     bool constructed : 1;

include/pybind11/pybind11.h

@@ -573,6 +573,33 @@ public:
 };
 
 NAMESPACE_BEGIN(detail)
+extern "C" inline PyObject *get_dict(PyObject *op, void *) {
+    auto *self = (instance<void> *) op;
+    if (!self->dict) {
+        self->dict = PyDict_New();
+    }
+    Py_XINCREF(self->dict);
+    return self->dict;
+}
+
+extern "C" inline int set_dict(PyObject *op, PyObject *dict, void *) {
+    if (!PyDict_Check(dict)) {
+        PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, not a '%.200s'",
+                     Py_TYPE(dict)->tp_name);
+        return -1;
+    }
+    auto *self = (instance<void> *) op;
+    Py_INCREF(dict);
+    Py_CLEAR(self->dict);
+    self->dict = dict;
+    return 0;
+}
+
+static PyGetSetDef generic_getset[] = {
+    {const_cast<char*>("__dict__"), get_dict, set_dict, nullptr, nullptr},
+    {nullptr, nullptr, nullptr, nullptr, nullptr}
+};
+
 /// Generic support for creating new Python heap types
 class generic_type : public object {
     template <typename...> friend class class_;
@@ -684,6 +711,15 @@ protected:
 #endif
         type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC;
 
+        /* Support dynamic attributes */
+        if (rec->dynamic_attr) {
+            type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_GC;
+            type->ht_type.tp_dictoffset = offsetof(instance_essentials<void>, dict);
+            type->ht_type.tp_getset = generic_getset;
+            type->ht_type.tp_traverse = traverse;
+            type->ht_type.tp_clear = clear;
+        }
+
         type->ht_type.tp_doc = tp_doc;
 
         if (PyType_Ready(&type->ht_type) < 0)
@@ -785,10 +821,24 @@ protected:
 
             if (self->weakrefs)
                 PyObject_ClearWeakRefs((PyObject *) self);
+
+            Py_CLEAR(self->dict);
         }
         Py_TYPE(self)->tp_free((PyObject*) self);
     }
 
+    static int traverse(PyObject *op, visitproc visit, void *arg) {
+        auto *self = (instance<void> *) op;
+        Py_VISIT(self->dict);
+        return 0;
+    }
+
+    static int clear(PyObject *op) {
+        auto *self = (instance<void> *) op;
+        Py_CLEAR(self->dict);
+        return 0;
+    }
+
     void install_buffer_funcs(
             buffer_info *(*get_buffer)(PyObject *, void *),
             void *get_buffer_data) {

tests/test_methods_and_attributes.cpp

@@ -53,6 +53,12 @@ public:
     int value = 0;
 };
 
+class DynamicClass {
+public:
+    DynamicClass() { print_default_created(this); }
+    ~DynamicClass() { print_destroyed(this); }
+};
+
 test_initializer methods_and_attributes([](py::module &m) {
     py::class_<ExampleMandA>(m, "ExampleMandA")
         .def(py::init<>())
@@ -81,4 +87,7 @@ test_initializer methods_and_attributes([](py::module &m) {
         .def("__str__", &ExampleMandA::toString)
         .def_readwrite("value", &ExampleMandA::value)
         ;
+
+    py::class_<DynamicClass>(m, "DynamicClass", py::dynamic_attr())
+        .def(py::init());
 });

tests/test_methods_and_attributes.py

+import pytest
 from pybind11_tests import ExampleMandA, ConstructorStats
 
 
@@ -44,3 +45,67 @@ def test_methods_and_attributes():
     assert cstats.move_constructions >= 1
     assert cstats.copy_assignments == 0
     assert cstats.move_assignments == 0
+
+
+def test_dynamic_attributes():
+    from pybind11_tests import DynamicClass
+
+    instance = DynamicClass()
+    assert not hasattr(instance, "foo")
+    assert "foo" not in dir(instance)
+
+    # Dynamically add attribute
+    instance.foo = 42
+    assert hasattr(instance, "foo")
+    assert instance.foo == 42
+    assert "foo" in dir(instance)
+
+    # __dict__ should be accessible and replaceable
+    assert "foo" in instance.__dict__
+    instance.__dict__ = {"bar": True}
+    assert not hasattr(instance, "foo")
+    assert hasattr(instance, "bar")
+
+    with pytest.raises(TypeError) as excinfo:
+        instance.__dict__ = []
+    assert str(excinfo.value) == "__dict__ must be set to a dictionary, not a 'list'"
+
+    cstats = ConstructorStats.get(DynamicClass)
+    assert cstats.alive() == 1
+    del instance
+    assert cstats.alive() == 0
+
+    # Derived classes should work as well
+    class Derived(DynamicClass):
+        pass
+
+    derived = Derived()
+    derived.foobar = 100
+    assert derived.foobar == 100
+
+    assert cstats.alive() == 1
+    del derived
+    assert cstats.alive() == 0
+
+
+def test_cyclic_gc():
+    from pybind11_tests import DynamicClass
+
+    # One object references itself
+    instance = DynamicClass()
+    instance.circular_reference = instance
+
+    cstats = ConstructorStats.get(DynamicClass)
+    assert cstats.alive() == 1
+    del instance
+    assert cstats.alive() == 0
+
+    # Two object reference each other
+    i1 = DynamicClass()
+    i2 = DynamicClass()
+    i1.cycle = i2
+    i2.cycle = i1
+
+    assert cstats.alive() == 2
+    del i1, i2
+    assert cstats.alive() == 0

tests/test_pickling.cpp

@@ -24,6 +24,14 @@ private:
     int m_extra2 = 0;
 };
 
+class PickleableWithDict {
+public:
+    PickleableWithDict(const std::string &value) : value(value) { }
+
+    std::string value;
+    int extra;
+};
+
 test_initializer pickling([](py::module &m) {
     py::class_<Pickleable>(m, "Pickleable")
         .def(py::init<std::string>())
@@ -48,4 +56,26 @@ test_initializer pickling([](py::module &m) {
             p.setExtra1(t[1].cast<int>());
             p.setExtra2(t[2].cast<int>());
         });
+
+    py::class_<PickleableWithDict>(m, "PickleableWithDict", py::dynamic_attr())
+        .def(py::init<std::string>())
+        .def_readwrite("value", &PickleableWithDict::value)
+        .def_readwrite("extra", &PickleableWithDict::extra)
+        .def("__getstate__", [](py::object self) {
+            /* Also include __dict__ in state */
+            return py::make_tuple(self.attr("value"), self.attr("extra"), self.attr("__dict__"));
+        })
+        .def("__setstate__", [](py::object self, py::tuple t) {
+            if (t.size() != 3)
+                throw std::runtime_error("Invalid state!");
+            /* Cast and construct */
+            auto& p = self.cast<PickleableWithDict&>();
+            new (&p) Pickleable(t[0].cast<std::string>());
+
+            /* Assign C++ state */
+            p.extra = t[1].cast<int>();
+
+            /* Assign Python state */
+            self.attr("__dict__") = t[2];
+        });
 });

tests/test_pickling.py

@@ -3,10 +3,10 @@ try:
 except ImportError:
     import pickle
 
-from pybind11_tests import Pickleable
-
 
 def test_roundtrip():
+    from pybind11_tests import Pickleable
+
     p = Pickleable("test_value")
     p.setExtra1(15)
     p.setExtra2(48)
@@ -16,3 +16,17 @@ def test_roundtrip():
     assert p2.value() == p.value()
     assert p2.extra1() == p.extra1()
     assert p2.extra2() == p.extra2()
+
+
+def test_roundtrip_with_dict():
+    from pybind11_tests import PickleableWithDict
+
+    p = PickleableWithDict("test_value")
+    p.extra = 15
+    p.dynamic = "Attribute"
+
+    data = pickle.dumps(p, pickle.HIGHEST_PROTOCOL)
+    p2 = pickle.loads(data)
+    assert p2.value == p.value
+    assert p2.extra == p.extra
+    assert p2.dynamic == p.dynamic