Simple micro-benchmark ("ubench") comparing runtime performance for several holders.

Tested using github.com/rwgk/pybind11_scons and Google-internal build system.
Sorry, no cmake support at the moment.

First results: https://docs.google.com/spreadsheets/d/1InapCYws2Gt-stmFf_Bwl33eOMo3aLE_gc9adveY7RU/edit#gid=0

ubench/holder_comparison.cpp

+#include <pybind11/smart_holder.h>
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+namespace hc { // holder comparison
+
+template <int Serial>
+struct number_bucket {
+    std::vector<double> data;
+
+    explicit number_bucket(std::size_t data_size = 0) : data(data_size, 1.0) {}
+
+    double sum() const {
+        std::size_t n   = 0;
+        double s        = 0;
+        const double *a = &*data.begin();
+        const double *e = &*data.end();
+        while (a != e) {
+            s += *a++;
+            n++;
+        }
+        if (n != data.size()) {
+            throw std::runtime_error("Internal consistency failure (sum).");
+        }
+        return s;
+    }
+
+    std::size_t add(const number_bucket &other) {
+        if (other.data.size() != data.size()) {
+            throw std::runtime_error("Incompatible data sizes.");
+        }
+        std::size_t n   = 0;
+        double *a       = &*data.begin();
+        const double *e = &*data.end();
+        const double *b = &*other.data.begin();
+        while (a != e) {
+            *a++ += *b++;
+            n++;
+        }
+        return n;
+    }
+
+private:
+    number_bucket(const number_bucket &) = delete;
+    number_bucket(number_bucket &&)      = delete;
+    number_bucket &operator=(const number_bucket &) = delete;
+    number_bucket &operator=(number_bucket &&) = delete;
+};
+
+namespace py = pybind11;
+
+template <typename WrappedType, typename HolderType>
+void wrap_number_bucket(py::module m, const char *class_name) {
+    py::class_<WrappedType, HolderType>(m, class_name)
+        .def(py::init<std::size_t>(), py::arg("data_size") = 0)
+        .def("sum", &WrappedType::sum)
+        .def("add", &WrappedType::add, py::arg("other"));
+}
+
+template <typename T>
+class padded_unique_ptr {
+    std::unique_ptr<T> ptr;
+    char padding[sizeof(py::smart_holder) - sizeof(std::unique_ptr<T>)];
+
+public:
+    padded_unique_ptr(T *p) : ptr(p) {}
+    T *get() { return ptr.get(); }
+};
+
+using nb_up = number_bucket<0>;
+using nb_sp = number_bucket<1>;
+using nb_pu = number_bucket<2>;
+using nb_sh = number_bucket<3>;
+
+static_assert(sizeof(padded_unique_ptr<nb_pu>) == sizeof(py::smart_holder),
+              "Unexpected sizeof mismatch.");
+
+} // namespace hc
+
+PYBIND11_DECLARE_HOLDER_TYPE(T, hc::padded_unique_ptr<T>);
+
+PYBIND11_SMART_POINTER_HOLDER_TYPE_CASTERS(hc::nb_up, std::unique_ptr<hc::nb_up>)
+PYBIND11_SMART_POINTER_HOLDER_TYPE_CASTERS(hc::nb_sp, std::shared_ptr<hc::nb_sp>)
+PYBIND11_SMART_POINTER_HOLDER_TYPE_CASTERS(hc::nb_pu, hc::padded_unique_ptr<hc::nb_pu>)
+PYBIND11_SMART_HOLDER_TYPE_CASTERS(hc::nb_sh)
+
+PYBIND11_MODULE(pybind11_ubench_holder_comparison, m) {
+    using namespace hc;
+    wrap_number_bucket<nb_up, std::unique_ptr<nb_up>>(m, "number_bucket_up");
+    wrap_number_bucket<nb_sp, std::shared_ptr<nb_sp>>(m, "number_bucket_sp");
+    wrap_number_bucket<nb_pu, padded_unique_ptr<nb_pu>>(m, "number_bucket_pu");
+    wrap_number_bucket<nb_sh, py::smart_holder>(m, "number_bucket_sh");
+}

ubench/holder_comparison.py

+# -*- coding: utf-8 -*-
+"""Simple comparison of holder performances, relative to unique_ptr holder."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import pybind11_ubench_holder_comparison as m
+
+import collections
+import sys
+import time
+
+
+def pflush(*args, **kwargs):
+    result = print(*args, **kwargs)
+    # Using "file" here because it is the name of the built-in keyword argument.
+    file = kwargs.get("file", sys.stdout)  # pylint: disable=redefined-builtin
+    file.flush()  # file object must have a flush method.
+    return result
+
+
+def run(args):
+    if not args:
+        size_exponent_min = 0
+        size_exponent_max = 16
+        size_exponent_step = 4
+        call_repetitions_first_pass = 100
+        call_repetitions_target_elapsed_secs = 0.1
+        num_samples = 10
+        selected_holder_type = "*"
+    else:
+        assert len(args) == 7, (
+            "size_exponent_min size_exponent_max size_exponent_step"
+            " call_repetitions_first_pass call_repetitions_target_elapsed_secs"
+            " num_samples selected_holder_type"
+        )
+        size_exponent_min = int(args[0])
+        size_exponent_max = int(args[1])
+        size_exponent_step = int(args[2])
+        call_repetitions_first_pass = int(args[3])
+        call_repetitions_target_elapsed_secs = float(args[4])
+        num_samples = int(args[5])
+        selected_holder_type = args[6]
+    pflush(
+        "command-line arguments:",
+        size_exponent_min,
+        size_exponent_max,
+        size_exponent_step,
+        call_repetitions_first_pass,
+        "%.3f" % call_repetitions_target_elapsed_secs,
+        num_samples,
+        selected_holder_type,
+    )
+
+    for size_exponent in range(
+        size_exponent_min, size_exponent_max + 1, size_exponent_step
+    ):
+        data_size = 2 ** size_exponent
+        pflush(data_size, "data_size")
+        ratios = collections.defaultdict(list)
+        call_repetitions_dynamic = None
+        for _ in range(num_samples):
+            row_0 = None
+            for nb_label, nb_type in [
+                ("up", m.number_bucket_up),
+                ("sp", m.number_bucket_sp),
+                ("pu", m.number_bucket_pu),
+                ("sh", m.number_bucket_sh),
+            ]:
+                if selected_holder_type != "*" and nb_label != selected_holder_type:
+                    continue
+                nb1 = nb_type(data_size)
+                nb2 = nb_type(data_size)
+                if call_repetitions_dynamic is None:
+                    assert int(round(nb1.sum())) == data_size
+                    t0 = time.time()
+                    for _ in range(call_repetitions_first_pass):
+                        nb1.sum()
+                    td_sum = time.time() - t0
+                    call_repetitions_dynamic = max(
+                        call_repetitions_first_pass,
+                        int(
+                            call_repetitions_target_elapsed_secs
+                            * call_repetitions_first_pass
+                            / max(td_sum, 1.0e-6)
+                        )
+                        + 1,
+                    )
+                    pflush(call_repetitions_dynamic, "call_repetitions_dynamic")
+                assert int(round(nb1.sum())) == data_size
+                t0 = time.time()
+                for _ in range(call_repetitions_dynamic):
+                    nb1.sum()
+                td_sum = time.time() - t0
+                assert nb1.add(nb2) == data_size
+                t0 = time.time()
+                for _ in range(call_repetitions_dynamic):
+                    nb1.add(nb2)
+                td_add = time.time() - t0
+                row = [td_sum, td_add]
+                if row_0 is None:
+                    pflush("     Sum   Add  ratS  ratA")
+                    row_0 = row
+                else:
+                    for curr, prev in zip(row, row_0):
+                        if prev:
+                            rat = curr / prev
+                        else:
+                            rat = -1
+                        row.append(curr / prev)
+                    ratios[nb_label + "_ratS"].append(row[-2])
+                    ratios[nb_label + "_ratA"].append(row[-1])
+                pflush(nb_label, " ".join(["%.3f" % v for v in row]))
+        pflush("          Min  Mean   Max")
+        for key, rat in ratios.items():
+            print(key, "%5.3f %5.3f %5.3f" % (min(rat), sum(rat) / len(rat), max(rat)))
+
+
+if __name__ == "__main__":
+    run(args=sys.argv[1:])