Add support for positional args with args/kwargs

This commit rewrites the function dispatcher code to support mixing
regular arguments with py::args/py::kwargs arguments.  It also
simplifies the argument loader noticeably as it no longer has to worry
about args/kwargs: all of that is now sorted out in the dispatcher,
which now simply appends a tuple/dict if the function takes
py::args/py::kwargs, then passes all the arguments in a vector.

When the argument loader hit a py::args or py::kwargs, it doesn't do
anything special: it just calls the appropriate type_caster just like it
does for any other argument (thus removing the previous special cases
for args/kwargs).

Switching to passing arguments in a single std::vector instead of a pair
of tuples also makes things simpler, both in the dispatch and the
argument_loader: since this argument list is strictly pybind-internal
(i.e. it never goes to Python) we have no particular reason to use a
Python tuple here.

Some (intentional) restrictions:
- you may not bind a function that has args/kwargs somewhere other than
  the end (this somewhat matches Python, and keeps the dispatch code a
  little cleaner by being able to not worry about where to inject the
  args/kwargs in the argument list).
- If you specify an argument both positionally and via a keyword
  argument, you get a TypeError alerting you to this (as you do in
  Python).

docs/advanced/functions.rst

@@ -256,16 +256,21 @@ Such functions can also be created using pybind11:
    m.def("generic", &generic);
    m.def("generic", &generic);
 
 
 The class ``py::args`` derives from ``py::tuple`` and ``py::kwargs`` derives
 The class ``py::args`` derives from ``py::tuple`` and ``py::kwargs`` derives
-from ``py::dict``. Note that the ``kwargs`` argument is invalid if no keyword
-arguments were actually provided. Please refer to the other examples for
-details on how to iterate over these, and on how to cast their entries into
-C++ objects. A demonstration is also available in
-``tests/test_kwargs_and_defaults.cpp``.
+from ``py::dict``.
 
 
-.. warning::
+You may also use just one or the other, and may combine these with other
+arguments as long as the ``py::args`` and ``py::kwargs`` arguments are the last
+arguments accepted by the function.
+
+Please refer to the other examples for details on how to iterate over these,
+and on how to cast their entries into C++ objects. A demonstration is also
+available in ``tests/test_kwargs_and_defaults.cpp``.
+
+.. note::
 
 
-   Unlike Python, pybind11 does not allow combining normal parameters with the
-   ``args`` / ``kwargs`` special parameters.
+    When combining \*args or \*\*kwargs with :ref:`keyword_args` you should
+    *not* include ``py::arg`` tags for the ``py::args`` and ``py::kwargs``
+    arguments.
 
 
 Default arguments revisited
 Default arguments revisited
 ===========================
 ===========================

include/pybind11/attr.h

@@ -69,7 +69,7 @@ struct undefined_t;
 template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> struct op_;
 template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> struct op_;
 template <typename... Args> struct init;
 template <typename... Args> struct init;
 template <typename... Args> struct init_alias;
 template <typename... Args> struct init_alias;
-inline void keep_alive_impl(int Nurse, int Patient, handle args, handle ret);
+inline void keep_alive_impl(int Nurse, int Patient, function_arguments args, handle ret);
 
 
 /// Internal data structure which holds metadata about a keyword argument
 /// Internal data structure which holds metadata about a keyword argument
 struct argument_record {
 struct argument_record {
@@ -100,7 +100,7 @@ struct function_record {
     std::vector<argument_record> args;
     std::vector<argument_record> args;
 
 
     /// Pointer to lambda function which converts arguments and performs the actual call
     /// Pointer to lambda function which converts arguments and performs the actual call
-    handle (*impl) (function_record *, handle, handle, handle) = nullptr;
+    handle (*impl) (function_record *, function_arguments, handle) = nullptr;
 
 
     /// Storage for the wrapped function pointer and captured data, if any
     /// Storage for the wrapped function pointer and captured data, if any
     void *data[3] = { };
     void *data[3] = { };
@@ -129,7 +129,7 @@ struct function_record {
     /// True if this is a method
     /// True if this is a method
     bool is_method : 1;
     bool is_method : 1;
 
 
-    /// Number of arguments
+    /// Number of arguments (including py::args and/or py::kwargs, if present)
     uint16_t nargs;
     uint16_t nargs;
 
 
     /// Python method object
     /// Python method object
@@ -233,8 +233,8 @@ template <typename T> struct process_attribute_default {
     /// Default implementation: do nothing
     /// Default implementation: do nothing
     static void init(const T &, function_record *) { }
     static void init(const T &, function_record *) { }
     static void init(const T &, type_record *) { }
     static void init(const T &, type_record *) { }
-    static void precall(handle) { }
-    static void postcall(handle, handle) { }
+    static void precall(function_arguments) { }
+    static void postcall(function_arguments, handle) { }
 };
 };
 
 
 /// Process an attribute specifying the function's name
 /// Process an attribute specifying the function's name
@@ -362,13 +362,13 @@ struct process_attribute<arithmetic> : process_attribute_default<arithmetic> {};
  */
  */
 template <int Nurse, int Patient> struct process_attribute<keep_alive<Nurse, Patient>> : public process_attribute_default<keep_alive<Nurse, Patient>> {
 template <int Nurse, int Patient> struct process_attribute<keep_alive<Nurse, Patient>> : public process_attribute_default<keep_alive<Nurse, Patient>> {
     template <int N = Nurse, int P = Patient, enable_if_t<N != 0 && P != 0, int> = 0>
     template <int N = Nurse, int P = Patient, enable_if_t<N != 0 && P != 0, int> = 0>
-    static void precall(handle args) { keep_alive_impl(Nurse, Patient, args, handle()); }
+    static void precall(function_arguments args) { keep_alive_impl(Nurse, Patient, args, handle()); }
     template <int N = Nurse, int P = Patient, enable_if_t<N != 0 && P != 0, int> = 0>
     template <int N = Nurse, int P = Patient, enable_if_t<N != 0 && P != 0, int> = 0>
-    static void postcall(handle, handle) { }
+    static void postcall(function_arguments, handle) { }
     template <int N = Nurse, int P = Patient, enable_if_t<N == 0 || P == 0, int> = 0>
     template <int N = Nurse, int P = Patient, enable_if_t<N == 0 || P == 0, int> = 0>
-    static void precall(handle) { }
+    static void precall(function_arguments) { }
     template <int N = Nurse, int P = Patient, enable_if_t<N == 0 || P == 0, int> = 0>
     template <int N = Nurse, int P = Patient, enable_if_t<N == 0 || P == 0, int> = 0>
-    static void postcall(handle args, handle ret) { keep_alive_impl(Nurse, Patient, args, ret); }
+    static void postcall(function_arguments args, handle ret) { keep_alive_impl(Nurse, Patient, args, ret); }
 };
 };
 
 
 /// Recursively iterate over variadic template arguments
 /// Recursively iterate over variadic template arguments
@@ -381,11 +381,11 @@ template <typename... Args> struct process_attributes {
         int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::init(args, r), 0) ... };
         int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::init(args, r), 0) ... };
         ignore_unused(unused);
         ignore_unused(unused);
     }
     }
-    static void precall(handle fn_args) {
+    static void precall(function_arguments fn_args) {
         int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::precall(fn_args), 0) ... };
         int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::precall(fn_args), 0) ... };
         ignore_unused(unused);
         ignore_unused(unused);
     }
     }
-    static void postcall(handle fn_args, handle fn_ret) {
+    static void postcall(function_arguments fn_args, handle fn_ret) {
         int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::postcall(fn_args, fn_ret), 0) ... };
         int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::postcall(fn_args, fn_ret), 0) ... };
         ignore_unused(unused);
         ignore_unused(unused);
     }
     }
@@ -395,8 +395,8 @@ template <typename... Args> struct process_attributes {
 template <typename... Extra,
 template <typename... Extra,
           size_t named = constexpr_sum(std::is_base_of<arg, Extra>::value...),
           size_t named = constexpr_sum(std::is_base_of<arg, Extra>::value...),
           size_t self  = constexpr_sum(std::is_same<is_method, Extra>::value...)>
           size_t self  = constexpr_sum(std::is_same<is_method, Extra>::value...)>
-constexpr bool expected_num_args(size_t nargs) {
-    return named == 0 || (self + named) == nargs;
+constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) {
+    return named == 0 || (self + named + has_args + has_kwargs) == nargs;
 }
 }
 
 
 NAMESPACE_END(detail)
 NAMESPACE_END(detail)

include/pybind11/cast.h

@@ -1245,22 +1245,42 @@ constexpr arg operator"" _a(const char *name, size_t) { return arg(name); }
 
 
 NAMESPACE_BEGIN(detail)
 NAMESPACE_BEGIN(detail)
 
 
+// forward declaration
+struct function_record;
+
+// Helper struct to only allow py::args and/or py::kwargs at the end of the function arguments
+template <bool args, bool kwargs, bool args_kwargs_are_last> struct assert_args_kwargs_must_be_last {
+    static constexpr bool has_args = args, has_kwargs = kwargs;
+    static_assert(args_kwargs_are_last, "py::args/py::kwargs are only permitted as the last argument(s) of a function");
+};
+template <typename... T> struct args_kwargs_must_be_last;
+template <typename T1, typename... Tmore> struct args_kwargs_must_be_last<T1, Tmore...>
+    : args_kwargs_must_be_last<Tmore...> {};
+template <typename... T> struct args_kwargs_must_be_last<args, T...>
+    : assert_args_kwargs_must_be_last<true, false, sizeof...(T) == 0> {};
+template <typename... T> struct args_kwargs_must_be_last<kwargs, T...>
+    : assert_args_kwargs_must_be_last<false, true, sizeof...(T) == 0> {};
+template <typename... T> struct args_kwargs_must_be_last<args, kwargs, T...>
+    : assert_args_kwargs_must_be_last<true, true, sizeof...(T) == 0> {};
+template <> struct args_kwargs_must_be_last<> : assert_args_kwargs_must_be_last<false, false, true> {};
+
+using function_arguments = const std::vector<handle> &;
+
 /// Helper class which loads arguments for C++ functions called from Python
 /// Helper class which loads arguments for C++ functions called from Python
 template <typename... Args>
 template <typename... Args>
 class argument_loader {
 class argument_loader {
-    using itypes = type_list<intrinsic_t<Args>...>;
     using indices = make_index_sequence<sizeof...(Args)>;
     using indices = make_index_sequence<sizeof...(Args)>;
 
 
-public:
-    argument_loader() : value() {} // Helps gcc-7 properly initialize value
+    using check_args_kwargs = args_kwargs_must_be_last<intrinsic_t<Args>...>;
 
 
-    static constexpr auto has_kwargs = std::is_same<itypes, type_list<args, kwargs>>::value;
-    static constexpr auto has_args = has_kwargs || std::is_same<itypes, type_list<args>>::value;
+public:
+    static constexpr bool has_kwargs = check_args_kwargs::has_kwargs;
+    static constexpr bool has_args = check_args_kwargs::has_args;
 
 
     static PYBIND11_DESCR arg_names() { return detail::concat(make_caster<Args>::name()...); }
     static PYBIND11_DESCR arg_names() { return detail::concat(make_caster<Args>::name()...); }
 
 
-    bool load_args(handle args, handle kwargs) {
-        return load_impl(args, kwargs, itypes{});
+    bool load_args(function_arguments args) {
+        return load_impl_sequence(args, indices{});
     }
     }
 
 
     template <typename Return, typename Func>
     template <typename Return, typename Func>
@@ -1275,26 +1295,12 @@ public:
     }
     }
 
 
 private:
 private:
-    bool load_impl(handle args_, handle, type_list<args>) {
-        std::get<0>(value).load(args_, true);
-        return true;
-    }
-
-    bool load_impl(handle args_, handle kwargs_, type_list<args, kwargs>) {
-        std::get<0>(value).load(args_, true);
-        std::get<1>(value).load(kwargs_, true);
-        return true;
-    }
-
-    bool load_impl(handle args, handle, ... /* anything else */) {
-        return load_impl_sequence(args, indices{});
-    }
 
 
-    static bool load_impl_sequence(handle, index_sequence<>) { return true; }
+    static bool load_impl_sequence(function_arguments, index_sequence<>) { return true; }
 
 
     template <size_t... Is>
     template <size_t... Is>
-    bool load_impl_sequence(handle src, index_sequence<Is...>) {
-        for (bool r : {std::get<Is>(value).load(PyTuple_GET_ITEM(src.ptr(), Is), true)...})
+    bool load_impl_sequence(function_arguments args, index_sequence<Is...>) {
+        for (bool r : {std::get<Is>(value).load(args[Is], true)...})
             if (!r)
             if (!r)
                 return false;
                 return false;
         return true;
         return true;

include/pybind11/pytypes.h

@@ -149,14 +149,14 @@ public:
         preferable to use the `object` class which derives from `handle` and calls
         preferable to use the `object` class which derives from `handle` and calls
         this function automatically. Returns a reference to itself.
         this function automatically. Returns a reference to itself.
     \endrst */
     \endrst */
-    const handle& inc_ref() const { Py_XINCREF(m_ptr); return *this; }
+    const handle& inc_ref() const & { Py_XINCREF(m_ptr); return *this; }
 
 
     /** \rst
     /** \rst
         Manually decrease the reference count of the Python object. Usually, it is
         Manually decrease the reference count of the Python object. Usually, it is
         preferable to use the `object` class which derives from `handle` and calls
         preferable to use the `object` class which derives from `handle` and calls
         this function automatically. Returns a reference to itself.
         this function automatically. Returns a reference to itself.
     \endrst */
     \endrst */
-    const handle& dec_ref() const { Py_XDECREF(m_ptr); return *this; }
+    const handle& dec_ref() const & { Py_XDECREF(m_ptr); return *this; }
 
 
     /** \rst
     /** \rst
         Attempt to cast the Python object into the given C++ type. A `cast_error`
         Attempt to cast the Python object into the given C++ type. A `cast_error`

tests/test_kwargs_and_defaults.cpp

@@ -28,6 +28,27 @@ py::tuple args_kwargs_function(py::args args, py::kwargs kwargs) {
     return py::make_tuple(args, kwargs);
     return py::make_tuple(args, kwargs);
 }
 }
 
 
+py::tuple mixed_plus_args(int i, double j, py::args args) {
+    return py::make_tuple(i, j, args);
+}
+
+py::tuple mixed_plus_kwargs(int i, double j, py::kwargs kwargs) {
+    return py::make_tuple(i, j, kwargs);
+}
+
+py::tuple mixed_plus_args_kwargs(int i, double j, py::args args, py::kwargs kwargs) {
+    return py::make_tuple(i, j, args, kwargs);
+}
+
+// pybind11 won't allow these to be bound: args and kwargs, if present, must be at the end.
+void bad_args1(py::args, int) {}
+void bad_args2(py::kwargs, int) {}
+void bad_args3(py::kwargs, py::args) {}
+void bad_args4(py::args, int, py::kwargs) {}
+void bad_args5(py::args, py::kwargs, int) {}
+void bad_args6(py::args, py::args) {}
+void bad_args7(py::kwargs, py::kwargs) {}
+
 struct KWClass {
 struct KWClass {
     void foo(int, float) {}
     void foo(int, float) {}
 };
 };
@@ -53,4 +74,20 @@ test_initializer arg_keywords_and_defaults([](py::module &m) {
     py::class_<KWClass>(m, "KWClass")
     py::class_<KWClass>(m, "KWClass")
         .def("foo0", &KWClass::foo)
         .def("foo0", &KWClass::foo)
         .def("foo1", &KWClass::foo, "x"_a, "y"_a);
         .def("foo1", &KWClass::foo, "x"_a, "y"_a);
+
+    m.def("mixed_plus_args", &mixed_plus_args);
+    m.def("mixed_plus_kwargs", &mixed_plus_kwargs);
+    m.def("mixed_plus_args_kwargs", &mixed_plus_args_kwargs);
+
+    m.def("mixed_plus_args_kwargs_defaults", &mixed_plus_args_kwargs,
+            py::arg("i") = 1, py::arg("j") = 3.14159);
+
+    // Uncomment these to test that the static_assert is indeed working:
+//    m.def("bad_args1", &bad_args1);
+//    m.def("bad_args2", &bad_args2);
+//    m.def("bad_args3", &bad_args3);
+//    m.def("bad_args4", &bad_args4);
+//    m.def("bad_args5", &bad_args5);
+//    m.def("bad_args6", &bad_args6);
+//    m.def("bad_args7", &bad_args7);
 });
 });

tests/test_kwargs_and_defaults.py

@@ -55,3 +55,52 @@ def test_arg_and_kwargs():
     args = 'a1', 'a2'
     args = 'a1', 'a2'
     kwargs = dict(arg3='a3', arg4=4)
     kwargs = dict(arg3='a3', arg4=4)
     assert args_kwargs_function(*args, **kwargs) == (args, kwargs)
     assert args_kwargs_function(*args, **kwargs) == (args, kwargs)
+
+
+def test_mixed_args_and_kwargs(msg):
+    from pybind11_tests import (mixed_plus_args, mixed_plus_kwargs, mixed_plus_args_kwargs,
+                                mixed_plus_args_kwargs_defaults)
+    mpa = mixed_plus_args
+    mpk = mixed_plus_kwargs
+    mpak = mixed_plus_args_kwargs
+    mpakd = mixed_plus_args_kwargs_defaults
+
+    assert mpa(1, 2.5, 4, 99.5, None) == (1, 2.5, (4, 99.5, None))
+    assert mpa(1, 2.5) == (1, 2.5, ())
+    with pytest.raises(TypeError) as excinfo:
+        assert mpa(1)
+    assert msg(excinfo.value) == """
+        mixed_plus_args(): incompatible function arguments. The following argument types are supported:
+            1. (arg0: int, arg1: float, *args) -> tuple
+
+        Invoked with: 1
+    """  # noqa: E501
+    with pytest.raises(TypeError) as excinfo:
+        assert mpa()
+    assert msg(excinfo.value) == """
+        mixed_plus_args(): incompatible function arguments. The following argument types are supported:
+            1. (arg0: int, arg1: float, *args) -> tuple
+
+        Invoked with:
+    """  # noqa: E501
+
+    assert mpk(-2, 3.5, pi=3.14159, e=2.71828) == (-2, 3.5, {'e': 2.71828, 'pi': 3.14159})
+    assert mpak(7, 7.7, 7.77, 7.777, 7.7777, minusseven=-7) == (
+        7, 7.7, (7.77, 7.777, 7.7777), {'minusseven': -7})
+    assert mpakd() == (1, 3.14159, (), {})
+    assert mpakd(3) == (3, 3.14159, (), {})
+    assert mpakd(j=2.71828) == (1, 2.71828, (), {})
+    assert mpakd(k=42) == (1, 3.14159, (), {'k': 42})
+    assert mpakd(1, 1, 2, 3, 5, 8, then=13, followedby=21) == (
+        1, 1, (2, 3, 5, 8), {'then': 13, 'followedby': 21})
+    # Arguments specified both positionally and via kwargs is an error:
+    with pytest.raises(TypeError) as excinfo:
+        assert mpakd(1, i=1)
+    assert msg(excinfo.value) == """
+        mixed_plus_args_kwargs_defaults(): got multiple values for argument 'i'
+    """
+    with pytest.raises(TypeError) as excinfo:
+        assert mpakd(1, 2, j=1)
+    assert msg(excinfo.value) == """
+        mixed_plus_args_kwargs_defaults(): got multiple values for argument 'j'
+    """