Provide better type hints for a variety of generic types (#4259)

* Provide better type hints for a variety of generic types

* Makes better documentation
* tuple, dict, list, set, function

* Move to py::typing

* style: pre-commit fixes

* Update copyright line with correct year and actual author. The author information was copy-pasted from the git log output.

---------

Co-authored-by: Ralf W. Grosse-Kunstleve <rwgk@google.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CMakeLists.txt

@@ -141,7 +141,8 @@ set(PYBIND11_HEADERS
     include/pybind11/stl.h
     include/pybind11/stl.h
     include/pybind11/stl_bind.h
     include/pybind11/stl_bind.h
     include/pybind11/stl/filesystem.h
     include/pybind11/stl/filesystem.h
-    include/pybind11/type_caster_pyobject_ptr.h)
+    include/pybind11/type_caster_pyobject_ptr.h
+    include/pybind11/typing.h)
 
 
 # Compare with grep and warn if mismatched
 # Compare with grep and warn if mismatched
 if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)
 if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)

docs/advanced/misc.rst

@@ -398,3 +398,32 @@ before they are used as a parameter or return type of a function:
         pyFoo.def(py::init<const ns::Bar&>());
         pyFoo.def(py::init<const ns::Bar&>());
         pyBar.def(py::init<const ns::Foo&>());
         pyBar.def(py::init<const ns::Foo&>());
     }
     }
+
+Setting inner type hints in docstrings
+======================================
+
+When you use pybind11 wrappers for ``list``, ``dict``, and other generic python
+types, the docstring will just display the generic type. You can convey the
+inner types in the docstring by using a special 'typed' version of the generic
+type.
+
+.. code-block:: cpp
+
+    PYBIND11_MODULE(example, m) {
+        m.def("pass_list_of_str", [](py::typing::List<py::str> arg) {
+            // arg can be used just like py::list
+        ));
+    }
+
+The resulting docstring will be ``pass_list_of_str(arg0: list[str]) -> None``.
+
+The following special types are available in ``pybind11/typing.h``:
+
+* ``py::Tuple<Args...>``
+* ``py::Dict<K, V>``
+* ``py::List<V>``
+* ``py::Set<V>``
+* ``py::Callable<Signature>``
+
+.. warning:: Just like in python, these are merely hints. They don't actually
+             enforce the types of their contents at runtime or compile time.

include/pybind11/typing.h

+/*
+    pybind11/typing.h: Convenience wrapper classes for basic Python types
+    with more explicit annotations.
+
+    Copyright (c) 2023 Dustin Spicuzza <dustin@virtualroadside.com>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "detail/common.h"
+#include "cast.h"
+#include "pytypes.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(typing)
+
+/*
+    The following types can be used to direct pybind11-generated docstrings
+    to have have more explicit types (e.g., `list[str]` instead of `list`).
+    Just use these in place of existing types.
+
+    There is no additional enforcement of types at runtime.
+*/
+
+template <typename... Types>
+class Tuple : public tuple {
+    using tuple::tuple;
+};
+
+template <typename K, typename V>
+class Dict : public dict {
+    using dict::dict;
+};
+
+template <typename T>
+class List : public list {
+    using list::list;
+};
+
+template <typename T>
+class Set : public set {
+    using set::set;
+};
+
+template <typename Signature>
+class Callable;
+
+template <typename Return, typename... Args>
+class Callable<Return(Args...)> : public function {
+    using function::function;
+};
+
+PYBIND11_NAMESPACE_END(typing)
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+template <typename... Types>
+struct handle_type_name<typing::Tuple<Types...>> {
+    static constexpr auto name
+        = const_name("tuple[") + concat(make_caster<Types>::name...) + const_name("]");
+};
+
+template <>
+struct handle_type_name<typing::Tuple<>> {
+    // PEP 484 specifies this syntax for an empty tuple
+    static constexpr auto name = const_name("tuple[()]");
+};
+
+template <typename K, typename V>
+struct handle_type_name<typing::Dict<K, V>> {
+    static constexpr auto name = const_name("dict[") + make_caster<K>::name + const_name(", ")
+                                 + make_caster<V>::name + const_name("]");
+};
+
+template <typename T>
+struct handle_type_name<typing::List<T>> {
+    static constexpr auto name = const_name("list[") + make_caster<T>::name + const_name("]");
+};
+
+template <typename T>
+struct handle_type_name<typing::Set<T>> {
+    static constexpr auto name = const_name("set[") + make_caster<T>::name + const_name("]");
+};
+
+template <typename Return, typename... Args>
+struct handle_type_name<typing::Callable<Return(Args...)>> {
+    using retval_type = conditional_t<std::is_same<Return, void>::value, void_type, Return>;
+    static constexpr auto name = const_name("Callable[[") + concat(make_caster<Args>::name...)
+                                 + const_name("], ") + make_caster<retval_type>::name
+                                 + const_name("]");
+};
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

tests/extra_python_package/test_files.py

@@ -44,6 +44,7 @@ main_headers = {
     "include/pybind11/stl.h",
     "include/pybind11/stl.h",
     "include/pybind11/stl_bind.h",
     "include/pybind11/stl_bind.h",
     "include/pybind11/type_caster_pyobject_ptr.h",
     "include/pybind11/type_caster_pyobject_ptr.h",
+    "include/pybind11/typing.h",
 }
 }
 
 
 detail_headers = {
 detail_headers = {

tests/test_pytypes.cpp

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
     BSD-style license that can be found in the LICENSE file.
 */
 */
 
 
+#include <pybind11/typing.h>
+
 #include "pybind11_tests.h"
 #include "pybind11_tests.h"
 
 
 #include <utility>
 #include <utility>
@@ -820,4 +822,12 @@ TEST_SUBMODULE(pytypes, m) {
         a >>= b;
         a >>= b;
         return a;
         return a;
     });
     });
+
+    m.def("annotate_tuple_float_str", [](const py::typing::Tuple<py::float_, py::str> &) {});
+    m.def("annotate_tuple_empty", [](const py::typing::Tuple<> &) {});
+    m.def("annotate_dict_str_int", [](const py::typing::Dict<py::str, int> &) {});
+    m.def("annotate_list_int", [](const py::typing::List<int> &) {});
+    m.def("annotate_set_str", [](const py::typing::Set<std::string> &) {});
+    m.def("annotate_fn",
+          [](const py::typing::Callable<int(py::typing::List<py::str>, py::str)> &) {});
 }
 }

tests/test_pytypes.py

@@ -896,3 +896,38 @@ def test_inplace_lshift(a, b):
 def test_inplace_rshift(a, b):
 def test_inplace_rshift(a, b):
     expected = a >> b
     expected = a >> b
     assert m.inplace_rshift(a, b) == expected
     assert m.inplace_rshift(a, b) == expected
+
+
+def test_tuple_nonempty_annotations(doc):
+    assert (
+        doc(m.annotate_tuple_float_str)
+        == "annotate_tuple_float_str(arg0: tuple[float, str]) -> None"
+    )
+
+
+def test_tuple_empty_annotations(doc):
+    assert (
+        doc(m.annotate_tuple_empty) == "annotate_tuple_empty(arg0: tuple[()]) -> None"
+    )
+
+
+def test_dict_annotations(doc):
+    assert (
+        doc(m.annotate_dict_str_int)
+        == "annotate_dict_str_int(arg0: dict[str, int]) -> None"
+    )
+
+
+def test_list_annotations(doc):
+    assert doc(m.annotate_list_int) == "annotate_list_int(arg0: list[int]) -> None"
+
+
+def test_set_annotations(doc):
+    assert doc(m.annotate_set_str) == "annotate_set_str(arg0: set[str]) -> None"
+
+
+def test_fn_annotations(doc):
+    assert (
+        doc(m.annotate_fn)
+        == "annotate_fn(arg0: Callable[[list[str], str], int]) -> None"
+    )