feat: typing support for helpers (#2588)

* feat: basic typing support

* docs: mention syncing as suggested by @rwgk

* docs: update changelog

* docs: copy of warning in limitations

.pre-commit-config.yaml

@@ -34,8 +34,10 @@ repos:
   rev: 20.8b1
   hooks:
   - id: black
+    # By default, this ignores pyi files, though black supports them
+    types: [text]
     # Not all Python files are Blacked, yet
-    files: ^(setup.py|pybind11|tests/extra)
+    files: ^(setup.py|pybind11|tests/extra|tools).*\.pyi?$
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
@@ -60,6 +62,17 @@ repos:
     types: [file]
     files: (\.cmake|CMakeLists.txt)(.in)?$
 
+# Check static types with mypy
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v0.790
+  hooks:
+  - id: mypy
+    # The default Python type ignores .pyi files, so let's rerun if detected
+    types: [text]
+    files: ^pybind11.*\.pyi?$
+    # Running per-file misbehaves a bit, so just run on all files, it's fast
+    pass_filenames: false
+
 # Checks the manifest for missing files (native support)
 - repo: https://github.com/mgedmin/check-manifest
   rev: "0.43"

MANIFEST.in

 recursive-include pybind11/include/pybind11 *.h
 recursive-include pybind11 *.py
+recursive-include pybind11 py.typed
+recursive-include pybind11 *.pyi
 include pybind11/share/cmake/pybind11/*.cmake
 include LICENSE README.rst pyproject.toml setup.py setup.cfg

docs/changelog.rst

@@ -72,7 +72,6 @@ API changes:
 * Public constructors for ``py::module_`` have been deprecated; please use
   ``pybind11::module_::create_extension_module`` if you were using the public
   constructor (fairly rare after ``PYBIND11_MODULE`` was introduced).
-  **Provisional in 2.6.0rc1.**
   `#2552 <https://github.com/pybind/pybind11/pull/2552>`_
 
 * ``PYBIND11_OVERLOAD*`` macros and ``get_overload`` function replaced by
@@ -102,6 +101,9 @@ Packaging / building improvements:
   * ``pybind11-config`` is another way to write ``python -m pybind11`` if you
     have your PATH set up.
 
+  * Added external typing support to the helper module, code from
+    ``import pybind11`` can now be type checked.
+    `#2588 <https://github.com/pybind/pybind11/pull/2588>`_
 
 * Minimum CMake required increased to 3.4.
   `#2338 <https://github.com/pybind/pybind11/pull/2338>`_ and

docs/limitations.rst

@@ -51,3 +51,20 @@ clean, well written patch would likely be accepted to solve them.
 - The ``cpptest`` does not run on Windows with Python 3.8 or newer, due to DLL
   loader changes. User code that is correctly installed should not be affected.
   `#2560 <https://github.com/pybind/pybind11/issue/2560>`_
+
+Python 3.9.0 warning
+^^^^^^^^^^^^^^^^^^^^
+
+Combining older versions of pybind11 (< 2.6.0) with Python on 3.9.0 will
+trigger undefined behavior that typically manifests as crashes during
+interpreter shutdown (but could also destroy your data. **You have been
+warned**).
+
+This issue has been
+`fixed in Python <https://github.com/python/cpython/pull/22670>`_.  As a
+mitigation until 3.9.1 is released and commonly used, pybind11 (2.6.0 or newer)
+includes a temporary workaround specifically when Python 3.9.0 is detected at
+runtime, leaking about 50 bytes of memory when a callback function is garbage
+collected. For reference; the pybind11 test suite has about 2,000 such
+callbacks, but only 49 are garbage collected before the end-of-process. Wheels
+built with Python 3.9.0 will correctly avoid the leak when run in Python 3.9.1.

docs/upgrade.rst

@@ -25,7 +25,6 @@ C++ language rules change again.
 
 The public constructors of ``py::module_`` have been deprecated. Use
 ``PYBIND11_MODULE`` or ``module_::create_extension_module`` instead.
-**Provisional in 2.6.0rc1.**
 
 An error is now thrown when ``__init__`` is forgotten on subclasses. This was
 incorrect before, but was not checked. Add a call to ``__init__`` if it is

pybind11/__main__.py

@@ -9,6 +9,7 @@ from .commands import get_include, get_cmake_dir
 
 
 def print_includes():
+    # type: () -> None
     dirs = [
         sysconfig.get_path("include"),
         sysconfig.get_path("platinclude"),
@@ -18,13 +19,15 @@ def print_includes():
     # Make unique but preserve order
     unique_dirs = []
     for d in dirs:
-        if d not in unique_dirs:
+        if d and d not in unique_dirs:
             unique_dirs.append(d)
 
     print(" ".join("-I" + d for d in unique_dirs))
 
 
 def main():
+    # type: () -> None
+
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--includes",

pybind11/_version.pyi

+from typing import Union, Tuple
+
+def _to_int(s: str) -> Union[int, str]: ...
+
+__version__: str
+version_info: Tuple[Union[int, str], ...]

pybind11/commands.py

@@ -6,12 +6,14 @@ DIR = os.path.abspath(os.path.dirname(__file__))
 
 
 def get_include(user=False):
+    # type: (bool) -> str
     installed_path = os.path.join(DIR, "include")
     source_path = os.path.join(os.path.dirname(DIR), "include")
     return installed_path if os.path.exists(installed_path) else source_path
 
 
 def get_cmake_dir():
+    # type: () -> str
     cmake_installed_path = os.path.join(DIR, "share", "cmake", "pybind11")
     if os.path.exists(cmake_installed_path):
         return cmake_installed_path

pybind11/setup_helpers.py

@@ -33,6 +33,12 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 """
 
+# IMPORTANT: If you change this file in the pybind11 repo, also review
+# setup_helpers.pyi for matching changes.
+#
+# If you copy this file in, you don't
+# need the .pyi file; it's just an interface file for static type checkers.
+
 import contextlib
 import os
 import shutil

pybind11/setup_helpers.pyi

+# IMPORTANT: Should stay in sync with setup_helpers.py (mostly checked by CI /
+# pre-commit).
+
+from typing import Any, Iterator, Optional, Type, TypeVar, Union
+from types import TracebackType
+
+from distutils.command.build_ext import build_ext as _build_ext  # type: ignore
+from distutils.extension import Extension as _Extension
+import distutils.ccompiler
+import contextlib
+
+WIN: bool
+PY2: bool
+MACOS: bool
+STD_TMPL: str
+
+class Pybind11Extension(_Extension):
+    def _add_cflags(self, *flags: str) -> None: ...
+    def _add_lflags(self, *flags: str) -> None: ...
+    def __init__(
+        self, *args: Any, cxx_std: int = 0, language: str = "c++", **kwargs: Any
+    ) -> None: ...
+    @property
+    def cxx_std(self) -> int: ...
+    @cxx_std.setter
+    def cxx_std(self, level: int) -> None: ...
+
+@contextlib.contextmanager
+def tmp_chdir() -> Iterator[str]: ...
+def has_flag(compiler: distutils.ccompiler.CCompiler, flag: str) -> bool: ...
+def auto_cpp_level(compiler: distutils.ccompiler.CCompiler) -> Union[int, str]: ...
+
+class build_ext(_build_ext):  # type: ignore
+    def build_extensions(self) -> None: ...
+
+T = TypeVar("T", bound="ParallelCompile")
+
+class ParallelCompile:
+    def __init__(
+        self, envvar: Optional[str] = None, default: int = 0, max: int = 0
+    ): ...
+    def function(self) -> Any: ...
+    def install(self: T) -> T: ...
+    def __enter__(self: T) -> T: ...
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None: ...

setup.cfg

@@ -64,3 +64,7 @@ ignore =
     N813
     # Black conflict
     W503, E203
+
+[mypy]
+files = pybind11
+strict = True

tests/extra_python_package/test_files.py

@@ -58,8 +58,11 @@ py_files = {
     "__init__.py",
     "__main__.py",
     "_version.py",
+    "_version.pyi",
     "commands.py",
+    "py.typed",
     "setup_helpers.py",
+    "setup_helpers.pyi",
 }
 
 headers = main_headers | detail_headers

tools/libsize.py

@@ -19,7 +19,7 @@ if not os.path.exists(lib):
 
 libsize = os.path.getsize(lib)
 
-print("------", os.path.basename(lib), "file size:", libsize, end='')
+print("------", os.path.basename(lib), "file size:", libsize, end="")
 
 if os.path.exists(save):
     with open(save) as sf:
@@ -34,5 +34,5 @@ if os.path.exists(save):
 else:
     print()
 
-with open(save, 'w') as sf:
+with open(save, "w") as sf:
     sf.write(str(libsize))

tools/setup_main.py.in

@@ -19,6 +19,7 @@ setup(
         "pybind11.share.cmake.pybind11",
     ],
     package_data={
+        "pybind11": ["py.typed", "*.pyi"],
         "pybind11.include.pybind11": ["*.h"],
         "pybind11.include.pybind11.detail": ["*.h"],
         "pybind11.share.cmake.pybind11": ["*.cmake"],