Skip to content

P
pybind11
Commit 9b3a2000 by Pieter P Committed by GitHub
Options

fix(cmake): improved cross-compilation support (#5083) 

* fix(cmake): do not use Python::Interpreter when cross-compiling

* chore: apply cmake-format to pybind11NewTools.cmake

* fix(cmake): do not look for Python Interpreter component when cross-compiling

* feat(cmake): guess Python extension suffix

* fix: add pybind11GuessPythonExtSuffix.cmake to packaging test

* Use PYBIND11_CROSSCOMPILING instead of CMAKE_CROSSCOMPILING

* refactor: require PYBIND11_USE_CROSSCOMPILING

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Henry Schreiner <henryschreineriii@gmail.com>
parent b9794be4
Showing with 310 additions and 20 deletions
CMakeLists.txt
......@@ -116,6 +116,7 @@ option(PYBIND11_NUMPY_1_ONLY
set(PYBIND11_INTERNALS_VERSION
""
CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF)
if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
......@@ -299,6 +300,7 @@ if(PYBIND11_INSTALL)
tools/pybind11Common.cmake
tools/pybind11Tools.cmake
tools/pybind11NewTools.cmake
tools/pybind11GuessPythonExtSuffix.cmake
DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
if(NOT PYBIND11_EXPORT_NAME)
......
tests/extra_python_package/test_files.py
......@@ -73,6 +73,7 @@ cmake_files = {
"share/cmake/pybind11/pybind11Common.cmake",
"share/cmake/pybind11/pybind11Config.cmake",
"share/cmake/pybind11/pybind11ConfigVersion.cmake",
"share/cmake/pybind11/pybind11GuessPythonExtSuffix.cmake",
"share/cmake/pybind11/pybind11NewTools.cmake",
"share/cmake/pybind11/pybind11Targets.cmake",
"share/cmake/pybind11/pybind11Tools.cmake",
......
tools/FindPythonLibsNew.cmake
......@@ -205,7 +205,7 @@ endif()
# Make sure the Python has the same pointer-size as the chosen compiler
# Skip if CMAKE_SIZEOF_VOID_P is not defined
# This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN)
if(NOT CMAKE_CROSSCOMPILING
if(NOT _PYBIND11_CROSSCOMPILING
AND CMAKE_SIZEOF_VOID_P
AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}"))
if(PythonLibsNew_FIND_REQUIRED)
......
tools/pybind11Common.cmake
......@@ -42,6 +42,16 @@ set(pybind11_INCLUDE_DIRS
"${pybind11_INCLUDE_DIR}"
CACHE INTERNAL "Include directory for pybind11 (Python not requested)")
if(CMAKE_CROSSCOMPILING AND PYBIND11_USE_CROSSCOMPILING)
set(_PYBIND11_CROSSCOMPILING
ON
CACHE INTERNAL "")
else()
set(_PYBIND11_CROSSCOMPILING
OFF
CACHE INTERNAL "")
endif()
# --------------------- Shared targets ----------------------------
# Build an interface library target:
......@@ -195,7 +205,7 @@ endif()
# --------------------- pybind11_find_import -------------------------------
if(NOT _pybind11_nopython)
if(NOT _pybind11_nopython AND NOT _PYBIND11_CROSSCOMPILING)
# Check to see if modules are importable. Use REQUIRED to force an error if
# one of the modules is not found. <package_name>_FOUND will be set if the
# package was found (underscores replace dashes if present). QUIET will hide
......
tools/pybind11GuessPythonExtSuffix.cmake 0 → 100644
cmake_minimum_required(VERSION 3.5)
function(pybind11_guess_python_module_extension python)
# The SETUPTOOLS_EXT_SUFFIX environment variable takes precedence:
if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ENV{SETUPTOOLS_EXT_SUFFIX})
message(
STATUS
"Getting Python extension suffix from ENV{SETUPTOOLS_EXT_SUFFIX}: $ENV{SETUPTOOLS_EXT_SUFFIX}"
)
set(PYTHON_MODULE_EXT_SUFFIX
"$ENV{SETUPTOOLS_EXT_SUFFIX}"
CACHE
STRING
"Extension suffix for Python extension modules (Initialized from SETUPTOOLS_EXT_SUFFIX)")
endif()
# If that didn't work, use the Python_SOABI variable:
if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ${python}_SOABI)
message(
STATUS "Determining Python extension suffix based on ${python}_SOABI: ${${python}_SOABI}")
# The final extension depends on the system
set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}")
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
set(_PY_BUILD_EXTENSION ".pyd")
endif()
# If the SOABI already has an extension, use it as the full suffix
# (used for debug versions of Python on Windows)
if(${python}_SOABI MATCHES "\\.")
set(PYTHON_MODULE_EXT_SUFFIX "${${python}_SOABI}")
# If the SOABI is empty, this is usually a bug, but we generate a
# correct extension anyway, which is the best we can do
elseif("${${python}_SOABI}" STREQUAL "")
message(
WARNING
"${python}_SOABI is defined but empty. You may want to set PYTHON_MODULE_EXT_SUFFIX explicitly."
)
set(PYTHON_MODULE_EXT_SUFFIX "${_PY_BUILD_EXTENSION}")
# Otherwise, add the system-dependent extension to it
else()
set(PYTHON_MODULE_EXT_SUFFIX ".${${python}_SOABI}${_PY_BUILD_EXTENSION}")
endif()
endif()
# If we could not deduce the extension suffix, unset the results:
if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_DEBUG_POSTFIX PARENT_SCOPE)
unset(PYTHON_MODULE_EXTENSION PARENT_SCOPE)
unset(PYTHON_IS_DEBUG PARENT_SCOPE)
return()
endif()
# Sanity checks:
if(${python}_SOABI AND NOT (PYTHON_MODULE_EXT_SUFFIX STREQUAL ${python}_SOABI
OR PYTHON_MODULE_EXT_SUFFIX MATCHES "\\.${${python}_SOABI}\\."))
message(
WARNING
"Python extension suffix (${PYTHON_MODULE_EXT_SUFFIX}) does not match ${python}_SOABI (${${python}_SOABI})."
)
endif()
# Separate file name postfix from extension: (https://github.com/pybind/pybind11/issues/4699)
get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
get_filename_component(_PYTHON_MODULE_EXTENSION "${PYTHON_MODULE_EXT_SUFFIX}" EXT)
# Try to deduce the debug ABI from the extension suffix:
if(NOT DEFINED _PYTHON_IS_DEBUG)
if(_PYTHON_MODULE_EXTENSION MATCHES "^\\.(cpython-|cp|pypy)[0-9]+dm?-"
OR _PYTHON_MODULE_DEBUG_POSTFIX MATCHES "^_d")
set(_PYTHON_IS_DEBUG On)
else()
set(_PYTHON_IS_DEBUG Off)
endif()
endif()
# Return results
set(PYTHON_MODULE_DEBUG_POSTFIX
"${_PYTHON_MODULE_DEBUG_POSTFIX}"
PARENT_SCOPE)
set(PYTHON_MODULE_EXTENSION
"${_PYTHON_MODULE_EXTENSION}"
PARENT_SCOPE)
set(PYTHON_IS_DEBUG
"${_PYTHON_IS_DEBUG}"
PARENT_SCOPE)
endfunction()
tools/pybind11NewTools.cmake
......@@ -32,6 +32,13 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
set(Python_ROOT_DIR "$ENV{pythonLocation}")
endif()
# Interpreter should not be found when cross-compiling
if(_PYBIND11_CROSSCOMPILING)
set(_pybind11_interp_component "")
else()
set(_pybind11_interp_component Interpreter)
endif()
# Development.Module support (required for manylinux) started in 3.18
if(CMAKE_VERSION VERSION_LESS 3.18)
set(_pybind11_dev_component Development)
......@@ -48,7 +55,8 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
endif()
endif()
find_package(Python 3.6 REQUIRED COMPONENTS Interpreter ${_pybind11_dev_component}
find_package(
Python 3.6 REQUIRED COMPONENTS ${_pybind11_interp_component} ${_pybind11_dev_component}
${_pybind11_quiet} ${_pybind11_global_keyword})
# If we are in submodule mode, export the Python targets to global targets.
......@@ -59,7 +67,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
if(TARGET Python::Python)
set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE)
endif()
if(TARGET Python::Interpreter)
set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
endif()
if(TARGET Python::Module)
set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE)
endif()
......@@ -100,28 +110,30 @@ if(PYBIND11_MASTER_PROJECT)
endif()
endif()
# If a user finds Python, they may forget to include the Interpreter component
# and the following two steps require it. It is highly recommended by CMake
# when finding development libraries anyway, so we will require it.
if(NOT DEFINED ${_Python}_EXECUTABLE)
if(NOT _PYBIND11_CROSSCOMPILING)
# If a user finds Python, they may forget to include the Interpreter component
# and the following two steps require it. It is highly recommended by CMake
# when finding development libraries anyway, so we will require it.
if(NOT DEFINED ${_Python}_EXECUTABLE)
message(
FATAL_ERROR
"${_Python} was found without the Interpreter component. Pybind11 requires this component.")
"${_Python} was found without the Interpreter component. Pybind11 requires this component."
)
endif()
endif()
if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
PYBIND11_PYTHON_EXECUTABLE_LAST)
# Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
unset(PYTHON_IS_DEBUG CACHE)
unset(PYTHON_MODULE_EXTENSION CACHE)
endif()
endif()
set(PYBIND11_PYTHON_EXECUTABLE_LAST
set(PYBIND11_PYTHON_EXECUTABLE_LAST
"${${_Python}_EXECUTABLE}"
CACHE INTERNAL "Python executable during the last CMake run")
if(NOT DEFINED PYTHON_IS_DEBUG)
if(NOT DEFINED PYTHON_IS_DEBUG)
# Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
execute_process(
COMMAND "${${_Python}_EXECUTABLE}" "-c"
......@@ -130,11 +142,11 @@ if(NOT DEFINED PYTHON_IS_DEBUG)
set(PYTHON_IS_DEBUG
"${_PYTHON_IS_DEBUG}"
CACHE INTERNAL "Python debug status")
endif()
endif()
# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
# required for PyPy3 (as of 7.3.1)
if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
# required for PyPy3 (as of 7.3.1)
if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
execute_process(
COMMAND
"${${_Python}_EXECUTABLE}" "-c"
......@@ -145,9 +157,9 @@ if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFI
if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
message(
FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'"
"package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}"
)
FATAL_ERROR
"pybind11 could not query the module file extension, likely the 'distutils'"
"package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}")
endif()
# This needs to be available for the pybind11_extension function
......@@ -164,6 +176,24 @@ if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFI
"${_PYTHON_MODULE_EXTENSION}"
CACHE INTERNAL "")
endif()
endif()
else()
if(NOT DEFINED PYTHON_IS_DEBUG
OR NOT DEFINED PYTHON_MODULE_EXTENSION
OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake")
pybind11_guess_python_module_extension("${_Python}")
endif()
# When cross-compiling, we cannot query the Python interpreter, so we require
# the user to set these variables explicitly.
if(NOT DEFINED PYTHON_IS_DEBUG
OR NOT DEFINED PYTHON_MODULE_EXTENSION
OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
message(
FATAL_ERROR
"When cross-compiling, you should set the PYTHON_IS_DEBUG, PYTHON_MODULE_EXTENSION and PYTHON_MODULE_DEBUG_POSTFIX \
variables appropriately before loading pybind11 (e.g. in your CMake toolchain file)")
endif()
endif()
# Python debug libraries expose slightly different objects before 3.8
......
tools/test-pybind11GuessPythonExtSuffix.cmake 0 → 100644
cmake_minimum_required(VERSION 3.5)
# Tests for pybind11_guess_python_module_extension
# Run using `cmake -P tools/test-pybind11GuessPythonExtSuffix.cmake`
include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake")
macro(expect_streq actual expected)
if(NOT "${actual}" STREQUAL "${expected}")
message(SEND_ERROR "Fail\n *** actual: '${actual}'\n *** expected: '${expected}'")
endif()
endmacro()
macro(expect_false actual)
if("${actual}")
message(SEND_ERROR "Fail\n *** actual: '${actual}'\n *** expected: false")
endif()
endmacro()
macro(expect_true actual)
if(NOT "${actual}")
message(SEND_ERROR "Fail\n *** actual: '${actual}'\n *** expected: true")
endif()
endmacro()
# Windows
set(CMAKE_SYSTEM_NAME "Windows")
set(CMAKE_SHARED_MODULE_SUFFIX ".dll")
set(Python3_SOABI "")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".pyd")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "cp311-win_arm64")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_arm64.pyd")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "cp311d-win_arm64")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311d-win_arm64.pyd")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_true("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "pypy310-pp73-win_amd64")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310-pp73-win_amd64.pyd")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "_d.cp311-win_amd64.pyd") # This is a quirk of FindPython3
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_amd64.pyd")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "_d")
expect_true("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
unset(Python3_SOABI)
set(ENV{SETUPTOOLS_EXT_SUFFIX} ".cp39-win_arm64.pyd") # Set by cibuildwheel
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp39-win_arm64.pyd")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
unset(ENV{SETUPTOOLS_EXT_SUFFIX})
set(Python3_SOABI "cp311-win_arm64")
set(ENV{SETUPTOOLS_EXT_SUFFIX} "") # Should not be used
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_arm64.pyd")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
unset(ENV{SETUPTOOLS_EXT_SUFFIX})
# macOS
set(CMAKE_SYSTEM_NAME "Darwin")
set(CMAKE_SHARED_MODULE_SUFFIX ".so")
set(Python3_SOABI "")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".so")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "cpython-312-darwin")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312-darwin.so")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "cpython-312d-darwin")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312d-darwin.so")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_true("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
# Linux
set(CMAKE_SYSTEM_NAME "Linux")
set(CMAKE_SHARED_MODULE_SUFFIX ".so")
set(Python3_SOABI "")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".so")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "cpython-312-arm-linux-gnueabihf")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312-arm-linux-gnueabihf.so")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "cpython-312d-arm-linux-gnueabihf")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312d-arm-linux-gnueabihf.so")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_true("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "pypy310-pp73-x86_64-linux-gnu")
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310-pp73-x86_64-linux-gnu.so")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_false("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
set(Python3_SOABI "pypy310d-pp73-x86_64-linux-gnu")
# TODO: I'm not sure if this is the right SOABI for PyPy debug builds
pybind11_guess_python_module_extension("Python3")
expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310d-pp73-x86_64-linux-gnu.so")
expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
expect_true("${PYTHON_IS_DEBUG}")
unset(PYTHON_MODULE_EXT_SUFFIX)
unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment