Move status_from_py_exc from Google-internal location to pybind11_abseil/compat.

Preparation for changing pybind11 handling of callbacks with `Status`, `StatusOr` returns to capture C++ exceptions.

PiperOrigin-RevId: 596700611

pybind11_abseil/CMakeLists.txt

+add_subdirectory(compat)
 add_subdirectory(cpp_capsule_tools)
 add_subdirectory(cpp_capsule_tools)
 
 
 # absl_casters ============================================================
 # absl_casters ============================================================

pybind11_abseil/compat/BUILD

+licenses(["notice"])
+
+package(
+    default_visibility = ["//visibility:private"],
+)
+
+cc_library(
+    name = "py_base_utilities",
+    srcs = ["py_base_utilities.cc"],
+    hdrs = ["py_base_utilities.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_google_absl//absl/log:absl_check",
+        "@com_google_absl//absl/log:absl_log",
+        "@com_google_absl//absl/strings",
+        "@local_config_python//:python_headers",  # buildcleaner: keep
+    ],
+)
+
+cc_library(
+    name = "status_from_core_py_exc",
+    srcs = ["status_from_core_py_exc.cc"],
+    hdrs = ["status_from_core_py_exc.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":py_base_utilities",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/status",
+        "@local_config_python//:python_headers",  # buildcleaner: keep
+    ],
+)
+
+cc_library(
+    name = "status_from_py_exc",
+    srcs = ["status_from_py_exc.cc"],
+    hdrs = ["status_from_py_exc.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":py_base_utilities",
+        ":status_from_core_py_exc",
+        "//pybind11_abseil/cpp_capsule_tools:raw_ptr_from_capsule",
+        "@com_google_absl//absl/log:absl_check",
+        "@com_google_absl//absl/log:absl_log",
+        "@com_google_absl//absl/status",
+        "@local_config_python//:python_headers",  # buildcleaner: keep
+    ],
+)

pybind11_abseil/compat/CMakeLists.txt

+# py_base_utilities ============================================================
+
+add_library(py_base_utilities STATIC py_base_utilities.cc)
+add_library(pybind11_abseil::compat::py_base_utilities ALIAS py_base_utilities)
+
+target_include_directories(py_base_utilities
+                           INTERFACE $<BUILD_INTERFACE:${TOP_LEVEL_DIR}>)
+
+target_link_libraries(py_base_utilities PUBLIC absl::strings absl::string_view)
+
+# status_from_core_py_exc ======================================================
+
+add_library(status_from_core_py_exc STATIC status_from_core_py_exc.cc)
+add_library(pybind11_abseil::compat::status_from_core_py_exc ALIAS
+            status_from_core_py_exc)
+
+target_include_directories(status_from_core_py_exc
+                           INTERFACE $<BUILD_INTERFACE:${TOP_LEVEL_DIR}>)
+
+target_link_libraries(status_from_core_py_exc PUBLIC py_base_utilities
+                                                     absl::status)
+
+# status_from_py_exc ===========================================================
+
+add_library(status_from_py_exc STATIC status_from_py_exc.cc)
+add_library(pybind11_abseil::compat::status_from_py_exc ALIAS
+            status_from_py_exc)
+
+target_include_directories(status_from_py_exc
+                           INTERFACE $<BUILD_INTERFACE:${TOP_LEVEL_DIR}>)
+
+target_link_libraries(
+  status_from_py_exc PUBLIC py_base_utilities status_from_core_py_exc
+                            void_ptr_from_capsule absl::status)

pybind11_abseil/compat/README.md

+Note
+----
+
+The code here
+
+* only depends on Python and abseil-cpp.
+
+* It does not depend on pybind11, mainly for compatibility with Google-internal
+  policies: use of C++ exception handling is strongly discouraged / banned.
+
+  See also:
+
+  * https://google.github.io/styleguide/cppguide.html#Exceptions

pybind11_abseil/compat/py_base_utilities.cc

+#include "pybind11_abseil/compat/py_base_utilities.h"
+
+#include <Python.h>
+
+#include <string>
+
+#include "absl/log/absl_check.h"
+#include "absl/log/absl_log.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+
+namespace pybind11_abseil::compat::py_base_utilities {
+
+const char* ClassName(PyObject* obj) {
+  if (PyType_Check(obj)) {
+    return reinterpret_cast<PyTypeObject*>(obj)->tp_name;
+  }
+  return Py_TYPE(obj)->tp_name;
+}
+
+std::string PyStrAsStdString(PyObject* str_object) {
+  Py_ssize_t value_size = 0;
+  const char* value = PyUnicode_AsUTF8AndSize(str_object, &value_size);
+  ABSL_CHECK(value != nullptr) << "FAILED: PyUnicode_AsUTF8AndSize()";
+  return std::string(value, value_size);
+}
+
+PyExcFetchMaybeErrOccurred::PyExcFetchMaybeErrOccurred() {
+  if (PyErr_Occurred()) {
+    PyErr_Fetch(&p_type_, &p_value_, &p_traceback_);
+  }
+}
+
+PyExcFetchGivenErrOccurred::PyExcFetchGivenErrOccurred() {
+  ABSL_CHECK(Type()) << "PyErr_Occurred() was false at entry.";
+}
+
+PyExcFetchMaybeErrOccurred::~PyExcFetchMaybeErrOccurred() {
+  Py_XDECREF(p_type_);
+  Py_XDECREF(p_value_);
+  Py_XDECREF(p_traceback_);
+}
+
+void PyExcFetchGivenErrOccurred::NormalizeException() {
+  PyErr_NormalizeException(&p_type_, &p_value_, &p_traceback_);
+}
+
+bool PyExcFetchGivenErrOccurred::Matches(PyObject* exc) const {
+  ABSL_CHECK(p_type_ != nullptr);
+  return PyErr_GivenExceptionMatches(p_type_, exc);
+}
+
+std::string PyExcFetchMaybeErrOccurred::FlatMessage() const {
+  if (p_type_ == nullptr) {
+    return "PyErr_Occurred() FALSE";
+  }
+  ABSL_CHECK(p_value_ != nullptr) << ClassName(p_type_);
+  PyObject* str = PyObject_Str(p_value_);
+  if (str == nullptr) {
+    ABSL_LOG(FATAL) << "FAILED (while processing " << ClassName(p_type_)
+                    << "): PyObject_Str(p_value_) ["
+                    << PyExcFetchMaybeErrOccurred().FlatMessage() << "]";
+  }
+  Py_ssize_t utf8_str_size = 0;
+  const char* utf8_str = PyUnicode_AsUTF8AndSize(str, &utf8_str_size);
+  if (utf8_str == nullptr) {
+    ABSL_LOG(FATAL) << "FAILED (while processing " << ClassName(p_type_)
+                    << "): PyUnicode_AsUTF8AndSize() ["
+                    << PyExcFetchMaybeErrOccurred().FlatMessage() << "]";
+  }
+  auto msg = absl::StrCat(ClassName(p_type_), ": ",
+                          absl::string_view(utf8_str, utf8_str_size));
+  Py_DECREF(str);
+  return msg;
+}
+
+PyObject* ImportModuleOrDie(const char* fq_mod) {
+  PyObject* imported_mod = PyImport_ImportModule(fq_mod);
+  if (imported_mod == nullptr || PyErr_Occurred()) {
+    ABSL_LOG(FATAL) << "FAILED: PyImport_ImportModule(\"" << fq_mod << "\") ["
+                    << PyExcFetchMaybeErrOccurred().FlatMessage() << "]";
+  }
+  return imported_mod;
+}
+
+PyObject* ImportObjectOrDie(const char* fq_mod, const char* mod_attr) {
+  PyObject* imported_mod = ImportModuleOrDie(fq_mod);
+  PyObject* imported_obj = PyObject_GetAttrString(imported_mod, mod_attr);
+  Py_DECREF(imported_mod);
+  if (imported_obj == nullptr || PyErr_Occurred()) {
+    ABSL_LOG(FATAL) << "FAILED: PyObject_GetAttrString(\"" << mod_attr
+                    << "\") [" << PyExcFetchMaybeErrOccurred().FlatMessage()
+                    << "]";
+  }
+  return imported_obj;
+}
+
+PyObject* ImportModuleOrReturnNone(const char* fq_mod) {
+  ABSL_CHECK(!PyErr_Occurred());
+  PyObject* imported_mod = PyImport_ImportModule(fq_mod);
+  if (PyErr_Occurred()) {
+    ABSL_CHECK(imported_mod == nullptr);
+    PyErr_Clear();
+    Py_RETURN_NONE;
+  }
+  ABSL_CHECK(imported_mod != nullptr);
+  return imported_mod;
+}
+
+PyObject* ImportObjectOrReturnNone(const char* fq_mod, const char* mod_attr) {
+  PyObject* imported_mod = ImportModuleOrReturnNone(fq_mod);
+  if (imported_mod == Py_None) {
+    return imported_mod;
+  }
+  PyObject* imported_obj = PyObject_GetAttrString(imported_mod, mod_attr);
+  Py_DECREF(imported_mod);
+  if (PyErr_Occurred()) {
+    ABSL_CHECK(imported_obj == nullptr);
+    PyErr_Clear();
+    Py_RETURN_NONE;
+  }
+  ABSL_CHECK(imported_obj != nullptr);
+  return imported_obj;
+}
+
+}  // namespace pybind11_abseil::compat::py_base_utilities

pybind11_abseil/compat/py_base_utilities.h

+#ifndef PYBIND11_ABSEIL_COMPAT_PY_BASE_UTILITIES_H_
+#define PYBIND11_ABSEIL_COMPAT_PY_BASE_UTILITIES_H_
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <string>
+
+namespace pybind11_abseil::compat::py_base_utilities {
+
+// Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class).
+const char* ClassName(PyObject* obj);
+
+std::string PyStrAsStdString(PyObject* str_object);
+
+class PyExcFetchMaybeErrOccurred {
+ public:
+  PyExcFetchMaybeErrOccurred();
+  ~PyExcFetchMaybeErrOccurred();
+
+  // Rule of Five completeness:
+  PyExcFetchMaybeErrOccurred(const PyExcFetchMaybeErrOccurred& other) = delete;
+  PyExcFetchMaybeErrOccurred(PyExcFetchMaybeErrOccurred&& other) = delete;
+  PyExcFetchMaybeErrOccurred& operator=(
+      const PyExcFetchMaybeErrOccurred& other) = delete;
+  PyExcFetchMaybeErrOccurred& operator=(PyExcFetchMaybeErrOccurred&& other) =
+      delete;
+
+  std::string FlatMessage() const;
+
+  PyObject* Type() const { return p_type_; }
+  PyObject* Value() const { return p_value_; }
+  PyObject* TraceBack() const { return p_traceback_; }
+
+ private:
+  PyObject* p_type_ = nullptr;
+  PyObject* p_value_ = nullptr;
+  PyObject* p_traceback_ = nullptr;
+
+  friend class PyExcFetchGivenErrOccurred;
+};
+
+class PyExcFetchGivenErrOccurred : public PyExcFetchMaybeErrOccurred {
+ public:
+  PyExcFetchGivenErrOccurred();
+
+  // WARNING: Calling this method has the potential to mask bugs.
+  //          This problem will go away with Python 3.12:
+  //          https://github.com/python/cpython/issues/102594
+  void NormalizeException();
+
+  bool Matches(PyObject* exc) const;
+};
+
+PyObject* ImportModuleOrDie(const char* fq_mod);
+PyObject* ImportObjectOrDie(const char* fq_mod, const char* mod_attr);
+
+PyObject* ImportModuleOrReturnNone(const char* fq_mod);
+PyObject* ImportObjectOrReturnNone(const char* fq_mod, const char* mod_attr);
+
+}  // namespace pybind11_abseil::compat::py_base_utilities
+
+#endif  // PYBIND11_ABSEIL_COMPAT_PY_BASE_UTILITIES_H_

pybind11_abseil/compat/status_from_core_py_exc.cc

+#include "pybind11_abseil/compat/status_from_core_py_exc.h"
+
+#include <Python.h>
+
+#include <string>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/status/status.h"
+#include "pybind11_abseil/compat/py_base_utilities.h"
+
+namespace pybind11_abseil::compat {
+
+namespace {
+
+using PyExceptionStatusCodeMapType =
+    absl::flat_hash_map<PyObject*, absl::StatusCode>;
+
+const PyExceptionStatusCodeMapType& GetPyExceptionStatusCodeMap() {
+  // When making changes here, please review
+  // tests/status_from_py_exc_testing_test.py:TAB_StatusFromFetchedExc
+  static const auto* kPyExcStatusCodeMap = new PyExceptionStatusCodeMapType(
+      {{PyExc_MemoryError, absl::StatusCode::kResourceExhausted},
+       {PyExc_NotImplementedError, absl::StatusCode::kUnimplemented},
+       {PyExc_KeyboardInterrupt, absl::StatusCode::kAborted},
+       {PyExc_SystemError, absl::StatusCode::kInternal},
+       {PyExc_SyntaxError, absl::StatusCode::kInternal},
+       {PyExc_TypeError, absl::StatusCode::kInvalidArgument},
+       {PyExc_ValueError, absl::StatusCode::kOutOfRange},
+       {PyExc_LookupError, absl::StatusCode::kNotFound}});
+  return *kPyExcStatusCodeMap;
+}
+
+}  // namespace
+
+absl::Status StatusFromFetchedExc(
+    const py_base_utilities::PyExcFetchGivenErrOccurred& fetched) {
+  std::string message = fetched.FlatMessage();
+  const PyExceptionStatusCodeMapType& pyexc_status_code_map =
+      GetPyExceptionStatusCodeMap();
+  for (const auto& it : pyexc_status_code_map) {
+    if (fetched.Matches(it.first)) {
+      return absl::Status(it.second, message);
+    }
+  }
+  return absl::UnknownError(message);
+}
+
+}  // namespace pybind11_abseil::compat

pybind11_abseil/compat/status_from_core_py_exc.h

+#ifndef PYBIND11_ABSEIL_COMPAT_STATUS_FROM_CORE_PY_EXC_H_
+#define PYBIND11_ABSEIL_COMPAT_STATUS_FROM_CORE_PY_EXC_H_
+
+#include "absl/status/status.h"
+#include "pybind11_abseil/compat/py_base_utilities.h"
+
+namespace pybind11_abseil::compat {
+
+absl::Status StatusFromFetchedExc(
+    const py_base_utilities::PyExcFetchGivenErrOccurred& fetched);
+
+}  // namespace pybind11_abseil::compat
+
+#endif  // PYBIND11_ABSEIL_COMPAT_STATUS_FROM_CORE_PY_EXC_H_

pybind11_abseil/compat/status_from_py_exc.cc

+#include "pybind11_abseil/compat/status_from_py_exc.h"
+
+#include <Python.h>
+
+#include "absl/log/absl_check.h"
+#include "absl/log/absl_log.h"
+#include "absl/status/status.h"
+#include "pybind11_abseil/compat/py_base_utilities.h"
+#include "pybind11_abseil/compat/status_from_core_py_exc.h"
+#include "pybind11_abseil/cpp_capsule_tools/raw_ptr_from_capsule.h"
+
+namespace pybind11_abseil::compat {
+
+absl::Status StatusFromFetchedStatusNotOk(
+    const py_base_utilities::PyExcFetchGivenErrOccurred& fetched) {
+  ABSL_CHECK(fetched.Value() != nullptr);
+  PyObject* py_status = nullptr;
+  bool py_status_owned = false;
+  if (PyTuple_Check(fetched.Value())) {
+    Py_ssize_t size = PyTuple_Size(fetched.Value());
+    ABSL_CHECK(size == 1) << "Unexpected tuple size from PyErr_Fetch(): "
+                          << size;
+    py_status = PyTuple_GetItem(fetched.Value(), 0);
+  } else {
+    py_status = PyObject_GetAttrString(fetched.Value(), "status");
+    py_status_owned = true;
+  }
+  if (py_status == nullptr) {
+    ABSL_LOG(FATAL) << "FAILED: Retrieving `StatusNotOk` `status` attribute "
+                       "from fetched Python exception ["
+                    << fetched.FlatMessage() << "]";
+  }
+  if (py_status == Py_None) {
+    ABSL_LOG(FATAL) << "FAILED: `StatusNotOk` `status` attribute from fetched "
+                       "Python exception is `None` ["
+                    << fetched.FlatMessage() << "]";
+  }
+  auto statusor_raw_ptr =
+      pybind11_abseil::cpp_capsule_tools::RawPtrFromCapsule<absl::Status>(
+          py_status, "::absl::Status", "as_absl_Status");
+  if (!statusor_raw_ptr.ok()) {
+    ABSL_LOG(FATAL)
+        << "FAILED: `StatusNotOk` `status` attribute from fetched Python "
+           "exception cannot be converted to an `absl::Status` object ["
+        << fetched.FlatMessage() << "]";
+  }
+  if (py_status_owned) {
+    Py_DECREF(py_status);
+  }
+  return *(statusor_raw_ptr.value());
+}
+
+namespace {
+
+PyObject* PyStatusNotOkOrNone() {
+  static PyObject* kImportedObj = nullptr;
+  if (kImportedObj == nullptr) {
+    kImportedObj = py_base_utilities::ImportObjectOrReturnNone(
+        "google3.third_party.pybind11_abseil.status", "StatusNotOk");
+  }
+  return kImportedObj;
+}
+
+}  // namespace
+
+absl::Status StatusFromPyExcGivenErrOccurred(bool normalize_exception) {
+  // Fetching exc IMMEDIATELY to ensure it does not accidentally get clobbered
+  // by Python C API calls while it is being processed (e.g. b/216844827).
+  py_base_utilities::PyExcFetchGivenErrOccurred fetched;
+
+  // Regarding "OrNone" in PyStatusNotOkOrNone():
+  // If StatusNotOk was not imported somewhere else already, it cannot possibly
+  // have been used to raise the exception to be matched here.
+  if (fetched.Matches(PyStatusNotOkOrNone())) {
+    return StatusFromFetchedStatusNotOk(fetched);
+  }
+
+  if (normalize_exception) {
+    fetched.NormalizeException();
+  }
+  return StatusFromFetchedExc(fetched);
+}
+
+absl::Status StatusFromPyExcMaybeErrOccurred(bool normalize_exception) {
+  if (!PyErr_Occurred()) {
+    return absl::OkStatus();
+  }
+  return StatusFromPyExcGivenErrOccurred(normalize_exception);
+}
+
+}  // namespace pybind11_abseil::compat

pybind11_abseil/compat/status_from_py_exc.h

+#ifndef PYBIND11_ABSEIL_COMPAT_STATUS_FROM_PY_EXC_H_
+#define PYBIND11_ABSEIL_COMPAT_STATUS_FROM_PY_EXC_H_
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <string>
+
+#include "absl/status/status.h"
+#include "pybind11_abseil/compat/py_base_utilities.h"
+
+namespace pybind11_abseil::compat {
+
+// Caller must ensure that the fetched exc type is "error.StatusNotOk".
+absl::Status StatusFromFetchedStatusNotOk(
+    const py_base_utilities::PyExcFetchGivenErrOccurred& fetched);
+
+// WARNING: `normalize_exception = true` has the potential to mask bugs.
+//          This problem will go away with Python 3.12:
+//          https://github.com/python/cpython/issues/102594
+absl::Status StatusFromPyExcGivenErrOccurred(bool normalize_exception = false);
+absl::Status StatusFromPyExcMaybeErrOccurred(bool normalize_exception = false);
+
+}  // namespace pybind11_abseil::compat
+
+#endif  // PYBIND11_ABSEIL_COMPAT_STATUS_FROM_PY_EXC_H_