Skip to content

P
pybind11
Commit 3c3533b4 by Wenzel Jakob Committed by GitHub
Options

Merge pull request #308 from aldanor/recarray 

py::dtype, buffer protocol improvements, structured types support
parents 09f40e01 bccbc10a
Showing with 1076 additions and 184 deletions
.gitignore
......@@ -3,7 +3,7 @@ CMakeFiles
Makefile
cmake_install.cmake
.DS_Store
/example/example.so
/example/example*.so
/example/example.cpython*.so
/example/example.pyd
/example/example*.dll
......@@ -31,3 +31,4 @@ MANIFEST
.DS_Store
/dist
/build
/cmake/
README.md
......@@ -106,8 +106,9 @@ Tomasz Miąsko,
Dean Moldovan,
Ben Pritchard,
Jason Rhinelander,
Boris Schäling, and
Pim Schellart.
Boris Schäling,
Pim Schellart,
Ivan Smirnov.
### License
......
docs/advanced.rst
......@@ -1224,12 +1224,12 @@ completely avoid copy operations with Python expressions like
py::class_<Matrix>(m, "Matrix")
.def_buffer([](Matrix &m) -> py::buffer_info {
return py::buffer_info(
m.data(), /* Pointer to buffer */
sizeof(float), /* Size of one scalar */
py::format_descriptor<float>::value, /* Python struct-style format descriptor */
2, /* Number of dimensions */
{ m.rows(), m.cols() }, /* Buffer dimensions */
{ sizeof(float) * m.rows(), /* Strides (in bytes) for each index */
m.data(), /* Pointer to buffer */
sizeof(float), /* Size of one scalar */
py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
2, /* Number of dimensions */
{ m.rows(), m.cols() }, /* Buffer dimensions */
{ sizeof(float) * m.rows(), /* Strides (in bytes) for each index */
sizeof(float) }
);
});
......@@ -1273,7 +1273,7 @@ buffer objects (e.g. a NumPy matrix).
py::buffer_info info = b.request();
/* Some sanity checks ... */
if (info.format != py::format_descriptor<Scalar>::value)
if (info.format != py::format_descriptor<Scalar>::format())
throw std::runtime_error("Incompatible format: expected a double array!");
if (info.ndim != 2)
......@@ -1299,7 +1299,7 @@ as follows:
m.data(), /* Pointer to buffer */
sizeof(Scalar), /* Size of one scalar */
/* Python struct-style format descriptor */
py::format_descriptor<Scalar>::value,
py::format_descriptor<Scalar>::format(),
/* Number of dimensions */
2,
/* Buffer dimensions */
......@@ -1358,6 +1358,30 @@ template paramenter, and it ensures that non-conforming arguments are converted
into an array satisfying the specified requirements instead of trying the next
function overload.
NumPy structured types
======================
In order for ``py::array_t`` to work with structured (record) types, we first need
to register the memory layout of the type. This can be done via ``PYBIND11_NUMPY_DTYPE``
macro which expects the type followed by field names:
.. code-block:: cpp
struct A {
int x;
double y;
};
struct B {
int z;
A a;
};
PYBIND11_NUMPY_DTYPE(A, x, y);
PYBIND11_NUMPY_DTYPE(B, z, a);
/* now both A and B can be used as template arguments to py::array_t */
Vectorizing functions
=====================
......@@ -1433,17 +1457,11 @@ simply using ``vectorize``).
if (buf1.ndim != 1 || buf2.ndim != 1)
throw std::runtime_error("Number of dimensions must be one");
if (buf1.shape[0] != buf2.shape[0])
if (buf1.size != buf2.size)
throw std::runtime_error("Input shapes must match");
auto result = py::array(py::buffer_info(
nullptr, /* Pointer to data (nullptr -> ask NumPy to allocate!) */
sizeof(double), /* Size of one item */
py::format_descriptor<double>::value, /* Buffer format */
buf1.ndim, /* How many dimensions? */
{ buf1.shape[0] }, /* Number of elements for each dimension */
{ sizeof(double) } /* Strides for each dimension */
));
/* No pointer is passed, so NumPy will allocate the buffer */
auto result = py::array_t<double>(buf1.size);
auto buf3 = result.request();
......@@ -1830,4 +1848,3 @@ is always ``none``).
// Evaluate the statements in an separate Python file on disk
py::eval_file("script.py", scope);
docs/changelog.rst
......@@ -27,6 +27,23 @@ Breaking changes queued for v2.0.0 (Not yet released)
* ``make_iterator()`` improvements for better compatibility with various types
(now uses prefix increment operator)
* ``arg()`` now accepts a wider range of argument types for default values
* Added support for registering structured dtypes via ``PYBIND11_NUMPY_DTYPE()`` macro.
* Added ``PYBIND11_STR_TYPE`` macro which maps to the ``builtins.str`` type.
* Added a simplified ``buffer_info`` constructor for 1-dimensional buffers.
* Format descriptor strings should now be accessed via ``format_descriptor::format()``
(for compatibility purposes, the old syntax ``format_descriptor::value`` will still
work for non-structured data types).
* Added a class wrapping NumPy array descriptors: ``dtype``.
* Added buffer/NumPy support for ``char[N]`` and ``std::array<char, N>`` types.
* ``array`` gained new constructors accepting dtype objects.
* Added constructors for ``array`` and ``array_t`` explicitly accepting shape and
strides; if strides are not provided, they are deduced assuming C-contiguity.
Also added simplified constructors for 1-dimensional case.
* Added constructors for ``str`` from ``bytes`` and for ``bytes`` from ``str``.
This will do the UTF-8 decoding/encoding as required.
* Added constructors for ``str`` and ``bytes`` from zero-terminated char pointers,
and from char pointers and length.
* Added ``memoryview`` wrapper type which is constructible from ``buffer_info``.
* Various minor improvements of library internals (no user-visible changes)
1.8.1 (July 12, 2016)
......
example/CMakeLists.txt
......@@ -26,6 +26,7 @@ set(PYBIND11_EXAMPLES
example-stl-binder-vector.cpp
example-eval.cpp
example-custom-exceptions.cpp
example-numpy-dtypes.cpp
issues.cpp
)
......@@ -65,4 +66,3 @@ foreach(VALUE ${PYBIND11_EXAMPLES})
string(REGEX REPLACE "^(.+).cpp$" "\\1" EXAMPLE_NAME "${VALUE}")
add_test(NAME ${EXAMPLE_NAME} COMMAND ${RUN_TEST} ${EXAMPLE_NAME})
endforeach()
example/example-buffers.cpp
......@@ -81,7 +81,7 @@ void init_ex_buffers(py::module &m) {
/// Construct from a buffer
.def("__init__", [](Matrix &v, py::buffer b) {
py::buffer_info info = b.request();
if (info.format != py::format_descriptor<float>::value || info.ndim != 2)
if (info.format != py::format_descriptor<float>::format() || info.ndim != 2)
throw std::runtime_error("Incompatible buffer format!");
new (&v) Matrix(info.shape[0], info.shape[1]);
memcpy(v.data(), info.ptr, sizeof(float) * v.rows() * v.cols());
......@@ -104,12 +104,12 @@ void init_ex_buffers(py::module &m) {
/// Provide buffer access
.def_buffer([](Matrix &m) -> py::buffer_info {
return py::buffer_info(
m.data(), /* Pointer to buffer */
sizeof(float), /* Size of one scalar */
py::format_descriptor<float>::value, /* Python struct-style format descriptor */
2, /* Number of dimensions */
{ m.rows(), m.cols() }, /* Buffer dimensions */
{ sizeof(float) * m.rows(), /* Strides (in bytes) for each index */
m.data(), /* Pointer to buffer */
sizeof(float), /* Size of one scalar */
py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
2, /* Number of dimensions */
{ m.rows(), m.cols() }, /* Buffer dimensions */
{ sizeof(float) * m.rows(), /* Strides (in bytes) for each index */
sizeof(float) }
);
})
......
example/example-numpy-dtypes.cpp 0 → 100644
/*
example/example-numpy-dtypes.cpp -- Structured and compound NumPy dtypes
Copyright (c) 2016 Ivan Smirnov
All rights reserved. Use of this source code is governed by a
BSD-style license that can be found in the LICENSE file.
*/
#include "example.h"
#include <pybind11/numpy.h>
#include <cstdint>
#include <iostream>
#ifdef __GNUC__
#define PYBIND11_PACKED(cls) cls __attribute__((__packed__))
#else
#define PYBIND11_PACKED(cls) __pragma(pack(push, 1)) cls __pragma(pack(pop))
#endif
namespace py = pybind11;
struct SimpleStruct {
bool x;
uint32_t y;
float z;
};
std::ostream& operator<<(std::ostream& os, const SimpleStruct& v) {
return os << "s:" << v.x << "," << v.y << "," << v.z;
}
PYBIND11_PACKED(struct PackedStruct {
bool x;
uint32_t y;
float z;
});
std::ostream& operator<<(std::ostream& os, const PackedStruct& v) {
return os << "p:" << v.x << "," << v.y << "," << v.z;
}
PYBIND11_PACKED(struct NestedStruct {
SimpleStruct a;
PackedStruct b;
});
std::ostream& operator<<(std::ostream& os, const NestedStruct& v) {
return os << "n:a=" << v.a << ";b=" << v.b;
}
struct PartialStruct {
bool x;
uint32_t y;
float z;
uint64_t dummy2;
};
struct PartialNestedStruct {
uint64_t dummy1;
PartialStruct a;
uint64_t dummy2;
};
struct UnboundStruct { };
struct StringStruct {
char a[3];
std::array<char, 3> b;
};
std::ostream& operator<<(std::ostream& os, const StringStruct& v) {
os << "a='";
for (size_t i = 0; i < 3 && v.a[i]; i++) os << v.a[i];
os << "',b='";
for (size_t i = 0; i < 3 && v.b[i]; i++) os << v.b[i];
return os << "'";
}
template <typename T>
py::array mkarray_via_buffer(size_t n) {
return py::array(py::buffer_info(nullptr, sizeof(T),
py::format_descriptor<T>::format(),
1, { n }, { sizeof(T) }));
}
template <typename S>
py::array_t<S, 0> create_recarray(size_t n) {
auto arr = mkarray_via_buffer<S>(n);
auto req = arr.request();
auto ptr = static_cast<S*>(req.ptr);
for (size_t i = 0; i < n; i++) {
ptr[i].x = i % 2 != 0; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f;
}
return arr;
}
std::string get_format_unbound() {
return py::format_descriptor<UnboundStruct>::format();
}
py::array_t<NestedStruct, 0> create_nested(size_t n) {
auto arr = mkarray_via_buffer<NestedStruct>(n);
auto req = arr.request();
auto ptr = static_cast<NestedStruct*>(req.ptr);
for (size_t i = 0; i < n; i++) {
ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f;
ptr[i].b.x = (i + 1) % 2 != 0; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f;
}
return arr;
}
py::array_t<PartialNestedStruct, 0> create_partial_nested(size_t n) {
auto arr = mkarray_via_buffer<PartialNestedStruct>(n);
auto req = arr.request();
auto ptr = static_cast<PartialNestedStruct*>(req.ptr);
for (size_t i = 0; i < n; i++) {
ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f;
}
return arr;
}
py::array_t<StringStruct, 0> create_string_array(bool non_empty) {
auto arr = mkarray_via_buffer<StringStruct>(non_empty ? 4 : 0);
if (non_empty) {
auto req = arr.request();
auto ptr = static_cast<StringStruct*>(req.ptr);
for (size_t i = 0; i < req.size * req.itemsize; i++)
static_cast<char*>(req.ptr)[i] = 0;
ptr[1].a[0] = 'a'; ptr[1].b[0] = 'a';
ptr[2].a[0] = 'a'; ptr[2].b[0] = 'a';
ptr[3].a[0] = 'a'; ptr[3].b[0] = 'a';
ptr[2].a[1] = 'b'; ptr[2].b[1] = 'b';
ptr[3].a[1] = 'b'; ptr[3].b[1] = 'b';
ptr[3].a[2] = 'c'; ptr[3].b[2] = 'c';
}
return arr;
}
template <typename S>
void print_recarray(py::array_t<S, 0> arr) {
auto req = arr.request();
auto ptr = static_cast<S*>(req.ptr);
for (size_t i = 0; i < req.size; i++)
std::cout << ptr[i] << std::endl;
}
void print_format_descriptors() {
std::cout << py::format_descriptor<SimpleStruct>::format() << std::endl;
std::cout << py::format_descriptor<PackedStruct>::format() << std::endl;
std::cout << py::format_descriptor<NestedStruct>::format() << std::endl;
std::cout << py::format_descriptor<PartialStruct>::format() << std::endl;
std::cout << py::format_descriptor<PartialNestedStruct>::format() << std::endl;
std::cout << py::format_descriptor<StringStruct>::format() << std::endl;
}
void print_dtypes() {
std::cout << (std::string) py::dtype::of<SimpleStruct>().str() << std::endl;
std::cout << (std::string) py::dtype::of<PackedStruct>().str() << std::endl;
std::cout << (std::string) py::dtype::of<NestedStruct>().str() << std::endl;
std::cout << (std::string) py::dtype::of<PartialStruct>().str() << std::endl;
std::cout << (std::string) py::dtype::of<PartialNestedStruct>().str() << std::endl;
std::cout << (std::string) py::dtype::of<StringStruct>().str() << std::endl;
}
py::array_t<int32_t, 0> test_array_ctors(int i) {
using arr_t = py::array_t<int32_t, 0>;
std::vector<int32_t> data { 1, 2, 3, 4, 5, 6 };
std::vector<size_t> shape { 3, 2 };
std::vector<size_t> strides { 8, 4 };
auto ptr = data.data();
auto vptr = (void *) ptr;
auto dtype = py::dtype("int32");
py::buffer_info buf_ndim1(vptr, 4, "i", 6);
py::buffer_info buf_ndim1_null(nullptr, 4, "i", 6);
py::buffer_info buf_ndim2(vptr, 4, "i", 2, shape, strides);
py::buffer_info buf_ndim2_null(nullptr, 4, "i", 2, shape, strides);
auto fill = [](py::array arr) {
auto req = arr.request();
for (int i = 0; i < 6; i++) ((int32_t *) req.ptr)[i] = i + 1;
return arr;
};
switch (i) {
// shape: (3, 2)
case 10: return arr_t(shape, strides, ptr);
case 11: return py::array(shape, strides, ptr);
case 12: return py::array(dtype, shape, strides, vptr);
case 13: return arr_t(shape, ptr);
case 14: return py::array(shape, ptr);
case 15: return py::array(dtype, shape, vptr);
case 16: return arr_t(buf_ndim2);
case 17: return py::array(buf_ndim2);
// shape: (3, 2) - post-fill
case 20: return fill(arr_t(shape, strides));
case 21: return py::array(shape, strides, ptr); // can't have nullptr due to templated ctor
case 22: return fill(py::array(dtype, shape, strides));
case 23: return fill(arr_t(shape));
case 24: return py::array(shape, ptr); // can't have nullptr due to templated ctor
case 25: return fill(py::array(dtype, shape));
case 26: return fill(arr_t(buf_ndim2_null));
case 27: return fill(py::array(buf_ndim2_null));
// shape: (6, )
case 30: return arr_t(6, ptr);
case 31: return py::array(6, ptr);
case 32: return py::array(dtype, 6, vptr);
case 33: return arr_t(buf_ndim1);
case 34: return py::array(buf_ndim1);
// shape: (6, )
case 40: return fill(arr_t(6));
case 41: return py::array(6, ptr); // can't have nullptr due to templated ctor
case 42: return fill(py::array(dtype, 6));
case 43: return fill(arr_t(buf_ndim1_null));
case 44: return fill(py::array(buf_ndim1_null));
}
return arr_t();
}
py::list test_dtype_ctors() {
py::list list;
list.append(py::dtype("int32"));
list.append(py::dtype(std::string("float64")));
list.append(py::dtype::from_args(py::str("bool")));
py::list names, offsets, formats;
py::dict dict;
names.append(py::str("a")); names.append(py::str("b")); dict["names"] = names;
offsets.append(py::int_(1)); offsets.append(py::int_(10)); dict["offsets"] = offsets;
formats.append(py::dtype("int32")); formats.append(py::dtype("float64")); dict["formats"] = formats;
dict["itemsize"] = py::int_(20);
list.append(py::dtype::from_args(dict));
list.append(py::dtype(names, formats, offsets, 20));
list.append(py::dtype(py::buffer_info((void *) 0, 1, "I", 1)));
list.append(py::dtype(py::buffer_info((void *) 0, 1, "T{i:a:f:b:}", 1)));
return list;
}
py::list test_dtype_methods() {
py::list list;
auto dt1 = py::dtype::of<int32_t>();
auto dt2 = py::dtype::of<SimpleStruct>();
list.append(dt1); list.append(dt2);
list.append(py::bool_(dt1.has_fields())); list.append(py::bool_(dt2.has_fields()));
list.append(py::int_(dt1.itemsize())); list.append(py::int_(dt2.itemsize()));
return list;
}
void init_ex_numpy_dtypes(py::module &m) {
PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z);
PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z);
PYBIND11_NUMPY_DTYPE(NestedStruct, a, b);
PYBIND11_NUMPY_DTYPE(PartialStruct, x, y, z);
PYBIND11_NUMPY_DTYPE(PartialNestedStruct, a);
PYBIND11_NUMPY_DTYPE(StringStruct, a, b);
m.def("create_rec_simple", &create_recarray<SimpleStruct>);
m.def("create_rec_packed", &create_recarray<PackedStruct>);
m.def("create_rec_nested", &create_nested);
m.def("create_rec_partial", &create_recarray<PartialStruct>);
m.def("create_rec_partial_nested", &create_partial_nested);
m.def("print_format_descriptors", &print_format_descriptors);
m.def("print_rec_simple", &print_recarray<SimpleStruct>);
m.def("print_rec_packed", &print_recarray<PackedStruct>);
m.def("print_rec_nested", &print_recarray<NestedStruct>);
m.def("print_dtypes", &print_dtypes);
m.def("get_format_unbound", &get_format_unbound);
m.def("create_string_array", &create_string_array);
m.def("print_string_array", &print_recarray<StringStruct>);
m.def("test_array_ctors", &test_array_ctors);
m.def("test_dtype_ctors", &test_dtype_ctors);
m.def("test_dtype_methods", &test_dtype_methods);
}
#undef PYBIND11_PACKED
example/example-numpy-dtypes.py 0 → 100644
#!/usr/bin/env python
from __future__ import print_function
import numpy as np
from example import (
create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors,
print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound,
create_rec_partial, create_rec_partial_nested, create_string_array, print_string_array,
test_array_ctors, test_dtype_ctors, test_dtype_methods
)
def check_eq(arr, data, dtype):
np.testing.assert_equal(arr, np.array(data, dtype=dtype))
try:
get_format_unbound()
raise Exception
except RuntimeError as e:
assert 'unsupported buffer format' in str(e)
print_format_descriptors()
print_dtypes()
simple_dtype = np.dtype({'names': ['x', 'y', 'z'],
'formats': ['?', 'u4', 'f4'],
'offsets': [0, 4, 8]})
packed_dtype = np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')])
elements = [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)]
for func, dtype in [(create_rec_simple, simple_dtype), (create_rec_packed, packed_dtype)]:
arr = func(0)
assert arr.dtype == dtype
check_eq(arr, [], simple_dtype)
check_eq(arr, [], packed_dtype)
arr = func(3)
assert arr.dtype == dtype
check_eq(arr, elements, simple_dtype)
check_eq(arr, elements, packed_dtype)
if dtype == simple_dtype:
print_rec_simple(arr)
else:
print_rec_packed(arr)
arr = create_rec_partial(3)
print(arr.dtype)
partial_dtype = arr.dtype
assert '' not in arr.dtype.fields
assert partial_dtype.itemsize > simple_dtype.itemsize
check_eq(arr, elements, simple_dtype)
check_eq(arr, elements, packed_dtype)
arr = create_rec_partial_nested(3)
print(arr.dtype)
assert '' not in arr.dtype.fields
assert '' not in arr.dtype.fields['a'][0].fields
assert arr.dtype.itemsize > partial_dtype.itemsize
np.testing.assert_equal(arr['a'], create_rec_partial(3))
nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)])
arr = create_rec_nested(0)
assert arr.dtype == nested_dtype
check_eq(arr, [], nested_dtype)
arr = create_rec_nested(3)
assert arr.dtype == nested_dtype
check_eq(arr, [((False, 0, 0.0), (True, 1, 1.5)),
((True, 1, 1.5), (False, 2, 3.0)),
((False, 2, 3.0), (True, 3, 4.5))], nested_dtype)
print_rec_nested(arr)
assert create_rec_nested.__doc__.strip().endswith('numpy.ndarray[NestedStruct]')
arr = create_string_array(True)
print(arr.dtype)
print_string_array(arr)
dtype = arr.dtype
assert arr['a'].tolist() == [b'', b'a', b'ab', b'abc']
assert arr['b'].tolist() == [b'', b'a', b'ab', b'abc']
arr = create_string_array(False)
assert dtype == arr.dtype
data = np.arange(1, 7, dtype='int32')
for i in range(8):
np.testing.assert_array_equal(test_array_ctors(10 + i), data.reshape((3, 2)))
np.testing.assert_array_equal(test_array_ctors(20 + i), data.reshape((3, 2)))
for i in range(5):
np.testing.assert_array_equal(test_array_ctors(30 + i), data)
np.testing.assert_array_equal(test_array_ctors(40 + i), data)
d1 = np.dtype({'names': ['a', 'b'], 'formats': ['int32', 'float64'],
'offsets': [1, 10], 'itemsize': 20})
d2 = np.dtype([('a', 'i4'), ('b', 'f4')])
assert test_dtype_ctors() == [np.dtype('int32'), np.dtype('float64'),
np.dtype('bool'), d1, d1, np.dtype('uint32'), d2]
assert test_dtype_methods() == [np.dtype('int32'), simple_dtype, False, True,
np.dtype('int32').itemsize, simple_dtype.itemsize]
example/example-numpy-dtypes.ref 0 → 100644
T{=?:x:3x=I:y:=f:z:}
T{=?:x:=I:y:=f:z:}
T{=T{=?:x:3x=I:y:=f:z:}:a:=T{=?:x:=I:y:=f:z:}:b:}
T{=?:x:3x=I:y:=f:z:12x}
T{8x=T{=?:x:3x=I:y:=f:z:12x}:a:8x}
T{=3s:a:=3s:b:}
{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':12}
[('x', '?'), ('y', '<u4'), ('z', '<f4')]
[('a', {'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':12}), ('b', [('x', '?'), ('y', '<u4'), ('z', '<f4')])]
{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}
{'names':['a'], 'formats':[{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}], 'offsets':[8], 'itemsize':40}
[('a', 'S3'), ('b', 'S3')]
s:0,0,0
s:1,1,1.5
s:0,2,3
p:0,0,0
p:1,1,1.5
p:0,2,3
{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}
{'names':['a'], 'formats':[{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}], 'offsets':[8], 'itemsize':40}
n:a=s:0,0,0;b=p:1,1,1.5
n:a=s:1,1,1.5;b=p:0,2,3
n:a=s:0,2,3;b=p:1,3,4.5
[('a', 'S3'), ('b', 'S3')]
a='',b=''
a='a',b='a'
a='ab',b='ab'
a='abc',b='abc'
example/example-python-types.cpp
......@@ -139,6 +139,22 @@ public:
throw std::runtime_error("This exception was intentionally thrown.");
}
py::bytes get_bytes_from_string() {
return (py::bytes) std::string("foo");
}
py::bytes get_bytes_from_str() {
return (py::bytes) py::str("bar", 3);
}
py::str get_str_from_string() {
return (py::str) std::string("baz");
}
py::str get_str_from_bytes() {
return (py::str) py::bytes("boo", 3);
}
static int value;
static const int value2;
};
......@@ -167,6 +183,10 @@ void init_ex_python_types(py::module &m) {
.def("pair_passthrough", &ExamplePythonTypes::pair_passthrough, "Return a pair in reversed order")
.def("tuple_passthrough", &ExamplePythonTypes::tuple_passthrough, "Return a triple in reversed order")
.def("throw_exception", &ExamplePythonTypes::throw_exception, "Throw an exception")
.def("get_bytes_from_string", &ExamplePythonTypes::get_bytes_from_string, "py::bytes from std::string")
.def("get_bytes_from_str", &ExamplePythonTypes::get_bytes_from_str, "py::bytes from py::str")
.def("get_str_from_string", &ExamplePythonTypes::get_str_from_string, "py::str from std::string")
.def("get_str_from_bytes", &ExamplePythonTypes::get_str_from_bytes, "py::str from py::bytes")
.def_static("new_instance", &ExamplePythonTypes::new_instance, "Return an instance")
.def_readwrite_static("value", &ExamplePythonTypes::value, "Static value member")
.def_readonly_static("value2", &ExamplePythonTypes::value2, "Static value member (readonly)")
......
example/example-python-types.py
......@@ -66,6 +66,11 @@ print("__module__(example.ExamplePythonTypes) = %s" % ExamplePythonTypes.__modul
print("__name__(example.ExamplePythonTypes.get_set) = %s" % ExamplePythonTypes.get_set.__name__)
print("__module__(example.ExamplePythonTypes.get_set) = %s" % ExamplePythonTypes.get_set.__module__)
print(instance.get_bytes_from_string().decode())
print(instance.get_bytes_from_str().decode())
print(instance.get_str_from_string().encode().decode())
print(instance.get_str_from_bytes().encode().decode())
from example import ConstructorStats
cstats = ConstructorStats.get(ExamplePythonTypes)
......
example/example-python-types.ref
......@@ -28,106 +28,106 @@ Help on class ExamplePythonTypes in module example
class EExxaammpplleePPyytthhoonnTTyyppeess(__builtin__.object)
| Example 2 documentation
|
|
| Methods defined here:
|
|
| ____iinniitt____(...)
| x.__init__(...) initializes x; see help(type(x)) for signature
|
|
| ggeett__aarrrraayy(...)
|
|
| Signature : (example.ExamplePythonTypes) -> List[unicode[2]]
| Return a C++ array
|
|
| ggeett__ddiicctt(...)
| Signature : (example.ExamplePythonTypes) -> dict
|
|
| Return a Python dictionary
|
|
| ggeett__ddiicctt__22(...)
|
|
| Signature : (example.ExamplePythonTypes) -> Dict[unicode, unicode]
| Return a C++ dictionary
|
|
| ggeett__lliisstt(...)
| Signature : (example.ExamplePythonTypes) -> list
|
|
| Return a Python list
|
|
| ggeett__lliisstt__22(...)
|
|
| Signature : (example.ExamplePythonTypes) -> List[unicode]
| Return a C++ list
|
|
| ggeett__sseett(...)
| Signature : (example.ExamplePythonTypes) -> set
|
|
| Return a Python set
|
|
| ggeett__sseett22(...)
| Signature : (example.ExamplePythonTypes) -> set
|
|
| Return a C++ set
|
|
| ppaaiirr__ppaasssstthhrroouugghh(...)
|
|
| Signature : (example.ExamplePythonTypes, Tuple[bool, unicode]) -> Tuple[unicode, bool]
| Return a pair in reversed order
|
|
| pprriinntt__aarrrraayy(...)
|
|
| Signature : (example.ExamplePythonTypes, List[unicode[2]]) -> None
| Print entries of a C++ array
|
|
| pprriinntt__ddiicctt(...)
|
|
| Signature : (example.ExamplePythonTypes, dict) -> None
| Print entries of a Python dictionary
|
|
| pprriinntt__ddiicctt__22(...)
|
|
| Signature : (example.ExamplePythonTypes, Dict[unicode, unicode]) -> None
| Print entries of a C++ dictionary
|
|
| pprriinntt__lliisstt(...)
|
|
| Signature : (example.ExamplePythonTypes, list) -> None
| Print entries of a Python list
|
|
| pprriinntt__lliisstt__22(...)
|
|
| Signature : (example.ExamplePythonTypes, List[unicode]) -> None
| Print entries of a C++ list
|
|
| pprriinntt__sseett(...)
|
|
| Signature : (example.ExamplePythonTypes, set) -> None
| Print entries of a Python set
|
|
| pprriinntt__sseett__22(...)
|
|
| Signature : (example.ExamplePythonTypes, Set[unicode]) -> None
| Print entries of a C++ set
|
|
| tthhrrooww__eexxcceeppttiioonn(...)
|
|
| Signature : (example.ExamplePythonTypes) -> None
| Throw an exception
|
|
| ttuuppllee__ppaasssstthhrroouugghh(...)
|
|
| Signature : (example.ExamplePythonTypes, Tuple[bool, unicode, int]) -> Tuple[int, unicode, bool]
| Return a triple in reversed order
|
|
| ----------------------------------------------------------------------
| Data and other attributes defined here:
|
|
| ____nneeww____ = <built-in method __new__ of example.ExamplePythonTypes__Meta object>
| T.__new__(S, ...) -> a new object with type S, a subtype of T
|
|
| nneeww__iinnssttaannccee = <built-in method new_instance of PyCapsule object>
| Signature : () -> example.ExamplePythonTypes
|
|
| Return an instance
__name__(example) = example
......@@ -135,6 +135,10 @@ __name__(example.ExamplePythonTypes) = ExamplePythonTypes
__module__(example.ExamplePythonTypes) = example
__name__(example.ExamplePythonTypes.get_set) = get_set
__module__(example.ExamplePythonTypes.get_set) = example
foo
bar
baz
boo
Instances not destroyed: 1
### ExamplePythonTypes @ 0x1045b80 destroyed
Instances not destroyed: 0
example/example.cpp
......@@ -29,6 +29,7 @@ void init_ex_inheritance(py::module &);
void init_ex_stl_binder_vector(py::module &);
void init_ex_eval(py::module &);
void init_ex_custom_exceptions(py::module &);
void init_ex_numpy_dtypes(py::module &);
void init_issues(py::module &);
#if defined(PYBIND11_TEST_EIGEN)
......@@ -72,6 +73,7 @@ PYBIND11_PLUGIN(example) {
init_ex_stl_binder_vector(m);
init_ex_eval(m);
init_ex_custom_exceptions(m);
init_ex_numpy_dtypes(m);
init_issues(m);
#if defined(PYBIND11_TEST_EIGEN)
......
example/run_test.py
......@@ -46,12 +46,11 @@ name = sys.argv[1]
try:
output_bytes = subprocess.check_output([sys.executable, "-u", name + ".py"],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if e.returncode == 99:
print('Test "%s" could not be run.' % name)
exit(0)
else:
raise
except subprocess.CalledProcessError as exc:
print('Test `{}` failed:\n{}\n'.format(name, '-' * 50))
print(exc.output.decode())
print('-' * 50)
sys.exit(1)
output = sanitize(output_bytes.decode('utf-8'))
reference = sanitize(open(name + '.ref', 'r').read())
......
include/pybind11/common.h
......@@ -95,6 +95,7 @@
#define PYBIND11_STRING_NAME "str"
#define PYBIND11_SLICE_OBJECT PyObject
#define PYBIND11_FROM_STRING PyUnicode_FromString
#define PYBIND11_STR_TYPE ::pybind11::str
#define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_base.ob_base.ob_type
#define PYBIND11_PLUGIN_IMPL(name) \
extern "C" PYBIND11_EXPORT PyObject *PyInit_##name()
......@@ -113,6 +114,7 @@
#define PYBIND11_STRING_NAME "unicode"
#define PYBIND11_SLICE_OBJECT PySliceObject
#define PYBIND11_FROM_STRING PyString_FromString
#define PYBIND11_STR_TYPE ::pybind11::bytes
#define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_type
#define PYBIND11_PLUGIN_IMPL(name) \
extern "C" PYBIND11_EXPORT PyObject *init##name()
......@@ -204,12 +206,13 @@ struct buffer_info {
void *ptr; // Pointer to the underlying storage
size_t itemsize; // Size of individual items in bytes
size_t size; // Total number of entries
std::string format; // For homogeneous buffers, this should be set to format_descriptor<T>::value
std::string format; // For homogeneous buffers, this should be set to format_descriptor<T>::format()
size_t ndim; // Number of dimensions
std::vector<size_t> shape; // Shape of the tensor (1 entry per dimension)
std::vector<size_t> strides; // Number of entries between adjacent entries (for each per dimension)
buffer_info() : ptr(nullptr), view(nullptr) {}
buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t ndim,
const std::vector<size_t> &shape, const std::vector<size_t> &strides)
: ptr(ptr), itemsize(itemsize), size(1), format(format),
......@@ -218,6 +221,10 @@ struct buffer_info {
size *= shape[i];
}
buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t size)
: buffer_info(ptr, itemsize, format, 1, std::vector<size_t> { size },
std::vector<size_t> { itemsize }) { }
buffer_info(Py_buffer *view)
: ptr(view->buf), itemsize((size_t) view->itemsize), size(1), format(view->format),
ndim((size_t) view->ndim), shape((size_t) view->ndim), strides((size_t) view->ndim), view(view) {
......@@ -231,6 +238,7 @@ struct buffer_info {
~buffer_info() {
if (view) { PyBuffer_Release(view); delete view; }
}
private:
Py_buffer *view = nullptr;
};
......@@ -323,14 +331,23 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error) /// Used internally
[[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const std::string &reason) { throw std::runtime_error(reason); }
/// Format strings for basic number types
#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor<t> { static constexpr const char *value = v; }
#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor<t> \
{ static constexpr const char* value = v; /* for backwards compatibility */ \
static constexpr const char* format() { return value; } }
template <typename T, typename SFINAE = void> struct format_descriptor { };
template <typename T> struct format_descriptor<T, typename std::enable_if<std::is_integral<T>::value>::type> {
static constexpr const char value[2] =
{ "bBhHiIqQ"[detail::log2(sizeof(T))*2 + (std::is_unsigned<T>::value ? 1 : 0)], '\0' };
static constexpr const char* format() { return value; }
};
template <typename T> constexpr const char format_descriptor<
T, typename std::enable_if<std::is_integral<T>::value>::type>::value[2];
PYBIND11_DECL_FMT(float, "f"); PYBIND11_DECL_FMT(double, "d"); PYBIND11_DECL_FMT(bool, "?");
PYBIND11_DECL_FMT(float, "f");
PYBIND11_DECL_FMT(double, "d");
PYBIND11_DECL_FMT(bool, "?");
NAMESPACE_END(pybind11)
include/pybind11/descr.h
......@@ -181,7 +181,7 @@ PYBIND11_NOINLINE inline descr concat(descr &&d) { return d; }
template <typename... Args> PYBIND11_NOINLINE descr concat(descr &&d, Args&&... args) { return std::move(d) + _(", ") + concat(std::forward<Args>(args)...); }
PYBIND11_NOINLINE inline descr type_descr(descr&& d) { return _("{") + std::move(d) + _("}"); }
#define PYBIND11_DESCR descr
#define PYBIND11_DESCR ::pybind11::detail::descr
#endif
NAMESPACE_END(detail)
......
include/pybind11/eigen.h
......@@ -133,7 +133,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_dense<Type>::value &&
/* Size of one scalar */
sizeof(Scalar),
/* Python struct-style format descriptor */
format_descriptor<Scalar>::value,
format_descriptor<Scalar>::format(),
/* Number of dimensions */
1,
/* Buffer dimensions */
......@@ -148,7 +148,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_dense<Type>::value &&
/* Size of one scalar */
sizeof(Scalar),
/* Python struct-style format descriptor */
format_descriptor<Scalar>::value,
format_descriptor<Scalar>::format(),
/* Number of dimensions */
isVector ? 1 : 2,
/* Buffer dimensions */
......@@ -233,7 +233,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
try {
obj = matrix_type(obj);
} catch (const error_already_set &) {
PyErr_Clear();
PyErr_Clear();
return false;
}
}
......@@ -276,7 +276,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
// Size of one scalar
sizeof(Scalar),
// Python struct-style format descriptor
format_descriptor<Scalar>::value,
format_descriptor<Scalar>::format(),
// Number of dimensions
1,
// Buffer dimensions
......@@ -291,7 +291,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
// Size of one scalar
sizeof(StorageIndex),
// Python struct-style format descriptor
format_descriptor<StorageIndex>::value,
format_descriptor<StorageIndex>::format(),
// Number of dimensions
1,
// Buffer dimensions
......@@ -306,7 +306,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
// Size of one scalar
sizeof(StorageIndex),
// Python struct-style format descriptor
format_descriptor<StorageIndex>::value,
format_descriptor<StorageIndex>::format(),
// Number of dimensions
1,
// Buffer dimensions
......
include/pybind11/numpy.h
......@@ -13,6 +13,11 @@
#include "complex.h"
#include <numeric>
#include <algorithm>
#include <array>
#include <cstdlib>
#include <cstring>
#include <sstream>
#include <initializer_list>
#if defined(_MSC_VER)
#pragma warning(push)
......@@ -20,123 +25,275 @@
#endif
NAMESPACE_BEGIN(pybind11)
namespace detail { template <typename type, typename SFINAE = void> struct npy_format_descriptor { }; }
namespace detail {
template <typename type, typename SFINAE = void> struct npy_format_descriptor { };
template <typename type> struct is_pod_struct;
struct npy_api {
enum constants {
NPY_C_CONTIGUOUS_ = 0x0001,
NPY_F_CONTIGUOUS_ = 0x0002,
NPY_ARRAY_FORCECAST_ = 0x0010,
NPY_ENSURE_ARRAY_ = 0x0040,
NPY_BOOL_ = 0,
NPY_BYTE_, NPY_UBYTE_,
NPY_SHORT_, NPY_USHORT_,
NPY_INT_, NPY_UINT_,
NPY_LONG_, NPY_ULONG_,
NPY_LONGLONG_, NPY_ULONGLONG_,
NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_,
NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_,
NPY_OBJECT_ = 17,
NPY_STRING_, NPY_UNICODE_, NPY_VOID_
};
class array : public buffer {
public:
struct API {
enum Entries {
API_PyArray_Type = 2,
API_PyArray_DescrFromType = 45,
API_PyArray_FromAny = 69,
API_PyArray_NewCopy = 85,
API_PyArray_NewFromDescr = 94,
NPY_C_CONTIGUOUS_ = 0x0001,
NPY_F_CONTIGUOUS_ = 0x0002,
NPY_ARRAY_FORCECAST_ = 0x0010,
NPY_ENSURE_ARRAY_ = 0x0040,
NPY_BOOL_ = 0,
NPY_BYTE_, NPY_UBYTE_,
NPY_SHORT_, NPY_USHORT_,
NPY_INT_, NPY_UINT_,
NPY_LONG_, NPY_ULONG_,
NPY_LONGLONG_, NPY_ULONGLONG_,
NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_,
NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_
};
static API lookup() {
module m = module::import("numpy.core.multiarray");
object c = (object) m.attr("_ARRAY_API");
static npy_api& get() {
static npy_api api = lookup();
return api;
}
bool PyArray_Check_(PyObject *obj) const {
return (bool) PyObject_TypeCheck(obj, PyArray_Type_);
}
bool PyArrayDescr_Check_(PyObject *obj) const {
return (bool) PyObject_TypeCheck(obj, PyArrayDescr_Type_);
}
PyObject *(*PyArray_DescrFromType_)(int);
PyObject *(*PyArray_NewFromDescr_)
(PyTypeObject *, PyObject *, int, Py_intptr_t *,
Py_intptr_t *, void *, int, PyObject *);
PyObject *(*PyArray_DescrNewFromType_)(int);
PyObject *(*PyArray_NewCopy_)(PyObject *, int);
PyTypeObject *PyArray_Type_;
PyTypeObject *PyArrayDescr_Type_;
PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *);
int (*PyArray_DescrConverter_) (PyObject *, PyObject **);
bool (*PyArray_EquivTypes_) (PyObject *, PyObject *);
int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *,
Py_ssize_t *, PyObject **, PyObject *);
private:
enum functions {
API_PyArray_Type = 2,
API_PyArrayDescr_Type = 3,
API_PyArray_DescrFromType = 45,
API_PyArray_FromAny = 69,
API_PyArray_NewCopy = 85,
API_PyArray_NewFromDescr = 94,
API_PyArray_DescrNewFromType = 9,
API_PyArray_DescrConverter = 174,
API_PyArray_EquivTypes = 182,
API_PyArray_GetArrayParamsFromObject = 278,
};
static npy_api lookup() {
module m = module::import("numpy.core.multiarray");
object c = (object) m.attr("_ARRAY_API");
#if PY_MAJOR_VERSION >= 3
void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr);
void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr);
#else
void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr);
void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr);
#endif
API api;
api.PyArray_Type_ = (decltype(api.PyArray_Type_)) api_ptr[API_PyArray_Type];
api.PyArray_DescrFromType_ = (decltype(api.PyArray_DescrFromType_)) api_ptr[API_PyArray_DescrFromType];
api.PyArray_FromAny_ = (decltype(api.PyArray_FromAny_)) api_ptr[API_PyArray_FromAny];
api.PyArray_NewCopy_ = (decltype(api.PyArray_NewCopy_)) api_ptr[API_PyArray_NewCopy];
api.PyArray_NewFromDescr_ = (decltype(api.PyArray_NewFromDescr_)) api_ptr[API_PyArray_NewFromDescr];
return api;
npy_api api;
#define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func];
DECL_NPY_API(PyArray_Type);
DECL_NPY_API(PyArrayDescr_Type);
DECL_NPY_API(PyArray_DescrFromType);
DECL_NPY_API(PyArray_FromAny);
DECL_NPY_API(PyArray_NewCopy);
DECL_NPY_API(PyArray_NewFromDescr);
DECL_NPY_API(PyArray_DescrNewFromType);
DECL_NPY_API(PyArray_DescrConverter);
DECL_NPY_API(PyArray_EquivTypes);
DECL_NPY_API(PyArray_GetArrayParamsFromObject);
#undef DECL_NPY_API
return api;
}
};
}
class dtype : public object {
public:
PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_);
dtype(const buffer_info &info) {
dtype descr(_dtype_from_pep3118()(PYBIND11_STR_TYPE(info.format)));
m_ptr = descr.strip_padding().release().ptr();
}
dtype(std::string format) {
m_ptr = from_args(pybind11::str(format)).release().ptr();
}
dtype(const char *format) : dtype(std::string(format)) { }
dtype(list names, list formats, list offsets, size_t itemsize) {
dict args;
args["names"] = names;
args["formats"] = formats;
args["offsets"] = offsets;
args["itemsize"] = int_(itemsize);
m_ptr = from_args(args).release().ptr();
}
static dtype from_args(object args) {
// This is essentially the same as calling np.dtype() constructor in Python
PyObject *ptr = nullptr;
if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr)
pybind11_fail("NumPy: failed to create structured dtype");
return object(ptr, false);
}
template <typename T> static dtype of() {
return detail::npy_format_descriptor<T>::dtype();
}
size_t itemsize() const {
return (size_t) attr("itemsize").cast<int_>();
}
bool has_fields() const {
return attr("fields").cast<object>().ptr() != Py_None;
}
std::string kind() const {
return (std::string) attr("kind").cast<pybind11::str>();
}
private:
static object _dtype_from_pep3118() {
static PyObject *obj = module::import("numpy.core._internal")
.attr("_dtype_from_pep3118").cast<object>().release().ptr();
return object(obj, true);
}
dtype strip_padding() {
// Recursively strip all void fields with empty names that are generated for
// padding fields (as of NumPy v1.11).
auto fields = attr("fields").cast<object>();
if (fields.ptr() == Py_None)
return *this;
struct field_descr { PYBIND11_STR_TYPE name; object format; int_ offset; };
std::vector<field_descr> field_descriptors;
auto items = fields.attr("items").cast<object>();
for (auto field : items()) {
auto spec = object(field, true).cast<tuple>();
auto name = spec[0].cast<pybind11::str>();
auto format = spec[1].cast<tuple>()[0].cast<dtype>();
auto offset = spec[1].cast<tuple>()[1].cast<int_>();
if (!len(name) && format.kind() == "V")
continue;
field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(), offset});
}
bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); }
std::sort(field_descriptors.begin(), field_descriptors.end(),
[](const field_descr& a, const field_descr& b) {
return (int) a.offset < (int) b.offset;
});
PyObject *(*PyArray_DescrFromType_)(int);
PyObject *(*PyArray_NewFromDescr_)
(PyTypeObject *, PyObject *, int, Py_intptr_t *,
Py_intptr_t *, void *, int, PyObject *);
PyObject *(*PyArray_NewCopy_)(PyObject *, int);
PyTypeObject *PyArray_Type_;
PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *);
};
list names, formats, offsets;
for (auto& descr : field_descriptors) {
names.append(descr.name);
formats.append(descr.format);
offsets.append(descr.offset);
}
return dtype(names, formats, offsets, itemsize());
}
};
PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check_)
class array : public buffer {
public:
PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_)
enum {
c_style = API::NPY_C_CONTIGUOUS_,
f_style = API::NPY_F_CONTIGUOUS_,
forcecast = API::NPY_ARRAY_FORCECAST_
c_style = detail::npy_api::NPY_C_CONTIGUOUS_,
f_style = detail::npy_api::NPY_F_CONTIGUOUS_,
forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_
};
template <typename Type> array(size_t size, const Type *ptr) {
API& api = lookup_api();
PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor<Type>::value);
if (descr == nullptr)
pybind11_fail("NumPy: unsupported buffer format!");
Py_intptr_t shape = (Py_intptr_t) size;
object tmp = object(api.PyArray_NewFromDescr_(
api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false);
if (ptr && tmp)
tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false);
array(const pybind11::dtype& dt, const std::vector<size_t>& shape,
const std::vector<size_t>& strides, void *ptr = nullptr) {
auto& api = detail::npy_api::get();
auto ndim = shape.size();
if (shape.size() != strides.size())
pybind11_fail("NumPy: shape ndim doesn't match strides ndim");
auto descr = dt;
object tmp(api.PyArray_NewFromDescr_(
api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(),
(Py_intptr_t *) strides.data(), ptr, 0, nullptr), false);
if (!tmp)
pybind11_fail("NumPy: unable to create array!");
if (ptr)
tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false);
m_ptr = tmp.release().ptr();
}
array(const buffer_info &info) {
API& api = lookup_api();
if ((info.format.size() < 1) || (info.format.size() > 2))
pybind11_fail("Unsupported buffer format!");
int fmt = (int) info.format[0];
if (info.format == "Zd") fmt = API::NPY_CDOUBLE_;
else if (info.format == "Zf") fmt = API::NPY_CFLOAT_;
array(const pybind11::dtype& dt, const std::vector<size_t>& shape, void *ptr = nullptr)
: array(dt, shape, default_strides(shape, dt.itemsize()), ptr) { }
PyObject *descr = api.PyArray_DescrFromType_(fmt);
if (descr == nullptr)
pybind11_fail("NumPy: unsupported buffer format '" + info.format + "'!");
object tmp(api.PyArray_NewFromDescr_(
api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0],
(Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false);
if (info.ptr && tmp)
tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false);
if (!tmp)
pybind11_fail("NumPy: unable to create array!");
m_ptr = tmp.release().ptr();
array(const pybind11::dtype& dt, size_t size, void *ptr = nullptr)
: array(dt, std::vector<size_t> { size }, ptr) { }
template<typename T> array(const std::vector<size_t>& shape,
const std::vector<size_t>& strides, T* ptr)
: array(pybind11::dtype::of<T>(), shape, strides, (void *) ptr) { }
template<typename T> array(const std::vector<size_t>& shape, T* ptr)
: array(shape, default_strides(shape, sizeof(T)), ptr) { }
template<typename T> array(size_t size, T* ptr)
: array(std::vector<size_t> { size }, ptr) { }
array(const buffer_info &info)
: array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { }
pybind11::dtype dtype() {
return attr("dtype").cast<pybind11::dtype>();
}
protected:
static API &lookup_api() {
static API api = API::lookup();
return api;
template <typename T, typename SFINAE> friend struct detail::npy_format_descriptor;
static std::vector<size_t> default_strides(const std::vector<size_t>& shape, size_t itemsize) {
auto ndim = shape.size();
std::vector<size_t> strides(ndim);
if (ndim) {
std::fill(strides.begin(), strides.end(), itemsize);
for (size_t i = 0; i < ndim - 1; i++)
for (size_t j = 0; j < ndim - 1 - i; j++)
strides[j] *= shape[ndim - 1 - i];
}
return strides;
}
};
template <typename T, int ExtraFlags = array::forcecast> class array_t : public array {
public:
PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr));
array_t() : array() { }
array_t(const buffer_info& info) : array(info) {}
array_t(const buffer_info& info) : array(info) { }
array_t(const std::vector<size_t>& shape, const std::vector<size_t>& strides, T* ptr = nullptr)
: array(shape, strides, ptr) { }
array_t(const std::vector<size_t>& shape, T* ptr = nullptr)
: array(shape, ptr) { }
array_t(size_t size, T* ptr = nullptr)
: array(size, ptr) { }
static bool is_non_null(PyObject *ptr) { return ptr != nullptr; }
static PyObject *ensure(PyObject *ptr) {
if (ptr == nullptr)
return nullptr;
API &api = lookup_api();
PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor<T>::value);
PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr);
auto& api = detail::npy_api::get();
PyObject *result = api.PyArray_FromAny_(ptr, pybind11::dtype::of<T>().release().ptr(), 0, 0,
detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr);
if (!result)
PyErr_Clear();
Py_DECREF(ptr);
......@@ -144,15 +301,47 @@ public:
}
};
template <typename T>
struct format_descriptor<T, typename std::enable_if<detail::is_pod_struct<T>::value>::type> {
static const char *format() { return detail::npy_format_descriptor<T>::format(); }
};
template <size_t N> struct format_descriptor<char[N]> {
static const char *format() { PYBIND11_DESCR s = detail::_<N>() + detail::_("s"); return s.text(); }
};
template <size_t N> struct format_descriptor<std::array<char, N>> {
static const char *format() { PYBIND11_DESCR s = detail::_<N>() + detail::_("s"); return s.text(); }
};
NAMESPACE_BEGIN(detail)
template <typename T> struct is_std_array : std::false_type { };
template <typename T, size_t N> struct is_std_array<std::array<T, N>> : std::true_type { };
template <typename T>
struct is_pod_struct {
enum { value = std::is_pod<T>::value && // offsetof only works correctly for POD types
!std::is_array<T>::value &&
!is_std_array<T>::value &&
!std::is_integral<T>::value &&
!std::is_same<T, float>::value &&
!std::is_same<T, double>::value &&
!std::is_same<T, bool>::value &&
!std::is_same<T, std::complex<float>>::value &&
!std::is_same<T, std::complex<double>>::value };
};
template <typename T> struct npy_format_descriptor<T, typename std::enable_if<std::is_integral<T>::value>::type> {
private:
constexpr static const int values[8] = {
array::API::NPY_BYTE_, array::API::NPY_UBYTE_, array::API::NPY_SHORT_, array::API::NPY_USHORT_,
array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ };
npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_,
npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_ };
public:
enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned<T>::value ? 1 : 0)] };
static pybind11::dtype dtype() {
if (auto ptr = npy_api::get().PyArray_DescrFromType_(value))
return object(ptr, true);
pybind11_fail("Unsupported buffer format!");
}
template <typename T2 = T, typename std::enable_if<std::is_signed<T2>::value, int>::type = 0>
static PYBIND11_DESCR name() { return _("int") + _<sizeof(T)*8>(); }
template <typename T2 = T, typename std::enable_if<!std::is_signed<T2>::value, int>::type = 0>
......@@ -162,12 +351,149 @@ template <typename T> constexpr const int npy_format_descriptor<
T, typename std::enable_if<std::is_integral<T>::value>::type>::values[8];
#define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor<Type> { \
enum { value = array::API::NumPyName }; \
enum { value = npy_api::NumPyName }; \
static pybind11::dtype dtype() { \
if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \
return object(ptr, true); \
pybind11_fail("Unsupported buffer format!"); \
} \
static PYBIND11_DESCR name() { return _(Name); } }
DECL_FMT(float, NPY_FLOAT_, "float32"); DECL_FMT(double, NPY_DOUBLE_, "float64"); DECL_FMT(bool, NPY_BOOL_, "bool");
DECL_FMT(std::complex<float>, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex<double>, NPY_CDOUBLE_, "complex128");
DECL_FMT(float, NPY_FLOAT_, "float32");
DECL_FMT(double, NPY_DOUBLE_, "float64");
DECL_FMT(bool, NPY_BOOL_, "bool");
DECL_FMT(std::complex<float>, NPY_CFLOAT_, "complex64");
DECL_FMT(std::complex<double>, NPY_CDOUBLE_, "complex128");
#undef DECL_FMT
#define DECL_CHAR_FMT \
static PYBIND11_DESCR name() { return _("S") + _<N>(); } \
static pybind11::dtype dtype() { \
PYBIND11_DESCR fmt = _("S") + _<N>(); \
return pybind11::dtype(fmt.text()); \
} \
static const char *format() { PYBIND11_DESCR s = _<N>() + _("s"); return s.text(); }
template <size_t N> struct npy_format_descriptor<char[N]> { DECL_CHAR_FMT };
template <size_t N> struct npy_format_descriptor<std::array<char, N>> { DECL_CHAR_FMT };
#undef DECL_CHAR_FMT
struct field_descriptor {
const char *name;
size_t offset;
size_t size;
const char *format;
dtype descr;
};
template <typename T>
struct npy_format_descriptor<T, typename std::enable_if<is_pod_struct<T>::value>::type> {
static PYBIND11_DESCR name() { return _("struct"); }
static pybind11::dtype dtype() {
if (!dtype_())
pybind11_fail("NumPy: unsupported buffer format!");
return object(dtype_(), true);
}
static const char* format() {
if (!dtype_())
pybind11_fail("NumPy: unsupported buffer format!");
return format_().c_str();
}
static void register_dtype(std::initializer_list<field_descriptor> fields) {
list names, formats, offsets;
for (auto field : fields) {
if (!field.descr)
pybind11_fail("NumPy: unsupported field dtype");
names.append(PYBIND11_STR_TYPE(field.name));
formats.append(field.descr);
offsets.append(int_(field.offset));
}
dtype_() = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr();
// There is an existing bug in NumPy (as of v1.11): trailing bytes are
// not encoded explicitly into the format string. This will supposedly
// get fixed in v1.12; for further details, see these:
// - https://github.com/numpy/numpy/issues/7797
// - https://github.com/numpy/numpy/pull/7798
// Because of this, we won't use numpy's logic to generate buffer format
// strings and will just do it ourselves.
std::vector<field_descriptor> ordered_fields(fields);
std::sort(ordered_fields.begin(), ordered_fields.end(),
[](const field_descriptor& a, const field_descriptor &b) {
return a.offset < b.offset;
});
size_t offset = 0;
std::ostringstream oss;
oss << "T{";
for (auto& field : ordered_fields) {
if (field.offset > offset)
oss << (field.offset - offset) << 'x';
// note that '=' is required to cover the case of unaligned fields
oss << '=' << field.format << ':' << field.name << ':';
offset = field.offset + field.size;
}
if (sizeof(T) > offset)
oss << (sizeof(T) - offset) << 'x';
oss << '}';
format_() = oss.str();
// Sanity check: verify that NumPy properly parses our buffer format string
auto& api = npy_api::get();
auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) }));
if (!api.PyArray_EquivTypes_(dtype_(), arr.dtype().ptr()))
pybind11_fail("NumPy: invalid buffer descriptor!");
}
private:
static inline PyObject*& dtype_() { static PyObject *ptr = nullptr; return ptr; }
static inline std::string& format_() { static std::string s; return s; }
};
// Extract name, offset and format descriptor for a struct field
#define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \
::pybind11::detail::field_descriptor { \
#Field, offsetof(Type, Field), sizeof(decltype(static_cast<Type*>(0)->Field)), \
::pybind11::format_descriptor<decltype(static_cast<Type*>(0)->Field)>::format(), \
::pybind11::detail::npy_format_descriptor<decltype(static_cast<Type*>(0)->Field)>::dtype() \
}
// The main idea of this macro is borrowed from https://github.com/swansontec/map-macro
// (C) William Swanson, Paul Fultz
#define PYBIND11_EVAL0(...) __VA_ARGS__
#define PYBIND11_EVAL1(...) PYBIND11_EVAL0 (PYBIND11_EVAL0 (PYBIND11_EVAL0 (__VA_ARGS__)))
#define PYBIND11_EVAL2(...) PYBIND11_EVAL1 (PYBIND11_EVAL1 (PYBIND11_EVAL1 (__VA_ARGS__)))
#define PYBIND11_EVAL3(...) PYBIND11_EVAL2 (PYBIND11_EVAL2 (PYBIND11_EVAL2 (__VA_ARGS__)))
#define PYBIND11_EVAL4(...) PYBIND11_EVAL3 (PYBIND11_EVAL3 (PYBIND11_EVAL3 (__VA_ARGS__)))
#define PYBIND11_EVAL(...) PYBIND11_EVAL4 (PYBIND11_EVAL4 (PYBIND11_EVAL4 (__VA_ARGS__)))
#define PYBIND11_MAP_END(...)
#define PYBIND11_MAP_OUT
#define PYBIND11_MAP_COMMA ,
#define PYBIND11_MAP_GET_END() 0, PYBIND11_MAP_END
#define PYBIND11_MAP_NEXT0(test, next, ...) next PYBIND11_MAP_OUT
#define PYBIND11_MAP_NEXT1(test, next) PYBIND11_MAP_NEXT0 (test, next, 0)
#define PYBIND11_MAP_NEXT(test, next) PYBIND11_MAP_NEXT1 (PYBIND11_MAP_GET_END test, next)
#ifdef _MSC_VER // MSVC is not as eager to expand macros, hence this workaround
#define PYBIND11_MAP_LIST_NEXT1(test, next) \
PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0))
#else
#define PYBIND11_MAP_LIST_NEXT1(test, next) \
PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)
#endif
#define PYBIND11_MAP_LIST_NEXT(test, next) \
PYBIND11_MAP_LIST_NEXT1 (PYBIND11_MAP_GET_END test, next)
#define PYBIND11_MAP_LIST0(f, t, x, peek, ...) \
f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST1) (f, t, peek, __VA_ARGS__)
#define PYBIND11_MAP_LIST1(f, t, x, peek, ...) \
f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST0) (f, t, peek, __VA_ARGS__)
// PYBIND11_MAP_LIST(f, t, a1, a2, ...) expands to f(t, a1), f(t, a2), ...
#define PYBIND11_MAP_LIST(f, t, ...) \
PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0))
#define PYBIND11_NUMPY_DTYPE(Type, ...) \
::pybind11::detail::npy_format_descriptor<Type>::register_dtype \
({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)})
template <class T>
using array_iterator = typename std::add_pointer<T>::type;
......@@ -348,7 +674,7 @@ struct vectorize_helper {
return cast(f(*((Args *) buffers[Index].ptr)...));
array result(buffer_info(nullptr, sizeof(Return),
format_descriptor<Return>::value,
format_descriptor<Return>::format(),
ndim, shape, strides));
buffer_info buf = result.request();
......
include/pybind11/pytypes.h
......@@ -339,12 +339,14 @@ inline iterator handle::begin() const { return iterator(PyObject_GetIter(ptr()),
inline iterator handle::end() const { return iterator(nullptr, false); }
inline detail::args_proxy handle::operator*() const { return detail::args_proxy(*this); }
class bytes;
class str : public object {
public:
PYBIND11_OBJECT_DEFAULT(str, object, detail::PyUnicode_Check_Permissive)
str(const std::string &s)
: object(PyUnicode_FromStringAndSize(s.c_str(), (ssize_t) s.length()), false) {
str(const char *c, size_t n)
: object(PyUnicode_FromStringAndSize(c, (ssize_t) n), false) {
if (!m_ptr) pybind11_fail("Could not allocate string object!");
}
......@@ -353,6 +355,10 @@ public:
if (!m_ptr) pybind11_fail("Could not allocate string object!");
}
str(const std::string &s) : str(s.data(), s.size()) { }
str(const bytes &b);
operator std::string() const {
object temp = *this;
if (PyUnicode_Check(m_ptr)) {
......@@ -362,8 +368,7 @@ public:
}
char *buffer;
ssize_t length;
int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length);
if (err == -1)
if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length))
pybind11_fail("Unable to extract string contents! (invalid type)");
return std::string(buffer, (size_t) length);
}
......@@ -382,21 +387,57 @@ class bytes : public object {
public:
PYBIND11_OBJECT_DEFAULT(bytes, object, PYBIND11_BYTES_CHECK)
bytes(const std::string &s)
: object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(s.data(), (ssize_t) s.size()), false) {
bytes(const char *c)
: object(PYBIND11_BYTES_FROM_STRING(c), false) {
if (!m_ptr) pybind11_fail("Could not allocate bytes object!");
}
bytes(const char *c, size_t n)
: object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(c, (ssize_t) n), false) {
if (!m_ptr) pybind11_fail("Could not allocate bytes object!");
}
bytes(const std::string &s) : bytes(s.data(), s.size()) { }
bytes(const pybind11::str &s);
operator std::string() const {
char *buffer;
ssize_t length;
int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length);
if (err == -1)
if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length))
pybind11_fail("Unable to extract bytes contents!");
return std::string(buffer, (size_t) length);
}
};
inline bytes::bytes(const pybind11::str &s) {
object temp = s;
if (PyUnicode_Check(s.ptr())) {
temp = object(PyUnicode_AsUTF8String(s.ptr()), false);
if (!temp)
pybind11_fail("Unable to extract string contents! (encoding issue)");
}
char *buffer;
ssize_t length;
if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length))
pybind11_fail("Unable to extract string contents! (invalid type)");
auto obj = object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(buffer, length), false);
if (!obj)
pybind11_fail("Could not allocate bytes object!");
m_ptr = obj.release().ptr();
}
inline str::str(const bytes& b) {
char *buffer;
ssize_t length;
if (PYBIND11_BYTES_AS_STRING_AND_SIZE(b.ptr(), &buffer, &length))
pybind11_fail("Unable to extract bytes contents!");
auto obj = object(PyUnicode_FromStringAndSize(buffer, (ssize_t) length), false);
if (!obj)
pybind11_fail("Could not allocate string object!");
m_ptr = obj.release().ptr();
}
class none : public object {
public:
PYBIND11_OBJECT(none, object, detail::PyNone_Check)
......@@ -570,6 +611,38 @@ public:
}
};
class memoryview : public object {
public:
memoryview(const buffer_info& info) {
static Py_buffer buf { };
// Py_buffer uses signed sizes, strides and shape!..
static std::vector<Py_ssize_t> py_strides { };
static std::vector<Py_ssize_t> py_shape { };
buf.buf = info.ptr;
buf.itemsize = (Py_ssize_t) info.itemsize;
buf.format = const_cast<char *>(info.format.c_str());
buf.ndim = (int) info.ndim;
buf.len = (Py_ssize_t) info.size;
py_strides.clear();
py_shape.clear();
for (size_t i = 0; i < info.ndim; ++i) {
py_strides.push_back((Py_ssize_t) info.strides[i]);
py_shape.push_back((Py_ssize_t) info.shape[i]);
}
buf.strides = py_strides.data();
buf.shape = py_shape.data();
buf.suboffsets = nullptr;
buf.readonly = false;
buf.internal = nullptr;
m_ptr = PyMemoryView_FromBuffer(&buf);
if (!m_ptr)
pybind11_fail("Unable to create memoryview from buffer descriptor");
}
PYBIND11_OBJECT_DEFAULT(memoryview, object, PyMemoryView_Check)
};
inline size_t len(handle h) {
ssize_t result = PyObject_Length(h.ptr());
if (result < 0)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment