1 /*
2 tests/test_numpy_array.cpp -- test core array functionality
3
4 Copyright (c) 2016 Ivan Smirnov <i.s.smirnov@gmail.com>
5
6 All rights reserved. Use of this source code is governed by a
7 BSD-style license that can be found in the LICENSE file.
8 */
9
10 #include "pybind11_tests.h"
11
12 #include <pybind11/numpy.h>
13 #include <pybind11/stl.h>
14
15 #include <cstdint>
16
17 // Size / dtype checks.
18 struct DtypeCheck {
19 py::dtype numpy{};
20 py::dtype pybind11{};
21 };
22
23 template <typename T>
get_dtype_check(const char * name)24 DtypeCheck get_dtype_check(const char* name) {
25 py::module_ np = py::module_::import("numpy");
26 DtypeCheck check{};
27 check.numpy = np.attr("dtype")(np.attr(name));
28 check.pybind11 = py::dtype::of<T>();
29 return check;
30 }
31
get_concrete_dtype_checks()32 std::vector<DtypeCheck> get_concrete_dtype_checks() {
33 return {
34 // Normalization
35 get_dtype_check<std::int8_t>("int8"),
36 get_dtype_check<std::uint8_t>("uint8"),
37 get_dtype_check<std::int16_t>("int16"),
38 get_dtype_check<std::uint16_t>("uint16"),
39 get_dtype_check<std::int32_t>("int32"),
40 get_dtype_check<std::uint32_t>("uint32"),
41 get_dtype_check<std::int64_t>("int64"),
42 get_dtype_check<std::uint64_t>("uint64")
43 };
44 }
45
46 struct DtypeSizeCheck {
47 std::string name{};
48 int size_cpp{};
49 int size_numpy{};
50 // For debugging.
51 py::dtype dtype{};
52 };
53
54 template <typename T>
get_dtype_size_check()55 DtypeSizeCheck get_dtype_size_check() {
56 DtypeSizeCheck check{};
57 check.name = py::type_id<T>();
58 check.size_cpp = sizeof(T);
59 check.dtype = py::dtype::of<T>();
60 check.size_numpy = check.dtype.attr("itemsize").template cast<int>();
61 return check;
62 }
63
get_platform_dtype_size_checks()64 std::vector<DtypeSizeCheck> get_platform_dtype_size_checks() {
65 return {
66 get_dtype_size_check<short>(),
67 get_dtype_size_check<unsigned short>(),
68 get_dtype_size_check<int>(),
69 get_dtype_size_check<unsigned int>(),
70 get_dtype_size_check<long>(),
71 get_dtype_size_check<unsigned long>(),
72 get_dtype_size_check<long long>(),
73 get_dtype_size_check<unsigned long long>(),
74 };
75 }
76
77 // Arrays.
78 using arr = py::array;
79 using arr_t = py::array_t<uint16_t, 0>;
80 static_assert(std::is_same<arr_t::value_type, uint16_t>::value, "");
81
data(const arr & a,Ix...index)82 template<typename... Ix> arr data(const arr& a, Ix... index) {
83 return arr(a.nbytes() - a.offset_at(index...), (const uint8_t *) a.data(index...));
84 }
85
data_t(const arr_t & a,Ix...index)86 template<typename... Ix> arr data_t(const arr_t& a, Ix... index) {
87 return arr(a.size() - a.index_at(index...), a.data(index...));
88 }
89
mutate_data(arr & a,Ix...index)90 template<typename... Ix> arr& mutate_data(arr& a, Ix... index) {
91 auto ptr = (uint8_t *) a.mutable_data(index...);
92 for (py::ssize_t i = 0; i < a.nbytes() - a.offset_at(index...); i++)
93 ptr[i] = (uint8_t) (ptr[i] * 2);
94 return a;
95 }
96
mutate_data_t(arr_t & a,Ix...index)97 template<typename... Ix> arr_t& mutate_data_t(arr_t& a, Ix... index) {
98 auto ptr = a.mutable_data(index...);
99 for (py::ssize_t i = 0; i < a.size() - a.index_at(index...); i++)
100 ptr[i]++;
101 return a;
102 }
103
index_at(const arr & a,Ix...idx)104 template<typename... Ix> py::ssize_t index_at(const arr& a, Ix... idx) { return a.index_at(idx...); }
index_at_t(const arr_t & a,Ix...idx)105 template<typename... Ix> py::ssize_t index_at_t(const arr_t& a, Ix... idx) { return a.index_at(idx...); }
offset_at(const arr & a,Ix...idx)106 template<typename... Ix> py::ssize_t offset_at(const arr& a, Ix... idx) { return a.offset_at(idx...); }
offset_at_t(const arr_t & a,Ix...idx)107 template<typename... Ix> py::ssize_t offset_at_t(const arr_t& a, Ix... idx) { return a.offset_at(idx...); }
at_t(const arr_t & a,Ix...idx)108 template<typename... Ix> py::ssize_t at_t(const arr_t& a, Ix... idx) { return a.at(idx...); }
mutate_at_t(arr_t & a,Ix...idx)109 template<typename... Ix> arr_t& mutate_at_t(arr_t& a, Ix... idx) { a.mutable_at(idx...)++; return a; }
110
111 #define def_index_fn(name, type) \
112 sm.def(#name, [](type a) { return name(a); }); \
113 sm.def(#name, [](type a, int i) { return name(a, i); }); \
114 sm.def(#name, [](type a, int i, int j) { return name(a, i, j); }); \
115 sm.def(#name, [](type a, int i, int j, int k) { return name(a, i, j, k); });
116
auxiliaries(T && r,T2 && r2)117 template <typename T, typename T2> py::handle auxiliaries(T &&r, T2 &&r2) {
118 if (r.ndim() != 2) throw std::domain_error("error: ndim != 2");
119 py::list l;
120 l.append(*r.data(0, 0));
121 l.append(*r2.mutable_data(0, 0));
122 l.append(r.data(0, 1) == r2.mutable_data(0, 1));
123 l.append(r.ndim());
124 l.append(r.itemsize());
125 l.append(r.shape(0));
126 l.append(r.shape(1));
127 l.append(r.size());
128 l.append(r.nbytes());
129 return l.release();
130 }
131
132 // note: declaration at local scope would create a dangling reference!
133 static int data_i = 42;
134
TEST_SUBMODULE(numpy_array,sm)135 TEST_SUBMODULE(numpy_array, sm) {
136 try { py::module_::import("numpy"); }
137 catch (...) { return; }
138
139 // test_dtypes
140 py::class_<DtypeCheck>(sm, "DtypeCheck")
141 .def_readonly("numpy", &DtypeCheck::numpy)
142 .def_readonly("pybind11", &DtypeCheck::pybind11)
143 .def("__repr__", [](const DtypeCheck& self) {
144 return py::str("<DtypeCheck numpy={} pybind11={}>").format(
145 self.numpy, self.pybind11);
146 });
147 sm.def("get_concrete_dtype_checks", &get_concrete_dtype_checks);
148
149 py::class_<DtypeSizeCheck>(sm, "DtypeSizeCheck")
150 .def_readonly("name", &DtypeSizeCheck::name)
151 .def_readonly("size_cpp", &DtypeSizeCheck::size_cpp)
152 .def_readonly("size_numpy", &DtypeSizeCheck::size_numpy)
153 .def("__repr__", [](const DtypeSizeCheck& self) {
154 return py::str("<DtypeSizeCheck name='{}' size_cpp={} size_numpy={} dtype={}>").format(
155 self.name, self.size_cpp, self.size_numpy, self.dtype);
156 });
157 sm.def("get_platform_dtype_size_checks", &get_platform_dtype_size_checks);
158
159 // test_array_attributes
160 sm.def("ndim", [](const arr& a) { return a.ndim(); });
161 sm.def("shape", [](const arr& a) { return arr(a.ndim(), a.shape()); });
162 sm.def("shape", [](const arr& a, py::ssize_t dim) { return a.shape(dim); });
163 sm.def("strides", [](const arr& a) { return arr(a.ndim(), a.strides()); });
164 sm.def("strides", [](const arr& a, py::ssize_t dim) { return a.strides(dim); });
165 sm.def("writeable", [](const arr& a) { return a.writeable(); });
166 sm.def("size", [](const arr& a) { return a.size(); });
167 sm.def("itemsize", [](const arr& a) { return a.itemsize(); });
168 sm.def("nbytes", [](const arr& a) { return a.nbytes(); });
169 sm.def("owndata", [](const arr& a) { return a.owndata(); });
170
171 // test_index_offset
172 def_index_fn(index_at, const arr&);
173 def_index_fn(index_at_t, const arr_t&);
174 def_index_fn(offset_at, const arr&);
175 def_index_fn(offset_at_t, const arr_t&);
176 // test_data
177 def_index_fn(data, const arr&);
178 def_index_fn(data_t, const arr_t&);
179 // test_mutate_data, test_mutate_readonly
180 def_index_fn(mutate_data, arr&);
181 def_index_fn(mutate_data_t, arr_t&);
182 def_index_fn(at_t, const arr_t&);
183 def_index_fn(mutate_at_t, arr_t&);
184
185 // test_make_c_f_array
186 sm.def("make_f_array", [] { return py::array_t<float>({ 2, 2 }, { 4, 8 }); });
187 sm.def("make_c_array", [] { return py::array_t<float>({ 2, 2 }, { 8, 4 }); });
188
189 // test_empty_shaped_array
190 sm.def("make_empty_shaped_array", [] { return py::array(py::dtype("f"), {}, {}); });
191 // test numpy scalars (empty shape, ndim==0)
192 sm.def("scalar_int", []() { return py::array(py::dtype("i"), {}, {}, &data_i); });
193
194 // test_wrap
195 sm.def("wrap", [](py::array a) {
196 return py::array(
197 a.dtype(),
198 {a.shape(), a.shape() + a.ndim()},
199 {a.strides(), a.strides() + a.ndim()},
200 a.data(),
201 a
202 );
203 });
204
205 // test_numpy_view
206 struct ArrayClass {
207 int data[2] = { 1, 2 };
208 ArrayClass() { py::print("ArrayClass()"); }
209 ~ArrayClass() { py::print("~ArrayClass()"); }
210 };
211 py::class_<ArrayClass>(sm, "ArrayClass")
212 .def(py::init<>())
213 .def("numpy_view", [](py::object &obj) {
214 py::print("ArrayClass::numpy_view()");
215 auto &a = obj.cast<ArrayClass&>();
216 return py::array_t<int>({2}, {4}, a.data, obj);
217 }
218 );
219
220 // test_cast_numpy_int64_to_uint64
221 sm.def("function_taking_uint64", [](uint64_t) { });
222
223 // test_isinstance
224 sm.def("isinstance_untyped", [](py::object yes, py::object no) {
225 return py::isinstance<py::array>(yes) && !py::isinstance<py::array>(no);
226 });
227 sm.def("isinstance_typed", [](py::object o) {
228 return py::isinstance<py::array_t<double>>(o) && !py::isinstance<py::array_t<int>>(o);
229 });
230
231 // test_constructors
232 sm.def("default_constructors", []() {
233 return py::dict(
234 "array"_a=py::array(),
235 "array_t<int32>"_a=py::array_t<std::int32_t>(),
236 "array_t<double>"_a=py::array_t<double>()
237 );
238 });
239 sm.def("converting_constructors", [](py::object o) {
240 return py::dict(
241 "array"_a=py::array(o),
242 "array_t<int32>"_a=py::array_t<std::int32_t>(o),
243 "array_t<double>"_a=py::array_t<double>(o)
244 );
245 });
246
247 // test_overload_resolution
248 sm.def("overloaded", [](py::array_t<double>) { return "double"; });
249 sm.def("overloaded", [](py::array_t<float>) { return "float"; });
250 sm.def("overloaded", [](py::array_t<int>) { return "int"; });
251 sm.def("overloaded", [](py::array_t<unsigned short>) { return "unsigned short"; });
252 sm.def("overloaded", [](py::array_t<long long>) { return "long long"; });
253 sm.def("overloaded", [](py::array_t<std::complex<double>>) { return "double complex"; });
254 sm.def("overloaded", [](py::array_t<std::complex<float>>) { return "float complex"; });
255
256 sm.def("overloaded2", [](py::array_t<std::complex<double>>) { return "double complex"; });
257 sm.def("overloaded2", [](py::array_t<double>) { return "double"; });
258 sm.def("overloaded2", [](py::array_t<std::complex<float>>) { return "float complex"; });
259 sm.def("overloaded2", [](py::array_t<float>) { return "float"; });
260
261 // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works.
262
263 // Only accept the exact types:
264 sm.def("overloaded3", [](py::array_t<int>) { return "int"; }, py::arg{}.noconvert());
265 sm.def("overloaded3", [](py::array_t<double>) { return "double"; }, py::arg{}.noconvert());
266
267 // Make sure we don't do unsafe coercion (e.g. float to int) when not using forcecast, but
268 // rather that float gets converted via the safe (conversion to double) overload:
269 sm.def("overloaded4", [](py::array_t<long long, 0>) { return "long long"; });
270 sm.def("overloaded4", [](py::array_t<double, 0>) { return "double"; });
271
272 // But we do allow conversion to int if forcecast is enabled (but only if no overload matches
273 // without conversion)
274 sm.def("overloaded5", [](py::array_t<unsigned int>) { return "unsigned int"; });
275 sm.def("overloaded5", [](py::array_t<double>) { return "double"; });
276
277 // test_greedy_string_overload
278 // Issue 685: ndarray shouldn't go to std::string overload
279 sm.def("issue685", [](std::string) { return "string"; });
280 sm.def("issue685", [](py::array) { return "array"; });
281 sm.def("issue685", [](py::object) { return "other"; });
282
283 // test_array_unchecked_fixed_dims
284 sm.def("proxy_add2", [](py::array_t<double> a, double v) {
285 auto r = a.mutable_unchecked<2>();
286 for (py::ssize_t i = 0; i < r.shape(0); i++)
287 for (py::ssize_t j = 0; j < r.shape(1); j++)
288 r(i, j) += v;
289 }, py::arg{}.noconvert(), py::arg());
290
291 sm.def("proxy_init3", [](double start) {
292 py::array_t<double, py::array::c_style> a({ 3, 3, 3 });
293 auto r = a.mutable_unchecked<3>();
294 for (py::ssize_t i = 0; i < r.shape(0); i++)
295 for (py::ssize_t j = 0; j < r.shape(1); j++)
296 for (py::ssize_t k = 0; k < r.shape(2); k++)
297 r(i, j, k) = start++;
298 return a;
299 });
300 sm.def("proxy_init3F", [](double start) {
301 py::array_t<double, py::array::f_style> a({ 3, 3, 3 });
302 auto r = a.mutable_unchecked<3>();
303 for (py::ssize_t k = 0; k < r.shape(2); k++)
304 for (py::ssize_t j = 0; j < r.shape(1); j++)
305 for (py::ssize_t i = 0; i < r.shape(0); i++)
306 r(i, j, k) = start++;
307 return a;
308 });
309 sm.def("proxy_squared_L2_norm", [](py::array_t<double> a) {
310 auto r = a.unchecked<1>();
311 double sumsq = 0;
312 for (py::ssize_t i = 0; i < r.shape(0); i++)
313 sumsq += r[i] * r(i); // Either notation works for a 1D array
314 return sumsq;
315 });
316
317 sm.def("proxy_auxiliaries2", [](py::array_t<double> a) {
318 auto r = a.unchecked<2>();
319 auto r2 = a.mutable_unchecked<2>();
320 return auxiliaries(r, r2);
321 });
322
323 sm.def("proxy_auxiliaries1_const_ref", [](py::array_t<double> a) {
324 const auto &r = a.unchecked<1>();
325 const auto &r2 = a.mutable_unchecked<1>();
326 return r(0) == r2(0) && r[0] == r2[0];
327 });
328
329 sm.def("proxy_auxiliaries2_const_ref", [](py::array_t<double> a) {
330 const auto &r = a.unchecked<2>();
331 const auto &r2 = a.mutable_unchecked<2>();
332 return r(0, 0) == r2(0, 0);
333 });
334
335 // test_array_unchecked_dyn_dims
336 // Same as the above, but without a compile-time dimensions specification:
337 sm.def("proxy_add2_dyn", [](py::array_t<double> a, double v) {
338 auto r = a.mutable_unchecked();
339 if (r.ndim() != 2) throw std::domain_error("error: ndim != 2");
340 for (py::ssize_t i = 0; i < r.shape(0); i++)
341 for (py::ssize_t j = 0; j < r.shape(1); j++)
342 r(i, j) += v;
343 }, py::arg{}.noconvert(), py::arg());
344 sm.def("proxy_init3_dyn", [](double start) {
345 py::array_t<double, py::array::c_style> a({ 3, 3, 3 });
346 auto r = a.mutable_unchecked();
347 if (r.ndim() != 3) throw std::domain_error("error: ndim != 3");
348 for (py::ssize_t i = 0; i < r.shape(0); i++)
349 for (py::ssize_t j = 0; j < r.shape(1); j++)
350 for (py::ssize_t k = 0; k < r.shape(2); k++)
351 r(i, j, k) = start++;
352 return a;
353 });
354 sm.def("proxy_auxiliaries2_dyn", [](py::array_t<double> a) {
355 return auxiliaries(a.unchecked(), a.mutable_unchecked());
356 });
357
358 sm.def("array_auxiliaries2", [](py::array_t<double> a) {
359 return auxiliaries(a, a);
360 });
361
362 // test_array_failures
363 // Issue #785: Uninformative "Unknown internal error" exception when constructing array from empty object:
364 sm.def("array_fail_test", []() { return py::array(py::object()); });
365 sm.def("array_t_fail_test", []() { return py::array_t<double>(py::object()); });
366 // Make sure the error from numpy is being passed through:
367 sm.def("array_fail_test_negative_size", []() { int c = 0; return py::array(-1, &c); });
368
369 // test_initializer_list
370 // Issue (unnumbered; reported in #788): regression: initializer lists can be ambiguous
371 sm.def("array_initializer_list1", []() { return py::array_t<float>(1); }); // { 1 } also works, but clang warns about it
372 sm.def("array_initializer_list2", []() { return py::array_t<float>({ 1, 2 }); });
373 sm.def("array_initializer_list3", []() { return py::array_t<float>({ 1, 2, 3 }); });
374 sm.def("array_initializer_list4", []() { return py::array_t<float>({ 1, 2, 3, 4 }); });
375
376 // test_array_resize
377 // reshape array to 2D without changing size
378 sm.def("array_reshape2", [](py::array_t<double> a) {
379 const auto dim_sz = (py::ssize_t)std::sqrt(a.size());
380 if (dim_sz * dim_sz != a.size())
381 throw std::domain_error("array_reshape2: input array total size is not a squared integer");
382 a.resize({dim_sz, dim_sz});
383 });
384
385 // resize to 3D array with each dimension = N
386 sm.def("array_resize3", [](py::array_t<double> a, size_t N, bool refcheck) {
387 a.resize({N, N, N}, refcheck);
388 });
389
390 // test_array_create_and_resize
391 // return 2D array with Nrows = Ncols = N
392 sm.def("create_and_resize", [](size_t N) {
393 py::array_t<double> a;
394 a.resize({N, N});
395 std::fill(a.mutable_data(), a.mutable_data() + a.size(), 42.);
396 return a;
397 });
398
399 sm.def("index_using_ellipsis", [](py::array a) {
400 return a[py::make_tuple(0, py::ellipsis(), 0)];
401 });
402
403 // test_argument_conversions
404 sm.def("accept_double",
405 [](py::array_t<double, 0>) {},
406 py::arg("a"));
407 sm.def("accept_double_forcecast",
408 [](py::array_t<double, py::array::forcecast>) {},
409 py::arg("a"));
410 sm.def("accept_double_c_style",
411 [](py::array_t<double, py::array::c_style>) {},
412 py::arg("a"));
413 sm.def("accept_double_c_style_forcecast",
414 [](py::array_t<double, py::array::forcecast | py::array::c_style>) {},
415 py::arg("a"));
416 sm.def("accept_double_f_style",
417 [](py::array_t<double, py::array::f_style>) {},
418 py::arg("a"));
419 sm.def("accept_double_f_style_forcecast",
420 [](py::array_t<double, py::array::forcecast | py::array::f_style>) {},
421 py::arg("a"));
422 sm.def("accept_double_noconvert",
423 [](py::array_t<double, 0>) {},
424 "a"_a.noconvert());
425 sm.def("accept_double_forcecast_noconvert",
426 [](py::array_t<double, py::array::forcecast>) {},
427 "a"_a.noconvert());
428 sm.def("accept_double_c_style_noconvert",
429 [](py::array_t<double, py::array::c_style>) {},
430 "a"_a.noconvert());
431 sm.def("accept_double_c_style_forcecast_noconvert",
432 [](py::array_t<double, py::array::forcecast | py::array::c_style>) {},
433 "a"_a.noconvert());
434 sm.def("accept_double_f_style_noconvert",
435 [](py::array_t<double, py::array::f_style>) {},
436 "a"_a.noconvert());
437 sm.def("accept_double_f_style_forcecast_noconvert",
438 [](py::array_t<double, py::array::forcecast | py::array::f_style>) {},
439 "a"_a.noconvert());
440 }
441