1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/python/lib/core/py_seq_tensor.h"
17
18 #include "tensorflow/c/eager/tfe_context_internal.h"
19 #include "tensorflow/c/eager/tfe_tensorhandle_internal.h"
20 #include "tensorflow/c/tensor_interface.h"
21 #include "tensorflow/c/tf_tensor_internal.h"
22 #include "tensorflow/core/framework/tensor.h"
23 #include "tensorflow/core/framework/types.h"
24 #include "tensorflow/core/lib/core/errors.h"
25 #include "tensorflow/core/lib/strings/str_util.h"
26 #include "tensorflow/core/platform/errors.h"
27 #include "tensorflow/core/platform/macros.h"
28 #include "tensorflow/core/platform/tstring.h"
29 #include "tensorflow/core/platform/types.h"
30 #include "tensorflow/python/lib/core/ndarray_tensor.h"
31 #include "tensorflow/python/lib/core/ndarray_tensor_bridge.h"
32 #include "tensorflow/python/lib/core/numpy.h"
33 #include "tensorflow/python/lib/core/py_util.h"
34 #include "tensorflow/python/lib/core/safe_ptr.h"
35
36 namespace tensorflow {
37 namespace {
38
PyIsInstance(PyObject * obj,PyTypeObject * t)39 inline bool PyIsInstance(PyObject* obj, PyTypeObject* t) {
40 return PyObject_IsInstance(obj, reinterpret_cast<PyObject*>(t));
41 }
42
PyType(PyObject * obj)43 inline PyObject* PyType(PyObject* obj) {
44 return reinterpret_cast<PyObject*>(obj->ob_type);
45 }
46
IsPyString(PyObject * obj)47 bool IsPyString(PyObject* obj) {
48 return PyBytes_Check(obj) || PyUnicode_Check(obj);
49 }
50
IsPyInt(PyObject * obj)51 bool IsPyInt(PyObject* obj) {
52 #if PY_MAJOR_VERSION >= 3
53 return PyLong_Check(obj) ||
54 PyIsInstance(obj, &PyIntegerArrType_Type); // NumPy integers
55 #else
56 return PyInt_Check(obj) || PyLong_Check(obj) ||
57 PyIsInstance(obj, &PyIntegerArrType_Type); // NumPy integers
58 #endif
59 }
60
IsPyDouble(PyObject * obj)61 bool IsPyDouble(PyObject* obj) {
62 return PyIsInstance(obj, &PyDoubleArrType_Type); // NumPy double type.
63 }
64
IsNumpyHalf(PyObject * obj)65 bool IsNumpyHalf(PyObject* obj) {
66 return PyIsInstance(obj, &PyHalfArrType_Type);
67 }
68
IsPyFloat(PyObject * obj)69 bool IsPyFloat(PyObject* obj) {
70 return PyFloat_Check(obj) ||
71 PyIsInstance(obj, &PyFloatingArrType_Type); // NumPy float types
72 }
73
74 struct ConverterState {
75 // The inferred tensor shape.
76 gtl::InlinedVector<int64, 4> inferred_shape;
77
78 // The inferred tensor data type.
79 DataType inferred_dtype;
80
81 // The following fields are used by ZeroDimArrayToScalar.
82 // We cache the last result of the check for a zero dimensional array: the
83 // function is called many times in a conversion, and most of the time is
84 // to check for the same type. This cache can reduce the conversion time by
85 // about 25%.
86 PyTypeObject* last_zerodim_type;
87 bool last_zerodim_check;
88
ConverterStatetensorflow::__anone25f2a6a0111::ConverterState89 ConverterState() : inferred_dtype(DT_INVALID), last_zerodim_type(nullptr) {}
90 };
91
92 // If the input is a zero dimensional PyArray return it converted to a scalar.
93 // Otherwise return the input and increment its reference count.
94 // Users must Py_DECREF the output of this method.
ZeroDimArrayToScalar(PyObject * obj,ConverterState * state)95 PyObject* ZeroDimArrayToScalar(PyObject* obj, ConverterState* state) {
96 auto type = Py_TYPE(obj);
97 auto pyarray_obj = reinterpret_cast<PyArrayObject*>(obj);
98 if (type != state->last_zerodim_type) {
99 state->last_zerodim_type = type;
100 state->last_zerodim_check =
101 PyObject_TypeCheck(obj, &PyArray_Type) &&
102 !PyObject_TypeCheck(obj, &PyGenericArrType_Type);
103 }
104
105 if (state->last_zerodim_check && PyArray_NDIM(pyarray_obj) == 0) {
106 obj = PyArray_ToScalar(PyArray_DATA(pyarray_obj), pyarray_obj);
107 } else {
108 Py_INCREF(obj);
109 }
110 return obj;
111 }
112
113 // Sets *elem to a NEW reference to an element in seq on success.
114 // REQUIRES: PySequence_Check(seq) && PySequence_Length(seq) > 0.
SampleElementFromSequence(PyObject * seq,PyObject ** elem)115 Status SampleElementFromSequence(PyObject* seq, PyObject** elem) {
116 *elem = PySequence_GetItem(seq, 0);
117 if (*elem != nullptr) return Status::OK();
118 // seq may implement the sequence protocol (i.e., implement __getitem__)
119 // but may legitimately not have a 0-th element (__getitem__(self, 0)
120 // raises a KeyError). For example:
121 // seq = pandas.Series([0, 1, 2], index=[2, 4, 6])
122 //
123 // We don't actually care for the element at key 0, any element will do
124 // for inferring the element types. All elements are expected to
125 // have the same type, and this will be validated when converting
126 // to an EagerTensor.
127 PyErr_Clear();
128 Safe_PyObjectPtr iter(PyObject_GetIter(seq));
129 if (PyErr_Occurred()) {
130 return errors::InvalidArgument("Cannot infer dtype of a ",
131 Py_TYPE(seq)->tp_name,
132 " object: ", PyExceptionFetch());
133 }
134 *elem = PyIter_Next(iter.get());
135 if (PyErr_Occurred()) {
136 return errors::InvalidArgument(
137 "Cannot infer dtype of a ", Py_TYPE(seq)->tp_name,
138 " object, as iter(<object>).next() failed: ", PyExceptionFetch());
139 }
140 if (*elem == nullptr) {
141 return errors::InvalidArgument("Cannot infer dtype of a ",
142 Py_TYPE(seq)->tp_name,
143 " object since it is an empty sequence");
144 }
145 return Status::OK();
146 }
147
148 tstring PyRepr(PyObject* obj);
149 bool IsPyDimension(PyObject* obj);
150
InferShapeAndType(PyObject * obj,ConverterState * state)151 Status InferShapeAndType(PyObject* obj, ConverterState* state) {
152 std::vector<Safe_PyObjectPtr> refs_to_clean;
153 while (true) {
154 // Convert any zero dimensional numpy arrays to scalars first of all.
155 // We also have to make sure a reference to the safe_obj is kept.
156 obj = ZeroDimArrayToScalar(obj, state);
157 refs_to_clean.push_back(make_safe(obj));
158 // We test strings first, in case a string is considered a sequence.
159 if (IsPyString(obj)) {
160 state->inferred_dtype = DT_STRING;
161 } else if (PySequence_Check(obj)) {
162 auto length = PySequence_Length(obj);
163 if (length > 0) {
164 state->inferred_shape.push_back(length);
165 PyObject* elem = nullptr;
166 TF_RETURN_IF_ERROR(SampleElementFromSequence(obj, &elem));
167 obj = elem;
168 refs_to_clean.push_back(make_safe(obj));
169 continue;
170 } else if (length == 0) {
171 state->inferred_shape.push_back(length);
172 state->inferred_dtype = DT_INVALID; // Invalid dtype for empty tensors.
173 } else {
174 // The sequence does not have a valid length (PySequence_Length < 0).
175 if (PyErr_Occurred()) {
176 // PySequence_Length failed and set an exception. Fetch the message
177 // and convert it to a failed status.
178 return errors::InvalidArgument(PyExceptionFetch());
179 } else {
180 // This is almost certainly dead code: PySequence_Length failed but
181 // did not set an exception.
182 return errors::InvalidArgument(
183 "Attempted to convert an invalid sequence to a Tensor.");
184 }
185 }
186 } else if (IsPyDouble(obj)) {
187 state->inferred_dtype = DT_DOUBLE;
188 } else if (IsNumpyHalf(obj)) {
189 state->inferred_dtype = DT_HALF;
190 } else if (IsPyFloat(obj)) {
191 state->inferred_dtype = DT_FLOAT;
192 } else if (PyBool_Check(obj) || PyIsInstance(obj, &PyBoolArrType_Type)) {
193 // Have to test for bool before int, since IsInt(True/False) == true.
194 state->inferred_dtype = DT_BOOL;
195 } else if (IsPyInt(obj)) {
196 state->inferred_dtype = DT_INT64;
197 } else if (IsPyDimension(obj)) {
198 state->inferred_dtype = DT_INT64;
199 } else if (PyComplex_Check(obj) ||
200 PyIsInstance(obj, &PyComplexFloatingArrType_Type)) { // NumPy
201 state->inferred_dtype = DT_COMPLEX128;
202 } else {
203 return errors::InvalidArgument("Attempt to convert a value (",
204 PyRepr(obj),
205 ") with an unsupported type (",
206 PyRepr(PyType(obj)), ") to a Tensor.");
207 }
208 return Status::OK();
209 }
210 }
211
212 // Error messages
213
214 const char ErrorConverting[] =
215 "Error while converting Python sequence to Tensor.";
216 const char ErrorRectangular[] =
217 "Can't convert non-rectangular Python sequence to Tensor.";
218 const char ErrorMixedTypes[] =
219 "Can't convert Python sequence with mixed types to Tensor.";
220 const char ErrorOutOfRange[] =
221 "Can't convert Python sequence with out-of-range integer to Tensor.";
222 const char ErrorOutOfRangeDouble[] =
223 "Can't convert Python sequence with a value out of range for a "
224 "double-precision float.";
225 const char ErrorConvertingUnicodeString[] =
226 "Error converting unicode string while converting Python sequence to "
227 "Tensor.";
228 const char ErrorFoundInt64[] =
229 "Can't convert Python sequence with out-of-range integer to int32 Tensor.";
230 const char ErrorFoundFloat[] =
231 "Can't convert Python sequence with floating point values to integer "
232 "Tensor.";
233
234 // Defines a converter that recursively converts an object into
235 // an array of type T using the conversion function defined by the
236 // traits class in a ConvertScalar function.
237 //
238 // Note that these helper functions require shape.dims() >= 1.
239 template <class T>
240 struct ConverterTraits {
241 static const tensorflow::DataType kTypeEnum;
242 static const char* ConvertScalar(PyObject* v, T* out);
243 };
244
245 template <class T>
246 struct Converter {
Helpertensorflow::__anone25f2a6a0111::Converter247 static const char* Helper(PyObject* obj, int depth, ConverterState* state,
248 T** buf) {
249 if (TF_PREDICT_FALSE(obj == nullptr)) {
250 return ErrorConverting;
251 }
252
253 Safe_PyObjectPtr seq = make_safe(PySequence_Fast(obj, ""));
254 if (TF_PREDICT_FALSE(seq == nullptr)) return ErrorRectangular;
255
256 const int64 s = state->inferred_shape[depth];
257 if (TF_PREDICT_FALSE(s != PySequence_Fast_GET_SIZE(seq.get()))) {
258 return ErrorRectangular;
259 }
260
261 if (state->inferred_shape.size() - depth > 1) {
262 /* Iterate over outer dim, and recursively convert each element. */
263 for (int64 i = 0; i < s; ++i) {
264 const char* error = Helper(PySequence_Fast_GET_ITEM(seq.get(), i),
265 depth + 1, state, buf);
266 if (TF_PREDICT_FALSE(error != nullptr)) return error;
267 }
268 } else {
269 PyObject** l = PySequence_Fast_ITEMS(seq.get());
270 for (int64 i = 0; i < s; ++i) {
271 auto scalar = ZeroDimArrayToScalar(l[i], state);
272 const char* error = ConverterTraits<T>::ConvertScalar(scalar, *buf);
273 Py_DECREF(scalar);
274 if (TF_PREDICT_FALSE(error != nullptr)) return error;
275 ++*buf;
276 }
277 }
278 return nullptr;
279 }
280
Converttensorflow::__anone25f2a6a0111::Converter281 static Status Convert(TFE_Context* ctx, PyObject* obj, ConverterState* state,
282 TFE_TensorHandle** h, const char** error) {
283 // TODO(josh11b): Allocator & attributes
284 AbstractTensorInterface* t;
285 if (state->inferred_shape.empty()) { /* Scalar case */
286 T value;
287 auto scalar = ZeroDimArrayToScalar(obj, state);
288 *error = ConverterTraits<T>::ConvertScalar(scalar, &value);
289 Py_DECREF(scalar);
290 if (*error != nullptr) return errors::InvalidArgument(*error);
291 t = ConverterTraits<T>::CreateScalar(ctx, value);
292 if (t == nullptr) {
293 return errors::Internal("Cannot create tensor.");
294 }
295 } else {
296 t = ConverterTraits<T>::CreateTensor(ctx, state->inferred_shape);
297 if (t == nullptr) {
298 return errors::Internal("Cannot create tensor.");
299 }
300 if (t->NumElements() > 0) {
301 T* buf = static_cast<T*>(t->Data());
302 *error = Helper(obj, 0, state, &buf);
303 if (*error != nullptr) {
304 t->Release();
305 return errors::InvalidArgument(*error);
306 }
307 }
308 }
309 *h = tensorflow::wrap(tensorflow::unwrap(ctx)->CreateLocalHandle(t));
310 t->Release();
311 return Status::OK();
312 }
313 };
314
315 // Int support
316
317 template <>
318 struct ConverterTraits<int64> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits319 static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, int64 value) {
320 return tensorflow::unwrap(ctx)->CreateInt64Scalar(value);
321 }
322
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits323 static AbstractTensorInterface* CreateTensor(
324 TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
325 return tensorflow::unwrap(ctx)->CreateTensor(DT_INT64, dim_sizes);
326 }
327
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits328 static const char* ConvertScalar(PyObject* v, int64* out) {
329 #if PY_MAJOR_VERSION < 3
330 if (TF_PREDICT_TRUE(PyInt_Check(v))) {
331 *out = PyInt_AS_LONG(v);
332 return nullptr;
333 }
334 #endif
335 if (TF_PREDICT_TRUE(PyLong_Check(v) || IsPyDimension(v))) {
336 int overflow = 0;
337 // Have to use LongLong for 64 bits, since long is 32 bits on Windows.
338 *out = PyLong_AsLongLongAndOverflow(v, &overflow);
339 if (TF_PREDICT_FALSE(overflow)) return ErrorOutOfRange;
340 return nullptr;
341 }
342 if (PyIsInstance(v, &PyIntegerArrType_Type)) { // NumPy integers
343 #if PY_MAJOR_VERSION < 3
344 Safe_PyObjectPtr as_int = make_safe(PyNumber_Int(v));
345 #else
346 Safe_PyObjectPtr as_int = make_safe(PyNumber_Long(v));
347 #endif
348 return ConvertScalar(as_int.get(), out);
349 }
350 if (IsPyFloat(v)) return ErrorFoundFloat;
351 return ErrorMixedTypes;
352 }
353 };
354
355 typedef Converter<int64> Int64Converter;
356
357 template <>
358 struct ConverterTraits<uint64> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits359 static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, uint64 value) {
360 return tensorflow::unwrap(ctx)->CreateUint64Scalar(value);
361 }
362
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits363 static AbstractTensorInterface* CreateTensor(
364 TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
365 return tensorflow::unwrap(ctx)->CreateTensor(DT_UINT64, dim_sizes);
366 }
367
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits368 static const char* ConvertScalar(PyObject* v, uint64* out) {
369 #if PY_MAJOR_VERSION < 3
370 if (TF_PREDICT_TRUE(PyInt_Check(v))) {
371 *out = PyInt_AsUnsignedLongLongMask(v);
372 return nullptr;
373 }
374 #endif
375 if (TF_PREDICT_TRUE(PyLong_Check(v) || IsPyDimension(v))) {
376 *out = PyLong_AsUnsignedLongLong(v);
377 return nullptr;
378 }
379 if (PyIsInstance(v, &PyIntegerArrType_Type)) { // NumPy integers
380 #if PY_MAJOR_VERSION < 3
381 Safe_PyObjectPtr as_int = make_safe(PyNumber_Int(v));
382 #else
383 Safe_PyObjectPtr as_int = make_safe(PyNumber_Long(v));
384 #endif
385 return ConvertScalar(as_int.get(), out);
386 }
387 if (IsPyFloat(v)) return ErrorFoundFloat;
388 return ErrorMixedTypes;
389 }
390 };
391
392 typedef Converter<uint64> UInt64Converter;
393
394 template <>
395 struct ConverterTraits<int32> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits396 static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, int32 value) {
397 return tensorflow::unwrap(ctx)->CreateInt32Scalar(value);
398 }
399
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits400 static AbstractTensorInterface* CreateTensor(
401 TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
402 return tensorflow::unwrap(ctx)->CreateTensor(DT_INT32, dim_sizes);
403 }
404
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits405 static const char* ConvertScalar(PyObject* v, int32* out) {
406 int64 i;
407 #if PY_MAJOR_VERSION < 3
408 if (TF_PREDICT_TRUE(PyInt_Check(v))) {
409 i = PyInt_AS_LONG(v);
410 } else
411 #endif
412 if (PyLong_Check(v) || IsPyDimension(v)) {
413 int overflow = 0;
414 // Have to use LongLong for 64 bits, since long is 32 bits on Windows.
415 i = PyLong_AsLongLongAndOverflow(v, &overflow);
416 if (TF_PREDICT_FALSE(overflow)) return ErrorOutOfRange;
417 } else if (PyIsInstance(v, &PyIntegerArrType_Type)) { // NumPy integers
418 #if PY_MAJOR_VERSION < 3
419 Safe_PyObjectPtr as_int = make_safe(PyNumber_Int(v));
420 #else
421 Safe_PyObjectPtr as_int = make_safe(PyNumber_Long(v));
422 #endif
423 return ConvertScalar(as_int.get(), out);
424 } else if (IsPyFloat(v)) {
425 return ErrorFoundFloat;
426 } else {
427 return ErrorMixedTypes;
428 }
429 *out = static_cast<uint32>(static_cast<uint64>(i));
430 // Check for 32-bit overflow.
431 if (TF_PREDICT_FALSE(i != *out)) return ErrorFoundInt64;
432 return nullptr;
433 }
434 };
435
436 typedef Converter<int32> Int32Converter;
437
438 // Floating-point support
439
440 // Returns `true` if `out` overflows when converted from `as_double`.
441 template <class T>
CheckForOverflow(double as_double,T * out)442 static inline bool CheckForOverflow(double as_double, T* out) {
443 return (sizeof(T) < sizeof(double) && std::isinf(*out) &&
444 std::isfinite(as_double));
445 }
446
447 // There is no `std::isinf` that takes `Eigen::half` as argument but Eigen
448 // provides `Eigen::half_impl::isinf` instead.
449 template <>
CheckForOverflow(double as_double,Eigen::half * out)450 inline bool CheckForOverflow<Eigen::half>(double as_double, Eigen::half* out) {
451 return (sizeof(Eigen::half) < sizeof(double) &&
452 Eigen::half_impl::isinf(*out) && std::isfinite(as_double));
453 }
454
455 template <class T>
ConvertOneFloat(PyObject * v,T * out)456 static const char* ConvertOneFloat(PyObject* v, T* out) {
457 if (PyErr_Occurred()) {
458 return nullptr;
459 }
460 if (TF_PREDICT_TRUE(PyFloat_Check(v))) {
461 const double as_double = PyFloat_AS_DOUBLE(v);
462 *out = static_cast<T>(as_double);
463 // Check for overflow
464 if (TF_PREDICT_FALSE(CheckForOverflow<T>(as_double, out))) {
465 return ErrorOutOfRangeDouble;
466 }
467 return nullptr;
468 }
469 #if PY_MAJOR_VERSION < 3
470 if (PyInt_Check(v)) {
471 *out = static_cast<T>(PyInt_AS_LONG(v));
472 return nullptr;
473 }
474 #endif
475 if (PyLong_Check(v)) {
476 *out = static_cast<T>(PyLong_AsDouble(v));
477 if (PyErr_Occurred()) return ErrorOutOfRangeDouble;
478 return nullptr;
479 }
480 if (PyIsInstance(v, &PyFloatingArrType_Type)) { // NumPy float types
481 Safe_PyObjectPtr as_float = make_safe(PyNumber_Float(v));
482 if (PyErr_Occurred()) {
483 return nullptr;
484 }
485 return ConvertOneFloat<T>(as_float.get(), out);
486 }
487 if (PyIsInstance(v, &PyIntegerArrType_Type)) { // NumPy integers
488 #if PY_MAJOR_VERSION < 3
489 Safe_PyObjectPtr as_int = make_safe(PyNumber_Int(v));
490 #else
491 Safe_PyObjectPtr as_int = make_safe(PyNumber_Long(v));
492 #endif
493 if (PyErr_Occurred()) {
494 return nullptr;
495 }
496 return ConvertOneFloat<T>(as_int.get(), out);
497 }
498 return ErrorMixedTypes;
499 }
500
501 template <>
502 struct ConverterTraits<float> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits503 static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, float value) {
504 return tensorflow::unwrap(ctx)->CreateFloatScalar(value);
505 }
506
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits507 static AbstractTensorInterface* CreateTensor(
508 TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
509 return tensorflow::unwrap(ctx)->CreateTensor(DT_FLOAT, dim_sizes);
510 }
511
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits512 static const char* ConvertScalar(PyObject* v, float* out) {
513 return ConvertOneFloat<float>(v, out);
514 }
515 };
516
517 template <>
518 struct ConverterTraits<double> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits519 static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, double value) {
520 return tensorflow::unwrap(ctx)->CreateDoubleScalar(value);
521 }
522
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits523 static AbstractTensorInterface* CreateTensor(
524 TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
525 return tensorflow::unwrap(ctx)->CreateTensor(DT_DOUBLE, dim_sizes);
526 }
527
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits528 static const char* ConvertScalar(PyObject* v, double* out) {
529 return ConvertOneFloat<double>(v, out);
530 }
531 };
532
533 typedef Converter<double> DoubleConverter;
534 typedef Converter<float> FloatConverter;
535
536 template <>
537 struct ConverterTraits<Eigen::half> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits538 static AbstractTensorInterface* CreateScalar(TFE_Context* ctx,
539 Eigen::half value) {
540 return tensorflow::unwrap(ctx)->CreateHalfScalar(value);
541 }
542
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits543 static AbstractTensorInterface* CreateTensor(
544 TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
545 return tensorflow::unwrap(ctx)->CreateTensor(DT_HALF, dim_sizes);
546 }
547
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits548 static const char* ConvertScalar(PyObject* v, Eigen::half* out) {
549 return ConvertOneFloat<Eigen::half>(v, out);
550 }
551 };
552
553 typedef Converter<Eigen::half> NumpyHalfConverter;
554
555 // String support
556
557 template <>
558 struct ConverterTraits<tstring> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits559 static AbstractTensorInterface* CreateScalar(TFE_Context* ctx,
560 tstring value) {
561 return tensorflow::unwrap(ctx)->CreateStringScalar(value);
562 }
563
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits564 static AbstractTensorInterface* CreateTensor(
565 TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
566 return tensorflow::unwrap(ctx)->CreateTensor(DT_STRING, dim_sizes);
567 }
568
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits569 static const char* ConvertScalar(PyObject* v, tstring* out) {
570 if (PyBytes_Check(v)) {
571 out->assign(PyBytes_AS_STRING(v), PyBytes_GET_SIZE(v));
572 return nullptr;
573 }
574 if (PyUnicode_Check(v)) {
575 #if PY_MAJOR_VERSION >= 3
576 Py_ssize_t size;
577 const char* str = PyUnicode_AsUTF8AndSize(v, &size);
578 if (str == nullptr) return ErrorConvertingUnicodeString;
579 out->assign(str, size);
580 return nullptr;
581 #else
582 PyObject* py_str = PyUnicode_AsUTF8String(v);
583 if (py_str == nullptr) return ErrorConvertingUnicodeString;
584 out->assign(PyBytes_AS_STRING(py_str), PyBytes_GET_SIZE(py_str));
585 Py_DECREF(py_str);
586 return nullptr;
587 #endif
588 }
589 return ErrorMixedTypes;
590 }
591 };
592
593 typedef Converter<tstring> StringConverter;
594
595 // Converts Python object `c` that should hold a Python string into a
596 // C++ string in *out. Returns nullptr on success, or a message on error.
597 // Defined below, but forward declared here for use in PyRepr.
PyRepr(PyObject * obj)598 tstring PyRepr(PyObject* obj) {
599 if (obj == nullptr) {
600 return "<null>";
601 }
602 Safe_PyObjectPtr repr_obj = make_safe(PyObject_Repr(obj));
603 if (repr_obj) {
604 tstring repr_str;
605 if (ConverterTraits<tstring>::ConvertScalar(repr_obj.get(), &repr_str) ==
606 nullptr) {
607 return repr_str;
608 }
609 }
610 return "<error computing repr()>";
611 }
612
IsPyDimension(PyObject * obj)613 bool IsPyDimension(PyObject* obj) {
614 const char* tp_name = obj->ob_type->tp_name;
615 if (strcmp(tp_name, "Dimension") != 0) return false;
616 bool ret = str_util::EndsWith(
617 PyRepr(PyType(obj)),
618 "tensorflow.python.framework.tensor_shape.Dimension'>");
619 return ret;
620 }
621
622 // Complex support
623
624 template <>
625 struct ConverterTraits<complex128> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits626 static AbstractTensorInterface* CreateScalar(TFE_Context* ctx,
627 complex128 value) {
628 return tensorflow::unwrap(ctx)->CreateComplex128Scalar(value);
629 }
630
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits631 static AbstractTensorInterface* CreateTensor(
632 TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
633 return tensorflow::unwrap(ctx)->CreateTensor(DT_COMPLEX128, dim_sizes);
634 }
635
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits636 static const char* ConvertScalar(PyObject* v, complex128* out) {
637 if (PyComplex_Check(v)) {
638 *out = complex128(PyComplex_RealAsDouble(v), PyComplex_ImagAsDouble(v));
639 return nullptr;
640 } else if (PyIsInstance(v, &PyComplexFloatingArrType_Type)) { // NumPy
641 auto as_complex = PyComplex_AsCComplex(v);
642 *out = complex128(as_complex.real, as_complex.imag);
643 return nullptr;
644 }
645 double as_double;
646 auto error = ConvertOneFloat<double>(v, &as_double);
647 if (error != nullptr) return error;
648 *out = complex128(as_double, 0.0);
649 return nullptr;
650 }
651 };
652
653 typedef Converter<complex128> Complex128Converter;
654
655 // Bool support
656
657 template <>
658 struct ConverterTraits<bool> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits659 static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, bool value) {
660 return tensorflow::unwrap(ctx)->CreateBoolScalar(value);
661 }
662
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits663 static AbstractTensorInterface* CreateTensor(
664 TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
665 return tensorflow::unwrap(ctx)->CreateTensor(DT_BOOL, dim_sizes);
666 }
667
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits668 static const char* ConvertScalar(PyObject* v, bool* out) {
669 if (v == Py_True) {
670 *out = true;
671 } else if (v == Py_False) {
672 *out = false;
673 } else if (PyIsInstance(v, &PyBoolArrType_Type)) { // NumPy
674 *out = PyObject_IsTrue(v);
675 } else {
676 return ErrorMixedTypes;
677 }
678 return nullptr;
679 }
680 };
681
682 typedef Converter<bool> BoolConverter;
683
684 // Convert a Python numpy.ndarray object to a TFE_TensorHandle.
685 // The two may share underlying storage so changes to one may reflect in the
686 // other.
NumpyToTFE_TensorHandle(TFE_Context * ctx,PyObject * obj)687 TFE_TensorHandle* NumpyToTFE_TensorHandle(TFE_Context* ctx, PyObject* obj) {
688 Safe_TF_TensorPtr tf_tensor = make_safe(static_cast<TF_Tensor*>(nullptr));
689 Status status = tensorflow::NdarrayToTensor(ctx, obj, &tf_tensor);
690
691 if (TF_PREDICT_FALSE(!status.ok())) {
692 PyErr_SetString(PyExc_ValueError,
693 tensorflow::strings::StrCat(
694 "Failed to convert a NumPy array to a Tensor (",
695 status.error_message(), ").")
696 .c_str());
697 return nullptr;
698 }
699
700 return tensorflow::wrap(
701 tensorflow::unwrap(ctx)->CreateLocalHandle(tf_tensor->tensor));
702 }
703
704 } // namespace
705
706 // TODO(b/147743551): This function handles enough conversions to justify
707 // promoting to something like PyObjectToTensorHandle.
708 // TODO(b/147828820): Handle Tensors properly.
PySeqToTFE_TensorHandle(TFE_Context * ctx,PyObject * obj,DataType dtype)709 TFE_TensorHandle* PySeqToTFE_TensorHandle(TFE_Context* ctx, PyObject* obj,
710 DataType dtype) {
711 // Shortcut: __array__ objects (such as Pandas data frames).
712 // These objects are efficiently handled by Numpy. We transform them into
713 // Numpy arrays and handle them in the Numpy case below. Note that Tensors
714 // implement the __array__ function, and will be handled in this shortcut.
715 Safe_PyObjectPtr array =
716 make_safe(PyArray_FromArrayAttr(obj, nullptr, nullptr));
717 if (array == nullptr) {
718 return nullptr;
719 }
720 if (array.get() == Py_NotImplemented) {
721 // The Py_NotImplemented returned from PyArray_FromArrayAttr is not
722 // Py_INCREF'ed, so we don't want the Safe_PyObjectPtr to Py_DECREF it.
723 array.release();
724 } else {
725 // PyArray_FromArrayAttr ensures that `array` is a PyArrayObject, so all
726 // we have to do is replace `obj` with it and continue.
727 obj = array.get();
728 }
729
730 // Shortcut: Numpy arrays.
731 if (PyArray_Check(obj)) {
732 int desired_np_dtype = -1;
733 if (dtype != tensorflow::DT_INVALID) {
734 if (!tensorflow::TF_DataType_to_PyArray_TYPE(
735 static_cast<TF_DataType>(dtype), &desired_np_dtype)
736 .ok()) {
737 PyErr_SetString(
738 PyExc_TypeError,
739 tensorflow::strings::StrCat("Invalid dtype argument value ", dtype)
740 .c_str());
741 return nullptr;
742 }
743 }
744
745 PyArrayObject* array = reinterpret_cast<PyArrayObject*>(obj);
746 int array_dtype = PyArray_TYPE(array);
747
748 Safe_PyObjectPtr safe_value(nullptr);
749 // Use Numpy to convert between types if needed.
750 if ((desired_np_dtype >= 0 && desired_np_dtype != array_dtype) ||
751 !PyArray_ISCARRAY(array)) {
752 int new_dtype = desired_np_dtype >= 0 ? desired_np_dtype : array_dtype;
753 safe_value = tensorflow::make_safe(
754 PyArray_FromAny(obj, PyArray_DescrFromType(new_dtype), 0, 0,
755 NPY_ARRAY_CARRAY_RO | NPY_ARRAY_FORCECAST, nullptr));
756 if (PyErr_Occurred()) return nullptr;
757 if (safe_value == nullptr) {
758 PyErr_SetString(PyExc_ValueError, "Error while casting a numpy value");
759 }
760 obj = safe_value.get();
761 }
762 return NumpyToTFE_TensorHandle(ctx, obj);
763 }
764
765 ConverterState state;
766 Status status = InferShapeAndType(obj, &state);
767 if (!status.ok()) {
768 PyErr_SetString(PyExc_ValueError, status.error_message().c_str());
769 return nullptr;
770 }
771 DataType requested_dtype = DT_INVALID;
772 if (dtype != DT_INVALID) {
773 requested_dtype = dtype;
774 }
775
776 // NOTE(josh11b): If don't successfully convert to the requested type,
777 // we just try instead to create a tensor of the inferred type and
778 // let the caller convert it to the requested type using a cast
779 // operation.
780 const char* error = nullptr;
781 TFE_TensorHandle* handle = nullptr;
782 status = errors::Unimplemented("Missing Python -> Tensor conversion for ",
783 DataTypeString(state.inferred_dtype));
784 switch (requested_dtype) {
785 case DT_FLOAT:
786 status = FloatConverter::Convert(ctx, obj, &state, &handle, &error);
787 break;
788
789 case DT_DOUBLE:
790 status = DoubleConverter::Convert(ctx, obj, &state, &handle, &error);
791 break;
792
793 case DT_HALF:
794 status = NumpyHalfConverter::Convert(ctx, obj, &state, &handle, &error);
795 break;
796
797 case DT_INT64:
798 status = Int64Converter::Convert(ctx, obj, &state, &handle, &error);
799 break;
800
801 case DT_INT32:
802 status = Int32Converter::Convert(ctx, obj, &state, &handle, &error);
803 break;
804
805 case DT_UINT64:
806 status = UInt64Converter::Convert(ctx, obj, &state, &handle, &error);
807 break;
808
809 case DT_COMPLEX128:
810 status = Complex128Converter::Convert(ctx, obj, &state, &handle, &error);
811 break;
812
813 case DT_STRING:
814 status = StringConverter::Convert(ctx, obj, &state, &handle, &error);
815 break;
816
817 case DT_BOOL:
818 status = BoolConverter::Convert(ctx, obj, &state, &handle, &error);
819 break;
820
821 default:
822 break;
823 }
824 if (status.ok()) return handle;
825
826 switch (state.inferred_dtype) {
827 case DT_FLOAT:
828 // TODO(josh11b): Handle mixed floats and complex numbers?
829 if (requested_dtype == DT_INVALID) {
830 // TensorFlow uses float32s to represent floating point numbers
831 // by default (for space and speed over using doubles).
832 status = FloatConverter::Convert(ctx, obj, &state, &handle, &error);
833 } else {
834 // We are going to do a cast to the user's requested dtype
835 // after this. We use doubles for this intermediate result so
836 // we don't lose precision that might be representable in the
837 // final type.
838 status = DoubleConverter::Convert(ctx, obj, &state, &handle, &error);
839 }
840 break;
841
842 case DT_DOUBLE:
843 status = DoubleConverter::Convert(ctx, obj, &state, &handle, &error);
844 break;
845
846 case DT_HALF:
847 status = NumpyHalfConverter::Convert(ctx, obj, &state, &handle, &error);
848 break;
849
850 case DT_INT64:
851 if (requested_dtype == DT_INVALID) {
852 status = Int32Converter::Convert(ctx, obj, &state, &handle, &error);
853 if (error == ErrorFoundInt64) {
854 status = Int64Converter::Convert(ctx, obj, &state, &handle, &error);
855 }
856 if (error == ErrorFoundFloat) {
857 status = FloatConverter::Convert(ctx, obj, &state, &handle, &error);
858 }
859 // TODO(josh11b): May also want to fall back to using doubles if
860 // error == ErrorOutOfRange?
861 } else {
862 status = Int64Converter::Convert(ctx, obj, &state, &handle, &error);
863 if (error == ErrorFoundFloat) {
864 status = DoubleConverter::Convert(ctx, obj, &state, &handle, &error);
865 }
866 }
867 break;
868
869 case DT_STRING:
870 status = StringConverter::Convert(ctx, obj, &state, &handle, &error);
871 break;
872
873 case DT_COMPLEX128:
874 status = Complex128Converter::Convert(ctx, obj, &state, &handle, &error);
875 break;
876
877 case DT_BOOL:
878 status = BoolConverter::Convert(ctx, obj, &state, &handle, &error);
879 break;
880
881 case DT_INVALID: // Only occurs for empty tensors.
882 {
883 AbstractTensorInterface* t = tensorflow::unwrap(ctx)->CreateTensor(
884 requested_dtype == DT_INVALID ? DT_FLOAT : requested_dtype,
885 state.inferred_shape);
886 auto* result =
887 tensorflow::wrap(tensorflow::unwrap(ctx)->CreateLocalHandle(t));
888 t->Release();
889 return result;
890 }
891
892 default:
893 break;
894 }
895
896 if (!status.ok()) {
897 PyErr_SetString(PyExc_ValueError, status.error_message().c_str());
898 return nullptr;
899 }
900
901 return handle;
902 }
903
904 } // namespace tensorflow
905