• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #include "tensorflow/python/lib/core/py_seq_tensor.h"
17 
18 #include "tensorflow/c/eager/tfe_context_internal.h"
19 #include "tensorflow/c/eager/tfe_tensorhandle_internal.h"
20 #include "tensorflow/c/tensor_interface.h"
21 #include "tensorflow/c/tf_tensor_internal.h"
22 #include "tensorflow/core/framework/tensor.h"
23 #include "tensorflow/core/framework/types.h"
24 #include "tensorflow/core/lib/core/errors.h"
25 #include "tensorflow/core/lib/strings/str_util.h"
26 #include "tensorflow/core/platform/errors.h"
27 #include "tensorflow/core/platform/macros.h"
28 #include "tensorflow/core/platform/tstring.h"
29 #include "tensorflow/core/platform/types.h"
30 #include "tensorflow/python/lib/core/ndarray_tensor.h"
31 #include "tensorflow/python/lib/core/ndarray_tensor_bridge.h"
32 #include "tensorflow/python/lib/core/numpy.h"
33 #include "tensorflow/python/lib/core/py_util.h"
34 #include "tensorflow/python/lib/core/safe_ptr.h"
35 
36 namespace tensorflow {
37 namespace {
38 
PyIsInstance(PyObject * obj,PyTypeObject * t)39 inline bool PyIsInstance(PyObject* obj, PyTypeObject* t) {
40   return PyObject_IsInstance(obj, reinterpret_cast<PyObject*>(t));
41 }
42 
PyType(PyObject * obj)43 inline PyObject* PyType(PyObject* obj) {
44   return reinterpret_cast<PyObject*>(obj->ob_type);
45 }
46 
IsPyString(PyObject * obj)47 bool IsPyString(PyObject* obj) {
48   return PyBytes_Check(obj) || PyUnicode_Check(obj);
49 }
50 
IsPyInt(PyObject * obj)51 bool IsPyInt(PyObject* obj) {
52 #if PY_MAJOR_VERSION >= 3
53   return PyLong_Check(obj) ||
54          PyIsInstance(obj, &PyIntegerArrType_Type);  // NumPy integers
55 #else
56   return PyInt_Check(obj) || PyLong_Check(obj) ||
57          PyIsInstance(obj, &PyIntegerArrType_Type);  // NumPy integers
58 #endif
59 }
60 
IsPyDouble(PyObject * obj)61 bool IsPyDouble(PyObject* obj) {
62   return PyIsInstance(obj, &PyDoubleArrType_Type);  // NumPy double type.
63 }
64 
IsNumpyHalf(PyObject * obj)65 bool IsNumpyHalf(PyObject* obj) {
66   return PyIsInstance(obj, &PyHalfArrType_Type);
67 }
68 
IsPyFloat(PyObject * obj)69 bool IsPyFloat(PyObject* obj) {
70   return PyFloat_Check(obj) ||
71          PyIsInstance(obj, &PyFloatingArrType_Type);  // NumPy float types
72 }
73 
74 struct ConverterState {
75   // The inferred tensor shape.
76   gtl::InlinedVector<int64, 4> inferred_shape;
77 
78   // The inferred tensor data type.
79   DataType inferred_dtype;
80 
81   // The following fields are used by ZeroDimArrayToScalar.
82   // We cache the last result of the check for a zero dimensional array: the
83   // function is called many times in a conversion, and most of the time is
84   // to check for the same type. This cache can reduce the conversion time by
85   // about 25%.
86   PyTypeObject* last_zerodim_type;
87   bool last_zerodim_check;
88 
ConverterStatetensorflow::__anone25f2a6a0111::ConverterState89   ConverterState() : inferred_dtype(DT_INVALID), last_zerodim_type(nullptr) {}
90 };
91 
92 // If the input is a zero dimensional PyArray return it converted to a scalar.
93 // Otherwise return the input and increment its reference count.
94 // Users must Py_DECREF the output of this method.
ZeroDimArrayToScalar(PyObject * obj,ConverterState * state)95 PyObject* ZeroDimArrayToScalar(PyObject* obj, ConverterState* state) {
96   auto type = Py_TYPE(obj);
97   auto pyarray_obj = reinterpret_cast<PyArrayObject*>(obj);
98   if (type != state->last_zerodim_type) {
99     state->last_zerodim_type = type;
100     state->last_zerodim_check =
101         PyObject_TypeCheck(obj, &PyArray_Type) &&
102         !PyObject_TypeCheck(obj, &PyGenericArrType_Type);
103   }
104 
105   if (state->last_zerodim_check && PyArray_NDIM(pyarray_obj) == 0) {
106     obj = PyArray_ToScalar(PyArray_DATA(pyarray_obj), pyarray_obj);
107   } else {
108     Py_INCREF(obj);
109   }
110   return obj;
111 }
112 
113 // Sets *elem to a NEW reference to an element in seq on success.
114 // REQUIRES: PySequence_Check(seq) && PySequence_Length(seq) > 0.
SampleElementFromSequence(PyObject * seq,PyObject ** elem)115 Status SampleElementFromSequence(PyObject* seq, PyObject** elem) {
116   *elem = PySequence_GetItem(seq, 0);
117   if (*elem != nullptr) return Status::OK();
118   // seq may implement the sequence protocol (i.e., implement __getitem__)
119   // but may legitimately not have a 0-th element (__getitem__(self, 0)
120   // raises a KeyError). For example:
121   // seq = pandas.Series([0, 1, 2], index=[2, 4, 6])
122   //
123   // We don't actually care for the element at key 0, any element will do
124   // for inferring the element types. All elements are expected to
125   // have the same type, and this will be validated when converting
126   // to an EagerTensor.
127   PyErr_Clear();
128   Safe_PyObjectPtr iter(PyObject_GetIter(seq));
129   if (PyErr_Occurred()) {
130     return errors::InvalidArgument("Cannot infer dtype of a ",
131                                    Py_TYPE(seq)->tp_name,
132                                    " object: ", PyExceptionFetch());
133   }
134   *elem = PyIter_Next(iter.get());
135   if (PyErr_Occurred()) {
136     return errors::InvalidArgument(
137         "Cannot infer dtype of a ", Py_TYPE(seq)->tp_name,
138         " object, as iter(<object>).next() failed: ", PyExceptionFetch());
139   }
140   if (*elem == nullptr) {
141     return errors::InvalidArgument("Cannot infer dtype of a ",
142                                    Py_TYPE(seq)->tp_name,
143                                    " object since it is an empty sequence");
144   }
145   return Status::OK();
146 }
147 
148 tstring PyRepr(PyObject* obj);
149 bool IsPyDimension(PyObject* obj);
150 
InferShapeAndType(PyObject * obj,ConverterState * state)151 Status InferShapeAndType(PyObject* obj, ConverterState* state) {
152   std::vector<Safe_PyObjectPtr> refs_to_clean;
153   while (true) {
154     // Convert any zero dimensional numpy arrays to scalars first of all.
155     // We also have to make sure a reference to the safe_obj is kept.
156     obj = ZeroDimArrayToScalar(obj, state);
157     refs_to_clean.push_back(make_safe(obj));
158     // We test strings first, in case a string is considered a sequence.
159     if (IsPyString(obj)) {
160       state->inferred_dtype = DT_STRING;
161     } else if (PySequence_Check(obj)) {
162       auto length = PySequence_Length(obj);
163       if (length > 0) {
164         state->inferred_shape.push_back(length);
165         PyObject* elem = nullptr;
166         TF_RETURN_IF_ERROR(SampleElementFromSequence(obj, &elem));
167         obj = elem;
168         refs_to_clean.push_back(make_safe(obj));
169         continue;
170       } else if (length == 0) {
171         state->inferred_shape.push_back(length);
172         state->inferred_dtype = DT_INVALID;  // Invalid dtype for empty tensors.
173       } else {
174         // The sequence does not have a valid length (PySequence_Length < 0).
175         if (PyErr_Occurred()) {
176           // PySequence_Length failed and set an exception. Fetch the message
177           // and convert it to a failed status.
178           return errors::InvalidArgument(PyExceptionFetch());
179         } else {
180           // This is almost certainly dead code: PySequence_Length failed but
181           // did not set an exception.
182           return errors::InvalidArgument(
183               "Attempted to convert an invalid sequence to a Tensor.");
184         }
185       }
186     } else if (IsPyDouble(obj)) {
187       state->inferred_dtype = DT_DOUBLE;
188     } else if (IsNumpyHalf(obj)) {
189       state->inferred_dtype = DT_HALF;
190     } else if (IsPyFloat(obj)) {
191       state->inferred_dtype = DT_FLOAT;
192     } else if (PyBool_Check(obj) || PyIsInstance(obj, &PyBoolArrType_Type)) {
193       // Have to test for bool before int, since IsInt(True/False) == true.
194       state->inferred_dtype = DT_BOOL;
195     } else if (IsPyInt(obj)) {
196       state->inferred_dtype = DT_INT64;
197     } else if (IsPyDimension(obj)) {
198       state->inferred_dtype = DT_INT64;
199     } else if (PyComplex_Check(obj) ||
200                PyIsInstance(obj, &PyComplexFloatingArrType_Type)) {  // NumPy
201       state->inferred_dtype = DT_COMPLEX128;
202     } else {
203       return errors::InvalidArgument("Attempt to convert a value (",
204                                      PyRepr(obj),
205                                      ") with an unsupported type (",
206                                      PyRepr(PyType(obj)), ") to a Tensor.");
207     }
208     return Status::OK();
209   }
210 }
211 
212 // Error messages
213 
214 const char ErrorConverting[] =
215     "Error while converting Python sequence to Tensor.";
216 const char ErrorRectangular[] =
217     "Can't convert non-rectangular Python sequence to Tensor.";
218 const char ErrorMixedTypes[] =
219     "Can't convert Python sequence with mixed types to Tensor.";
220 const char ErrorOutOfRange[] =
221     "Can't convert Python sequence with out-of-range integer to Tensor.";
222 const char ErrorOutOfRangeDouble[] =
223     "Can't convert Python sequence with a value out of range for a "
224     "double-precision float.";
225 const char ErrorConvertingUnicodeString[] =
226     "Error converting unicode string while converting Python sequence to "
227     "Tensor.";
228 const char ErrorFoundInt64[] =
229     "Can't convert Python sequence with out-of-range integer to int32 Tensor.";
230 const char ErrorFoundFloat[] =
231     "Can't convert Python sequence with floating point values to integer "
232     "Tensor.";
233 
234 // Defines a converter that recursively converts an object into
235 // an array of type T using the conversion function defined by the
236 // traits class in a ConvertScalar function.
237 //
238 // Note that these helper functions require shape.dims() >= 1.
239 template <class T>
240 struct ConverterTraits {
241   static const tensorflow::DataType kTypeEnum;
242   static const char* ConvertScalar(PyObject* v, T* out);
243 };
244 
245 template <class T>
246 struct Converter {
Helpertensorflow::__anone25f2a6a0111::Converter247   static const char* Helper(PyObject* obj, int depth, ConverterState* state,
248                             T** buf) {
249     if (TF_PREDICT_FALSE(obj == nullptr)) {
250       return ErrorConverting;
251     }
252 
253     Safe_PyObjectPtr seq = make_safe(PySequence_Fast(obj, ""));
254     if (TF_PREDICT_FALSE(seq == nullptr)) return ErrorRectangular;
255 
256     const int64 s = state->inferred_shape[depth];
257     if (TF_PREDICT_FALSE(s != PySequence_Fast_GET_SIZE(seq.get()))) {
258       return ErrorRectangular;
259     }
260 
261     if (state->inferred_shape.size() - depth > 1) {
262       /* Iterate over outer dim, and recursively convert each element. */
263       for (int64 i = 0; i < s; ++i) {
264         const char* error = Helper(PySequence_Fast_GET_ITEM(seq.get(), i),
265                                    depth + 1, state, buf);
266         if (TF_PREDICT_FALSE(error != nullptr)) return error;
267       }
268     } else {
269       PyObject** l = PySequence_Fast_ITEMS(seq.get());
270       for (int64 i = 0; i < s; ++i) {
271         auto scalar = ZeroDimArrayToScalar(l[i], state);
272         const char* error = ConverterTraits<T>::ConvertScalar(scalar, *buf);
273         Py_DECREF(scalar);
274         if (TF_PREDICT_FALSE(error != nullptr)) return error;
275         ++*buf;
276       }
277     }
278     return nullptr;
279   }
280 
Converttensorflow::__anone25f2a6a0111::Converter281   static Status Convert(TFE_Context* ctx, PyObject* obj, ConverterState* state,
282                         TFE_TensorHandle** h, const char** error) {
283     // TODO(josh11b): Allocator & attributes
284     AbstractTensorInterface* t;
285     if (state->inferred_shape.empty()) { /* Scalar case */
286       T value;
287       auto scalar = ZeroDimArrayToScalar(obj, state);
288       *error = ConverterTraits<T>::ConvertScalar(scalar, &value);
289       Py_DECREF(scalar);
290       if (*error != nullptr) return errors::InvalidArgument(*error);
291       t = ConverterTraits<T>::CreateScalar(ctx, value);
292       if (t == nullptr) {
293         return errors::Internal("Cannot create tensor.");
294       }
295     } else {
296       t = ConverterTraits<T>::CreateTensor(ctx, state->inferred_shape);
297       if (t == nullptr) {
298         return errors::Internal("Cannot create tensor.");
299       }
300       if (t->NumElements() > 0) {
301         T* buf = static_cast<T*>(t->Data());
302         *error = Helper(obj, 0, state, &buf);
303         if (*error != nullptr) {
304           t->Release();
305           return errors::InvalidArgument(*error);
306         }
307       }
308     }
309     *h = tensorflow::wrap(tensorflow::unwrap(ctx)->CreateLocalHandle(t));
310     t->Release();
311     return Status::OK();
312   }
313 };
314 
315 // Int support
316 
317 template <>
318 struct ConverterTraits<int64> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits319   static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, int64 value) {
320     return tensorflow::unwrap(ctx)->CreateInt64Scalar(value);
321   }
322 
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits323   static AbstractTensorInterface* CreateTensor(
324       TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
325     return tensorflow::unwrap(ctx)->CreateTensor(DT_INT64, dim_sizes);
326   }
327 
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits328   static const char* ConvertScalar(PyObject* v, int64* out) {
329 #if PY_MAJOR_VERSION < 3
330     if (TF_PREDICT_TRUE(PyInt_Check(v))) {
331       *out = PyInt_AS_LONG(v);
332       return nullptr;
333     }
334 #endif
335     if (TF_PREDICT_TRUE(PyLong_Check(v) || IsPyDimension(v))) {
336       int overflow = 0;
337       // Have to use LongLong for 64 bits, since long is 32 bits on Windows.
338       *out = PyLong_AsLongLongAndOverflow(v, &overflow);
339       if (TF_PREDICT_FALSE(overflow)) return ErrorOutOfRange;
340       return nullptr;
341     }
342     if (PyIsInstance(v, &PyIntegerArrType_Type)) {  // NumPy integers
343 #if PY_MAJOR_VERSION < 3
344       Safe_PyObjectPtr as_int = make_safe(PyNumber_Int(v));
345 #else
346       Safe_PyObjectPtr as_int = make_safe(PyNumber_Long(v));
347 #endif
348       return ConvertScalar(as_int.get(), out);
349     }
350     if (IsPyFloat(v)) return ErrorFoundFloat;
351     return ErrorMixedTypes;
352   }
353 };
354 
355 typedef Converter<int64> Int64Converter;
356 
357 template <>
358 struct ConverterTraits<uint64> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits359   static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, uint64 value) {
360     return tensorflow::unwrap(ctx)->CreateUint64Scalar(value);
361   }
362 
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits363   static AbstractTensorInterface* CreateTensor(
364       TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
365     return tensorflow::unwrap(ctx)->CreateTensor(DT_UINT64, dim_sizes);
366   }
367 
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits368   static const char* ConvertScalar(PyObject* v, uint64* out) {
369 #if PY_MAJOR_VERSION < 3
370     if (TF_PREDICT_TRUE(PyInt_Check(v))) {
371       *out = PyInt_AsUnsignedLongLongMask(v);
372       return nullptr;
373     }
374 #endif
375     if (TF_PREDICT_TRUE(PyLong_Check(v) || IsPyDimension(v))) {
376       *out = PyLong_AsUnsignedLongLong(v);
377       return nullptr;
378     }
379     if (PyIsInstance(v, &PyIntegerArrType_Type)) {  // NumPy integers
380 #if PY_MAJOR_VERSION < 3
381       Safe_PyObjectPtr as_int = make_safe(PyNumber_Int(v));
382 #else
383       Safe_PyObjectPtr as_int = make_safe(PyNumber_Long(v));
384 #endif
385       return ConvertScalar(as_int.get(), out);
386     }
387     if (IsPyFloat(v)) return ErrorFoundFloat;
388     return ErrorMixedTypes;
389   }
390 };
391 
392 typedef Converter<uint64> UInt64Converter;
393 
394 template <>
395 struct ConverterTraits<int32> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits396   static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, int32 value) {
397     return tensorflow::unwrap(ctx)->CreateInt32Scalar(value);
398   }
399 
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits400   static AbstractTensorInterface* CreateTensor(
401       TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
402     return tensorflow::unwrap(ctx)->CreateTensor(DT_INT32, dim_sizes);
403   }
404 
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits405   static const char* ConvertScalar(PyObject* v, int32* out) {
406     int64 i;
407 #if PY_MAJOR_VERSION < 3
408     if (TF_PREDICT_TRUE(PyInt_Check(v))) {
409       i = PyInt_AS_LONG(v);
410     } else
411 #endif
412         if (PyLong_Check(v) || IsPyDimension(v)) {
413       int overflow = 0;
414       // Have to use LongLong for 64 bits, since long is 32 bits on Windows.
415       i = PyLong_AsLongLongAndOverflow(v, &overflow);
416       if (TF_PREDICT_FALSE(overflow)) return ErrorOutOfRange;
417     } else if (PyIsInstance(v, &PyIntegerArrType_Type)) {  // NumPy integers
418 #if PY_MAJOR_VERSION < 3
419       Safe_PyObjectPtr as_int = make_safe(PyNumber_Int(v));
420 #else
421       Safe_PyObjectPtr as_int = make_safe(PyNumber_Long(v));
422 #endif
423       return ConvertScalar(as_int.get(), out);
424     } else if (IsPyFloat(v)) {
425       return ErrorFoundFloat;
426     } else {
427       return ErrorMixedTypes;
428     }
429     *out = static_cast<uint32>(static_cast<uint64>(i));
430     // Check for 32-bit overflow.
431     if (TF_PREDICT_FALSE(i != *out)) return ErrorFoundInt64;
432     return nullptr;
433   }
434 };
435 
436 typedef Converter<int32> Int32Converter;
437 
438 // Floating-point support
439 
440 // Returns `true` if `out` overflows when converted from `as_double`.
441 template <class T>
CheckForOverflow(double as_double,T * out)442 static inline bool CheckForOverflow(double as_double, T* out) {
443   return (sizeof(T) < sizeof(double) && std::isinf(*out) &&
444           std::isfinite(as_double));
445 }
446 
447 // There is no `std::isinf` that takes `Eigen::half` as argument but Eigen
448 // provides `Eigen::half_impl::isinf` instead.
449 template <>
CheckForOverflow(double as_double,Eigen::half * out)450 inline bool CheckForOverflow<Eigen::half>(double as_double, Eigen::half* out) {
451   return (sizeof(Eigen::half) < sizeof(double) &&
452           Eigen::half_impl::isinf(*out) && std::isfinite(as_double));
453 }
454 
455 template <class T>
ConvertOneFloat(PyObject * v,T * out)456 static const char* ConvertOneFloat(PyObject* v, T* out) {
457   if (PyErr_Occurred()) {
458     return nullptr;
459   }
460   if (TF_PREDICT_TRUE(PyFloat_Check(v))) {
461     const double as_double = PyFloat_AS_DOUBLE(v);
462     *out = static_cast<T>(as_double);
463     // Check for overflow
464     if (TF_PREDICT_FALSE(CheckForOverflow<T>(as_double, out))) {
465       return ErrorOutOfRangeDouble;
466     }
467     return nullptr;
468   }
469 #if PY_MAJOR_VERSION < 3
470   if (PyInt_Check(v)) {
471     *out = static_cast<T>(PyInt_AS_LONG(v));
472     return nullptr;
473   }
474 #endif
475   if (PyLong_Check(v)) {
476     *out = static_cast<T>(PyLong_AsDouble(v));
477     if (PyErr_Occurred()) return ErrorOutOfRangeDouble;
478     return nullptr;
479   }
480   if (PyIsInstance(v, &PyFloatingArrType_Type)) {  // NumPy float types
481     Safe_PyObjectPtr as_float = make_safe(PyNumber_Float(v));
482     if (PyErr_Occurred()) {
483       return nullptr;
484     }
485     return ConvertOneFloat<T>(as_float.get(), out);
486   }
487   if (PyIsInstance(v, &PyIntegerArrType_Type)) {  // NumPy integers
488 #if PY_MAJOR_VERSION < 3
489     Safe_PyObjectPtr as_int = make_safe(PyNumber_Int(v));
490 #else
491     Safe_PyObjectPtr as_int = make_safe(PyNumber_Long(v));
492 #endif
493     if (PyErr_Occurred()) {
494       return nullptr;
495     }
496     return ConvertOneFloat<T>(as_int.get(), out);
497   }
498   return ErrorMixedTypes;
499 }
500 
501 template <>
502 struct ConverterTraits<float> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits503   static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, float value) {
504     return tensorflow::unwrap(ctx)->CreateFloatScalar(value);
505   }
506 
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits507   static AbstractTensorInterface* CreateTensor(
508       TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
509     return tensorflow::unwrap(ctx)->CreateTensor(DT_FLOAT, dim_sizes);
510   }
511 
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits512   static const char* ConvertScalar(PyObject* v, float* out) {
513     return ConvertOneFloat<float>(v, out);
514   }
515 };
516 
517 template <>
518 struct ConverterTraits<double> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits519   static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, double value) {
520     return tensorflow::unwrap(ctx)->CreateDoubleScalar(value);
521   }
522 
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits523   static AbstractTensorInterface* CreateTensor(
524       TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
525     return tensorflow::unwrap(ctx)->CreateTensor(DT_DOUBLE, dim_sizes);
526   }
527 
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits528   static const char* ConvertScalar(PyObject* v, double* out) {
529     return ConvertOneFloat<double>(v, out);
530   }
531 };
532 
533 typedef Converter<double> DoubleConverter;
534 typedef Converter<float> FloatConverter;
535 
536 template <>
537 struct ConverterTraits<Eigen::half> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits538   static AbstractTensorInterface* CreateScalar(TFE_Context* ctx,
539                                                Eigen::half value) {
540     return tensorflow::unwrap(ctx)->CreateHalfScalar(value);
541   }
542 
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits543   static AbstractTensorInterface* CreateTensor(
544       TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
545     return tensorflow::unwrap(ctx)->CreateTensor(DT_HALF, dim_sizes);
546   }
547 
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits548   static const char* ConvertScalar(PyObject* v, Eigen::half* out) {
549     return ConvertOneFloat<Eigen::half>(v, out);
550   }
551 };
552 
553 typedef Converter<Eigen::half> NumpyHalfConverter;
554 
555 // String support
556 
557 template <>
558 struct ConverterTraits<tstring> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits559   static AbstractTensorInterface* CreateScalar(TFE_Context* ctx,
560                                                tstring value) {
561     return tensorflow::unwrap(ctx)->CreateStringScalar(value);
562   }
563 
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits564   static AbstractTensorInterface* CreateTensor(
565       TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
566     return tensorflow::unwrap(ctx)->CreateTensor(DT_STRING, dim_sizes);
567   }
568 
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits569   static const char* ConvertScalar(PyObject* v, tstring* out) {
570     if (PyBytes_Check(v)) {
571       out->assign(PyBytes_AS_STRING(v), PyBytes_GET_SIZE(v));
572       return nullptr;
573     }
574     if (PyUnicode_Check(v)) {
575 #if PY_MAJOR_VERSION >= 3
576       Py_ssize_t size;
577       const char* str = PyUnicode_AsUTF8AndSize(v, &size);
578       if (str == nullptr) return ErrorConvertingUnicodeString;
579       out->assign(str, size);
580       return nullptr;
581 #else
582       PyObject* py_str = PyUnicode_AsUTF8String(v);
583       if (py_str == nullptr) return ErrorConvertingUnicodeString;
584       out->assign(PyBytes_AS_STRING(py_str), PyBytes_GET_SIZE(py_str));
585       Py_DECREF(py_str);
586       return nullptr;
587 #endif
588     }
589     return ErrorMixedTypes;
590   }
591 };
592 
593 typedef Converter<tstring> StringConverter;
594 
595 // Converts Python object `c` that should hold a Python string into a
596 // C++ string in *out.  Returns nullptr on success, or a message on error.
597 // Defined below, but forward declared here for use in PyRepr.
PyRepr(PyObject * obj)598 tstring PyRepr(PyObject* obj) {
599   if (obj == nullptr) {
600     return "<null>";
601   }
602   Safe_PyObjectPtr repr_obj = make_safe(PyObject_Repr(obj));
603   if (repr_obj) {
604     tstring repr_str;
605     if (ConverterTraits<tstring>::ConvertScalar(repr_obj.get(), &repr_str) ==
606         nullptr) {
607       return repr_str;
608     }
609   }
610   return "<error computing repr()>";
611 }
612 
IsPyDimension(PyObject * obj)613 bool IsPyDimension(PyObject* obj) {
614   const char* tp_name = obj->ob_type->tp_name;
615   if (strcmp(tp_name, "Dimension") != 0) return false;
616   bool ret = str_util::EndsWith(
617       PyRepr(PyType(obj)),
618       "tensorflow.python.framework.tensor_shape.Dimension'>");
619   return ret;
620 }
621 
622 // Complex support
623 
624 template <>
625 struct ConverterTraits<complex128> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits626   static AbstractTensorInterface* CreateScalar(TFE_Context* ctx,
627                                                complex128 value) {
628     return tensorflow::unwrap(ctx)->CreateComplex128Scalar(value);
629   }
630 
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits631   static AbstractTensorInterface* CreateTensor(
632       TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
633     return tensorflow::unwrap(ctx)->CreateTensor(DT_COMPLEX128, dim_sizes);
634   }
635 
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits636   static const char* ConvertScalar(PyObject* v, complex128* out) {
637     if (PyComplex_Check(v)) {
638       *out = complex128(PyComplex_RealAsDouble(v), PyComplex_ImagAsDouble(v));
639       return nullptr;
640     } else if (PyIsInstance(v, &PyComplexFloatingArrType_Type)) {  // NumPy
641       auto as_complex = PyComplex_AsCComplex(v);
642       *out = complex128(as_complex.real, as_complex.imag);
643       return nullptr;
644     }
645     double as_double;
646     auto error = ConvertOneFloat<double>(v, &as_double);
647     if (error != nullptr) return error;
648     *out = complex128(as_double, 0.0);
649     return nullptr;
650   }
651 };
652 
653 typedef Converter<complex128> Complex128Converter;
654 
655 // Bool support
656 
657 template <>
658 struct ConverterTraits<bool> {
CreateScalartensorflow::__anone25f2a6a0111::ConverterTraits659   static AbstractTensorInterface* CreateScalar(TFE_Context* ctx, bool value) {
660     return tensorflow::unwrap(ctx)->CreateBoolScalar(value);
661   }
662 
CreateTensortensorflow::__anone25f2a6a0111::ConverterTraits663   static AbstractTensorInterface* CreateTensor(
664       TFE_Context* ctx, absl::Span<const int64> dim_sizes) {
665     return tensorflow::unwrap(ctx)->CreateTensor(DT_BOOL, dim_sizes);
666   }
667 
ConvertScalartensorflow::__anone25f2a6a0111::ConverterTraits668   static const char* ConvertScalar(PyObject* v, bool* out) {
669     if (v == Py_True) {
670       *out = true;
671     } else if (v == Py_False) {
672       *out = false;
673     } else if (PyIsInstance(v, &PyBoolArrType_Type)) {  // NumPy
674       *out = PyObject_IsTrue(v);
675     } else {
676       return ErrorMixedTypes;
677     }
678     return nullptr;
679   }
680 };
681 
682 typedef Converter<bool> BoolConverter;
683 
684 // Convert a Python numpy.ndarray object to a TFE_TensorHandle.
685 // The two may share underlying storage so changes to one may reflect in the
686 // other.
NumpyToTFE_TensorHandle(TFE_Context * ctx,PyObject * obj)687 TFE_TensorHandle* NumpyToTFE_TensorHandle(TFE_Context* ctx, PyObject* obj) {
688   Safe_TF_TensorPtr tf_tensor = make_safe(static_cast<TF_Tensor*>(nullptr));
689   Status status = tensorflow::NdarrayToTensor(ctx, obj, &tf_tensor);
690 
691   if (TF_PREDICT_FALSE(!status.ok())) {
692     PyErr_SetString(PyExc_ValueError,
693                     tensorflow::strings::StrCat(
694                         "Failed to convert a NumPy array to a Tensor (",
695                         status.error_message(), ").")
696                         .c_str());
697     return nullptr;
698   }
699 
700   return tensorflow::wrap(
701       tensorflow::unwrap(ctx)->CreateLocalHandle(tf_tensor->tensor));
702 }
703 
704 }  // namespace
705 
706 // TODO(b/147743551): This function handles enough conversions to justify
707 // promoting to something like PyObjectToTensorHandle.
708 // TODO(b/147828820): Handle Tensors properly.
PySeqToTFE_TensorHandle(TFE_Context * ctx,PyObject * obj,DataType dtype)709 TFE_TensorHandle* PySeqToTFE_TensorHandle(TFE_Context* ctx, PyObject* obj,
710                                           DataType dtype) {
711   // Shortcut: __array__ objects (such as Pandas data frames).
712   // These objects are efficiently handled by Numpy. We transform them into
713   // Numpy arrays and handle them in the Numpy case below. Note that Tensors
714   // implement the __array__ function, and will be handled in this shortcut.
715   Safe_PyObjectPtr array =
716       make_safe(PyArray_FromArrayAttr(obj, nullptr, nullptr));
717   if (array == nullptr) {
718     return nullptr;
719   }
720   if (array.get() == Py_NotImplemented) {
721     // The Py_NotImplemented returned from PyArray_FromArrayAttr is not
722     // Py_INCREF'ed, so we don't want the Safe_PyObjectPtr to Py_DECREF it.
723     array.release();
724   } else {
725     // PyArray_FromArrayAttr ensures that `array` is a PyArrayObject, so all
726     // we have to do is replace `obj` with it and continue.
727     obj = array.get();
728   }
729 
730   // Shortcut: Numpy arrays.
731   if (PyArray_Check(obj)) {
732     int desired_np_dtype = -1;
733     if (dtype != tensorflow::DT_INVALID) {
734       if (!tensorflow::TF_DataType_to_PyArray_TYPE(
735                static_cast<TF_DataType>(dtype), &desired_np_dtype)
736                .ok()) {
737         PyErr_SetString(
738             PyExc_TypeError,
739             tensorflow::strings::StrCat("Invalid dtype argument value ", dtype)
740                 .c_str());
741         return nullptr;
742       }
743     }
744 
745     PyArrayObject* array = reinterpret_cast<PyArrayObject*>(obj);
746     int array_dtype = PyArray_TYPE(array);
747 
748     Safe_PyObjectPtr safe_value(nullptr);
749     // Use Numpy to convert between types if needed.
750     if ((desired_np_dtype >= 0 && desired_np_dtype != array_dtype) ||
751         !PyArray_ISCARRAY(array)) {
752       int new_dtype = desired_np_dtype >= 0 ? desired_np_dtype : array_dtype;
753       safe_value = tensorflow::make_safe(
754           PyArray_FromAny(obj, PyArray_DescrFromType(new_dtype), 0, 0,
755                           NPY_ARRAY_CARRAY_RO | NPY_ARRAY_FORCECAST, nullptr));
756       if (PyErr_Occurred()) return nullptr;
757       if (safe_value == nullptr) {
758         PyErr_SetString(PyExc_ValueError, "Error while casting a numpy value");
759       }
760       obj = safe_value.get();
761     }
762     return NumpyToTFE_TensorHandle(ctx, obj);
763   }
764 
765   ConverterState state;
766   Status status = InferShapeAndType(obj, &state);
767   if (!status.ok()) {
768     PyErr_SetString(PyExc_ValueError, status.error_message().c_str());
769     return nullptr;
770   }
771   DataType requested_dtype = DT_INVALID;
772   if (dtype != DT_INVALID) {
773     requested_dtype = dtype;
774   }
775 
776   // NOTE(josh11b): If don't successfully convert to the requested type,
777   // we just try instead to create a tensor of the inferred type and
778   // let the caller convert it to the requested type using a cast
779   // operation.
780   const char* error = nullptr;
781   TFE_TensorHandle* handle = nullptr;
782   status = errors::Unimplemented("Missing Python -> Tensor conversion for ",
783                                  DataTypeString(state.inferred_dtype));
784   switch (requested_dtype) {
785     case DT_FLOAT:
786       status = FloatConverter::Convert(ctx, obj, &state, &handle, &error);
787       break;
788 
789     case DT_DOUBLE:
790       status = DoubleConverter::Convert(ctx, obj, &state, &handle, &error);
791       break;
792 
793     case DT_HALF:
794       status = NumpyHalfConverter::Convert(ctx, obj, &state, &handle, &error);
795       break;
796 
797     case DT_INT64:
798       status = Int64Converter::Convert(ctx, obj, &state, &handle, &error);
799       break;
800 
801     case DT_INT32:
802       status = Int32Converter::Convert(ctx, obj, &state, &handle, &error);
803       break;
804 
805     case DT_UINT64:
806       status = UInt64Converter::Convert(ctx, obj, &state, &handle, &error);
807       break;
808 
809     case DT_COMPLEX128:
810       status = Complex128Converter::Convert(ctx, obj, &state, &handle, &error);
811       break;
812 
813     case DT_STRING:
814       status = StringConverter::Convert(ctx, obj, &state, &handle, &error);
815       break;
816 
817     case DT_BOOL:
818       status = BoolConverter::Convert(ctx, obj, &state, &handle, &error);
819       break;
820 
821     default:
822       break;
823   }
824   if (status.ok()) return handle;
825 
826   switch (state.inferred_dtype) {
827     case DT_FLOAT:
828       // TODO(josh11b): Handle mixed floats and complex numbers?
829       if (requested_dtype == DT_INVALID) {
830         // TensorFlow uses float32s to represent floating point numbers
831         // by default (for space and speed over using doubles).
832         status = FloatConverter::Convert(ctx, obj, &state, &handle, &error);
833       } else {
834         // We are going to do a cast to the user's requested dtype
835         // after this.  We use doubles for this intermediate result so
836         // we don't lose precision that might be representable in the
837         // final type.
838         status = DoubleConverter::Convert(ctx, obj, &state, &handle, &error);
839       }
840       break;
841 
842     case DT_DOUBLE:
843       status = DoubleConverter::Convert(ctx, obj, &state, &handle, &error);
844       break;
845 
846     case DT_HALF:
847       status = NumpyHalfConverter::Convert(ctx, obj, &state, &handle, &error);
848       break;
849 
850     case DT_INT64:
851       if (requested_dtype == DT_INVALID) {
852         status = Int32Converter::Convert(ctx, obj, &state, &handle, &error);
853         if (error == ErrorFoundInt64) {
854           status = Int64Converter::Convert(ctx, obj, &state, &handle, &error);
855         }
856         if (error == ErrorFoundFloat) {
857           status = FloatConverter::Convert(ctx, obj, &state, &handle, &error);
858         }
859         // TODO(josh11b): May also want to fall back to using doubles if
860         // error == ErrorOutOfRange?
861       } else {
862         status = Int64Converter::Convert(ctx, obj, &state, &handle, &error);
863         if (error == ErrorFoundFloat) {
864           status = DoubleConverter::Convert(ctx, obj, &state, &handle, &error);
865         }
866       }
867       break;
868 
869     case DT_STRING:
870       status = StringConverter::Convert(ctx, obj, &state, &handle, &error);
871       break;
872 
873     case DT_COMPLEX128:
874       status = Complex128Converter::Convert(ctx, obj, &state, &handle, &error);
875       break;
876 
877     case DT_BOOL:
878       status = BoolConverter::Convert(ctx, obj, &state, &handle, &error);
879       break;
880 
881     case DT_INVALID:  // Only occurs for empty tensors.
882     {
883       AbstractTensorInterface* t = tensorflow::unwrap(ctx)->CreateTensor(
884           requested_dtype == DT_INVALID ? DT_FLOAT : requested_dtype,
885           state.inferred_shape);
886       auto* result =
887           tensorflow::wrap(tensorflow::unwrap(ctx)->CreateLocalHandle(t));
888       t->Release();
889       return result;
890     }
891 
892     default:
893       break;
894   }
895 
896   if (!status.ok()) {
897     PyErr_SetString(PyExc_ValueError, status.error_message().c_str());
898     return nullptr;
899   }
900 
901   return handle;
902 }
903 
904 }  // namespace tensorflow
905