1# Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Defines functions common to multiple feature column files.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import six 22 23from tensorflow.python.framework import dtypes 24from tensorflow.python.framework import ops 25from tensorflow.python.ops import array_ops 26from tensorflow.python.ops import math_ops 27from tensorflow.python.util import nest 28 29 30def sequence_length_from_sparse_tensor(sp_tensor, num_elements=1): 31 """Returns a [batch_size] Tensor with per-example sequence length.""" 32 with ops.name_scope(None, 'sequence_length') as name_scope: 33 row_ids = sp_tensor.indices[:, 0] 34 column_ids = sp_tensor.indices[:, 1] 35 # Add one to convert column indices to element length 36 column_ids += array_ops.ones_like(column_ids) 37 # Get the number of elements we will have per example/row 38 seq_length = math_ops.segment_max(column_ids, segment_ids=row_ids) 39 40 # The raw values are grouped according to num_elements; 41 # how many entities will we have after grouping? 42 # Example: orig tensor [[1, 2], [3]], col_ids = (0, 1, 1), 43 # row_ids = (0, 0, 1), seq_length = [2, 1]. If num_elements = 2, 44 # these will get grouped, and the final seq_length is [1, 1] 45 seq_length = math_ops.cast( 46 math_ops.ceil(seq_length / num_elements), dtypes.int64) 47 48 # If the last n rows do not have ids, seq_length will have shape 49 # [batch_size - n]. Pad the remaining values with zeros. 50 n_pad = array_ops.shape(sp_tensor)[:1] - array_ops.shape(seq_length)[:1] 51 padding = array_ops.zeros(n_pad, dtype=seq_length.dtype) 52 return array_ops.concat([seq_length, padding], axis=0, name=name_scope) 53 54 55def assert_string_or_int(dtype, prefix): 56 if (dtype != dtypes.string) and (not dtype.is_integer): 57 raise ValueError( 58 '{} dtype must be string or integer. dtype: {}.'.format(prefix, dtype)) 59 60 61def assert_key_is_string(key): 62 if not isinstance(key, six.string_types): 63 raise ValueError( 64 'key must be a string. Got: type {}. Given key: {}.'.format( 65 type(key), key)) 66 67 68def check_default_value(shape, default_value, dtype, key): 69 """Returns default value as tuple if it's valid, otherwise raises errors. 70 71 This function verifies that `default_value` is compatible with both `shape` 72 and `dtype`. If it is not compatible, it raises an error. If it is compatible, 73 it casts default_value to a tuple and returns it. `key` is used only 74 for error message. 75 76 Args: 77 shape: An iterable of integers specifies the shape of the `Tensor`. 78 default_value: If a single value is provided, the same value will be applied 79 as the default value for every item. If an iterable of values is 80 provided, the shape of the `default_value` should be equal to the given 81 `shape`. 82 dtype: defines the type of values. Default value is `tf.float32`. Must be a 83 non-quantized, real integer or floating point type. 84 key: Column name, used only for error messages. 85 86 Returns: 87 A tuple which will be used as default value. 88 89 Raises: 90 TypeError: if `default_value` is an iterable but not compatible with `shape` 91 TypeError: if `default_value` is not compatible with `dtype`. 92 ValueError: if `dtype` is not convertible to `tf.float32`. 93 """ 94 if default_value is None: 95 return None 96 97 if isinstance(default_value, int): 98 return _create_tuple(shape, default_value) 99 100 if isinstance(default_value, float) and dtype.is_floating: 101 return _create_tuple(shape, default_value) 102 103 if callable(getattr(default_value, 'tolist', None)): # Handles numpy arrays 104 default_value = default_value.tolist() 105 106 if nest.is_sequence(default_value): 107 if not _is_shape_and_default_value_compatible(default_value, shape): 108 raise ValueError( 109 'The shape of default_value must be equal to given shape. ' 110 'default_value: {}, shape: {}, key: {}'.format( 111 default_value, shape, key)) 112 # Check if the values in the list are all integers or are convertible to 113 # floats. 114 is_list_all_int = all( 115 isinstance(v, int) for v in nest.flatten(default_value)) 116 is_list_has_float = any( 117 isinstance(v, float) for v in nest.flatten(default_value)) 118 if is_list_all_int: 119 return _as_tuple(default_value) 120 if is_list_has_float and dtype.is_floating: 121 return _as_tuple(default_value) 122 raise TypeError('default_value must be compatible with dtype. ' 123 'default_value: {}, dtype: {}, key: {}'.format( 124 default_value, dtype, key)) 125 126 127def _create_tuple(shape, value): 128 """Returns a tuple with given shape and filled with value.""" 129 if shape: 130 return tuple([_create_tuple(shape[1:], value) for _ in range(shape[0])]) 131 return value 132 133 134def _as_tuple(value): 135 if not nest.is_sequence(value): 136 return value 137 return tuple([_as_tuple(v) for v in value]) 138 139 140def _is_shape_and_default_value_compatible(default_value, shape): 141 """Verifies compatibility of shape and default_value.""" 142 # Invalid condition: 143 # * if default_value is not a scalar and shape is empty 144 # * or if default_value is an iterable and shape is not empty 145 if nest.is_sequence(default_value) != bool(shape): 146 return False 147 if not shape: 148 return True 149 if len(default_value) != shape[0]: 150 return False 151 for i in range(shape[0]): 152 if not _is_shape_and_default_value_compatible(default_value[i], shape[1:]): 153 return False 154 return True 155