1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for SoftmaxOp and LogSoftmaxOp.""" 16 17import unittest 18 19import numpy as np 20 21 22from tensorflow.python.framework import constant_op 23from tensorflow.python.framework import dtypes 24from tensorflow.python.framework import errors_impl 25from tensorflow.python.ops import array_ops 26from tensorflow.python.ops import math_ops 27from tensorflow.python.ops import nn_ops 28from tensorflow.python.platform import test 29from tensorflow.python.platform import tf_logging as logging 30 31 32class SoftmaxTest(test.TestCase): 33 34 def _npSoftmax(self, features, dim=-1, log=False): 35 if dim == -1: 36 dim = len(features.shape) - 1 37 one_only_on_dim = list(features.shape) 38 one_only_on_dim[dim] = 1 39 is_fp16 = features.dtype == np.float16 40 if is_fp16: 41 # Do the compute in fp32 and cast the input back to fp32. 42 features = features.astype(np.float32) 43 e = np.exp(features - np.reshape( 44 np.amax( 45 features, axis=dim), one_only_on_dim)) 46 softmax = e / np.reshape(np.sum(e, axis=dim), one_only_on_dim) 47 if log: 48 res = np.log(softmax) 49 else: 50 res = softmax 51 if is_fp16: 52 res = res.astype(np.float16) 53 return res 54 55 def _testSoftmax(self, 56 np_features, 57 dim=-1, 58 log=False, 59 dtype=None, 60 use_gpu=False): 61 # A previous version of the code checked the op name rather than the op type 62 # to distinguish between log and non-log. Use an arbitrary name to catch 63 # this bug in future. 64 name = "arbitrary" 65 np_softmax = self._npSoftmax(np_features, dim=dim, log=log) 66 with self.cached_session(use_gpu=use_gpu): 67 if dtype is not None: 68 np_features = math_ops.cast(np_features, dtype=dtype) 69 70 if log: 71 tf_softmax = nn_ops.log_softmax(np_features, axis=dim, name=name) 72 else: 73 tf_softmax = nn_ops.softmax(np_features, axis=dim, name=name) 74 out = self.evaluate(tf_softmax) 75 self.assertAllCloseAccordingToType(np_softmax, out) 76 self.assertShapeEqual(np_softmax, tf_softmax) 77 if not log and dtype is None: 78 # Bonus check: the softmaxes should add to one in dimension dim. 79 sum_along_dim = np.sum(out, axis=dim) 80 self.assertAllCloseAccordingToType( 81 np.ones(sum_along_dim.shape), sum_along_dim) 82 83 def _testAll(self, features, dtype=None): 84 self._testSoftmax(features, dtype=dtype, use_gpu=True) 85 self._testSoftmax(features, dtype=dtype, log=True, use_gpu=True) 86 self._testOverflow(use_gpu=True) 87 88 def testNpSoftmax(self): 89 features = [[1., 1., 1., 1.], [1., 2., 3., 4.]] 90 # Batch 0: All exps are 1. The expected result is 91 # Softmaxes = [0.25, 0.25, 0.25, 0.25] 92 # LogSoftmaxes = [-1.386294, -1.386294, -1.386294, -1.386294] 93 # 94 # Batch 1: 95 # exps = [1., 2.718, 7.389, 20.085] 96 # sum = 31.192 97 # Softmaxes = exps / sum = [0.0320586, 0.08714432, 0.23688282, 0.64391426] 98 # LogSoftmaxes = [-3.44019 , -2.44019 , -1.44019 , -0.44019] 99 np_sm = self._npSoftmax(np.array(features)) 100 self.assertAllClose( 101 np.array([[0.25, 0.25, 0.25, 0.25], 102 [0.0320586, 0.08714432, 0.23688282, 0.64391426]]), 103 np_sm, 104 rtol=1.e-5, 105 atol=1.e-5) 106 np_lsm = self._npSoftmax(np.array(features), log=True) 107 self.assertAllClose( 108 np.array([[-1.386294, -1.386294, -1.386294, -1.386294], 109 [-3.4401897, -2.4401897, -1.4401897, -0.4401897]]), 110 np_lsm, 111 rtol=1.e-5, 112 atol=1.e-5) 113 114 def _testOverflow(self, use_gpu=False): 115 if use_gpu: 116 type = np.float32 # pylint: disable=redefined-builtin 117 else: 118 type = np.float64 # pylint: disable=redefined-builtin 119 max = np.finfo(type).max # pylint: disable=redefined-builtin 120 features = np.array([[1., 1., 1., 1.], [max, 1., 2., 3.]]).astype(type) 121 with self.cached_session(use_gpu=use_gpu): 122 tf_log_softmax = nn_ops.log_softmax(features) 123 out = self.evaluate(tf_log_softmax) 124 self.assertAllClose( 125 np.array([[-1.386294, -1.386294, -1.386294, -1.386294], 126 [0, -max, -max, -max]]), 127 out, 128 rtol=1.e-5, 129 atol=1.e-5) 130 131 def testFloat(self): 132 self._testAll( 133 np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float32)) 134 135 @unittest.skipUnless(test.is_built_with_gpu_support(), 136 "Test only applicable when running on GPUs") 137 def testFloatGPU(self): 138 if test.is_gpu_available(cuda_only=True): 139 rows = [2**x + np.random.randint(0, 16) for x in range(1, 4)] 140 cols = [2**x + np.random.randint(0, 16) for x in range(1, 4)] 141 for row, col in zip(rows, cols): 142 logging.info("Testing softmax float dtype in shape [%d, %d]", row, col) 143 data = np.random.rand(row, col) 144 self._testAll(data.astype(np.float32)) 145 146 def testHalf(self): 147 self._testAll( 148 np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float16)) 149 150 @unittest.skipUnless(test.is_built_with_gpu_support(), 151 "Test only applicable when running on GPUs") 152 def testHalfGPU(self): 153 if test.is_gpu_available(cuda_only=True): 154 rows = [2**x + np.random.randint(0, 16) for x in range(1, 4)] 155 cols = [2**x + np.random.randint(0, 16) for x in range(1, 4)] 156 for row, col in zip(rows, cols): 157 logging.info("Testing softmax half dtype in shape [%d, %d]", row, col) 158 data = np.random.rand(row, col) 159 self._testAll(data.astype(np.float16)) 160 161 def testDouble(self): 162 self._testSoftmax( 163 np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float64)) 164 self._testOverflow() 165 166 @unittest.skipUnless(test.is_built_with_gpu_support(), 167 "Test only applicable when running on GPUs") 168 def testDoubleGPU(self): 169 if test.is_gpu_available(cuda_only=True): 170 rows = [2**x + np.random.randint(0, 16) for x in range(1, 4)] 171 cols = [2**x + np.random.randint(0, 16) for x in range(1, 4)] 172 for row, col in zip(rows, cols): 173 logging.info("Testing softmax float dtype in shape [%d, %d]", row, col) 174 data = np.random.rand(row, col) 175 self._testAll(data.astype(np.float64)) 176 177 def testBfloat16(self): 178 self._testAll( 179 np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float32), 180 dtype=dtypes.bfloat16) 181 182 def test1DTensorAsInput(self): 183 self._testSoftmax( 184 np.array([3., 2., 3., 9.]).astype(np.float64), use_gpu=False) 185 self._testOverflow(use_gpu=False) 186 187 def test1DTensorAsInputNoReshape(self): 188 self._testSoftmax( 189 np.array([3., 2., 3., 9.]).astype(np.float64), use_gpu=False) 190 self._testOverflow(use_gpu=False) 191 192 def test3DTensorAsInput(self): 193 self._testSoftmax( 194 np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]], 195 [[2., 3., 4., 5.], [6., 7., 8., 9.]], 196 [[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32), 197 use_gpu=False) 198 self._testOverflow(use_gpu=False) 199 200 def test3DTensorAsInputNoReshape(self): 201 self._testSoftmax( 202 np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]], 203 [[2., 3., 4., 5.], [6., 7., 8., 9.]], 204 [[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32), 205 use_gpu=False) 206 self._testOverflow(use_gpu=False) 207 208 def testAlongFirstDimension(self): 209 self._testSoftmax( 210 np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]], 211 [[2., 3., 4., 5.], [6., 7., 8., 9.]], 212 [[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32), 213 dim=0, 214 use_gpu=False) 215 self._testOverflow(use_gpu=False) 216 217 def testAlongSecondDimension(self): 218 self._testSoftmax( 219 np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]], 220 [[2., 3., 4., 5.], [6., 7., 8., 9.]], 221 [[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32), 222 dim=1, 223 use_gpu=False) 224 self._testOverflow(use_gpu=False) 225 226 def testAlongNegativeDimension(self): 227 self._testSoftmax( 228 np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]], 229 [[2., 3., 4., 5.], [6., 7., 8., 9.]], 230 [[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32), 231 dim=-2, 232 use_gpu=False) 233 self._testOverflow(use_gpu=False) 234 235 def testShapeInference(self): 236 op = nn_ops.softmax([[[1., 1., 1., 1.], [1., 2., 3., 4.]], 237 [[2., 3., 4., 5.], [6., 7., 8., 9.]], 238 [[5., 4., 3., 2.], [1., 2., 3., 4.]]]) 239 self.assertEqual([3, 2, 4], op.get_shape()) 240 241 def testEmptyInput(self): 242 x = array_ops.ones(shape=[0, 3], dtype=dtypes.float32) 243 y = np.zeros(shape=[0, 3], dtype=np.float32) 244 self.assertEqual(0, self.evaluate(array_ops.size(x))) 245 self.assertAllEqual(y, self.evaluate(nn_ops.softmax(x, axis=0))) 246 247 def testDimTooLarge(self): 248 with self.cached_session(): 249 # Use placeholder to make sure we get runtime error instead of shape 250 # inference error. 251 dim = array_ops.placeholder_with_default(100, shape=[]) 252 with self.assertRaises(errors_impl.InvalidArgumentError): 253 nn_ops.softmax([1., 2., 3., 4.], axis=dim).eval() 254 255 def testInvalidAxis(self): 256 # Test case for GitHub issue 22793. 257 with self.cached_session(): 258 ones = array_ops.ones(shape=[2, 3]) 259 with self.assertRaises(errors_impl.InvalidArgumentError): 260 nn_ops.softmax(ones, axis=2).eval() 261 262 def testLargeDims(self): 263 # Make sure that we properly handle large inputs. See 264 # https://github.com/tensorflow/tensorflow/issues/4425 for details 265 for dims in [129, 256]: 266 ones = np.random.rand(dims, dims).astype(np.float32) 267 np_softmax = self._npSoftmax(ones) 268 269 for use_gpu in [True, False]: 270 with self.cached_session(use_gpu=use_gpu): 271 x = constant_op.constant(ones) 272 y = nn_ops.softmax(x) 273 tf_softmax = self.evaluate(y) 274 self.assertAllClose(tf_softmax, np_softmax) 275 276 277if __name__ == "__main__": 278 test.main() 279