1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Functional tests for BiasAdd.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import numpy as np 22 23from tensorflow.python.framework import constant_op 24from tensorflow.python.framework import dtypes 25from tensorflow.python.framework import test_util 26from tensorflow.python.ops import array_ops 27from tensorflow.python.ops import gradient_checker 28from tensorflow.python.ops import gradients_impl 29from tensorflow.python.ops import nn_ops 30import tensorflow.python.ops.nn_grad # pylint: disable=unused-import 31from tensorflow.python.platform import test 32 33 34class BiasAddTestBase(test.TestCase): 35 36 def _npBias(self, inputs, bias): 37 assert len(bias.shape) == 1 38 assert inputs.shape[-1] == bias.shape[0] 39 return inputs + bias.reshape(([1] * 40 (len(inputs.shape) - 1)) + [bias.shape[0]]) 41 42 def testNpBias(self): 43 self.assertAllClose( 44 np.array([[11, 22, 33], [41, 52, 63]]), 45 self._npBias( 46 np.array([[10, 20, 30], [40, 50, 60]]), np.array([1, 2, 3]))) 47 48 def _testBias(self, np_inputs, np_bias, use_gpu=False): 49 np_val = self._npBias(np_inputs, np_bias) 50 with self.cached_session(use_gpu=use_gpu): 51 tf_val = nn_ops.bias_add(np_inputs, np_bias).eval() 52 self.assertAllCloseAccordingToType(np_val, tf_val) 53 54 def _AtLeast3d(self, np_value): 55 # fill the input value to at least 3-dimension 56 if np_value.ndim < 3: 57 return np.reshape(np_value, (1,) * (3 - np_value.ndim) + np_value.shape) 58 return np_value 59 60 def _NHWCToNCHW(self, np_value): 61 # fill the input value to at least 3-dimension 62 np_value = self._AtLeast3d(np_value) 63 # move the last dimension to second 64 np_dim = list(range(np_value.ndim)) 65 np_dim_new = list(np_dim[0:1]) + list(np_dim[-1:]) + list(np_dim[1:-1]) 66 return np.transpose(np_value, np_dim_new) 67 68 def _NCHWToNHWC(self, np_value): 69 assert len(np_value.shape) >= 3 70 np_dim = list(range(np_value.ndim)) 71 # move the second dimension to the last 72 np_dim_new = list(np_dim[0:1]) + list(np_dim[2:]) + list(np_dim[1:2]) 73 return np.transpose(np_value, np_dim_new) 74 75 def _testBiasNCHW(self, np_inputs, np_bias, use_gpu): 76 np_val = self._npBias(np_inputs, np_bias) 77 np_inputs = self._NHWCToNCHW(np_inputs) 78 with self.cached_session(use_gpu=use_gpu): 79 tf_val = nn_ops.bias_add(np_inputs, np_bias, data_format="NCHW").eval() 80 tf_val = self._NCHWToNHWC(tf_val) 81 self.assertAllCloseAccordingToType(self._AtLeast3d(np_val), tf_val) 82 83 def _testAll(self, np_inputs, np_bias): 84 self._testBias(np_inputs, np_bias, use_gpu=False) 85 self._testBiasNCHW(np_inputs, np_bias, use_gpu=False) 86 if np_inputs.dtype in [np.float16, np.float32, np.float64, np.int32]: 87 self._testBias(np_inputs, np_bias, use_gpu=True) 88 self._testBiasNCHW(np_inputs, np_bias, use_gpu=True) 89 90 @test_util.run_deprecated_v1 91 def testInputDims(self): 92 with self.assertRaises(ValueError): 93 nn_ops.bias_add([1, 2], [1]) 94 95 @test_util.run_deprecated_v1 96 def testBiasVec(self): 97 with self.assertRaises(ValueError): 98 nn_ops.bias_add( 99 array_ops.reshape([1, 2], shape=[1, 2]), 100 array_ops.reshape([1, 2], shape=[1, 2])) 101 102 @test_util.run_deprecated_v1 103 def testBiasInputsMatch(self): 104 with self.assertRaises(ValueError): 105 nn_ops.bias_add( 106 array_ops.reshape([1, 2], shape=[1, 2]), 107 array_ops.reshape([1], shape=[1])) 108 109 @test_util.run_deprecated_v1 110 def testIntTypes(self): 111 for t in [np.int8, np.int16, np.int32, np.int64]: 112 self._testAll( 113 np.array([[10, 20, 30], [40, 50, 60]]).astype(t), 114 np.array([1, 2, 3]).astype(t)) 115 116 @test_util.run_deprecated_v1 117 def testFloatTypes(self): 118 for t in [np.float16, np.float32, np.float64]: 119 self._testAll( 120 np.random.rand(4, 3, 3).astype(t), 121 np.random.rand(3).astype(t)) 122 123 @test_util.run_deprecated_v1 124 def test4DFloatTypes(self): 125 for t in [np.float16, np.float32, np.float64]: 126 self._testAll( 127 np.random.rand(4, 3, 2, 3).astype(t), 128 np.random.rand(3).astype(t)) 129 self._testAll( 130 np.random.rand(2048, 4, 4, 4).astype(t), 131 np.random.rand(4).astype(t)) 132 self._testAll( 133 np.random.rand(4, 4, 4, 2048).astype(t), 134 np.random.rand(2048).astype(t)) 135 136 @test_util.run_deprecated_v1 137 def test5DFloatTypes(self): 138 for t in [np.float16, np.float32, np.float64]: 139 self._testAll( 140 np.random.rand(4, 3, 2, 3, 4).astype(t), 141 np.random.rand(4).astype(t)) 142 143 def _testGradient(self, np_input, bias, dtype, data_format, use_gpu): 144 with self.cached_session(use_gpu=use_gpu): 145 if data_format == "NCHW": 146 np_input = self._NHWCToNCHW(np_input) 147 input_tensor = constant_op.constant( 148 np_input, shape=np_input.shape, dtype=dtype) 149 bias_tensor = constant_op.constant(bias, shape=bias.shape, dtype=dtype) 150 output_tensor = nn_ops.bias_add( 151 input_tensor, bias_tensor, data_format=data_format) 152 tensor_jacob_t, tensor_jacob_n = gradient_checker.compute_gradient( 153 input_tensor, np_input.shape, output_tensor, np_input.shape) 154 bias_jacob_t, bias_jacob_n = gradient_checker.compute_gradient( 155 bias_tensor, bias.shape, output_tensor, np_input.shape) 156 157 # Test gradient of BiasAddGrad 158 bias_add_grad = gradients_impl.gradients( 159 nn_ops.l2_loss(output_tensor), bias_tensor)[0] 160 grad_jacob_t, grad_jacob_n = gradient_checker.compute_gradient( 161 output_tensor, np_input.shape, bias_add_grad, bias.shape) 162 163 if dtype == np.float16: 164 # Compare fp16 analytical gradients to fp32 numerical gradients, 165 # since fp16 numerical gradients are too imprecise unless great 166 # care is taken with choosing the inputs and the delta. This is 167 # a weaker, but pragmatic, check (in particular, it does not test 168 # the op itself, only its gradient). 169 input_tensor = constant_op.constant( 170 np_input, shape=np_input.shape, dtype=np.float32) 171 bias_tensor = constant_op.constant( 172 bias, shape=bias.shape, dtype=np.float32) 173 output_tensor = nn_ops.bias_add( 174 input_tensor, bias_tensor, data_format=data_format) 175 _, tensor_jacob_n = gradient_checker.compute_gradient( 176 input_tensor, np_input.shape, output_tensor, np_input.shape) 177 _, bias_jacob_n = gradient_checker.compute_gradient( 178 bias_tensor, bias.shape, output_tensor, np_input.shape) 179 180 bias_add_grad = gradients_impl.gradients( 181 nn_ops.l2_loss(output_tensor), bias_tensor)[0] 182 _, grad_jacob_n = gradient_checker.compute_gradient( 183 output_tensor, np_input.shape, bias_add_grad, bias.shape) 184 185 threshold = 5e-3 186 if dtype == dtypes.float64: 187 threshold = 1e-10 188 self.assertAllClose(tensor_jacob_t, tensor_jacob_n, threshold, threshold) 189 self.assertAllClose(bias_jacob_t, bias_jacob_n, threshold, threshold) 190 self.assertAllClose(grad_jacob_t, grad_jacob_n, threshold, threshold) 191 192 @test_util.run_deprecated_v1 193 def testGradientTensor2D(self): 194 for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True): 195 for dtype in (dtypes.float16, dtypes.float32, dtypes.float64): 196 np_input = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 197 dtype=dtype.as_numpy_dtype).reshape(3, 2) 198 bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype) 199 self._testGradient(np_input, bias, dtype, data_format, use_gpu) 200 201 @test_util.run_deprecated_v1 202 def testGradientTensor3D(self): 203 for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True), 204 ("NCHW", False), ("NCHW", True)]: 205 for dtype in (dtypes.float16, dtypes.float32, dtypes.float64): 206 np_input = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 207 dtype=dtype.as_numpy_dtype).reshape(1, 3, 2) 208 bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype) 209 self._testGradient(np_input, bias, dtype, data_format, use_gpu) 210 211 @test_util.run_deprecated_v1 212 def testGradientTensor4D(self): 213 for (data_format, use_gpu) in [("NHWC", False)]: 214 for dtype in (dtypes.float16, dtypes.float32, dtypes.float64): 215 np_input = np.arange( 216 1.0, 49.0, 217 dtype=dtype.as_numpy_dtype).reshape([2, 3, 4, 2]).astype(np.float32) 218 bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype) 219 self._testGradient(np_input, bias, dtype, data_format, use_gpu) 220 np_input = np.arange( 221 1.0, 513.0, 222 dtype=dtype.as_numpy_dtype).reshape([64, 2, 2, 223 2]).astype(np.float32) 224 self._testGradient(np_input, bias, dtype, data_format, use_gpu) 225 np_input = np.arange( 226 1.0, 513.0, 227 dtype=dtype.as_numpy_dtype).reshape([2, 2, 2, 228 64]).astype(np.float32) 229 self._testGradient(np_input, 230 np.random.rand(64).astype(dtype.as_numpy_dtype), 231 dtype, data_format, use_gpu) 232 233 @test_util.run_deprecated_v1 234 def testGradientTensor5D(self): 235 for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True), 236 ("NCHW", False), ("NCHW", True)]: 237 for dtype in (dtypes.float16, dtypes.float32, dtypes.float64): 238 np_input = np.arange( 239 1.0, 49.0, 240 dtype=dtype.as_numpy_dtype).reshape([1, 2, 3, 4, 241 2]).astype(np.float32) 242 bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype) 243 self._testGradient(np_input, bias, dtype, data_format, use_gpu) 244 245 @test_util.run_deprecated_v1 246 def testEmpty(self): 247 np.random.seed(7) 248 for shape in (0, 0), (2, 0), (0, 2), (4, 3, 0), (4, 0, 3), (0, 4, 3): 249 self._testAll(np.random.randn(*shape), np.random.randn(shape[-1])) 250 251 @test_util.run_deprecated_v1 252 def testEmptyGradient(self): 253 for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True): 254 for shape in (0, 0), (2, 0), (0, 2): 255 self._testGradient( 256 np.random.randn(*shape), np.random.randn(shape[-1]), dtypes.float64, 257 data_format, use_gpu) 258 259 for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True), 260 ("NCHW", False), ("NCHW", True)]: 261 for shape in (4, 3, 0), (4, 0, 3), (0, 4, 3): 262 self._testGradient( 263 np.random.randn(*shape), np.random.randn(shape[-1]), dtypes.float64, 264 data_format, use_gpu) 265