1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for Relu and ReluGrad.""" 16 17import numpy as np 18 19from tensorflow.python.eager import backprop 20from tensorflow.python.framework import constant_op 21from tensorflow.python.framework import dtypes 22from tensorflow.python.framework import errors 23from tensorflow.python.framework import ops 24from tensorflow.python.framework import test_util 25from tensorflow.python.ops import gradient_checker_v2 26from tensorflow.python.ops import nn_ops 27from tensorflow.python.ops import random_ops 28from tensorflow.python.ops import variables 29import tensorflow.python.ops.nn_grad # pylint: disable=unused-import 30from tensorflow.python.platform import test 31from tensorflow.python.training import gradient_descent 32 33 34def _elu_grad_grad(activation): 35 if activation < 0: 36 return np.exp(activation) 37 return 0 38 39 40class ReluTest(test.TestCase): 41 42 def _npRelu(self, np_features): 43 return np.maximum(np_features, np.zeros(np_features.shape)) 44 45 def testNpRelu(self): 46 self.assertAllClose( 47 np.array([[0.0, 0.7, 0.0, 0.3, 0.0], [0.1, 0.0, 0.5, 0.0, 0.9]]), 48 self._npRelu( 49 np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 50 0.9]]))) 51 52 def _testRelu(self, np_features): 53 np_relu = self._npRelu(np_features) 54 tf_relu = nn_ops.relu(np_features) 55 self.assertAllClose(np_relu, tf_relu) 56 self.assertShapeEqual(np_relu, tf_relu) 57 58 def testNumbersCPU(self): 59 for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: 60 # Force execution on CPU even if a GPU kernel is available for the type. 61 with ops.device("/device:CPU:0"): 62 self._testRelu( 63 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 64 65 def testNumbersGPU(self): 66 if not test.is_gpu_available(): 67 self.skipTest("No GPU available") 68 for t in [np.float16, np.float32, np.float64]: 69 self._testRelu( 70 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 71 72 def testReluInt8x4GoodShape(self): 73 if not test.is_gpu_available(cuda_only=True): 74 self.skipTest("No GPU available") 75 inputs = np.array([[-50, 7, 23, 0], [-1, -5, 6, 11]]) 76 np_relu = self._npRelu(inputs) 77 tf_relu = nn_ops.relu(constant_op.constant(inputs, dtypes.qint8)) 78 self.assertAllClose(np_relu, tf_relu) 79 self.assertShapeEqual(np_relu, tf_relu) 80 81 @test_util.disable_xla("b/123338077") # Passes with XLA 82 def testReluInt8x4BadShape(self): 83 if not test.is_gpu_available(cuda_only=True): 84 self.skipTest("No GPU available") 85 inputs = constant_op.constant( 86 np.array([[-50, 7, 23], [0, 1, -5], [6, -2, 11]]), dtypes.qint8) 87 with self.assertRaisesRegex( 88 errors.InvalidArgumentError, 89 "Tensor size must be a multiple of 4 for Relu<qint8>. Got 9"): 90 self.evaluate(nn_ops.relu(inputs)) 91 92 inputs = constant_op.constant( 93 np.array([1, -2, 3, -4, 5, -6, 7, -8, 9, -8, 7, -6, 5, -4, 3, -2, 1]), 94 dtypes.qint8) 95 with self.assertRaisesRegex( 96 errors.InvalidArgumentError, 97 "Tensor size must be a multiple of 4 for Relu<qint8>. Got 17"): 98 self.evaluate(nn_ops.relu(inputs)) 99 100 def testNoElement(self): 101 self._testRelu(np.array([[], []], dtype=np.float32)) 102 103 @test_util.disable_xla("b/157978028: Does not yet pass with XLA") 104 def testNaNPropagation(self): 105 for t in [np.float16, np.float32, np.float64]: 106 self._testRelu(np.array([-1, np.nan, 1, np.nan]).astype(t)) 107 108 # The gradient test for ReLU is a bit tricky as the derivative is not well 109 # defined at around zero and we want to avoid that in terms of input values. 110 def testGradientFloat32(self): 111 with self.cached_session(): 112 x = np.asarray( 113 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 114 dtype=np.float32, 115 order="F") 116 err = gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient( 117 nn_ops.relu, [x], delta=1.0 / 1024)) 118 self.assertLess(err, 1e-6) 119 120 # The gradient test for ReLU is a bit tricky as the derivative is not well 121 # defined at around zero and we want to avoid that in terms of input values. 122 def testGradientFloat16(self): 123 with self.cached_session(): 124 x = np.asarray( 125 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 126 dtype=np.float16, 127 order="F") 128 err = gradient_checker_v2.max_error( 129 *gradient_checker_v2.compute_gradient(nn_ops.relu, [x])) 130 self.assertLess(err, 1e-6) 131 132 def testGradientFloat64(self): 133 with self.cached_session(): 134 x = np.asarray( 135 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 136 dtype=np.float64, 137 order="F") 138 err = gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient( 139 nn_ops.relu, [x], delta=1.0 / 1024)) 140 self.assertLess(err, 1e-15) 141 142 def testGradGradFloat32(self): 143 with self.cached_session(): 144 145 def f(x): 146 assert x.dtype == dtypes.float32 147 with backprop.GradientTape() as tape: 148 tape.watch(x) 149 y = nn_ops.relu(x) 150 return tape.gradient(y, x) 151 152 x = np.asarray( 153 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 154 dtype=np.float32, 155 order="F") 156 err = gradient_checker_v2.max_error( 157 *gradient_checker_v2.compute_gradient(f, [x], delta=1.0 / 1024)) 158 self.assertLess(err, 1e-4) 159 160 def testGradGradFloat64(self): 161 with self.cached_session(): 162 163 def f(x): 164 assert x.dtype == dtypes.float64 165 with backprop.GradientTape() as tape: 166 tape.watch(x) 167 y = nn_ops.relu(x) 168 return tape.gradient(y, x) 169 170 x = np.asarray( 171 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 172 dtype=np.float64, 173 order="F") 174 err = gradient_checker_v2.max_error( 175 *gradient_checker_v2.compute_gradient(f, [x], delta=1.0 / 1024)) 176 self.assertLess(err, 1e-10) 177 178 def testGradientScalar(self): 179 x = variables.Variable(100.) 180 181 def loss(): 182 return nn_ops.relu(x)**2 183 184 optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.25) 185 self.evaluate(variables.global_variables_initializer()) 186 self.evaluate(optimizer.minimize(loss)) 187 self.assertAllClose(x.read_value(), 50.0) 188 189 def testGradientNoElement(self): 190 with self.cached_session(): 191 192 def f(x): 193 with backprop.GradientTape() as tape: 194 tape.watch(x) 195 y = nn_ops.relu(x) 196 return tape.gradient(y, x) 197 198 x = np.asarray([[], []], dtype=np.float32) 199 z = list(gradient_checker_v2.compute_gradient(f, [x]))[0][0] 200 self.assertAllEqual(z, np.reshape(x, (0, 0))) 201 202 203class Relu6Test(test.TestCase): 204 205 def _npRelu6(self, np_features): 206 sixes = np.copy(np_features) 207 sixes.fill(6.0) 208 return np.minimum( 209 np.maximum(np_features, np.zeros(np_features.shape)), sixes) 210 211 def testNpRelu6(self): 212 self.assertAllClose( 213 np.array([[0.0, 0.7, 0.0, 0.3, 6.0], [0.1, 0.0, 6.0, 0.0, 0.9]]), 214 self._npRelu6( 215 np.array([[-0.9, 0.7, -0.5, 0.3, 6.0], [0.1, -0.3, 6.5, -0.7, 216 0.9]]))) 217 218 def _testRelu6(self, np_features): 219 np_relu6 = self._npRelu6(np_features) 220 tf_relu6 = nn_ops.relu6(np_features) 221 self.assertAllClose(np_relu6, tf_relu6) 222 self.assertShapeEqual(np_relu6, tf_relu6) 223 224 def testNumbersCPU(self): 225 for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: 226 # Force execution on CPU even if a GPU kernel is available for the type. 227 with ops.device("/device:CPU:0"): 228 self._testRelu6( 229 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 230 231 def testNumbersGPU(self): 232 if not test.is_gpu_available(): 233 self.skipTest("No GPU available") 234 for t in [np.float16, np.float64, np.double]: 235 self._testRelu6( 236 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 237 238 @test_util.disable_xla("b/157978028: Does not yet pass with XLA") 239 def testNaNPropagation(self): 240 for t in [np.float16, np.float32, np.float64]: 241 self._testRelu6(np.array([-1, np.nan, 1, 7, np.nan]).astype(t)) 242 243 # The gradient test for ReLU6 is a bit tricky as the derivative is 244 # not well defined at around zero and six and we want to avoid that 245 # in terms of input values. 246 def testGradientFloat32(self): 247 with self.cached_session(): 248 x = np.asarray( 249 [[-0.9, -0.7, -0.5, -0.3, -0.1], [6.1, 6.3, 6.5, 6.7, 6.9]], 250 dtype=np.float32, 251 order="F") 252 err = gradient_checker_v2.max_error( 253 *gradient_checker_v2.compute_gradient(nn_ops.relu6, [x])) 254 self.assertLess(err, 1e-4) 255 256 def testGradientFloat64(self): 257 with self.cached_session(): 258 x = np.asarray( 259 [[-0.9, -0.7, -0.5, -0.3, -0.1], [6.1, 6.3, 6.5, 6.7, 6.9]], 260 dtype=np.float64, 261 order="F") 262 err = gradient_checker_v2.max_error( 263 *gradient_checker_v2.compute_gradient(nn_ops.relu6, [x])) 264 self.assertLess(err, 1e-10) 265 266 267class LeakyReluTest(test.TestCase): 268 269 def _npLeakyRelu(self, np_features, alpha=0.1): 270 return np.maximum(np_features, alpha * np_features) 271 272 def testNpLeakyRelu(self): 273 self.assertAllClose( 274 np.array([[-0.09, 0.7, -0.05, 0.3, -0.01], 275 [0.1, -0.03, 0.5, -0.07, 0.9]]), 276 self._npLeakyRelu( 277 np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 278 0.9]]), 279 alpha=0.1)) 280 281 def _testLeakyRelu(self, np_features, alpha): 282 np_leaky_relu = self._npLeakyRelu(np_features, alpha) 283 tf_leaky_relu = nn_ops.leaky_relu(np_features, alpha) 284 self.assertAllCloseAccordingToType(np_leaky_relu, tf_leaky_relu) 285 self.assertShapeEqual(np_leaky_relu, tf_leaky_relu) 286 287 def testNumbersCPU(self): 288 for t in [ 289 np.int32, np.int64, np.float16, np.float32, np.float64, 290 dtypes.bfloat16.as_numpy_dtype 291 ]: 292 # Force execution on CPU even if a GPU kernel is available for the type. 293 with ops.device("/device:CPU:0"): 294 self._testLeakyRelu( 295 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), 296 alpha=0.2) 297 298 def testNumbersGPU(self): 299 if not test.is_gpu_available(): 300 self.skipTest("No GPU available") 301 for t in [np.float16, np.float32, np.float64]: 302 self._testLeakyRelu( 303 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), 304 alpha=0.1) 305 306 def testNaNPropagation(self): 307 for t in [np.float16, np.float32, np.float64]: 308 self._testLeakyRelu(np.array([-1, np.nan, 1, np.nan]).astype(t), 309 alpha=0.2) 310 311 # The gradient test for Leaky ReLU is a bit tricky as the derivative is not 312 # well defined at around zero and we want to avoid that in terms of input 313 # values. 314 def testGradientFloat32(self): 315 with self.cached_session(): 316 x = np.asarray( 317 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 318 dtype=np.float32, 319 order="F") 320 err = gradient_checker_v2.max_error( 321 *gradient_checker_v2.compute_gradient(nn_ops.leaky_relu, [x])) 322 self.assertLess(err, 1e-4) 323 324 def testGradientFloat64(self): 325 with self.cached_session(): 326 x = np.asarray( 327 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 328 dtype=np.float64, 329 order="F") 330 err = gradient_checker_v2.max_error( 331 *gradient_checker_v2.compute_gradient(nn_ops.leaky_relu, [x])) 332 self.assertLess(err, 1e-10) 333 334 def testGradGradFloat32(self): 335 with self.cached_session(): 336 337 def f(x): 338 assert x.dtype == dtypes.float32 339 with backprop.GradientTape() as tape: 340 tape.watch(x) 341 y = nn_ops.leaky_relu(x) 342 return tape.gradient(y, x) 343 344 x = np.asarray( 345 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 346 dtype=np.float32, 347 order="F") 348 err = gradient_checker_v2.max_error( 349 *gradient_checker_v2.compute_gradient(f, [x])) 350 self.assertLess(err, 1e-4) 351 352 def testGradGradFloat64(self): 353 with self.cached_session(): 354 355 def f(x): 356 assert x.dtype == dtypes.float64 357 with backprop.GradientTape() as tape: 358 tape.watch(x) 359 y = nn_ops.leaky_relu(x) 360 return tape.gradient(y, x) 361 362 x = np.asarray( 363 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 364 dtype=np.float64, 365 order="F") 366 err = gradient_checker_v2.max_error( 367 *gradient_checker_v2.compute_gradient(f, [x])) 368 self.assertLess(err, 1e-10) 369 370 def testGradientScalar(self): 371 x = variables.Variable(-100.) 372 373 def loss(): 374 return nn_ops.leaky_relu(x, 0.05)**2 375 376 optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2) 377 self.evaluate(variables.global_variables_initializer()) 378 self.evaluate(optimizer.minimize(loss)) 379 self.assertAllClose(x.read_value(), -99.9) 380 381 def testUnexpectedAlphaValue(self): 382 self.assertAllClose( 383 np.array([[-9.0, 0.7, -5.0, 0.3, -0.1], [0.1, -3.0, 0.5, -27.0, 0.9]]), 384 nn_ops.leaky_relu( 385 np.array([[-0.9, 0.7, -0.5, 0.3, -0.01], 386 [0.1, -0.3, 0.5, -2.7, 0.9]]), 387 alpha=10)) 388 self.assertAllClose( 389 np.array([[9.0, 0.7, 5.0, 0.3, 0.1], [0.1, 3.0, 0.5, 27.0, 0.9]]), 390 nn_ops.leaky_relu( 391 np.array([[-0.9, 0.7, -0.5, 0.3, -0.01], 392 [0.1, -0.3, 0.5, -2.7, 0.9]]), 393 alpha=-10)) 394 395 396class EluTest(test.TestCase): 397 398 def _npElu(self, np_features): 399 return np.where(np_features < 0, np.exp(np_features) - 1, np_features) 400 401 def testNpElu(self): 402 self.assertAllClose( 403 np.array([[-0.59343034025, 0.7, -0.39346934028, 0.3, -0.09516258196], 404 [0.1, -0.25918177931, 0.5, -0.5034146962, 0.9]]), 405 self._npElu( 406 np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 407 0.9]]))) 408 409 def _testElu(self, np_features): 410 np_elu = self._npElu(np_features) 411 tf_elu = nn_ops.elu(np_features) 412 self.assertAllCloseAccordingToType(np_elu, tf_elu) 413 self.assertShapeEqual(np_elu, tf_elu) 414 415 def testNumbersCPU(self): 416 for t in [np.float16, np.float32, np.float64]: 417 # Force execution on CPU even if a GPU kernel is available for the type. 418 with ops.device("/device:CPU:0"): 419 self._testElu( 420 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 421 422 def testNumbersGPU(self): 423 if not test.is_gpu_available(): 424 self.skipTest("No GPU available") 425 for t in [np.float16, np.float32, np.float64]: 426 self._testElu(np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 427 428 def testNaNPropagation(self): 429 for t in [np.float16, np.float32, np.float64]: 430 self._testElu(np.array([-1, np.nan, 1, np.nan]).astype(t)) 431 432 def testGradientFloat32(self): 433 with self.cached_session(): 434 x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] 435 x = np.asarray(x_val, dtype=np.float32, order="F") 436 err = gradient_checker_v2.max_error( 437 *gradient_checker_v2.compute_gradient(nn_ops.elu, [x])) 438 self.assertLess(err, 1e-4) 439 440 def testGradientFloat64(self): 441 with self.cached_session(): 442 x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] 443 x = np.asarray(x_val, dtype=np.float64, order="F") 444 err = gradient_checker_v2.max_error( 445 *gradient_checker_v2.compute_gradient(nn_ops.elu, [x])) 446 self.assertLess(err, 1e-6) 447 448 def testGradGrad(self): 449 with self.cached_session(): 450 451 def f(x): 452 with backprop.GradientTape(persistent=True) as tape: 453 tape.watch(x) 454 y = nn_ops.elu(x) 455 dy = tape.gradient(y, x) 456 return tape.gradient(dy, x) 457 458 for x in [-1., -0.5, 0.5, 1.]: 459 got = self.evaluate(f(constant_op.constant(x))) 460 want = _elu_grad_grad(x) 461 err = np.abs(got - want) 462 self.assertLess(err, 1e-4) 463 464 def testGradGradFloat32(self): 465 with self.cached_session(): 466 467 def f(x): 468 assert x.dtype == dtypes.float32 469 with backprop.GradientTape() as tape: 470 tape.watch(x) 471 y = nn_ops.elu(x) 472 return tape.gradient(y, x) 473 474 x = np.asarray( 475 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 476 dtype=np.float32, 477 order="F") 478 err = gradient_checker_v2.max_error( 479 *gradient_checker_v2.compute_gradient(f, [x])) 480 self.assertLess(err, 1e-4) 481 482 def testGradGradFloat64(self): 483 with self.cached_session(): 484 485 def f(x): 486 assert x.dtype == dtypes.float64 487 with backprop.GradientTape() as tape: 488 tape.watch(x) 489 y = nn_ops.elu(x) 490 return tape.gradient(y, x) 491 492 x = np.asarray( 493 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 494 dtype=np.float64, 495 order="F") 496 err = gradient_checker_v2.max_error( 497 *gradient_checker_v2.compute_gradient(f, [x])) 498 self.assertLess(err, 1e-6) 499 500 501class SeluTest(test.TestCase): 502 503 def _npSelu(self, np_features): 504 scale = 1.0507009873554804934193349852946 505 scale_alpha = 1.7580993408473768599402175208123 506 return np.where(np_features < 0, scale_alpha * (np.exp(np_features) - 1), 507 scale * np_features) 508 509 def testNpSelu(self): 510 self.assertAllClose( 511 np.array([[-1.0433095, 0.73549069, -0.6917582, 0.3152103, -0.16730527], 512 [0.1050701, -0.45566732, 0.5253505, -0.88505305, 0.9456309]]), 513 self._npSelu( 514 np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 515 0.9]]))) 516 517 def _testSelu(self, np_features): 518 np_selu = self._npSelu(np_features) 519 tf_selu = nn_ops.selu(np_features) 520 self.assertAllCloseAccordingToType(np_selu, tf_selu) 521 self.assertShapeEqual(np_selu, tf_selu) 522 523 def testNumbers(self): 524 for t in [np.float16, np.float32, np.float64]: 525 self._testSelu( 526 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 527 # Force executed on CPU in case GPU kernels are available. 528 with ops.device("/device:CPU:0"): 529 self._testSelu( 530 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 531 532 def testGradientFloat32(self): 533 with self.cached_session(): 534 x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] 535 x = np.asarray(x_val, dtype=np.float32, order="F") 536 err = gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient( 537 nn_ops.selu, [x], delta=1.0 / 1024)) 538 self.assertLess(err, 1e-4) 539 540 def testGradientFloat64(self): 541 with self.cached_session(): 542 x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] 543 x = np.asarray(x_val, dtype=np.float64, order="F") 544 err = gradient_checker_v2.max_error( 545 *gradient_checker_v2.compute_gradient(nn_ops.selu, [x])) 546 self.assertLess(err, 1e-6) 547 548 def testGradGradFloat32(self): 549 with self.cached_session(): 550 551 def f(x): 552 assert x.dtype == dtypes.float32 553 with backprop.GradientTape() as tape: 554 tape.watch(x) 555 y = nn_ops.selu(x) 556 return tape.gradient(y, x) 557 558 x = np.asarray( 559 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 560 dtype=np.float32, 561 order="F") 562 err = gradient_checker_v2.max_error( 563 *gradient_checker_v2.compute_gradient(f, [x], delta=1.0 / 1024)) 564 self.assertLess(err, 1e-4) 565 566 def testGradGradFloat64(self): 567 with self.cached_session(): 568 569 def f(x): 570 assert x.dtype == dtypes.float64 571 with backprop.GradientTape() as tape: 572 tape.watch(x) 573 y = nn_ops.selu(x) 574 return tape.gradient(y, x) 575 576 x = np.asarray( 577 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 578 dtype=np.float64, 579 order="F") 580 err = gradient_checker_v2.max_error( 581 *gradient_checker_v2.compute_gradient(f, [x])) 582 self.assertLess(err, 1e-6) 583 584 585class CreluTest(test.TestCase): 586 587 def testCreluShape(self): 588 f = random_ops.random_normal([50, 5, 7, 10]) 589 t = nn_ops.crelu(f) 590 self.assertEqual([50, 5, 7, 20], t.get_shape()) 591 592 def _testCrelu(self, np_features): 593 np_relu = np.maximum(np_features, np.zeros_like(np_features)) 594 np_neg_relu = np.maximum(-np_features, np.zeros_like(np_features)) 595 np_crelu = np.concatenate((np_relu, np_neg_relu), 596 len(np_features.shape) - 1) 597 598 tf_crelu = nn_ops.crelu(np_features) 599 600 self.assertAllClose(np_crelu, tf_crelu) 601 self.assertShapeEqual(np_crelu, tf_crelu) 602 603 def testNumbersCPU(self): 604 for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: 605 # Force execution on CPU even if a GPU kernel is available for the type. 606 with ops.device("/device:CPU:0"): 607 self._testCrelu( 608 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 609 610 def testNumbersGPU(self): 611 if not test.is_gpu_available(): 612 self.skipTest("No GPU available") 613 for t in [np.float16, np.float32, np.float64]: 614 self._testCrelu( 615 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 616 617 def testNumbersWithAxis0(self): 618 tf_crelu = nn_ops.crelu( 619 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=0) 620 np_crelu = np.array([[0, 7, 0, 3, 0], [1, 0, 5, 0, 9], [9, 0, 5, 0, 1], 621 [0, 3, 0, 7, 0]]) 622 self.assertAllEqual(np_crelu, tf_crelu) 623 624 def testNumbersWithAxis1(self): 625 tf_crelu = nn_ops.crelu( 626 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=1) 627 np_crelu = np.array([[0, 7, 0, 3, 0, 9, 0, 5, 0, 1], 628 [1, 0, 5, 0, 9, 0, 3, 0, 7, 0]]) 629 self.assertAllEqual(np_crelu, tf_crelu) 630 631 632if __name__ == "__main__": 633 test.main() 634