1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for Relu and ReluGrad.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import numpy as np 22 23from tensorflow.python.eager import backprop 24from tensorflow.python.framework import constant_op 25from tensorflow.python.framework import dtypes 26from tensorflow.python.framework import errors 27from tensorflow.python.framework import ops 28from tensorflow.python.framework import test_util 29from tensorflow.python.ops import gradient_checker_v2 30from tensorflow.python.ops import nn_ops 31from tensorflow.python.ops import random_ops 32from tensorflow.python.ops import variables 33import tensorflow.python.ops.nn_grad # pylint: disable=unused-import 34from tensorflow.python.platform import test 35from tensorflow.python.training import gradient_descent 36 37 38def _elu_grad_grad(activation): 39 if activation < 0: 40 return np.exp(activation) 41 return 0 42 43 44class ReluTest(test.TestCase): 45 46 def _npRelu(self, np_features): 47 return np.maximum(np_features, np.zeros(np_features.shape)) 48 49 def testNpRelu(self): 50 self.assertAllClose( 51 np.array([[0.0, 0.7, 0.0, 0.3, 0.0], [0.1, 0.0, 0.5, 0.0, 0.9]]), 52 self._npRelu( 53 np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 54 0.9]]))) 55 56 def _testRelu(self, np_features): 57 np_relu = self._npRelu(np_features) 58 tf_relu = nn_ops.relu(np_features) 59 self.assertAllClose(np_relu, tf_relu) 60 self.assertShapeEqual(np_relu, tf_relu) 61 62 def testNumbersCPU(self): 63 for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: 64 # Force execution on CPU even if a GPU kernel is available for the type. 65 with ops.device("/device:CPU:0"): 66 self._testRelu( 67 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 68 69 def testNumbersGPU(self): 70 if not test.is_gpu_available(): 71 self.skipTest("No GPU available") 72 for t in [np.float16, np.float32, np.float64]: 73 self._testRelu( 74 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 75 76 def testReluInt8x4GoodShape(self): 77 if not test.is_gpu_available(cuda_only=True): 78 self.skipTest("No GPU available") 79 inputs = np.array([[-50, 7, 23, 0], [-1, -5, 6, 11]]) 80 np_relu = self._npRelu(inputs) 81 tf_relu = nn_ops.relu(constant_op.constant(inputs, dtypes.qint8)) 82 self.assertAllClose(np_relu, tf_relu) 83 self.assertShapeEqual(np_relu, tf_relu) 84 85 @test_util.disable_xla("b/123338077") # Passes with XLA 86 def testReluInt8x4BadShape(self): 87 if not test.is_gpu_available(cuda_only=True): 88 self.skipTest("No GPU available") 89 inputs = constant_op.constant( 90 np.array([[-50, 7, 23], [0, 1, -5], [6, -2, 11]]), dtypes.qint8) 91 with self.assertRaisesRegex( 92 errors.InvalidArgumentError, 93 "Tensor size must be a multiple of 4 for Relu<qint8>. Got 9"): 94 self.evaluate(nn_ops.relu(inputs)) 95 96 inputs = constant_op.constant( 97 np.array([1, -2, 3, -4, 5, -6, 7, -8, 9, -8, 7, -6, 5, -4, 3, -2, 1]), 98 dtypes.qint8) 99 with self.assertRaisesRegex( 100 errors.InvalidArgumentError, 101 "Tensor size must be a multiple of 4 for Relu<qint8>. Got 17"): 102 self.evaluate(nn_ops.relu(inputs)) 103 104 def testNoElement(self): 105 self._testRelu(np.array([[], []], dtype=np.float32)) 106 107 @test_util.disable_xla("b/157978028: Does not yet pass with XLA") 108 def testNaNPropagation(self): 109 for t in [np.float16, np.float32, np.float64]: 110 self._testRelu(np.array([-1, np.nan, 1, np.nan]).astype(t)) 111 112 # The gradient test for ReLU is a bit tricky as the derivative is not well 113 # defined at around zero and we want to avoid that in terms of input values. 114 def testGradientFloat32(self): 115 with self.cached_session(): 116 x = np.asarray( 117 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 118 dtype=np.float32, 119 order="F") 120 err = gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient( 121 nn_ops.relu, [x], delta=1.0 / 1024)) 122 self.assertLess(err, 1e-6) 123 124 # The gradient test for ReLU is a bit tricky as the derivative is not well 125 # defined at around zero and we want to avoid that in terms of input values. 126 def testGradientFloat16(self): 127 with self.cached_session(): 128 x = np.asarray( 129 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 130 dtype=np.float16, 131 order="F") 132 err = gradient_checker_v2.max_error( 133 *gradient_checker_v2.compute_gradient(nn_ops.relu, [x])) 134 self.assertLess(err, 1e-6) 135 136 def testGradientFloat64(self): 137 with self.cached_session(): 138 x = np.asarray( 139 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 140 dtype=np.float64, 141 order="F") 142 err = gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient( 143 nn_ops.relu, [x], delta=1.0 / 1024)) 144 self.assertLess(err, 1e-15) 145 146 def testGradGradFloat32(self): 147 with self.cached_session(): 148 149 def f(x): 150 assert x.dtype == dtypes.float32 151 with backprop.GradientTape() as tape: 152 tape.watch(x) 153 y = nn_ops.relu(x) 154 return tape.gradient(y, x) 155 156 x = np.asarray( 157 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 158 dtype=np.float32, 159 order="F") 160 err = gradient_checker_v2.max_error( 161 *gradient_checker_v2.compute_gradient(f, [x], delta=1.0 / 1024)) 162 self.assertLess(err, 1e-4) 163 164 def testGradGradFloat64(self): 165 with self.cached_session(): 166 167 def f(x): 168 assert x.dtype == dtypes.float64 169 with backprop.GradientTape() as tape: 170 tape.watch(x) 171 y = nn_ops.relu(x) 172 return tape.gradient(y, x) 173 174 x = np.asarray( 175 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 176 dtype=np.float64, 177 order="F") 178 err = gradient_checker_v2.max_error( 179 *gradient_checker_v2.compute_gradient(f, [x], delta=1.0 / 1024)) 180 self.assertLess(err, 1e-10) 181 182 def testGradientScalar(self): 183 x = variables.Variable(100.) 184 185 def loss(): 186 return nn_ops.relu(x)**2 187 188 optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.25) 189 self.evaluate(variables.global_variables_initializer()) 190 self.evaluate(optimizer.minimize(loss)) 191 self.assertAllClose(x.read_value(), 50.0) 192 193 def testGradientNoElement(self): 194 with self.cached_session(): 195 196 def f(x): 197 with backprop.GradientTape() as tape: 198 tape.watch(x) 199 y = nn_ops.relu(x) 200 return tape.gradient(y, x) 201 202 x = np.asarray([[], []], dtype=np.float32) 203 z = list(gradient_checker_v2.compute_gradient(f, [x]))[0][0] 204 self.assertAllEqual(z, np.reshape(x, (0, 0))) 205 206 207class Relu6Test(test.TestCase): 208 209 def _npRelu6(self, np_features): 210 sixes = np.copy(np_features) 211 sixes.fill(6.0) 212 return np.minimum( 213 np.maximum(np_features, np.zeros(np_features.shape)), sixes) 214 215 def testNpRelu6(self): 216 self.assertAllClose( 217 np.array([[0.0, 0.7, 0.0, 0.3, 6.0], [0.1, 0.0, 6.0, 0.0, 0.9]]), 218 self._npRelu6( 219 np.array([[-0.9, 0.7, -0.5, 0.3, 6.0], [0.1, -0.3, 6.5, -0.7, 220 0.9]]))) 221 222 def _testRelu6(self, np_features): 223 np_relu6 = self._npRelu6(np_features) 224 tf_relu6 = nn_ops.relu6(np_features) 225 self.assertAllClose(np_relu6, tf_relu6) 226 self.assertShapeEqual(np_relu6, tf_relu6) 227 228 def testNumbersCPU(self): 229 for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: 230 # Force execution on CPU even if a GPU kernel is available for the type. 231 with ops.device("/device:CPU:0"): 232 self._testRelu6( 233 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 234 235 def testNumbersGPU(self): 236 if not test.is_gpu_available(): 237 self.skipTest("No GPU available") 238 for t in [np.float16, np.float, np.double]: 239 self._testRelu6( 240 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 241 242 @test_util.disable_xla("b/157978028: Does not yet pass with XLA") 243 def testNaNPropagation(self): 244 for t in [np.float16, np.float32, np.float64]: 245 self._testRelu6(np.array([-1, np.nan, 1, 7, np.nan]).astype(t)) 246 247 # The gradient test for ReLU6 is a bit tricky as the derivative is 248 # not well defined at around zero and six and we want to avoid that 249 # in terms of input values. 250 def testGradientFloat32(self): 251 with self.cached_session(): 252 x = np.asarray( 253 [[-0.9, -0.7, -0.5, -0.3, -0.1], [6.1, 6.3, 6.5, 6.7, 6.9]], 254 dtype=np.float32, 255 order="F") 256 err = gradient_checker_v2.max_error( 257 *gradient_checker_v2.compute_gradient(nn_ops.relu6, [x])) 258 self.assertLess(err, 1e-4) 259 260 def testGradientFloat64(self): 261 with self.cached_session(): 262 x = np.asarray( 263 [[-0.9, -0.7, -0.5, -0.3, -0.1], [6.1, 6.3, 6.5, 6.7, 6.9]], 264 dtype=np.float64, 265 order="F") 266 err = gradient_checker_v2.max_error( 267 *gradient_checker_v2.compute_gradient(nn_ops.relu6, [x])) 268 self.assertLess(err, 1e-10) 269 270 271class LeakyReluTest(test.TestCase): 272 273 def _npLeakyRelu(self, np_features, alpha=0.1): 274 return np.maximum(np_features, alpha * np_features) 275 276 def testNpLeakyRelu(self): 277 self.assertAllClose( 278 np.array([[-0.09, 0.7, -0.05, 0.3, -0.01], 279 [0.1, -0.03, 0.5, -0.07, 0.9]]), 280 self._npLeakyRelu( 281 np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 282 0.9]]), 283 alpha=0.1)) 284 285 def _testLeakyRelu(self, np_features, alpha): 286 np_leaky_relu = self._npLeakyRelu(np_features, alpha) 287 tf_leaky_relu = nn_ops.leaky_relu(np_features, alpha) 288 self.assertAllClose(np_leaky_relu, tf_leaky_relu) 289 self.assertShapeEqual(np_leaky_relu, tf_leaky_relu) 290 291 def testNumbersCPU(self): 292 for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: 293 # Force execution on CPU even if a GPU kernel is available for the type. 294 with ops.device("/device:CPU:0"): 295 self._testLeakyRelu( 296 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), 297 alpha=0.2) 298 299 def testNumbersGPU(self): 300 if not test.is_gpu_available(): 301 self.skipTest("No GPU available") 302 for t in [np.float16, np.float32, np.float64]: 303 self._testLeakyRelu( 304 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), 305 alpha=0.1) 306 307 def testNaNPropagation(self): 308 for t in [np.float16, np.float32, np.float64]: 309 self._testLeakyRelu(np.array([-1, np.nan, 1, np.nan]).astype(t), 310 alpha=0.2) 311 312 # The gradient test for Leaky ReLU is a bit tricky as the derivative is not 313 # well defined at around zero and we want to avoid that in terms of input 314 # values. 315 def testGradientFloat32(self): 316 with self.cached_session(): 317 x = np.asarray( 318 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 319 dtype=np.float32, 320 order="F") 321 err = gradient_checker_v2.max_error( 322 *gradient_checker_v2.compute_gradient(nn_ops.leaky_relu, [x])) 323 self.assertLess(err, 1e-4) 324 325 def testGradientFloat64(self): 326 with self.cached_session(): 327 x = np.asarray( 328 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 329 dtype=np.float64, 330 order="F") 331 err = gradient_checker_v2.max_error( 332 *gradient_checker_v2.compute_gradient(nn_ops.leaky_relu, [x])) 333 self.assertLess(err, 1e-10) 334 335 def testGradGradFloat32(self): 336 with self.cached_session(): 337 338 def f(x): 339 assert x.dtype == dtypes.float32 340 with backprop.GradientTape() as tape: 341 tape.watch(x) 342 y = nn_ops.leaky_relu(x) 343 return tape.gradient(y, x) 344 345 x = np.asarray( 346 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 347 dtype=np.float32, 348 order="F") 349 err = gradient_checker_v2.max_error( 350 *gradient_checker_v2.compute_gradient(f, [x])) 351 self.assertLess(err, 1e-4) 352 353 def testGradGradFloat64(self): 354 with self.cached_session(): 355 356 def f(x): 357 assert x.dtype == dtypes.float64 358 with backprop.GradientTape() as tape: 359 tape.watch(x) 360 y = nn_ops.leaky_relu(x) 361 return tape.gradient(y, x) 362 363 x = np.asarray( 364 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 365 dtype=np.float64, 366 order="F") 367 err = gradient_checker_v2.max_error( 368 *gradient_checker_v2.compute_gradient(f, [x])) 369 self.assertLess(err, 1e-10) 370 371 def testGradientScalar(self): 372 x = variables.Variable(-100.) 373 374 def loss(): 375 return nn_ops.leaky_relu(x, 0.05)**2 376 377 optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2) 378 self.evaluate(variables.global_variables_initializer()) 379 self.evaluate(optimizer.minimize(loss)) 380 self.assertAllClose(x.read_value(), -99.9) 381 382 def testUnexpectedAlphaValue(self): 383 self.assertAllClose( 384 np.array([[-9.0, 0.7, -5.0, 0.3, -0.1], [0.1, -3.0, 0.5, -27.0, 0.9]]), 385 nn_ops.leaky_relu( 386 np.array([[-0.9, 0.7, -0.5, 0.3, -0.01], 387 [0.1, -0.3, 0.5, -2.7, 0.9]]), 388 alpha=10)) 389 self.assertAllClose( 390 np.array([[9.0, 0.7, 5.0, 0.3, 0.1], [0.1, 3.0, 0.5, 27.0, 0.9]]), 391 nn_ops.leaky_relu( 392 np.array([[-0.9, 0.7, -0.5, 0.3, -0.01], 393 [0.1, -0.3, 0.5, -2.7, 0.9]]), 394 alpha=-10)) 395 396 397class EluTest(test.TestCase): 398 399 def _npElu(self, np_features): 400 return np.where(np_features < 0, np.exp(np_features) - 1, np_features) 401 402 def testNpElu(self): 403 self.assertAllClose( 404 np.array([[-0.59343034025, 0.7, -0.39346934028, 0.3, -0.09516258196], 405 [0.1, -0.25918177931, 0.5, -0.5034146962, 0.9]]), 406 self._npElu( 407 np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 408 0.9]]))) 409 410 def _testElu(self, np_features): 411 np_elu = self._npElu(np_features) 412 tf_elu = nn_ops.elu(np_features) 413 self.assertAllCloseAccordingToType(np_elu, tf_elu) 414 self.assertShapeEqual(np_elu, tf_elu) 415 416 def testNumbersCPU(self): 417 for t in [np.float16, np.float32, np.float64]: 418 # Force execution on CPU even if a GPU kernel is available for the type. 419 with ops.device("/device:CPU:0"): 420 self._testElu( 421 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 422 423 def testNumbersGPU(self): 424 if not test.is_gpu_available(): 425 self.skipTest("No GPU available") 426 for t in [np.float16, np.float32, np.float64]: 427 self._testElu(np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 428 429 def testNaNPropagation(self): 430 for t in [np.float16, np.float32, np.float64]: 431 self._testElu(np.array([-1, np.nan, 1, np.nan]).astype(t)) 432 433 def testGradientFloat32(self): 434 with self.cached_session(): 435 x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] 436 x = np.asarray(x_val, dtype=np.float32, order="F") 437 err = gradient_checker_v2.max_error( 438 *gradient_checker_v2.compute_gradient(nn_ops.elu, [x])) 439 self.assertLess(err, 1e-4) 440 441 def testGradientFloat64(self): 442 with self.cached_session(): 443 x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] 444 x = np.asarray(x_val, dtype=np.float64, order="F") 445 err = gradient_checker_v2.max_error( 446 *gradient_checker_v2.compute_gradient(nn_ops.elu, [x])) 447 self.assertLess(err, 1e-6) 448 449 def testGradGrad(self): 450 with self.cached_session(): 451 452 def f(x): 453 with backprop.GradientTape(persistent=True) as tape: 454 tape.watch(x) 455 y = nn_ops.elu(x) 456 dy = tape.gradient(y, x) 457 return tape.gradient(dy, x) 458 459 for x in [-1., -0.5, 0.5, 1.]: 460 got = self.evaluate(f(constant_op.constant(x))) 461 want = _elu_grad_grad(x) 462 err = np.abs(got - want) 463 self.assertLess(err, 1e-4) 464 465 def testGradGradFloat32(self): 466 with self.cached_session(): 467 468 def f(x): 469 assert x.dtype == dtypes.float32 470 with backprop.GradientTape() as tape: 471 tape.watch(x) 472 y = nn_ops.elu(x) 473 return tape.gradient(y, x) 474 475 x = np.asarray( 476 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 477 dtype=np.float32, 478 order="F") 479 err = gradient_checker_v2.max_error( 480 *gradient_checker_v2.compute_gradient(f, [x])) 481 self.assertLess(err, 1e-4) 482 483 def testGradGradFloat64(self): 484 with self.cached_session(): 485 486 def f(x): 487 assert x.dtype == dtypes.float64 488 with backprop.GradientTape() as tape: 489 tape.watch(x) 490 y = nn_ops.elu(x) 491 return tape.gradient(y, x) 492 493 x = np.asarray( 494 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 495 dtype=np.float64, 496 order="F") 497 err = gradient_checker_v2.max_error( 498 *gradient_checker_v2.compute_gradient(f, [x])) 499 self.assertLess(err, 1e-6) 500 501 502class SeluTest(test.TestCase): 503 504 def _npSelu(self, np_features): 505 scale = 1.0507009873554804934193349852946 506 scale_alpha = 1.7580993408473768599402175208123 507 return np.where(np_features < 0, scale_alpha * (np.exp(np_features) - 1), 508 scale * np_features) 509 510 def testNpSelu(self): 511 self.assertAllClose( 512 np.array([[-1.0433095, 0.73549069, -0.6917582, 0.3152103, -0.16730527], 513 [0.1050701, -0.45566732, 0.5253505, -0.88505305, 0.9456309]]), 514 self._npSelu( 515 np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 516 0.9]]))) 517 518 def _testSelu(self, np_features): 519 np_selu = self._npSelu(np_features) 520 tf_selu = nn_ops.selu(np_features) 521 self.assertAllCloseAccordingToType(np_selu, tf_selu) 522 self.assertShapeEqual(np_selu, tf_selu) 523 524 def testNumbers(self): 525 for t in [np.float16, np.float32, np.float64]: 526 self._testSelu( 527 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 528 # Force executed on CPU in case GPU kernels are available. 529 with ops.device("/device:CPU:0"): 530 self._testSelu( 531 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 532 533 def testGradientFloat32(self): 534 with self.cached_session(): 535 x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] 536 x = np.asarray(x_val, dtype=np.float32, order="F") 537 err = gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient( 538 nn_ops.selu, [x], delta=1.0 / 1024)) 539 self.assertLess(err, 1e-4) 540 541 def testGradientFloat64(self): 542 with self.cached_session(): 543 x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] 544 x = np.asarray(x_val, dtype=np.float64, order="F") 545 err = gradient_checker_v2.max_error( 546 *gradient_checker_v2.compute_gradient(nn_ops.selu, [x])) 547 self.assertLess(err, 1e-6) 548 549 def testGradGradFloat32(self): 550 with self.cached_session(): 551 552 def f(x): 553 assert x.dtype == dtypes.float32 554 with backprop.GradientTape() as tape: 555 tape.watch(x) 556 y = nn_ops.selu(x) 557 return tape.gradient(y, x) 558 559 x = np.asarray( 560 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 561 dtype=np.float32, 562 order="F") 563 err = gradient_checker_v2.max_error( 564 *gradient_checker_v2.compute_gradient(f, [x], delta=1.0 / 1024)) 565 self.assertLess(err, 1e-4) 566 567 def testGradGradFloat64(self): 568 with self.cached_session(): 569 570 def f(x): 571 assert x.dtype == dtypes.float64 572 with backprop.GradientTape() as tape: 573 tape.watch(x) 574 y = nn_ops.selu(x) 575 return tape.gradient(y, x) 576 577 x = np.asarray( 578 [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], 579 dtype=np.float64, 580 order="F") 581 err = gradient_checker_v2.max_error( 582 *gradient_checker_v2.compute_gradient(f, [x])) 583 self.assertLess(err, 1e-6) 584 585 586class CreluTest(test.TestCase): 587 588 def testCreluShape(self): 589 f = random_ops.random_normal([50, 5, 7, 10]) 590 t = nn_ops.crelu(f) 591 self.assertEqual([50, 5, 7, 20], t.get_shape()) 592 593 def _testCrelu(self, np_features): 594 np_relu = np.maximum(np_features, np.zeros_like(np_features)) 595 np_neg_relu = np.maximum(-np_features, np.zeros_like(np_features)) 596 np_crelu = np.concatenate((np_relu, np_neg_relu), 597 len(np_features.shape) - 1) 598 599 tf_crelu = nn_ops.crelu(np_features) 600 601 self.assertAllClose(np_crelu, tf_crelu) 602 self.assertShapeEqual(np_crelu, tf_crelu) 603 604 def testNumbersCPU(self): 605 for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: 606 # Force execution on CPU even if a GPU kernel is available for the type. 607 with ops.device("/device:CPU:0"): 608 self._testCrelu( 609 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 610 611 def testNumbersGPU(self): 612 if not test.is_gpu_available(): 613 self.skipTest("No GPU available") 614 for t in [np.float16, np.float32, np.float64]: 615 self._testCrelu( 616 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) 617 618 def testNumbersWithAxis0(self): 619 tf_crelu = nn_ops.crelu( 620 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=0) 621 np_crelu = np.array([[0, 7, 0, 3, 0], [1, 0, 5, 0, 9], [9, 0, 5, 0, 1], 622 [0, 3, 0, 7, 0]]) 623 self.assertAllEqual(np_crelu, tf_crelu) 624 625 def testNumbersWithAxis1(self): 626 tf_crelu = nn_ops.crelu( 627 np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=1) 628 np_crelu = np.array([[0, 7, 0, 3, 0, 9, 0, 5, 0, 1], 629 [1, 0, 5, 0, 9, 0, 3, 0, 7, 0]]) 630 self.assertAllEqual(np_crelu, tf_crelu) 631 632 633if __name__ == "__main__": 634 test.main() 635