1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for miscellaneous functionality in tensorflow.ops.nn.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import math 22 23from absl.testing import parameterized 24import numpy as np 25from six.moves import xrange # pylint: disable=redefined-builtin 26 27from tensorflow.python.eager import def_function 28from tensorflow.python.framework import constant_op 29from tensorflow.python.framework import dtypes 30from tensorflow.python.framework import ops 31from tensorflow.python.framework import tensor_spec 32from tensorflow.python.framework import test_util 33from tensorflow.python.ops import array_ops 34from tensorflow.python.ops import gradient_checker 35from tensorflow.python.ops import math_ops 36from tensorflow.python.ops import nn 37from tensorflow.python.ops import nn_impl 38from tensorflow.python.ops import nn_ops 39from tensorflow.python.ops import partitioned_variables 40from tensorflow.python.ops import variable_scope 41from tensorflow.python.ops import variables 42import tensorflow.python.ops.nn_grad # pylint: disable=unused-import 43from tensorflow.python.ops.nn_impl import _compute_sampled_logits 44from tensorflow.python.ops.ragged import ragged_factory_ops 45from tensorflow.python.platform import test as test_lib 46 47 48class ZeroFractionTest(test_lib.TestCase): 49 50 def _ZeroFraction(self, x): 51 assert x.shape 52 total_elements = np.prod(x.shape) 53 nonzeros = np.count_nonzero(x.flatten()) 54 return 1.0 - nonzeros / total_elements 55 56 @test_util.run_deprecated_v1 57 def testZeroFraction(self): 58 x_shape = [5, 17] 59 x_np = np.random.randint(0, 2, size=x_shape).astype(np.float32) 60 y_np = self._ZeroFraction(x_np) 61 62 x_tf = constant_op.constant(x_np) 63 x_tf.set_shape(x_shape) 64 y_tf = nn_impl.zero_fraction(x_tf) 65 y_tf_np = self.evaluate(y_tf) 66 67 eps = 1e-8 68 self.assertAllClose(y_tf_np, y_np, eps) 69 70 @test_util.run_deprecated_v1 71 def testZeroFractionEmpty(self): 72 x = np.zeros(0) 73 y = self.evaluate(nn_impl.zero_fraction(x)) 74 self.assertTrue(np.isnan(y)) 75 76 @test_util.run_deprecated_v1 77 def testZeroFraction2_27Zeros(self): 78 sparsity = nn_impl.zero_fraction( 79 array_ops.zeros([int(2**27 * 1.01)], dtype=dtypes.int8)) 80 self.assertAllClose(1.0, self.evaluate(sparsity)) 81 82 @test_util.run_deprecated_v1 83 def testZeroFraction2_27Ones(self): 84 sparsity = nn_impl.zero_fraction( 85 array_ops.ones([int(2**27 * 1.01)], dtype=dtypes.int8)) 86 self.assertAllClose(0.0, self.evaluate(sparsity)) 87 88 @test_util.run_deprecated_v1 89 def testUnknownSize(self): 90 value = array_ops.placeholder(dtype=dtypes.float32) 91 sparsity = nn_impl.zero_fraction(value) 92 with self.cached_session() as sess: 93 self.assertAllClose( 94 0.25, 95 sess.run(sparsity, {value: [[0., 1.], [0.3, 2.]]})) 96 97 98class SoftmaxTest(test_lib.TestCase, parameterized.TestCase): 99 100 def _softmax(self, x): 101 assert len(x.shape) == 2 102 m = x.max(1)[:, np.newaxis] 103 u = np.exp(x - m) 104 z = u.sum(1)[:, np.newaxis] 105 return u / z 106 107 @test_util.run_in_graph_and_eager_modes 108 def testSoftmax(self): 109 x_shape = [5, 10] 110 x_np = np.random.randn(*x_shape).astype(np.float32) 111 y_np = self._softmax(x_np) 112 x_tf = constant_op.constant(x_np) 113 y_tf = nn_ops.softmax_v2(x_tf) 114 y_tf_last_dim = nn_ops.softmax_v2(x_tf, 1) 115 y_tf_np = self.evaluate(y_tf) 116 y_tf_last_dim_np = self.evaluate(y_tf_last_dim) 117 eps = 1e-3 118 self.assertAllClose(y_tf_np, y_np, eps) 119 self.assertAllClose(y_tf_last_dim_np, y_np, eps) 120 121 def testSoftmaxAxes(self): 122 arr = np.linspace(0., 1, 12).reshape(3, 4) 123 x_neg_axis = nn_ops.softmax_v2(arr, axis=-2) 124 y_pos_axis = nn_ops.softmax_v2(arr, axis=0) 125 z_gt_axis = nn_ops.softmax_v2(arr, axis=0) 126 x_neg_axis_tf = self.evaluate(x_neg_axis) 127 y_pos_axis_tf = self.evaluate(y_pos_axis) 128 z_gt_axis_tf = self.evaluate(z_gt_axis) 129 eps = 1e-3 130 self.assertAllClose(x_neg_axis_tf, y_pos_axis_tf, eps) 131 self.assertAllClose(y_pos_axis_tf, z_gt_axis_tf, eps) 132 133 @parameterized.parameters(((5, 10),), ((2, 3, 4),)) 134 @test_util.run_deprecated_v1 135 def testGradient(self, x_shape): 136 x_np = np.random.randn(*x_shape).astype(np.float64) 137 with self.cached_session(): 138 x_tf = constant_op.constant(x_np) 139 y_tf = nn_ops.softmax_v2(x_tf) 140 err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, 141 x_shape) 142 eps = 2e-8 143 self.assertLess(err, eps) 144 145 146class LogPoissonLossTest(test_lib.TestCase): 147 148 def _log_poisson_loss(self, x, z, compute_full_loss=False): 149 lpl = np.exp(x) - z * x 150 if compute_full_loss: 151 stirling_approx = z * np.log(z) - z + 0.5 * np.log(2. * np.pi * z) 152 lpl += np.ma.masked_array(stirling_approx, mask=(z <= 1)).filled(0.) 153 return lpl 154 155 @test_util.run_in_graph_and_eager_modes 156 def testLogPoissonLoss(self): 157 x_shape = [5, 10] 158 x_np = np.random.randn(*x_shape).astype(np.float32) 159 z_np = np.random.randint(0, 5, size=x_shape).astype(np.float32) 160 y_np = self._log_poisson_loss(x_np, z_np, compute_full_loss=False) 161 y_np_stirling = self._log_poisson_loss(x_np, z_np, compute_full_loss=True) 162 y_tf = nn_impl.log_poisson_loss(z_np, x_np, compute_full_loss=False) 163 y_tf_stirling = nn_impl.log_poisson_loss(z_np, x_np, compute_full_loss=True) 164 y_tf_np = self.evaluate(y_tf) 165 y_tf_np_stirling = self.evaluate(y_tf_stirling) 166 eps = 1e-3 167 self.assertAllClose(y_tf_np, y_np, eps) 168 self.assertAllClose(y_tf_np_stirling, y_np_stirling, eps) 169 170 @test_util.run_deprecated_v1 171 def testGradient(self): 172 x_shape = [5, 10] 173 x_np = np.random.randn(*x_shape).astype(np.float64) 174 z_np = np.random.randint(0, 5, size=x_shape).astype(np.float64) 175 with self.cached_session(): 176 x_tf = constant_op.constant(x_np) 177 y_tf = nn_impl.log_poisson_loss(z_np, x_tf, compute_full_loss=False) 178 y_tf_stirling = nn_impl.log_poisson_loss( 179 z_np, x_tf, compute_full_loss=True) 180 err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, 181 x_shape) 182 err_stirling = gradient_checker.compute_gradient_error( 183 x_tf, x_shape, y_tf_stirling, x_shape) 184 eps = 1e-6 185 self.assertLess(err, eps) 186 self.assertLess(err_stirling, eps) 187 188 189class LogSoftmaxTest(test_lib.TestCase, parameterized.TestCase): 190 191 def _log_softmax(self, x): 192 assert len(x.shape) == 2 193 m = x.max(1)[:, np.newaxis] 194 u = x - m 195 return u - np.log(np.sum(np.exp(u), 1, keepdims=True)) 196 197 @test_util.run_in_graph_and_eager_modes 198 def testLogSoftmax(self): 199 x_shape = [5, 10] 200 x_np = np.random.randn(*x_shape).astype(np.float32) 201 y_np = self._log_softmax(x_np) 202 x_tf = constant_op.constant(x_np) 203 y_tf = nn_ops.log_softmax_v2(x_tf) 204 y_tf_np = self.evaluate(y_tf) 205 eps = 1e-3 206 self.assertAllClose(y_tf_np, y_np, eps) 207 208 def testLogSoftmaxAxes(self): 209 arr = np.linspace(0., 1, 12).reshape(3, 4) 210 x_neg_axis = nn_ops.log_softmax_v2(arr, axis=-2) 211 y_pos_axis = nn_ops.log_softmax_v2(arr, axis=0) 212 z_gt_axis = nn_ops.log_softmax_v2(arr, axis=0) 213 x_neg_axis_tf = self.evaluate(x_neg_axis) 214 y_pos_axis_tf = self.evaluate(y_pos_axis) 215 z_gt_axis_tf = self.evaluate(z_gt_axis) 216 eps = 1e-3 217 self.assertAllClose(x_neg_axis_tf, y_pos_axis_tf, eps) 218 self.assertAllClose(y_pos_axis_tf, z_gt_axis_tf, eps) 219 220 @parameterized.parameters(((5, 10),), ((2, 3, 4),)) 221 @test_util.run_deprecated_v1 222 def testGradient(self, x_shape): 223 x_np = np.random.randn(*x_shape).astype(np.float64) 224 with self.cached_session(): 225 x_tf = constant_op.constant(x_np) 226 y_tf = nn_ops.log_softmax_v2(x_tf) 227 err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, 228 x_shape) 229 eps = 1e-7 230 self.assertLess(err, eps) 231 232 233class L2LossTest(test_lib.TestCase): 234 235 @test_util.run_in_graph_and_eager_modes 236 def testL2Loss(self): 237 for dtype in [dtypes.float32, dtypes.float64]: 238 x = constant_op.constant( 239 [1.0, 0.0, 3.0, 2.0], shape=[2, 2], name="x", dtype=dtype) 240 l2loss = nn_ops.l2_loss(x) 241 value = self.evaluate(l2loss) 242 self.assertAllClose(7.0, value) 243 244 @test_util.run_deprecated_v1 245 def testGradient(self): 246 x_shape = [20, 7, 3] 247 np.random.seed(1) # Make it reproducible. 248 x_val = np.random.random_sample(x_shape).astype(np.float64) 249 with self.cached_session(): 250 x = constant_op.constant(x_val, name="x") 251 output = nn_ops.l2_loss(x) 252 err = gradient_checker.compute_gradient_error(x, x_shape, output, [1]) 253 print("L2Loss gradient err = %g " % err) 254 err_tolerance = 1e-10 255 self.assertLess(err, err_tolerance) 256 257 258class L2NormalizeTest(test_lib.TestCase): 259 260 def _l2Normalize(self, x, dim): 261 if isinstance(dim, list): 262 norm = np.linalg.norm(x, axis=tuple(dim)) 263 for d in dim: 264 norm = np.expand_dims(norm, d) 265 return x / norm 266 else: 267 norm = np.apply_along_axis(np.linalg.norm, dim, x) 268 return x / np.expand_dims(norm, dim) 269 270 @test_util.run_in_graph_and_eager_modes 271 def testL2Normalize(self): 272 x_shape = [20, 7, 3] 273 np.random.seed(1) 274 x_np = np.random.random_sample(x_shape).astype(np.float32) 275 for dim in range(len(x_shape)): 276 y_np = self._l2Normalize(x_np, dim) 277 x_tf = constant_op.constant(x_np, name="x") 278 y_tf = nn_impl.l2_normalize_v2(x_tf, dim) 279 self.assertAllClose(y_np, self.evaluate(y_tf)) 280 281 @test_util.run_in_graph_and_eager_modes 282 def testL2NormalizeDimArray(self): 283 x_shape = [20, 7, 3] 284 np.random.seed(1) 285 x_np = np.random.random_sample(x_shape).astype(np.float32) 286 dim = [1, 2] 287 y_np = self._l2Normalize(x_np, dim) 288 x_tf = constant_op.constant(x_np, name="x") 289 y_tf = nn_impl.l2_normalize_v2(x_tf, dim) 290 self.assertAllClose(y_np, self.evaluate(y_tf)) 291 292 @test_util.run_deprecated_v1 293 def testL2NormalizeGradient(self): 294 x_shape = [20, 7, 3] 295 np.random.seed(1) 296 x_np = np.random.random_sample(x_shape).astype(np.float64) 297 for dim in range(len(x_shape)): 298 with self.cached_session(): 299 x_tf = constant_op.constant(x_np, name="x") 300 y_tf = nn_impl.l2_normalize_v2(x_tf, dim) 301 err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, 302 x_shape) 303 print("L2Normalize gradient err = %g " % err) 304 self.assertLess(err, 1e-4) 305 306 307class DropoutTest(test_lib.TestCase): 308 309 def testDropout(self): 310 # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate 311 # that it is producing approximately the right number of ones over a large 312 # number of samples, based on the keep probability. 313 x_dim = 40 314 y_dim = 30 315 num_iter = 10 316 for keep_prob in [0.1, 0.5, 0.8]: 317 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 318 dropout = nn_ops.dropout(t, rate=(1 - keep_prob)) 319 final_count = 0 320 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 321 for _ in xrange(0, num_iter): 322 value = self.evaluate(dropout) 323 final_count += np.count_nonzero(value) 324 # Verifies that there are only two values: 0 and 1/keep_prob. 325 sorted_value = np.unique(np.sort(value)) 326 self.assertEqual(0, sorted_value[0]) 327 self.assertAllClose(1 / keep_prob, sorted_value[1]) 328 329 # Check that we are in the 15% error range 330 expected_count = x_dim * y_dim * keep_prob * num_iter 331 rel_error = math.fabs(final_count - expected_count) / expected_count 332 print(rel_error) 333 self.assertTrue(rel_error < 0.15) 334 335 def testShapedDropout(self): 336 # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate 337 # that it is producing approximately the right number of ones over a large 338 # number of samples, based on the keep probability. This time with shaped 339 # noise. 340 x_dim = 40 * 30 341 y_dim = 3 342 num_iter = 10 343 for keep_prob in [0.1, 0.5, 0.8]: 344 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 345 dropout = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, 1]) 346 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 347 final_count = 0 348 for _ in xrange(0, num_iter): 349 value = self.evaluate(dropout) 350 final_count += np.count_nonzero(value) 351 # Verifies that there are only two values: 0 and 1/keep_prob. 352 sorted_value = np.unique(np.sort(value)) 353 self.assertEqual(0, sorted_value[0]) 354 self.assertAllClose(1 / keep_prob, sorted_value[1]) 355 356 # Check that we are in the 15% error range 357 expected_count = x_dim * y_dim * keep_prob * num_iter 358 rel_error = math.fabs(final_count - expected_count) / expected_count 359 print(rel_error) 360 self.assertTrue(rel_error < 0.15) 361 362 def testShapedDropoutCorrelation(self): 363 # Runs a shaped dropout and tests that the correlations are correct. 364 x_dim = 40 365 y_dim = 30 366 num_iter = 10 367 for keep_prob in [0.1, 0.5, 0.8]: 368 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 369 dropout = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, 1]) 370 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 371 for _ in xrange(0, num_iter): 372 value = self.evaluate(dropout) 373 # Verifies that each y column as only one type of activation. 374 for i in xrange(x_dim): 375 sorted_value = np.unique(np.sort(value[i, :])) 376 self.assertEqual(sorted_value.size, 1) 377 378 @test_util.run_deprecated_v1 379 def testDropoutPlaceholderKeepProb(self): 380 # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate 381 # that it is producing approximately the right number of ones over a large 382 # number of samples, based on the keep probability. 383 x_dim = 40 384 y_dim = 30 385 num_iter = 10 386 for keep_prob in [0.1, 0.5, 0.8]: 387 with self.cached_session(): 388 t = constant_op.constant( 389 1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 390 keep_prob_placeholder = array_ops.placeholder(dtypes.float32) 391 dropout = nn_ops.dropout(t, keep_prob_placeholder) 392 final_count = 0 393 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 394 for _ in xrange(0, num_iter): 395 value = dropout.eval(feed_dict={keep_prob_placeholder: keep_prob}) 396 final_count += np.count_nonzero(value) 397 # Verifies that there are only two values: 0 and 1/keep_prob. 398 sorted_value = np.unique(np.sort(value)) 399 self.assertEqual(0, sorted_value[0]) 400 self.assertAllClose(1 / keep_prob, sorted_value[1]) 401 # Check that we are in the 15% error range 402 expected_count = x_dim * y_dim * keep_prob * num_iter 403 rel_error = math.fabs(final_count - expected_count) / expected_count 404 print(rel_error) 405 self.assertTrue(rel_error < 0.15) 406 407 @test_util.run_deprecated_v1 408 def testShapedDropoutUnknownShape(self): 409 x_dim = 40 410 y_dim = 30 411 keep_prob = 0.5 412 x = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 413 dropout_x = nn_ops.dropout( 414 x, 415 rate=(1 - keep_prob), 416 noise_shape=array_ops.placeholder(dtypes.int32)) 417 self.assertEqual(x.get_shape(), dropout_x.get_shape()) 418 419 def testPartialShapedDropout(self): 420 x_dim = 40 * 30 421 y_dim = 3 422 num_iter = 10 423 for keep_prob in [0.1, 0.5, 0.8]: 424 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 425 # Set noise_shape=[None, 1] which means [x_dim, 1]. 426 dropout = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[None, 1]) 427 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 428 final_count = 0 429 for _ in xrange(0, num_iter): 430 value = self.evaluate(dropout) 431 final_count += np.count_nonzero(value) 432 # Verifies that there are only two values: 0 and 1/keep_prob. 433 sorted_value = np.unique(np.sort(value)) 434 self.assertEqual(0, sorted_value[0]) 435 self.assertAllClose(1 / keep_prob, sorted_value[1]) 436 437 # Check that we are in the 15% error range 438 expected_count = x_dim * y_dim * keep_prob * num_iter 439 rel_error = math.fabs(final_count - expected_count) / expected_count 440 print(rel_error) 441 self.assertTrue(rel_error < 0.15) 442 443 @test_util.run_deprecated_v1 444 def testInvalidKeepProb(self): 445 x_dim = 40 446 y_dim = 30 447 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 448 with self.assertRaises(ValueError): 449 nn_ops.dropout(t, -1.0) 450 with self.assertRaises(ValueError): 451 nn_ops.dropout(t, 1.1) 452 with self.assertRaises(ValueError): 453 nn_ops.dropout(t, [0.0, 1.0]) 454 with self.assertRaises(ValueError): 455 nn_ops.dropout(t, array_ops.placeholder(dtypes.float64)) 456 with self.assertRaises(ValueError): 457 nn_ops.dropout(t, array_ops.placeholder(dtypes.float32, shape=[2])) 458 459 @test_util.run_deprecated_v1 460 def testInvalidRate(self): 461 x_dim = 40 462 y_dim = 30 463 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 464 with self.assertRaises(ValueError): 465 nn_ops.dropout_v2(t, -1.0) 466 with self.assertRaises(ValueError): 467 nn_ops.dropout_v2(t, 1.1) 468 with self.assertRaises(ValueError): 469 nn_ops.dropout_v2(t, [0.0, 1.0]) 470 471 def testLargeRate(self): 472 x_dim = 40 473 y_dim = 30 474 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 475 _ = nn_ops.dropout_v2(t, 0.9) 476 477 def testVariableRef(self): 478 x = variable_scope.get_variable("x", shape=[10, 10], dtype=dtypes.float32) 479 _ = nn_ops.dropout(x, keep_prob=0.1) 480 481 @test_util.run_deprecated_v1 482 def testShapedDropoutShapeError(self): 483 # Runs shaped dropout and verifies an error is thrown on misshapen noise. 484 x_dim = 40 485 y_dim = 30 486 keep_prob = 0.5 487 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 488 with self.assertRaises(ValueError): 489 _ = nn_ops.dropout( 490 t, rate=(1 - keep_prob), noise_shape=[x_dim, y_dim + 10]) 491 with self.assertRaises(ValueError): 492 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, y_dim, 5]) 493 with self.assertRaises(ValueError): 494 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim + 3]) 495 with self.assertRaises(ValueError): 496 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim]) 497 # test that broadcasting proceeds 498 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[y_dim]) 499 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[1, y_dim]) 500 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, 1]) 501 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[1, 1]) 502 503 def testNoDropout(self): 504 x = array_ops.zeros((5,)) 505 y = nn_ops.dropout(x, rate=0) 506 self.assertAllEqual(x, y) 507 508 y = nn_ops.dropout_v2(x, rate=0) 509 self.assertAllEqual(x, y) 510 511 def testDropoutWithIntegerInputs(self): 512 x = constant_op.constant([1, 1, 1, 1, 1]) 513 with self.assertRaises(ValueError): 514 _ = nn_ops.dropout(x, 0.5) 515 516 517class ComputeSampledLogitsTest(test_lib.TestCase): 518 519 def setUp(self): 520 self._eps = 1e-3 521 522 def _GenerateTestData(self, num_classes, dim, batch_size, num_true, labels, 523 sampled, subtract_log_q): 524 """Randomly generates input/output data for a single test case. 525 526 This function returns numpy constants for use in a test case. 527 528 Args: 529 num_classes: An int. The number of embedding classes in the test case. 530 dim: An int. The dimension of the embedding. 531 batch_size: An int. The batch size. 532 num_true: An int. The number of target classes per training example. 533 labels: A list of batch_size * num_true ints. The target classes. 534 sampled: A list of indices in [0, num_classes). 535 subtract_log_q: A bool corresponding to the parameter in 536 _compute_sampled_logits(). 537 538 Returns: 539 weights: Embedding weights to use as test input. It is a numpy array 540 of shape [num_classes, dim] 541 biases: Embedding biases to use as test input. It is a numpy array 542 of shape [num_classes]. 543 hidden_acts: Forward activations of the network to use as test input. 544 It is a numpy array of shape [batch_size, dim]. 545 sampled_vals: A tuple based on `sampled` to use as test input in the 546 format returned by a *_candidate_sampler function. 547 exp_logits: The output logits expected from _compute_sampled_logits(). 548 It is a numpy array of shape [batch_size, num_true + len(sampled)]. 549 exp_labels: The output labels expected from _compute_sampled_logits(). 550 It is a numpy array of shape [batch_size, num_true + len(sampled)]. 551 """ 552 weights = np.random.randn(num_classes, dim).astype(np.float32) 553 biases = np.random.randn(num_classes).astype(np.float32) 554 hidden_acts = np.random.randn(batch_size, dim).astype(np.float32) 555 556 true_exp = np.full([batch_size, 1], fill_value=0.5, dtype=np.float32) 557 sampled_exp = np.full([len(sampled)], fill_value=0.5, dtype=np.float32) 558 sampled_vals = (sampled, true_exp, sampled_exp) 559 560 sampled_w, sampled_b = weights[sampled], biases[sampled] 561 true_w, true_b = weights[labels], biases[labels] 562 563 true_logits = np.sum( 564 hidden_acts.reshape((batch_size, 1, dim)) * true_w.reshape( 565 (batch_size, num_true, dim)), 566 axis=2) 567 true_b = true_b.reshape((batch_size, num_true)) 568 true_logits += true_b 569 sampled_logits = np.dot(hidden_acts, sampled_w.T) + sampled_b 570 571 if subtract_log_q: 572 true_logits -= np.log(true_exp) 573 sampled_logits -= np.log(sampled_exp[np.newaxis, :]) 574 575 exp_logits = np.concatenate([true_logits, sampled_logits], axis=1) 576 exp_labels = np.hstack((np.ones_like(true_logits) / num_true, 577 np.zeros_like(sampled_logits))) 578 579 return weights, biases, hidden_acts, sampled_vals, exp_logits, exp_labels 580 581 def _ShardTestEmbeddings(self, weights, biases, num_shards): 582 """Shards the weights and biases returned by _GenerateTestData. 583 584 Args: 585 weights: The weights returned by _GenerateTestData. 586 biases: The biases returned by _GenerateTestData. 587 num_shards: The number of shards to create. 588 589 Returns: 590 sharded_weights: A list of size `num_shards` containing all the weights. 591 sharded_biases: A list of size `num_shards` containing all the biases. 592 """ 593 with ops.Graph().as_default() as g: 594 sharded_weights = variable_scope.get_variable( 595 "w", 596 partitioner=partitioned_variables.fixed_size_partitioner(num_shards), 597 initializer=constant_op.constant(weights)) 598 sharded_biases = variable_scope.get_variable( 599 "b", 600 partitioner=partitioned_variables.fixed_size_partitioner(num_shards), 601 initializer=constant_op.constant(biases)) 602 with self.session(graph=g) as sess: 603 variables.global_variables_initializer().run() 604 return self.evaluate([list(sharded_weights), list(sharded_biases)]) 605 606 def testShapes(self): 607 np.random.seed(0) 608 num_classes = 5 609 batch_size = 3 610 611 for num_true in range(1, 5): 612 labels = np.random.randint( 613 low=0, high=num_classes, size=batch_size * num_true) 614 (weights, biases, hidden_acts, sampled_vals, exp_logits, 615 exp_labels) = self._GenerateTestData( 616 num_classes=num_classes, 617 dim=10, 618 batch_size=batch_size, 619 num_true=num_true, 620 labels=labels, 621 sampled=[1, 0, 2, 3], 622 subtract_log_q=False) 623 logits_tensor, labels_tensor = _compute_sampled_logits( 624 weights=constant_op.constant(weights), 625 biases=constant_op.constant(biases), 626 labels=constant_op.constant( 627 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 628 inputs=constant_op.constant(hidden_acts), 629 num_sampled=4, 630 num_classes=num_classes, 631 num_true=num_true, 632 sampled_values=sampled_vals, 633 subtract_log_q=False, 634 remove_accidental_hits=False, 635 partition_strategy="div", 636 name="sampled_logits_basic_num_true_%d" % num_true) 637 got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) 638 self.assertEqual(exp_logits.shape, got_logits.shape, self._eps) 639 self.assertEqual(exp_labels.shape, got_labels.shape, self._eps) 640 641 def testBasic(self): 642 """Without accidental hit removal or subtract_log_q.""" 643 np.random.seed(0) 644 num_classes = 5 645 batch_size = 3 646 647 for num_true in range(1, 5): 648 labels = np.random.randint( 649 low=0, high=num_classes, size=batch_size * num_true) 650 (weights, biases, hidden_acts, sampled_vals, exp_logits, 651 exp_labels) = self._GenerateTestData( 652 num_classes=num_classes, 653 dim=10, 654 batch_size=batch_size, 655 num_true=num_true, 656 labels=labels, 657 sampled=[1, 0, 2, 3], 658 subtract_log_q=False) 659 logits_tensor, labels_tensor = _compute_sampled_logits( 660 weights=constant_op.constant(weights), 661 biases=constant_op.constant(biases), 662 labels=constant_op.constant( 663 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 664 inputs=constant_op.constant(hidden_acts), 665 num_sampled=4, 666 num_classes=num_classes, 667 num_true=num_true, 668 sampled_values=sampled_vals, 669 subtract_log_q=False, 670 remove_accidental_hits=False, 671 partition_strategy="div", 672 name="sampled_logits_basic_num_true_%d" % num_true) 673 got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) 674 self.assertAllClose(exp_logits, got_logits, self._eps) 675 self.assertAllClose(exp_labels, got_labels, self._eps) 676 677 def testAccidentalHitRemoval(self): 678 """With accidental hit removal, no subtract_log_q.""" 679 np.random.seed(0) 680 num_classes = 5 681 batch_size = 3 682 sampled = [1, 0, 2, 3] 683 684 for num_true in range(1, 5): 685 labels = np.random.randint( 686 low=0, high=num_classes, size=batch_size * num_true) 687 (weights, biases, hidden_acts, sampled_vals, _, 688 _) = self._GenerateTestData( 689 num_classes=num_classes, 690 dim=10, 691 batch_size=batch_size, 692 num_true=num_true, 693 labels=labels, 694 sampled=sampled, 695 subtract_log_q=False) 696 logits_tensor, _ = _compute_sampled_logits( 697 weights=constant_op.constant(weights), 698 biases=constant_op.constant(biases), 699 labels=constant_op.constant( 700 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 701 inputs=constant_op.constant(hidden_acts), 702 num_sampled=len(sampled), 703 num_classes=num_classes, 704 num_true=num_true, 705 sampled_values=sampled_vals, 706 subtract_log_q=False, 707 remove_accidental_hits=True, 708 partition_strategy="div", 709 name="sampled_logits_accidental_hit_removal_num_true_%d" % num_true) 710 # Test that the exponentiated logits of accidental hits are near 0. 711 # First we need to find the hits in this random test run: 712 labels_reshape = labels.reshape((batch_size, num_true)) 713 got_logits = self.evaluate(logits_tensor) 714 for row in xrange(batch_size): 715 row_labels = labels_reshape[row, :] 716 for col in xrange(len(sampled)): 717 if sampled[col] in row_labels: 718 # We need to add the num_true_test offset into logits_* 719 self.assertNear( 720 np.exp(got_logits[row, col + num_true]), 0., self._eps) 721 722 def testSubtractLogQ(self): 723 """With subtract_log_q, no accidental hit removal.""" 724 np.random.seed(0) 725 num_classes = 5 726 batch_size = 3 727 728 for num_true in range(1, 5): 729 labels = np.random.randint( 730 low=0, high=num_classes, size=batch_size * num_true) 731 (weights, biases, hidden_acts, sampled_vals, exp_logits, 732 exp_labels) = self._GenerateTestData( 733 num_classes=num_classes, 734 dim=10, 735 batch_size=batch_size, 736 num_true=num_true, 737 labels=labels, 738 sampled=[1, 0, 2, 3], 739 subtract_log_q=True) 740 logits_tensor, labels_tensor = _compute_sampled_logits( 741 weights=constant_op.constant(weights), 742 biases=constant_op.constant(biases), 743 labels=constant_op.constant( 744 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 745 inputs=constant_op.constant(hidden_acts), 746 num_sampled=4, 747 num_classes=num_classes, 748 num_true=num_true, 749 sampled_values=sampled_vals, 750 subtract_log_q=True, 751 remove_accidental_hits=False, 752 partition_strategy="div", 753 name="sampled_logits_subtract_log_q_num_true_%d" % num_true) 754 got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) 755 self.assertAllClose(exp_logits, got_logits, self._eps) 756 self.assertAllClose(exp_labels, got_labels, self._eps) 757 758 def testSharded(self): 759 """With sharded weights and sharded biases.""" 760 np.random.seed(0) 761 num_classes = 5 762 batch_size = 3 763 764 for num_true in range(1, 5): 765 labels = np.random.randint( 766 low=0, high=num_classes, size=batch_size * num_true) 767 (weights, biases, hidden_acts, sampled_vals, exp_logits, 768 exp_labels) = self._GenerateTestData( 769 num_classes=num_classes, 770 dim=10, 771 batch_size=batch_size, 772 num_true=num_true, 773 labels=labels, 774 sampled=[1, 0, 2, 3], 775 subtract_log_q=False) 776 weight_shards, bias_shards = self._ShardTestEmbeddings( 777 weights, biases, num_shards=3) 778 logits_tensor, labels_tensor = _compute_sampled_logits( 779 weights=[constant_op.constant(shard) for shard in weight_shards], 780 biases=[constant_op.constant(shard) for shard in bias_shards], 781 labels=constant_op.constant( 782 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 783 inputs=constant_op.constant(hidden_acts), 784 num_sampled=4, 785 num_classes=num_classes, 786 num_true=num_true, 787 sampled_values=sampled_vals, 788 subtract_log_q=False, 789 remove_accidental_hits=False, 790 partition_strategy="div", 791 name="sampled_logits_sharded_num_true_%d" % num_true) 792 got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) 793 self.assertAllClose(exp_logits, got_logits, self._eps) 794 self.assertAllClose(exp_labels, got_labels, self._eps) 795 796 def testNCELoss(self): 797 # A simple test to verify the numerics. 798 799 def _SigmoidCrossEntropyWithLogits(logits, targets): 800 # logits, targets: float arrays of the same shape. 801 assert logits.shape == targets.shape 802 pred = 1. / (1. + np.exp(-logits)) 803 eps = 0.0001 804 pred = np.minimum(np.maximum(pred, eps), 1 - eps) 805 return -targets * np.log(pred) - (1. - targets) * np.log(1. - pred) 806 807 np.random.seed(0) 808 num_classes = 5 809 batch_size = 3 810 labels = [0, 1, 2] 811 (weights, biases, hidden_acts, sampled_vals, exp_logits, 812 exp_labels) = self._GenerateTestData( 813 num_classes=num_classes, 814 dim=10, 815 batch_size=batch_size, 816 num_true=1, 817 labels=labels, 818 sampled=[1, 0, 2, 3], 819 subtract_log_q=True) 820 exp_nce_loss = np.sum( 821 _SigmoidCrossEntropyWithLogits(exp_logits, exp_labels), 1) 822 823 got_nce_loss = nn_impl.nce_loss_v2( 824 weights=constant_op.constant(weights), 825 biases=constant_op.constant(biases), 826 labels=constant_op.constant(labels, shape=(batch_size, 1)), 827 inputs=constant_op.constant(hidden_acts), 828 num_sampled=4, 829 num_classes=num_classes, 830 num_true=1, 831 sampled_values=sampled_vals) 832 833 self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4) 834 835 # Test with sharded weights and sharded biases. 836 weight_shards, bias_shards = self._ShardTestEmbeddings( 837 weights, biases, num_shards=3) 838 got_nce_loss = nn_impl.nce_loss_v2( 839 weights=[constant_op.constant(shard) for shard in weight_shards], 840 biases=[constant_op.constant(shard) for shard in bias_shards], 841 labels=constant_op.constant(labels, shape=(batch_size, 1)), 842 inputs=constant_op.constant(hidden_acts), 843 num_sampled=4, 844 num_classes=num_classes, 845 num_true=1, 846 sampled_values=sampled_vals) 847 848 self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4) 849 850 def testSampledSoftmaxLoss(self): 851 # A simple test to verify the numerics. 852 853 def _SoftmaxCrossEntropyWithLogits(logits, targets): 854 # logits, targets: float arrays of the same shape. 855 assert logits.shape == targets.shape 856 stable_exp_logits = np.exp( 857 logits - np.amax(logits, axis=1, keepdims=True)) 858 pred = stable_exp_logits / np.sum(stable_exp_logits, 1, keepdims=True) 859 return -np.sum(targets * np.log(pred + 1.0e-20), axis=1) 860 861 np.random.seed(0) 862 num_classes = 5 863 batch_size = 3 864 labels = [0, 1, 2] 865 (weights, biases, hidden_acts, sampled_vals, exp_logits, 866 exp_labels) = self._GenerateTestData( 867 num_classes=num_classes, 868 dim=10, 869 batch_size=batch_size, 870 num_true=1, 871 labels=labels, 872 sampled=[1, 0, 2, 3], 873 subtract_log_q=True) 874 exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits( 875 exp_logits, exp_labels) 876 877 got_sampled_softmax_loss = nn_impl.sampled_softmax_loss_v2( 878 weights=constant_op.constant(weights), 879 biases=constant_op.constant(biases), 880 labels=constant_op.constant(labels, shape=(batch_size, 1)), 881 inputs=constant_op.constant(hidden_acts), 882 num_sampled=4, 883 num_classes=num_classes, 884 num_true=1, 885 sampled_values=sampled_vals, 886 remove_accidental_hits=False) 887 888 self.assertAllClose(exp_sampled_softmax_loss, 889 self.evaluate(got_sampled_softmax_loss), 1e-4) 890 891 # Test with sharded weights and sharded biases. 892 weight_shards, bias_shards = self._ShardTestEmbeddings( 893 weights, biases, num_shards=3) 894 got_sampled_softmax_loss = nn_impl.sampled_softmax_loss_v2( 895 weights=[constant_op.constant(shard) for shard in weight_shards], 896 biases=[constant_op.constant(shard) for shard in bias_shards], 897 labels=constant_op.constant(labels, shape=(batch_size, 1)), 898 inputs=constant_op.constant(hidden_acts), 899 num_sampled=4, 900 num_classes=num_classes, 901 num_true=1, 902 sampled_values=sampled_vals, 903 remove_accidental_hits=False) 904 905 self.assertAllClose(exp_sampled_softmax_loss, 906 self.evaluate(got_sampled_softmax_loss), 1e-4) 907 908 def testSampledSoftmaxLossBf16(self): 909 # A simple test to verify the numerics for bfloat16. 910 def _SoftmaxCrossEntropyWithLogits(logits, targets): 911 # logits, targets: float arrays of the same shape. 912 assert logits.shape == targets.shape 913 stable_exp_logits = np.exp( 914 logits - np.amax(logits, axis=1, keepdims=True)) 915 pred = stable_exp_logits / np.sum(stable_exp_logits, 1, keepdims=True) 916 return -np.sum(targets * np.log(pred + 1.0e-20), axis=1) 917 918 np.random.seed(0) 919 num_classes = 5 920 batch_size = 3 921 labels = [0, 1, 2] 922 sampled = [1, 0, 2, 3] 923 (weights, biases, hidden_acts, _, exp_logits, 924 exp_labels) = self._GenerateTestData( 925 num_classes=num_classes, 926 dim=10, 927 batch_size=batch_size, 928 num_true=1, 929 labels=labels, 930 sampled=sampled, 931 subtract_log_q=True) 932 exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits( 933 exp_logits, exp_labels) 934 935 true_exp_bf16 = np.full([batch_size, 1], 936 fill_value=0.5, 937 dtype=dtypes.bfloat16.as_numpy_dtype) 938 sampled_exp_bf16 = np.full([len(sampled)], 939 fill_value=0.5, 940 dtype=dtypes.bfloat16.as_numpy_dtype) 941 sampled_vals_bf16 = (sampled, true_exp_bf16, sampled_exp_bf16) 942 943 got_sampled_softmax_loss = math_ops.cast( 944 nn_impl.sampled_softmax_loss_v2( 945 weights=constant_op.constant(weights, dtype=dtypes.bfloat16), 946 biases=constant_op.constant(biases, dtype=dtypes.bfloat16), 947 labels=constant_op.constant( 948 labels, shape=(batch_size, 1), dtype=dtypes.bfloat16), 949 inputs=constant_op.constant(hidden_acts, dtype=dtypes.bfloat16), 950 num_sampled=4, 951 num_classes=num_classes, 952 num_true=1, 953 sampled_values=sampled_vals_bf16, 954 remove_accidental_hits=False), dtypes.float32) 955 956 self.assertAllClose(exp_sampled_softmax_loss, 957 self.evaluate(got_sampled_softmax_loss), 1e-1) 958 959 960class CReluTest(test_lib.TestCase): 961 962 def test(self): 963 np.random.seed(1) # Make it reproducible. 964 x = np.random.randn(3, 4).astype(np.float32) 965 y = np.concatenate([x * (x > 0), -x * (x < 0)], axis=1) 966 967 z = self.evaluate(nn_ops.crelu(constant_op.constant(x))) 968 self.assertAllClose(y, z, 1e-4) 969 970 971class ReluTest(test_lib.TestCase): 972 973 def test(self): 974 np.random.seed(1) # Make it reproducible. 975 x = np.random.randn(3, 4).astype(np.float32) 976 y = np.maximum(x, 0.0) 977 978 z = self.evaluate(nn_ops.relu(constant_op.constant(x))) 979 self.assertAllEqual(y, z) 980 981 @test_util.run_deprecated_v1 982 def testNaNs(self): 983 # Test that relu(nan) = nan for various sizes. 984 for i in range(18): 985 x = np.zeros(i) + np.nan 986 with self.cached_session(): 987 z = nn_ops.relu(constant_op.constant(x)).eval() 988 self.assertTrue(np.isnan(z).all()) 989 990 991class LeakyReluTest(test_lib.TestCase): 992 993 def testRange(self): 994 batch_size = 3 995 height, width = 4, 4 996 np.random.seed(1) # Make it reproducible. 997 inputs = np.random.uniform(size=(batch_size, height, width, 3)).astype( 998 np.float32) 999 inputs = constant_op.constant(inputs) 1000 1001 outputs = nn_ops.leaky_relu(inputs) 1002 self.assertEquals(inputs.shape, outputs.shape) 1003 1004 inputs, outputs = self.evaluate([inputs, outputs]) 1005 1006 self.assertGreaterEqual(outputs.min(), 0.0) 1007 self.assertLessEqual(outputs.max(), 1.0) 1008 self.assertAllClose(inputs, outputs) 1009 1010 @test_util.run_deprecated_v1 1011 def testValues(self): 1012 for dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]: 1013 np_values = np.array([-2, -1, 0, 1, 2], dtype=dtype) 1014 outputs = nn_ops.leaky_relu(constant_op.constant(np_values)) 1015 1016 outputs = self.evaluate(outputs) 1017 1018 tol = 2e-3 if dtype == np.float16 else 1e-6 1019 self.assertAllClose( 1020 outputs, [-0.4, -0.2, 0.0, 1.0, 2.0], rtol=tol, atol=tol) 1021 1022 @test_util.run_deprecated_v1 1023 def testName(self): 1024 np_values = np.array([-2, -1, 0, 1, 2], dtype=np.float64) 1025 outputs_with_name_set = nn_ops.leaky_relu( 1026 constant_op.constant(np_values), 1027 name='test_relu_op') 1028 self.assertEqual(outputs_with_name_set.name, 'test_relu_op:0') 1029 outputs_without_name_set = nn_ops.leaky_relu( 1030 constant_op.constant(np_values)) 1031 self.assertEqual(outputs_without_name_set.name, 'LeakyRelu:0') 1032 1033 1034class SwishTest(test_lib.TestCase): 1035 1036 @test_util.run_deprecated_v1 1037 def testValues(self): 1038 np_values = np.array( 1039 [np.linspace(-7.0, 0.0, 100), 1040 np.linspace(0.0, 7.0, 100)], 1041 dtype=np.float32) 1042 tf_values = constant_op.constant(np_values) 1043 actual_tf_outputs = nn_impl.swish(tf_values) 1044 expected_tf_outputs = tf_values * math_ops.sigmoid(tf_values) 1045 1046 actual_outputs, expected_outputs = self.evaluate( 1047 [actual_tf_outputs, expected_tf_outputs]) 1048 1049 self.assertAllClose(actual_outputs, expected_outputs) 1050 1051 @test_util.run_deprecated_v1 1052 def testGradients(self): 1053 shape = [5, 3, 4] 1054 sigma = 5 1055 input_values = np.random.randn(*shape) * sigma 1056 x_tf = constant_op.constant(input_values) 1057 y_tf = nn_impl.swish(x_tf) 1058 with self.cached_session(): 1059 err = gradient_checker.compute_gradient_error(x_tf, shape, y_tf, shape) 1060 self.assertLess(err, 1e-4) 1061 1062 1063class MomentsTest(test_lib.TestCase): 1064 1065 def doOutputTest(self, 1066 input_shape, 1067 moments_axes, 1068 tol=1e-4, 1069 check_gradients=False): 1070 for mu in [0.0, 1.0, 1e3]: 1071 for sigma in [1.0, 0.1]: 1072 for keep_dims in [True, False]: 1073 input_values = np.random.rand(*input_shape) * sigma + mu 1074 expected_mean = np.mean( 1075 input_values, axis=moments_axes, keepdims=keep_dims) 1076 expected_var = np.var( 1077 input_values, axis=moments_axes, keepdims=keep_dims) 1078 with ops.Graph().as_default() as g: 1079 with self.session(graph=g) as sess: 1080 inputs = constant_op.constant( 1081 input_values, shape=input_shape, dtype=dtypes.float32) 1082 mean, variance = nn_impl.moments_v2( 1083 inputs, moments_axes, keepdims=keep_dims) 1084 1085 if check_gradients: 1086 err = gradient_checker.compute_gradient_error( 1087 inputs, input_shape, mean, mean.shape.as_list()) 1088 self.assertLess(err, 1e-3) 1089 err = gradient_checker.compute_gradient_error( 1090 inputs, input_shape, variance, variance.shape.as_list()) 1091 self.assertLess(err, 1e-3) 1092 1093 # Evaluate. 1094 [mean, variance] = self.evaluate([mean, variance]) 1095 # Make sure that there are no NaNs 1096 self.assertFalse(np.isnan(mean).any()) 1097 self.assertFalse(np.isnan(variance).any()) 1098 self.assertAllClose(mean, expected_mean, rtol=tol, atol=tol) 1099 self.assertAllClose(variance, expected_var, rtol=tol, atol=tol) 1100 1101 def testOutputAndGradient2DInput0(self): 1102 self.doOutputTest((10, 10), (0,), check_gradients=True) 1103 1104 def testOutputAndGradient2DInput01(self): 1105 self.doOutputTest((10, 10), (0, 1), check_gradients=True) 1106 1107 def testOutput2DInput0(self): 1108 self.doOutputTest((10, 300), (0,)) 1109 1110 def testOutput2DInput1(self): 1111 self.doOutputTest((10, 300), (1,)) 1112 1113 def testOutput2DInput01(self): 1114 self.doOutputTest((10, 300), (0, 1)) 1115 1116 def testOutput4DInput0(self): 1117 self.doOutputTest((10, 10, 10, 30), (0,)) 1118 1119 def testOutput4DInput1(self): 1120 self.doOutputTest((10, 10, 10, 30), (1,)) 1121 1122 def testOutput4DInput3(self): 1123 self.doOutputTest((10, 10, 10, 30), (3,)) 1124 1125 def testOutput4DInput012(self): 1126 self.doOutputTest((10, 10, 10, 30), (0, 1, 2)) 1127 1128 def testOutput4DInput123(self): 1129 self.doOutputTest((10, 10, 10, 30), (1, 2, 3)) 1130 1131 1132class DataFormatDimMapTest(test_lib.TestCase): 1133 1134 def _test(self, x_val, y_val_expected): 1135 x = constant_op.constant(x_val) 1136 y = nn_ops.data_format_dim_map(x) 1137 1138 y_val = self.evaluate(y) 1139 self.assertAllEqual(y_val, y_val_expected) 1140 1141 def test(self): 1142 self._test(0, 0) 1143 self._test(1, 2) 1144 self._test(2, 3) 1145 self._test(3, 1) 1146 self._test(-1, 1) 1147 self._test(-2, 3) 1148 self._test(-3, 2) 1149 self._test(-4, 0) 1150 self._test([1, 3], [2, 1]) 1151 self._test([1, 3, -2], [2, 1, 3]) 1152 self._test([1, -3, -2], [2, 2, 3]) 1153 self._test([[1, -3], [1, -1]], [[2, 2], [2, 1]]) 1154 1155 def testNHWCtoNCHW(self): 1156 x_val = [1, -3, -2] 1157 y_val_expected = [2, 2, 3] 1158 x = constant_op.constant(x_val) 1159 y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="NCHW") 1160 with test_util.use_gpu(): 1161 y_val = self.evaluate(y) 1162 self.assertAllEqual(y_val, y_val_expected) 1163 1164 def testNHWCtoHWNC(self): 1165 x_val = [-4, -3, -2, -1, 0, 1, 2, 3] 1166 y_val_expected = [2, 0, 1, 3, 2, 0, 1, 3] 1167 x = constant_op.constant(x_val) 1168 y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="HWNC") 1169 with test_util.use_gpu(): 1170 y_val = self.evaluate(y) 1171 self.assertAllEqual(y_val, y_val_expected) 1172 1173 def testNHWCtoWHCN(self): 1174 x_val = [-4, -3, -2, -1, 0, 1, 2, 3] 1175 y_val_expected = [3, 1, 0, 2, 3, 1, 0, 2] 1176 x = constant_op.constant(x_val) 1177 y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="WHCN") 1178 with test_util.use_gpu(): 1179 y_val = self.evaluate(y) 1180 self.assertAllEqual(y_val, y_val_expected) 1181 1182 def testArbitraryASCII(self): 1183 x_val = [-4, -3, -2, -1, 0, 1, 2, 3] 1184 y_val_expected = [3, 2, 1, 0, 3, 2, 1, 0] 1185 x = constant_op.constant(x_val) 1186 y = nn_ops.data_format_dim_map(x, src_format="qwer", dst_format="rewq") 1187 with test_util.use_gpu(): 1188 y_val = self.evaluate(y) 1189 self.assertAllEqual(y_val, y_val_expected) 1190 1191 1192class DataFormatVectorPermuteTest(test_lib.TestCase): 1193 1194 def testNHWCToNCHW(self): 1195 x_val = [7, 4, 9, 3] 1196 x = constant_op.constant(x_val) 1197 y = nn_ops.data_format_vec_permute(x) 1198 with test_util.use_gpu(): 1199 y_val = self.evaluate(y) 1200 self.assertAllEqual(y_val, [7, 3, 4, 9]) 1201 1202 def testNCHWToNHWC(self): 1203 x_val = [7, 4, 9, 3] 1204 x = constant_op.constant(x_val) 1205 y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC") 1206 with test_util.use_gpu(): 1207 y_val = self.evaluate(y) 1208 self.assertAllEqual(y_val, [7, 9, 3, 4]) 1209 1210 def testNHWCToHWNC(self): 1211 x_val = [7, 4, 9, 3] 1212 x = constant_op.constant(x_val) 1213 y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC") 1214 with test_util.use_gpu(): 1215 y_val = self.evaluate(y) 1216 self.assertAllEqual(y_val, [4, 9, 7, 3]) 1217 1218 def testHWNCToNHWC(self): 1219 x_val = [7, 4, 9, 3] 1220 x = constant_op.constant(x_val) 1221 y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC") 1222 with test_util.use_gpu(): 1223 y_val = self.evaluate(y) 1224 self.assertAllEqual(y_val, [9, 7, 4, 3]) 1225 1226 def testNHWCToNCHW2D(self): 1227 x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] 1228 x = constant_op.constant(x_val) 1229 y = nn_ops.data_format_vec_permute(x) 1230 with test_util.use_gpu(): 1231 y_val = self.evaluate(y) 1232 self.assertAllEqual(y_val, [[7, 4], [5, 1], [9, 3], [4, 5]]) 1233 1234 def testNHWCToHWNC2D(self): 1235 x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] 1236 x = constant_op.constant(x_val) 1237 y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC") 1238 with test_util.use_gpu(): 1239 y_val = self.evaluate(y) 1240 self.assertAllEqual(y_val, [[9, 3], [4, 5], [7, 4], [5, 1]]) 1241 1242 def testHWNCToNHWC2D(self): 1243 x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] 1244 x = constant_op.constant(x_val) 1245 y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC") 1246 with test_util.use_gpu(): 1247 y_val = self.evaluate(y) 1248 self.assertAllEqual(y_val, [[4, 5], [7, 4], [9, 3], [5, 1]]) 1249 1250 def testNCHWToNHWC2D(self): 1251 x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] 1252 x = constant_op.constant(x_val) 1253 y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC") 1254 with test_util.use_gpu(): 1255 y_val = self.evaluate(y) 1256 self.assertAllEqual(y_val, [[7, 4], [4, 5], [5, 1], [9, 3]]) 1257 1258 1259@test_util.run_all_in_graph_and_eager_modes 1260class AvgPoolTest(test_lib.TestCase): 1261 1262 def test1DTensor(self): 1263 x = array_ops.ones([3, 6, 5]) 1264 ksize = 2 1265 strides = 2 1266 1267 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1268 y2 = nn_ops.avg_pool1d(x, ksize, strides, "SAME") 1269 1270 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1271 1272 def test1DNumpy(self): 1273 # explicilty use float32 for ROCm, as MIOpen does not yet support float64 1274 # np.ones defaults to using float64 when dtype is not explicitly specified 1275 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1276 x = np.ones([3, 6, 5], dtype=dtype) 1277 ksize = 2 1278 strides = 2 1279 1280 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1281 y2 = nn_ops.avg_pool1d(x, ksize, strides, "SAME") 1282 1283 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1284 1285 def test1DNumpyWithGolden(self): 1286 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1287 x = np.array([[[3], [6], [5]], 1288 [[1], [0], [1]]], dtype=dtype) 1289 ksize = 2 1290 strides = 1 1291 y = nn_ops.avg_pool1d(x, ksize, strides, "SAME") 1292 expected_y = np.array([[[4.5], [5.5], [5.0]], 1293 [[0.5], [0.5], [1.0]]], dtype=dtype) 1294 self.assertAllEqual(self.evaluate(y), expected_y) 1295 1296 def test2DTensor(self): 1297 x = array_ops.ones([3, 6, 6, 5]) 1298 ksize = 2 1299 strides = 2 1300 1301 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1302 y2 = nn_ops.avg_pool(x, ksize, strides, "SAME") 1303 1304 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1305 1306 def test2DNumpy(self): 1307 # explicilty use float32 for ROCm, as MIOpen does not yet support float64 1308 # np.ones defaults to using float64 when dtype is not explicitly specified 1309 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1310 x = np.ones([3, 6, 6, 5], dtype=dtype) 1311 ksize = 2 1312 strides = 2 1313 1314 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1315 y2 = nn_ops.avg_pool(x, ksize, strides, "SAME") 1316 1317 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1318 1319 def test3DTensor(self): 1320 if test_lib.is_built_with_rocm(): 1321 self.skipTest("Pooling with 3D tensors is not supported in ROCm") 1322 x = array_ops.ones([3, 7, 6, 6, 5]) 1323 ksize = 2 1324 strides = 2 1325 1326 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1327 y2 = nn_ops.avg_pool3d(x, ksize, strides, "SAME") 1328 1329 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1330 1331 def test3DNumpy(self): 1332 if test_lib.is_built_with_rocm(): 1333 self.skipTest("Pooling with 3D tensors is not supported in ROCm") 1334 x = np.ones([3, 7, 6, 6, 5], dtype=np.float32) 1335 ksize = 2 1336 strides = 2 1337 1338 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1339 y2 = nn_ops.avg_pool3d(x, ksize, strides, "SAME") 1340 1341 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1342 1343 1344@test_util.run_all_in_graph_and_eager_modes 1345class MaxPoolTest(test_lib.TestCase): 1346 1347 def test1DTensor(self): 1348 x = array_ops.ones([3, 6, 5]) 1349 ksize = 2 1350 strides = 2 1351 1352 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1353 y2 = nn_ops.max_pool1d(x, ksize, strides, "SAME") 1354 1355 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1356 1357 def test1DNumpy(self): 1358 # explicilty use float32 for ROCm, as MIOpen does not yet support float64 1359 # np.ones defaults to using float64 when dtype is not explicitly specified 1360 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1361 x = np.ones([3, 6, 5], dtype=dtype) 1362 ksize = 2 1363 strides = 2 1364 1365 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1366 y2 = nn_ops.max_pool1d(x, ksize, strides, "SAME") 1367 1368 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1369 1370 def test1DNumpyWithGolden(self): 1371 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1372 x = np.array([[[3], [6], [5]], 1373 [[1], [0], [1]]], dtype=dtype) 1374 ksize = 2 1375 strides = 1 1376 y = nn_ops.max_pool1d(x, ksize, strides, "SAME") 1377 expected_y = np.array([[[6], [6], [5]], 1378 [[1], [1], [1]]], dtype=dtype) 1379 self.assertAllEqual(self.evaluate(y), expected_y) 1380 1381 def test2DTensor(self): 1382 x = array_ops.ones([3, 6, 6, 5]) 1383 ksize = 2 1384 strides = 2 1385 1386 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1387 y2 = nn_ops.max_pool(x, ksize, strides, "SAME") 1388 1389 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1390 1391 def test2DNumpy(self): 1392 # explicilty use float32 for ROCm, as MIOpen does not yet support float64 1393 # np.ones defaults to using float64 when dtype is not explicitly specified 1394 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1395 x = np.ones([3, 6, 6, 5], dtype=dtype) 1396 ksize = 2 1397 strides = 2 1398 1399 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1400 y2 = nn_ops.max_pool(x, ksize, strides, "SAME") 1401 1402 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1403 1404 def test3DTensor(self): 1405 if test_lib.is_built_with_rocm(): 1406 self.skipTest("Pooling with 3D tensors is not supported in ROCm") 1407 x = array_ops.ones([3, 7, 6, 6, 5]) 1408 ksize = 2 1409 strides = 2 1410 1411 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1412 y2 = nn_ops.max_pool3d(x, ksize, strides, "SAME") 1413 1414 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1415 1416 def test3DNumpy(self): 1417 if test_lib.is_built_with_rocm(): 1418 self.skipTest("Pooling with 3D tensors is not supported in ROCm") 1419 x = np.ones([3, 7, 6, 6, 5], dtype=np.float32) 1420 ksize = 2 1421 strides = 2 1422 1423 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1424 y2 = nn_ops.max_pool3d(x, ksize, strides, "SAME") 1425 1426 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1427 1428 def testIncorrectSizeInputSmall(self): 1429 x = array_ops.ones([3, 4]) 1430 with self.assertRaisesRegex( 1431 ValueError, "Input tensor must be of rank 3, 4 or 5 but was 2."): 1432 nn_ops.max_pool_v2(x, 2, 2, "SAME") 1433 1434 def testIncorrectSizeInput(self): 1435 x = array_ops.ones([3, 4, 1, 2, 1, 2]) 1436 with self.assertRaisesRegex( 1437 ValueError, "Input tensor must be of rank 3, 4 or 5 but was 6."): 1438 nn_ops.max_pool_v2(x, 2, 2, "SAME") 1439 1440 1441@test_util.run_all_in_graph_and_eager_modes 1442class ConvolutionTest(test_lib.TestCase): 1443 1444 def testUnknownSize(self): 1445 # explicilty use float32 for ROCm, as MIOpen does not yet support float64 1446 # np.ones defaults to using float64 when dtype is not explicitly specified 1447 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1448 x = tensor_spec.TensorSpec(None, dtypes.float32, name="x") 1449 k = np.ones([3, 6, 6, 5], dtype=dtype) 1450 1451 @def_function.function 1452 def F(value): 1453 return nn_ops.convolution(value, k, "SAME") 1454 1455 F.get_concrete_function(x) 1456 1457 1458class ConvTransposeTest(test_lib.TestCase): 1459 1460 def test1D(self): 1461 t = array_ops.ones([2, 4, 3]) 1462 v = array_ops.ones([2, 5, 3]) 1463 strides = 2 1464 1465 y1 = nn_ops.conv1d_transpose(t, v, [2, 8, 5], strides) 1466 y2 = nn_ops.conv_transpose(t, v, [2, 8, 5], strides) 1467 1468 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1469 1470 def test1DTensor(self): 1471 t = array_ops.ones([2, 4, 3]) 1472 v = array_ops.ones([2, 5, 3]) 1473 strides = 2 1474 1475 y1 = nn_ops.conv1d_transpose(t, v, [2, 8, 5], strides) 1476 y2 = nn_ops.conv_transpose(t, v, constant_op.constant([2, 8, 5]), strides) 1477 1478 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1479 1480 def test2D(self): 1481 t = array_ops.ones([2, 4, 4, 3]) 1482 v = array_ops.ones([2, 2, 5, 3]) 1483 strides = 2 1484 1485 y1 = nn_ops.conv2d_transpose_v2(t, v, [2, 8, 8, 5], strides) 1486 y2 = nn_ops.conv_transpose(t, v, [2, 8, 8, 5], strides) 1487 1488 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1489 1490 def test2DTensor(self): 1491 t = array_ops.ones([2, 4, 4, 3]) 1492 v = array_ops.ones([2, 2, 5, 3]) 1493 strides = 2 1494 1495 y1 = nn_ops.conv2d_transpose_v2(t, v, [2, 8, 8, 5], strides) 1496 y2 = nn_ops.conv_transpose(t, v, constant_op.constant([2, 8, 8, 5]), 1497 strides) 1498 1499 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1500 1501 def test3D(self): 1502 t = array_ops.ones([2, 4, 4, 4, 3]) 1503 v = array_ops.ones([2, 2, 2, 5, 3]) 1504 strides = 2 1505 1506 y1 = nn_ops.conv3d_transpose_v2(t, v, [2, 8, 8, 8, 5], strides) 1507 y2 = nn_ops.conv_transpose(t, v, [2, 8, 8, 8, 5], strides) 1508 1509 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1510 1511 def test3DTensor(self): 1512 t = array_ops.ones([2, 4, 4, 4, 3]) 1513 v = array_ops.ones([2, 2, 2, 5, 3]) 1514 strides = 2 1515 1516 y1 = nn_ops.conv3d_transpose_v2(t, v, [2, 8, 8, 8, 5], strides) 1517 y2 = nn_ops.conv_transpose(t, v, constant_op.constant([2, 8, 8, 8, 5]), 1518 strides) 1519 1520 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1521 1522 def testIncorrectSizeInputSmall(self): 1523 with self.assertRaisesRegex( 1524 ValueError, "output_shape must be of length 3, 4 or 5 but was 2."): 1525 nn_ops.conv_transpose(None, 2, [2, 3], "SAME") 1526 1527 def testIncorrectSizeInput(self): 1528 with self.assertRaisesRegex( 1529 ValueError, "output_shape must be of length 3, 4 or 5 but was 6."): 1530 nn_ops.conv_transpose(None, 2, [2, 3, 4, 2, 5, 1], "SAME") 1531 1532 def testTensorsNoShape(self): 1533 with self.assertRaisesRegex( 1534 ValueError, 1535 "output_shape must be a tensor or sized collection."): 1536 nn_ops.conv_transpose(None, None, None, None) 1537 1538 1539class RaggedEmbeddingTest(test_lib.TestCase): 1540 1541 def testRaggedTensor(self): 1542 weights = constant_op.constant([[0, 0, 0], [1, 1, 1], [2, 2, 2], [3, 3, 3]]) 1543 ragged_ids = ragged_factory_ops.constant([[1, 2, 3], [0], [1, 2]], 1544 ragged_rank=1) 1545 1546 embedded_ragged = nn.embedding_lookup_ragged(weights, ragged_ids) 1547 expected_output = ragged_factory_ops.constant( 1548 [[[1, 1, 1], [2, 2, 2], [3, 3, 3]], [[0, 0, 0]], [[1, 1, 1], [2, 2, 2]] 1549 ], 1550 ragged_rank=1) 1551 1552 self.assertAllEqual(expected_output, embedded_ragged) 1553 1554 def testMultipleRaggedDimTensor(self): 1555 weights = constant_op.constant([[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], 1556 [5, 5], [6, 6]]) 1557 ragged_ids = ragged_factory_ops.constant( 1558 [[[[3, 4], [0, 6]], []], [[[2, 1], [1, 0]], [[2, 5], [2, 3]]], [[[1, 0]] 1559 ]], 1560 ragged_rank=2) 1561 1562 embedded_ragged = nn.embedding_lookup_ragged(weights, ragged_ids) 1563 expected_output = ragged_factory_ops.constant( 1564 [[[[[3, 3], [4, 4]], [[0, 0], [6, 6]]], []], 1565 [[[[2, 2], [1, 1]], [[1, 1], [0, 0]]], 1566 [[[2, 2], [5, 5]], [[2, 2], [3, 3]]]], [[[[1, 1], [0, 0]]]]], 1567 ragged_rank=2) 1568 1569 self.assertAllEqual(expected_output, embedded_ragged) 1570 1571 def testMissingWeights(self): 1572 ragged_ids = ragged_factory_ops.constant([[1, 2, 3], [0], [1, 2]]) 1573 1574 with self.assertRaisesRegex(ValueError, 1575 "The embedding weights must be specified.*"): 1576 nn.embedding_lookup_ragged(None, ragged_ids) 1577 1578 def testEmptyWeights(self): 1579 ragged_ids = ragged_factory_ops.constant([[1, 2, 3], [0], [1, 2]]) 1580 1581 with self.assertRaisesRegex(ValueError, 1582 "The embedding weights should not be empty.*"): 1583 nn.embedding_lookup_ragged([], ragged_ids) 1584 1585 def testInvalidIndicesType(self): 1586 weights = constant_op.constant([[0, 0, 0], [1, 1, 1], [2, 2, 2]]) 1587 ragged_ids = ragged_factory_ops.constant([[1., 2., 3.], [1., 2.]]) 1588 1589 with self.assertRaisesRegex( 1590 ValueError, "The values contained by the inputs have type*"): 1591 nn.embedding_lookup_ragged(weights, ragged_ids) 1592 1593 1594if __name__ == "__main__": 1595 test_lib.main() 1596