1# Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for TensorFlow 2.0 layer behavior.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import copy 22import os 23import sys 24import traceback 25 26import numpy as np 27 28from tensorflow.python.eager import context 29from tensorflow.python.eager import def_function 30from tensorflow.python.framework import composite_tensor 31from tensorflow.python.framework import constant_op 32from tensorflow.python.framework import dtypes 33from tensorflow.python.framework import errors_impl 34from tensorflow.python.framework import ops 35from tensorflow.python.framework import sparse_tensor 36from tensorflow.python.framework import tensor_spec 37from tensorflow.python.framework import type_spec 38from tensorflow.python.keras import backend 39from tensorflow.python.keras import combinations 40from tensorflow.python.keras import keras_parameterized 41from tensorflow.python.keras import layers 42from tensorflow.python.keras import regularizers 43from tensorflow.python.keras import testing_utils 44from tensorflow.python.keras.engine import base_layer 45from tensorflow.python.keras.engine import input_layer 46from tensorflow.python.keras.engine import sequential 47from tensorflow.python.keras.engine import training as training_lib 48from tensorflow.python.keras.legacy_tf_layers import core as legacy_core 49from tensorflow.python.keras.optimizer_v2 import rmsprop 50from tensorflow.python.keras.utils import control_flow_util 51from tensorflow.python.module import module 52from tensorflow.python.ops import array_ops 53from tensorflow.python.ops import math_ops 54from tensorflow.python.ops import state_ops 55from tensorflow.python.ops import summary_ops_v2 56from tensorflow.python.ops import tensor_array_ops 57from tensorflow.python.ops import variables 58from tensorflow.python.ops.ragged import ragged_tensor 59from tensorflow.python.platform import gfile 60from tensorflow.python.platform import test 61from tensorflow.python.summary import summary_iterator 62from tensorflow.python.util import nest 63 64 65class DynamicLayer(base_layer.Layer): 66 67 def __init__(self, dynamic=False, **kwargs): 68 super(DynamicLayer, self).__init__(dynamic=dynamic, **kwargs) 69 70 def call(self, inputs): 71 samples = tensor_array_ops.TensorArray( 72 dtype=dtypes.float32, size=array_ops.shape(inputs)[0]) 73 for idx, sample in enumerate(inputs): 74 samples = samples.write(idx, math_ops.square(sample)) 75 return samples.stack() 76 77 def compute_output_shape(self, input_shape): 78 return input_shape 79 80 81class InvalidLayer(base_layer.Layer): 82 83 def call(self, inputs): 84 raise ValueError('You did something wrong!') 85 86 87class BaseLayerTest(keras_parameterized.TestCase): 88 89 @combinations.generate(combinations.keras_mode_combinations()) 90 def test_layer_instrumentation(self): 91 layer = layers.Add() 92 self.assertTrue(layer._instrumented_keras_api) 93 self.assertTrue(layer._instrumented_keras_layer_class) 94 self.assertFalse(layer._instrumented_keras_model_class) 95 self.assertTrue(base_layer.keras_api_gauge.get_cell('tf.keras.layers.Add')) 96 base_layer.keras_api_gauge.get_cell('tf.keras.layers.Add').set(False) 97 98 @combinations.generate(combinations.keras_model_type_combinations()) 99 def test_dynamic_layer(self): 100 model = testing_utils.get_model_from_layers([DynamicLayer(dynamic=True)], 101 input_shape=(3,)) 102 self.assertEqual(model.dynamic, True) 103 model.compile(rmsprop.RMSprop(0.001), loss='mse') 104 self.assertEqual(model.run_eagerly, True) 105 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 106 107 @combinations.generate(combinations.keras_model_type_combinations()) 108 def test_dynamic_layer_error(self): 109 # Functional Models hit the `dyanamic=True` error during construction. 110 # Subclass Models should just throw the original autograph error during 111 # execution. 112 raised_error = False 113 try: 114 model = testing_utils.get_model_from_layers([DynamicLayer()], 115 input_shape=(3,)) 116 model.compile(rmsprop.RMSprop(0.001), loss='mse') 117 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 118 except errors_impl.OperatorNotAllowedInGraphError as e: 119 if 'iterating over `tf.Tensor` is not allowed' in str(e): 120 raised_error = True 121 except TypeError as e: 122 if 'attempting to use Python control flow' in str(e): 123 raised_error = True 124 self.assertTrue(raised_error) 125 126 @combinations.generate(combinations.keras_model_type_combinations()) 127 def test_dynamic_layer_error_running_in_graph_mode(self): 128 with ops.get_default_graph().as_default(): 129 model = testing_utils.get_model_from_layers([DynamicLayer(dynamic=True)], 130 input_shape=(3,)) 131 self.assertEqual(model.dynamic, True) 132 # But then you cannot run the model since you're in a graph scope. 133 with self.assertRaisesRegex(ValueError, 134 'You must enable eager execution'): 135 model.compile(rmsprop.RMSprop(0.001), loss='mse') 136 137 def test_manual_compute_output_shape(self): 138 139 class BuildCounter(base_layer.Layer): 140 141 def __init__(self, *args, **kwargs): # pylint: disable=redefined-outer-name 142 super(BuildCounter, self).__init__(*args, **kwargs) 143 self.build_counter = 0 144 145 def build(self, input_shape): 146 self.build_counter += 1 147 self.build_shape = input_shape 148 149 def call(self, inputs): 150 return inputs 151 152 layer = BuildCounter(dtype=dtypes.float64) 153 output_shape = layer.compute_output_shape((None, 10)) 154 self.assertEqual(layer.build_counter, 1) 155 self.assertEqual(layer.build_shape.as_list(), [None, 10]) 156 self.assertEqual(output_shape.as_list(), [None, 10]) 157 output_signature = layer.compute_output_signature( 158 tensor_spec.TensorSpec(dtype=dtypes.float64, shape=[None, 10])) 159 self.assertEqual(layer.build_counter, 1) 160 self.assertEqual(layer.build_shape.as_list(), [None, 10]) 161 self.assertEqual(output_signature.dtype, dtypes.float64) 162 self.assertEqual(output_signature.shape.as_list(), [None, 10]) 163 layer(np.ones((5, 10))) 164 self.assertEqual(layer.build_counter, 1) 165 self.assertEqual(layer.build_shape.as_list(), [None, 10]) 166 167 def test_dynamic_layer_with_deferred_sequential_model(self): 168 model = sequential.Sequential([DynamicLayer(dynamic=True), layers.Dense(3)]) 169 self.assertEqual(model.dynamic, True) 170 model.compile(rmsprop.RMSprop(0.001), loss='mse') 171 self.assertEqual(model.run_eagerly, True) 172 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 173 174 def test_nested_dynamic_layers_in_eager_mode(self): 175 inputs = input_layer.Input((3,)) 176 outputs = DynamicLayer(dynamic=True)(inputs) 177 inner_model = training_lib.Model(inputs, outputs) 178 self.assertEqual(inner_model.dynamic, True) 179 180 inputs = input_layer.Input((3,)) 181 x = DynamicLayer(dynamic=True)(inputs) 182 outputs = inner_model(x) 183 184 model = training_lib.Model(inputs, outputs) 185 self.assertEqual(model.dynamic, True) 186 model.compile(rmsprop.RMSprop(0.001), loss='mse') 187 self.assertEqual(model.run_eagerly, True) 188 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 189 190 def test_dynamic_subclassed_model_no_shape_inference(self): 191 192 class MyModel(training_lib.Model): 193 194 def __init__(self): 195 super(MyModel, self).__init__(dynamic=True) 196 self.layer1 = layers.Dense(3) 197 self.layer2 = layers.Dense(3) 198 199 def call(self, inputs): 200 if math_ops.reduce_sum(inputs) > 0: 201 return self.layer1(inputs) 202 else: 203 return self.layer2(inputs) 204 205 model = MyModel() 206 self.assertEqual(model.dynamic, True) 207 model.compile(rmsprop.RMSprop(0.001), loss='mse') 208 self.assertEqual(model.run_eagerly, True) 209 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 210 self.assertEqual(model.outputs, None) 211 212 def test_dynamic_subclassed_model_with_shape_inference(self): 213 214 class MyModel(training_lib.Model): 215 216 def __init__(self): 217 super(MyModel, self).__init__(dynamic=True) 218 self.layer1 = layers.Dense(3) 219 self.layer2 = layers.Dense(3) 220 221 def call(self, inputs): 222 if math_ops.reduce_sum(inputs) > 0: 223 return self.layer1(inputs) 224 else: 225 return self.layer2(inputs) 226 227 def compute_output_shape(self, input_shape): 228 return tuple(input_shape[:-1].as_list()) + (3,) 229 230 model = MyModel() 231 self.assertEqual(model.dynamic, True) 232 model.compile(rmsprop.RMSprop(0.001), loss='mse') 233 x, y = np.random.random((2, 3)), np.random.random((2, 3)) 234 model.train_on_batch(x, y) 235 outputs = model(x) 236 self.assertEqual(outputs.shape.as_list(), [2, 3]) 237 238 def test_deepcopy(self): 239 bias_reg = lambda x: 1e-3 * math_ops.reduce_sum(x) 240 layer = layers.Conv2D(32, (3, 3), bias_regularizer=bias_reg) 241 # Call the Layer on data to generate regularize losses. 242 layer(array_ops.ones((1, 10, 10, 3))) 243 self.assertLen(layer.losses, 1) 244 new_layer = copy.deepcopy(layer) 245 self.assertEqual(new_layer.bias_regularizer, bias_reg) 246 self.assertEqual(layer.get_config(), new_layer.get_config()) 247 248 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 249 def test_invalid_forward_pass(self): 250 inputs = input_layer.Input((3,)) 251 with self.assertRaisesRegex(ValueError, 'You did something wrong!'): 252 _ = InvalidLayer()(inputs) 253 254 def test_no_legacy_model(self): 255 inputs = input_layer.Input((1,)) 256 legacy_dense_0 = legacy_core.Dense(1, name='legacy_dense_0') 257 legacy_dense_1 = legacy_core.Dense(1, name='legacy_dense_1') 258 259 layer = legacy_dense_0(inputs) 260 layer = layers.Dense(1)(layer) 261 layer = legacy_dense_1(layer) 262 263 expected_regex = (r'The following are legacy tf\.layers\.Layers:\n ' 264 '{}\n {}'.format(legacy_dense_0, legacy_dense_1)) 265 266 with self.assertRaisesRegex(TypeError, expected_regex): 267 _ = training_lib.Model(inputs=[inputs], outputs=[layer]) 268 269 model = training_lib.Model(inputs=[inputs], outputs=[inputs]) 270 with self.assertRaisesRegex(TypeError, expected_regex): 271 model._insert_layers([legacy_dense_0, legacy_dense_1]) 272 273 def test_no_legacy_sequential(self): 274 layer = [layers.Dense(1), legacy_core.Dense(1, name='legacy_dense_0')] 275 276 expected_regex = r'legacy tf\.layers\.Layers:\n {}'.format(layer[1]) 277 with self.assertRaisesRegex(TypeError, expected_regex): 278 _ = sequential.Sequential(layer) 279 280 with self.assertRaisesRegex(TypeError, expected_regex): 281 _ = sequential.Sequential([input_layer.Input(shape=(4,))] + layer) 282 283 model = sequential.Sequential() 284 with self.assertRaisesRegex(TypeError, expected_regex): 285 for l in layer: 286 model.add(l) 287 288 @combinations.generate( 289 combinations.times( 290 combinations.keras_model_type_combinations(), 291 combinations.combine(mode=['graph', 'eager']))) 292 def test_build_with_numpy_data(self): 293 model_layers = [ 294 layers.Dense(3, activation='relu', kernel_initializer='ones'), 295 layers.Dense(1, activation='sigmoid', kernel_initializer='ones') 296 ] 297 model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,)) 298 model(np.zeros((2, 4), dtype='float32')) 299 self.assertTrue(model.built) 300 301 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 302 def test_default_add_weight(self): 303 304 class TestLayer(base_layer.Layer): 305 306 def __init__(self): 307 super(TestLayer, self).__init__() 308 self.default_weight = self.add_weight() 309 self.weight_without_name = self.add_weight(shape=(3, 4)) 310 self.regularized_weight_without_name = self.add_weight( 311 shape=(3, 4), regularizer='l2') 312 313 layer = TestLayer() 314 self.assertEqual(layer.default_weight.shape.as_list(), []) 315 self.assertEqual(layer.weight_without_name.shape.as_list(), [3, 4]) 316 self.assertEqual(layer.default_weight.dtype.name, 'float32') 317 self.assertEqual(layer.weight_without_name.dtype.name, 'float32') 318 self.assertEqual(len(layer.losses), 1) 319 if not context.executing_eagerly(): 320 # Cannot access tensor.name in eager execution. 321 self.assertIn('Variable_2/Regularizer', layer.losses[0].name) 322 323 @combinations.generate(combinations.keras_mode_combinations(mode=['eager'])) 324 def test_learning_phase_freezing_for_layers(self): 325 326 class LearningPhaseLayer(base_layer.Layer): 327 328 def call(self, inputs): 329 return backend.in_train_phase(lambda: array_ops.ones_like(inputs), 330 lambda: array_ops.zeros_like(inputs)) 331 332 def get_learning_phase_value(): 333 model = sequential.Sequential([LearningPhaseLayer(input_shape=(1,))]) 334 model._run_eagerly = testing_utils.should_run_eagerly() 335 return np.sum(model(np.ones((1, 1)))) 336 337 self.assertEqual(get_learning_phase_value(), 0) 338 339 # Test scope. 340 with backend.learning_phase_scope(1): 341 self.assertEqual(get_learning_phase_value(), 1) 342 343 # The effects of the scope end after exiting it. 344 self.assertEqual(get_learning_phase_value(), 0) 345 346 # Test setting. 347 backend.set_learning_phase(1) 348 self.assertEqual(get_learning_phase_value(), 1) 349 backend.set_learning_phase(0) 350 self.assertEqual(get_learning_phase_value(), 0) 351 352 # Cannot be enabled with `run_eagerly=True`, see b/123904578 353 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 354 def test_layer_can_return_variable(self): 355 356 class ComputeSum(base_layer.Layer): 357 358 def __init__(self): 359 super(ComputeSum, self).__init__() 360 self.total = variables.Variable( 361 initial_value=array_ops.zeros((1, 1)), trainable=False) 362 if not context.executing_eagerly(): 363 backend.get_session().run(self.total.initializer) 364 365 def call(self, inputs): 366 self.total.assign_add(inputs) 367 return self.total 368 369 inputs = input_layer.Input(shape=(1,)) 370 model = training_lib.Model(inputs, ComputeSum()(inputs)) 371 model.predict(np.ones((1, 1))) 372 373 def _get_layer_with_training_arg(self): 374 375 class TrainingLayer(base_layer.Layer): 376 """A layer with a `training` argument in a defuned `call`.""" 377 378 @def_function.function 379 def call(self, inputs, training=None): 380 if training is None: 381 training = backend.learning_phase() 382 return control_flow_util.smart_cond( 383 training, lambda: array_ops.ones_like(inputs), 384 lambda: array_ops.zeros_like(inputs)) 385 386 return TrainingLayer() 387 388 # b/124459427: can't test with `run_eagerly=True` for now. 389 @combinations.generate( 390 combinations.times(combinations.keras_mode_combinations(), 391 combinations.keras_model_type_combinations())) 392 def test_training_arg_in_defun(self): 393 layer = self._get_layer_with_training_arg() 394 model = testing_utils.get_model_from_layers([layer], input_shape=(1,)) 395 model.compile(rmsprop.RMSprop(0.), 396 loss='mae') 397 history = model.fit(np.zeros((1, 1)), np.zeros((1, 1))) 398 self.assertEqual(history.history['loss'][0], 1.) 399 loss = model.evaluate(np.zeros((1, 1)), np.zeros((1, 1))) 400 self.assertEqual(loss, 0.) 401 402 # Test that the argument injection performed in `call` is not active 403 # when the argument is passed explicitly. 404 layer = self._get_layer_with_training_arg() 405 inputs = input_layer.Input(shape=(1,)) 406 # Pass `training` by name 407 outputs = layer(inputs, training=False) 408 model = training_lib.Model(inputs, outputs) 409 model.compile(rmsprop.RMSprop(0.), 410 loss='mae') 411 history = model.fit(np.zeros((1, 1)), np.zeros((1, 1))) 412 self.assertEqual(history.history['loss'][0], 0.) 413 414 @combinations.generate( 415 combinations.times(combinations.keras_mode_combinations(), 416 combinations.keras_model_type_combinations())) 417 def test_raw_variable_assignment(self): 418 419 class RawVariableLayer(base_layer.Layer): 420 421 def __init__(self, **kwargs): 422 super(RawVariableLayer, self).__init__(**kwargs) 423 # Test variables in nested structure. 424 self.var_list = [variables.Variable(1.), {'a': variables.Variable(2.)}] 425 426 def call(self, inputs): 427 return inputs * self.var_list[0] * self.var_list[1]['a'] 428 429 model = testing_utils.get_model_from_layers([RawVariableLayer()], 430 input_shape=(10,)) 431 model.compile( 432 'sgd', 433 'mse', 434 run_eagerly=testing_utils.should_run_eagerly()) 435 x, y = np.ones((10, 10)), np.ones((10, 10)) 436 # Checks that variables get initialized. 437 model.fit(x, y, batch_size=2, epochs=2) 438 439 @combinations.generate(combinations.combine(mode=['eager'])) 440 def test_composite_variable_assignment(self): 441 442 class Spec(type_spec.TypeSpec): 443 444 value_type = property(lambda self: CompositeVariable) 445 446 def _component_specs(self): 447 pass 448 449 def _serialize(self): 450 pass 451 452 def _to_components(self, value): 453 return value._variables 454 455 def _from_components(self, variable_list): 456 return CompositeVariable(variable_list) 457 458 class CompositeVariable(composite_tensor.CompositeTensor): 459 460 def __init__(self, variable_list): 461 self._variables = variable_list 462 463 @property 464 def _type_spec(self): 465 return Spec() 466 467 class CompositeVariableLayer(base_layer.Layer): 468 469 def __init__(self): 470 super().__init__() 471 self.composite_var = CompositeVariable( 472 [variables.Variable(1.), 473 variables.Variable(2.)]) 474 475 layer = CompositeVariableLayer() 476 self.assertLen(layer.weights, 2) 477 self.assertIsInstance(layer.weights[0], variables.Variable) 478 self.assertIsInstance(layer.weights[1], variables.Variable) 479 self.assertEqual(self.evaluate(layer.weights[0]), 1.) 480 self.assertEqual(self.evaluate(layer.weights[1]), 2.) 481 482 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 483 def test_layer_names(self): 484 inputs = input_layer.Input(shape=[2]) 485 add1 = inputs + inputs 486 add2 = layers.Add()([inputs, inputs]) 487 add3 = inputs + inputs 488 add4 = layers.Add()([inputs, inputs]) 489 model = training_lib.Model(inputs=[inputs], 490 outputs=[add1, add2, add3, add4]) 491 actual_names = [l.name for l in model.layers] 492 graph_names = [ 493 'input_1', 'tf_op_layer_add', 'add', 'tf_op_layer_add_2', 'add_1' 494 ] 495 eager_names = [ 496 'input_1', 'tf.__operators__.add', 'add', 'tf.__operators__.add_1', 497 'add_1' 498 ] 499 for actual, eager, graph in zip(actual_names, graph_names, eager_names): 500 self.assertIn(actual, {eager, graph}) 501 502 @combinations.generate(combinations.combine(mode=['eager'])) 503 def test_layer_names_after_loading(self): 504 backend.clear_session() 505 # Mimic loading a model that already contained add layers with 506 # name = 'add_1' and 'tf.__operators__.add' 507 layers.Add(name='add_1') 508 layers.Add(name='tf.__operators__.add') 509 510 inputs = input_layer.Input(shape=[2]) 511 add1 = inputs + inputs 512 add2 = layers.Add()([inputs, inputs]) 513 add3 = inputs + inputs 514 add4 = layers.Add()([inputs, inputs]) 515 model = training_lib.Model( 516 inputs=[inputs], outputs=[add1, add2, add3, add4]) 517 actual_names = [l.name for l in model.layers] 518 # The generated op layer names should have avoided layer names seen in 519 # the loaded model. (This avoiance should not apply to non-op-layers) 520 expected_names = [ 521 'input_1', 'tf.__operators__.add_1', 522 'add', 'tf.__operators__.add_2', 'add_1' 523 ] 524 self.assertAllEqual(actual_names, expected_names) 525 526 def test_add_trainable_weight_on_frozen_layer(self): 527 528 class TestLayer(base_layer.Layer): 529 530 def build(self, input_shape): 531 self.w = self.add_weight(shape=(), trainable=True) 532 533 def call(self, inputs): 534 return self.w * inputs 535 536 layer = TestLayer() 537 layer.trainable = False 538 layer.build(None) 539 layer.trainable = True 540 self.assertListEqual(layer.trainable_weights, [layer.w]) 541 542 @combinations.generate( 543 combinations.times(combinations.keras_mode_combinations(), 544 combinations.keras_model_type_combinations())) 545 def test_passing_initial_weights_values(self): 546 kernel_value = np.random.random((10, 2)) 547 layer_with_weights = layers.Dense(2, use_bias=False, weights=[kernel_value]) 548 549 model = testing_utils.get_model_from_layers([layer_with_weights], 550 input_shape=(10,)) 551 model.compile( 552 'sgd', 553 'mse', 554 run_eagerly=testing_utils.should_run_eagerly()) 555 inputs = np.random.random((3, 10)) 556 out = model.predict(inputs) 557 self.assertAllClose(model.layers[-1].get_weights()[0], kernel_value) 558 self.assertAllClose(out, np.dot(inputs, kernel_value)) 559 560 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 561 def test_set_weights_and_get_weights(self): 562 layer = layers.Dense(2) 563 layer.build((None, 10)) 564 kernel = np.random.random((10, 2)) 565 bias = np.random.random((2,)) 566 layer.set_weights([kernel, bias]) 567 weights = layer.get_weights() 568 self.assertEqual(len(weights), 2) 569 self.assertAllClose(weights[0], kernel) 570 self.assertAllClose(weights[1], bias) 571 with self.assertRaisesRegex(ValueError, 572 'but the layer was expecting 2 weights'): 573 layer.set_weights([1, 2, 3]) 574 with self.assertRaisesRegex(ValueError, 575 'not compatible with provided weight shape'): 576 layer.set_weights([kernel.T, bias]) 577 578 def test_get_config_error(self): 579 580 class MyLayer(base_layer.Layer): 581 582 def __init__(self, my_kwarg='default', **kwargs): 583 super(MyLayer, self).__init__(**kwargs) 584 self.my_kwarg = my_kwarg 585 586 # `__init__` includes kwargs but `get_config` is not overridden, so 587 # an error should be thrown: 588 with self.assertRaisesRegex(NotImplementedError, 'Layer MyLayer has'): 589 MyLayer('custom').get_config() 590 591 class MyLayerNew(base_layer.Layer): 592 593 def __init__(self, my_kwarg='default', **kwargs): 594 super(MyLayerNew, self).__init__(**kwargs) 595 self.my_kwarg = my_kwarg 596 597 def get_config(self): 598 config = super(MyLayerNew, self).get_config() 599 config['my_kwarg'] = self.my_kwarg 600 return config 601 602 # Test to make sure that error is not raised if the method call is 603 # from an overridden `get_config`: 604 self.assertEqual(MyLayerNew('custom').get_config()['my_kwarg'], 'custom') 605 606 class MyLayerNew2(base_layer.Layer): 607 608 def __init__(self, name='MyLayerName', dtype=None, **kwargs): # pylint:disable=redefined-outer-name 609 super(MyLayerNew2, self).__init__(name=name, dtype=dtype, **kwargs) 610 611 # Check that if the kwargs in `__init__` are base layer constructor 612 # arguments, no error is thrown: 613 self.assertEqual(MyLayerNew2(name='New').get_config()['name'], 'New') 614 615 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 616 def test_count_params(self): 617 dense = layers.Dense(16) 618 dense.build((None, 4)) 619 self.assertEqual(dense.count_params(), 16 * 4 + 16) 620 621 dense = layers.Dense(16) 622 with self.assertRaisesRegex(ValueError, 'call `count_params`'): 623 dense.count_params() 624 625 model = sequential.Sequential(layers.Dense(16)) 626 with self.assertRaisesRegex(ValueError, 'call `count_params`'): 627 model.count_params() 628 629 dense = layers.Dense(16, input_dim=4) 630 model = sequential.Sequential(dense) 631 self.assertEqual(model.count_params(), 16 * 4 + 16) 632 633 def test_super_not_called(self): 634 635 class CustomLayerNotCallingSuper(base_layer.Layer): 636 637 def __init__(self): 638 pass 639 640 layer = CustomLayerNotCallingSuper() 641 with self.assertRaisesRegex(RuntimeError, 'You must call `super()'): 642 layer(np.random.random((10, 2))) 643 644 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 645 def test_first_arg_not_called_inputs(self): 646 x, y = array_ops.ones((10, 1)), array_ops.ones((10, 1)) 647 648 class ArgLayer(base_layer.Layer): 649 650 def call(self, x, y): 651 return x + y 652 653 layer = ArgLayer() 654 out = self.evaluate(layer(x=x, y=y)) 655 self.assertAllClose(out, 2 * np.ones((10, 1))) 656 657 class KwargLayer(base_layer.Layer): 658 659 def call(self, x=None, y=None): 660 return x + y 661 662 layer = KwargLayer() 663 out = self.evaluate(layer(x=x, y=y)) 664 self.assertAllClose(out, 2 * np.ones((10, 1))) 665 666 with self.assertRaisesRegex(ValueError, 'must always be passed'): 667 layer(y=y) 668 669 class TFFunctionLayer(base_layer.Layer): 670 671 @def_function.function 672 def call(self, x, y=None): 673 if y is None: 674 return x 675 return x + y 676 677 layer = TFFunctionLayer() 678 out = self.evaluate(layer(x=x, y=y)) 679 self.assertAllClose(out, 2 * np.ones((10, 1))) 680 681 def test_build_input_shape(self): 682 683 class CustomLayer(base_layer.Layer): 684 685 def build(self, input_shape): 686 self.add_weight('w', shape=input_shape[1:]) 687 super(CustomLayer, self).build(input_shape) 688 689 layer = CustomLayer() 690 self.assertFalse(layer.built) 691 692 layer.build([None, 1, 2, 3]) 693 self.assertTrue(layer.built) 694 self.assertEqual([None, 1, 2, 3], layer._build_input_shape) 695 696 layer = CustomLayer() 697 layer(input_layer.Input((3,))) 698 self.assertTrue(layer.built) 699 self.assertEqual([None, 3], layer._build_input_shape.as_list()) 700 701 @combinations.generate(combinations.combine(mode=['eager'])) 702 def custom_layer_training_arg(self): 703 class CustomLayerNoTrainingArg(base_layer.Layer): 704 705 def __init__(self, nested_layer=None): 706 self._nested_layer = nested_layer or array_ops.identity 707 708 def call(self, inputs): 709 return self._nested_layer(inputs) 710 711 class CustomLayerDefaultTrainingMissing(base_layer.Layer): 712 713 def __init__(self, nested_layer=None): 714 self._nested_layer = nested_layer or array_ops.identity 715 716 def call(self, inputs, training): 717 if training: 718 return self._nested_layer(inputs) 719 else: 720 return self._nested_layer(inputs) * 0.5 721 722 class CustomLayerDefaultTrainingNone(base_layer.Layer): 723 724 def __init__(self, nested_layer=None): 725 self._nested_layer = nested_layer or array_ops.identity 726 727 def call(self, inputs, training=None): 728 if training: 729 return self._nested_layer(inputs) 730 else: 731 return self._nested_layer(inputs) * 0.5 732 733 class CustomLayerDefaultTrainingFalse(base_layer.Layer): 734 735 def __init__(self, nested_layer=None): 736 self._nested_layer = nested_layer or array_ops.identity 737 738 def call(self, inputs, training=False): 739 if training: 740 return self._nested_layer(inputs) 741 else: 742 return self._nested_layer(inputs) * 0.5 743 744 class CustomLayerDefaultTrainingTrue(base_layer.Layer): 745 746 def __init__(self, nested_layer=None): 747 self._nested_layer = nested_layer or array_ops.identity 748 749 def call(self, inputs, training=True): 750 if training: 751 return self._nested_layer(inputs) 752 else: 753 return self._nested_layer(inputs) * 0.5 754 755 x = array_ops.ones(shape=(1, 1)) 756 757 # If the layer signature doesn't specify a default training arg, 758 # run it in inference mode when to training arg is passed 759 # to __call__ 760 layer = CustomLayerDefaultTrainingMissing() 761 self.assertAllEqual(layer(x), x * 0.5) 762 self.assertAllEqual(layer(x, training=False), x * 0.5) 763 self.assertAllEqual(layer(x, training=True), x) 764 765 # If the layer signature specifies `False` as the default training arg, 766 # run it in inference mode when no training arg is passed 767 # to __call__ 768 layer = CustomLayerDefaultTrainingFalse() 769 self.assertAllEqual(layer(x), x * 0.5) 770 self.assertAllEqual(layer(x, training=False), x * 0.5) 771 self.assertAllEqual(layer(x, training=True), x) 772 773 # If the layer signature specifies `True` as the default training arg, 774 # explicitly run it in training mode when no training arg is passed 775 # to __call__ 776 layer = CustomLayerDefaultTrainingTrue() 777 self.assertAllEqual(layer(x), x) 778 self.assertAllEqual(layer(x, training=False), x * 0.5) 779 self.assertAllEqual(layer(x, training=True), x) 780 781 # Outer layers/models should set the training context implicitly for all 782 # nested layers, respecting whatever mode the outer layer was run with. 783 layer = CustomLayerDefaultTrainingTrue(CustomLayerDefaultTrainingFalse()) 784 # No outer value passed: use local defaults 785 self.assertAllEqual(layer(x), x * 0.25) # Use local default False 786 # Outer value passed: override local defaults 787 self.assertAllEqual(layer(x, training=False), x * 0.25) 788 self.assertAllEqual(layer(x, training=True), x) 789 790 layer = CustomLayerDefaultTrainingFalse(CustomLayerDefaultTrainingTrue()) 791 # No outer value passed: use local defaults 792 self.assertAllEqual(layer(x), x) # Use local default True 793 # Outer value passed: override local defaults 794 self.assertAllEqual(layer(x, training=False), x * 0.25) 795 self.assertAllEqual(layer(x, training=True), x) 796 797 # If the outer layer `call` doesn't take a training argument at all, 798 # it'll set the nested scope as None when no training arg is passed in. 799 # If a training arg is passed in it won't use it directly in `call`, but 800 # it will set the nested training mode. 801 layer = CustomLayerNoTrainingArg(CustomLayerDefaultTrainingTrue()) 802 self.assertAllEqual(layer(x), x) # Use local default True 803 self.assertAllEqual(layer(x, training=False), x * 0.5) 804 self.assertAllEqual(layer(x, training=True), x) 805 806 layer = CustomLayerDefaultTrainingNone(CustomLayerDefaultTrainingTrue()) 807 self.assertAllEqual(layer(x), x) # Use local default True 808 self.assertAllEqual(layer(x, training=False), x * 0.5) 809 self.assertAllEqual(layer(x, training=True), x) 810 811 def test_activity_regularizer_string(self): 812 813 class MyLayer(base_layer.Layer): 814 pass 815 816 layer = MyLayer(activity_regularizer='l2') 817 self.assertIsInstance(layer.activity_regularizer, regularizers.L2) 818 819 def test_tf_module_tracking(self): 820 821 class MyModule(module.Module): 822 823 def __init__(self): 824 super(MyModule, self).__init__() 825 self.v1 = variables.Variable(1., trainable=True, name='v1') 826 self.v2 = variables.Variable(2., trainable=False, name='v2') 827 828 def __call__(self, x): 829 return x * self.v1 * self.v2 830 831 class MyLayer(base_layer.Layer): 832 833 def __init__(self, **kwargs): 834 super(MyLayer, self).__init__(self, **kwargs) 835 self.my_modules = {} 836 self.my_modules['a'] = MyModule() 837 838 def call(self, x): 839 return self.my_modules['a'](x) 840 841 layer = MyLayer() 842 self.assertLen(layer.variables, 2) 843 self.assertLen(layer.trainable_variables, 1) 844 self.assertLen(layer.non_trainable_variables, 1) 845 846 layer.trainable = False 847 self.assertLen(layer.variables, 2) 848 self.assertLen(layer.trainable_variables, 0) 849 self.assertLen(layer.non_trainable_variables, 2) 850 851 class MyModel(training_lib.Model): 852 853 def __init__(self): 854 super(MyModel, self).__init__() 855 self.my_modules = [] 856 self.my_modules.append(MyModule()) 857 858 def call(self, x): 859 return self.my_modules[0](x) 860 861 model = MyModel() 862 self.assertLen(model.variables, 2) 863 self.assertLen(model.trainable_variables, 1) 864 self.assertLen(model.non_trainable_variables, 1) 865 866 model.trainable = False 867 self.assertLen(model.variables, 2) 868 self.assertLen(model.trainable_variables, 0) 869 self.assertLen(model.non_trainable_variables, 2) 870 871 872class SymbolicSupportTest(keras_parameterized.TestCase): 873 874 def test_using_symbolic_tensors_with_tf_ops(self): 875 # Single-input. 876 x = input_layer.Input((3,)) 877 math_ops.square(x) 878 879 # Multi-inputs. 880 x1, x2 = input_layer.Input((3,)), input_layer.Input((3,)) 881 array_ops.concat([x1, x2], axis=1) 882 883 # Mixing Keras symbolic tensors and graph tensors from the same graph works. 884 with backend.get_graph().as_default(): 885 x1 = input_layer.Input((3,)) 886 x2 = input_layer.Input((3,)) 887 math_ops.matmul(x1, x2) 888 889 # Creating same op type (matmul) multiple times in the Keras graph works. 890 x1 = input_layer.Input((3,)) 891 x2 = input_layer.Input((3,)) 892 math_ops.matmul(x1, x2) 893 894 def test_mixing_eager_and_graph_tensors(self): 895 with ops.Graph().as_default(): 896 x1 = array_ops.ones((3, 3)) 897 x2 = array_ops.ones((3, 3)) 898 with self.assertRaisesRegex(TypeError, 'Graph tensors'): 899 math_ops.matmul(x1, x2) 900 901 def test_mixing_numpy_arrays_and_graph_tensors(self): 902 with ops.Graph().as_default(): 903 x1 = array_ops.ones((3, 3)) 904 x2 = np.ones((3, 3), dtype='float32') 905 with self.assertRaisesRegex(TypeError, 'Graph tensors'): 906 math_ops.matmul(x1, x2) 907 908 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 909 def test_mixing_keras_symbolic_tensors_and_eager_tensors(self): 910 x1 = input_layer.Input((3,)) 911 x2 = array_ops.ones((3, 3)) 912 y = math_ops.matmul(x1, x2) 913 914 fn = backend.function(inputs=[x1], outputs=[y]) 915 x_val = np.random.random((3, 3)) 916 y_val = np.ones((3, 3)) 917 self.assertAllClose(fn([x_val])[0], 918 np.matmul(x_val, y_val), 919 atol=1e-5) 920 921 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 922 def test_mixing_keras_symbolic_tensors_and_numpy_arrays(self): 923 x1 = input_layer.Input((3,)) 924 x2 = np.ones((3, 3), dtype='float32') 925 y = math_ops.matmul(x1, x2) 926 927 fn = backend.function(inputs=[x1], outputs=[y]) 928 x_val = np.random.random((3, 3)) 929 y_val = np.ones((3, 3)) 930 self.assertAllClose(fn([x_val])[0], 931 np.matmul(x_val, y_val), 932 atol=1e-5) 933 934 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 935 def test_reraising_exception(self): 936 # When layer is not dynamic, we have some pattern matching during exception 937 # handling to detect when the user is trying to use python control flow. 938 # When an exception is thrown but the pattern doesn't match, we want to 939 # preserve the originating stack trace. An early implementation of this 940 # logic lost the stack trace. We test the correct behavior here. 941 942 class TypeErrorLayer(base_layer.Layer): 943 944 def call(self, inputs): 945 def easily_identifiable_name(): 946 raise TypeError('Non-matching TypeError message.') 947 easily_identifiable_name() 948 949 inputs = input_layer.Input((3,)) 950 951 try: 952 _ = TypeErrorLayer()(inputs) 953 except TypeError as e: 954 if hasattr(e, 'ag_error_metadata'): 955 self.assertIn('easily_identifiable_name', str(e)) 956 # See ErrorMetadataBase in autograph/pyct/errors.py 957 function_name = e.ag_error_metadata.translated_stack[-1].function_name 958 else: 959 tb = traceback.extract_tb(sys.exc_info()[2]) 960 last_entry = tb[-1] 961 function_name = last_entry[2] 962 self.assertEqual(function_name, 'easily_identifiable_name') 963 964 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 965 def test_summaries_in_tf_function(self): 966 if not context.executing_eagerly(): 967 return 968 969 class MyLayer(base_layer.Layer): 970 971 def call(self, inputs): 972 summary_ops_v2.scalar('mean', math_ops.reduce_mean(inputs)) 973 return inputs 974 975 tmp_dir = self.get_temp_dir() 976 writer = summary_ops_v2.create_file_writer_v2(tmp_dir) 977 with writer.as_default(step=1), summary_ops_v2.record_if(True): 978 my_layer = MyLayer() 979 x = array_ops.ones((10, 10)) 980 981 def my_fn(x): 982 return my_layer(x) 983 984 _ = my_fn(x) 985 986 event_file = gfile.Glob(os.path.join(tmp_dir, 'events*')) 987 self.assertLen(event_file, 1) 988 event_file = event_file[0] 989 tags = set() 990 for e in summary_iterator.summary_iterator(event_file): 991 for val in e.summary.value: 992 tags.add(val.tag) 993 self.assertEqual(set(['my_layer/mean']), tags) 994 995 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 996 def test_error_when_passing_non_tensor(self): 997 # layers that have an `input_spec` will raise an error when called on 998 # non-tensors. This covers all built-in layers. 999 layer = layers.Dense(3) 1000 x = object() 1001 with self.assertRaisesRegex(TypeError, r'should be tensors'): 1002 layer(x) 1003 1004 1005@combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1006class NestedTrackingTest(test.TestCase): 1007 1008 def test_nested_layer_variable_tracking(self): 1009 # Test that variables from nested sublayers are 1010 # being tracked by subclassed layers. 1011 1012 class MyLayer(base_layer.Layer): 1013 1014 def __init__(self): 1015 super(MyLayer, self).__init__() 1016 self.dense1 = layers.Dense(1) 1017 self.dense2 = layers.BatchNormalization() 1018 1019 def build(self, input_shape): 1020 self.v1 = self.add_weight('v1', shape=input_shape[1:].as_list()) 1021 self.v2 = variables.Variable( 1022 name='v2', 1023 initial_value=np.zeros(input_shape[1:].as_list(), dtype='float32'), 1024 trainable=False) 1025 1026 def call(self, inputs): 1027 x = self.dense1(inputs) + self.dense2(inputs) 1028 return x + self.v1 + self.v2 1029 1030 layer = MyLayer() 1031 inputs = input_layer.Input((1,)) 1032 _ = layer(inputs) 1033 1034 self.assertEqual(len(layer.weights), 8) 1035 self.assertEqual(len(layer.trainable_weights), 5) 1036 self.assertEqual(len(layer.non_trainable_weights), 3) 1037 1038 layer.dense1.trainable = False 1039 self.assertEqual(len(layer.weights), 8) 1040 self.assertEqual(len(layer.trainable_weights), 3) 1041 self.assertEqual(len(layer.non_trainable_weights), 5) 1042 1043 layer.trainable = False 1044 self.assertEqual(len(layer.weights), 8) 1045 self.assertEqual(len(layer.trainable_weights), 0) 1046 self.assertEqual(len(layer.non_trainable_weights), 8) 1047 self.assertEqual( 1048 {id(v) for v in [layer.dense1, layer.dense2, layer.v1, layer.v2]}, 1049 {id(v) for _, v in layer._checkpoint_dependencies}) 1050 1051 def test_nested_layer_updates_losses_tracking(self): 1052 # Test that updates and losses from nested sublayers are 1053 # being tracked by subclassed layers. 1054 1055 class UpdateAndLossLayer(base_layer.Layer): 1056 1057 def build(self, _): 1058 self.v1 = self.add_weight('v1', shape=()) 1059 1060 def call(self, inputs): 1061 self.add_loss(math_ops.reduce_sum(inputs)) 1062 self.add_update(state_ops.assign_add(self.v1, 1)) 1063 return inputs + 1 1064 1065 class MyLayer(base_layer.Layer): 1066 1067 def build(self, _): 1068 self.v1 = self.add_weight('v1', shape=()) 1069 1070 def __init__(self): 1071 super(MyLayer, self).__init__() 1072 self.ul1 = UpdateAndLossLayer() 1073 self.ul2 = UpdateAndLossLayer() 1074 1075 def call(self, inputs): 1076 self.add_loss(math_ops.reduce_sum(inputs)) 1077 self.add_update(state_ops.assign_add(self.v1, 1)) 1078 x = self.ul1(inputs) 1079 return self.ul2(x) 1080 1081 layer = MyLayer() 1082 1083 if context.executing_eagerly(): 1084 inputs = array_ops.ones((3, 1)) 1085 _ = layer(inputs) 1086 self.assertEqual(len(layer.losses), 3) 1087 self.assertLen(layer.get_losses_for(None), 3) 1088 else: 1089 inputs = input_layer.Input((1,)) 1090 _ = layer(inputs) 1091 self.assertEqual(len(layer.losses), 3) 1092 self.assertEqual(len(layer.updates), 3) 1093 self.assertLen(layer.get_losses_for(None), 3) 1094 1095 def test_attribute_reassignment(self): 1096 l = base_layer.Layer() 1097 l.a = base_layer.Layer() 1098 l.a = [] 1099 l.a = variables.Variable(1.) 1100 l.a = base_layer.Layer() 1101 last_assignment = base_layer.Layer() 1102 l.a = last_assignment 1103 l.b = variables.Variable(1.) 1104 del l.b 1105 l.c = base_layer.Layer() 1106 del l.c 1107 l.d = last_assignment 1108 del l.d 1109 sublayers = list(l._flatten_layers(include_self=False, recursive=False)) 1110 self.assertEqual([last_assignment], sublayers) 1111 self.assertEqual([], l.trainable_weights) 1112 self.assertEqual([], l.non_trainable_weights) 1113 self.assertEqual([], l.weights) 1114 del l.a 1115 self.assertEqual([], l._self_tracked_trackables) 1116 1117 def test_layer_class_not_tracked_as_sublayer(self): 1118 # See https://github.com/tensorflow/tensorflow/issues/27431 for details. 1119 1120 class LayerWithClassAttribute(base_layer.Layer): 1121 1122 def __init__(self): 1123 super(LayerWithClassAttribute, self).__init__() 1124 self.layer_fn = layers.Dense 1125 1126 layer = LayerWithClassAttribute() 1127 self.assertEmpty(layer.variables) 1128 self.assertEmpty(layer.submodules) 1129 1130 def test_layer_call_fn_args(self): 1131 1132 class NonDefunLayer(base_layer.Layer): 1133 1134 def call(self, inputs, a, mask, b=None, training=None): 1135 return inputs 1136 1137 class DefunLayer(base_layer.Layer): 1138 1139 @def_function.function 1140 def call(self, x, mask, a, training=None, b=None): 1141 return x 1142 1143 nondefun_layer = NonDefunLayer() 1144 self.assertEqual(nondefun_layer._call_fn_args, 1145 ['inputs', 'a', 'mask', 'b', 'training']) 1146 defun_layer = DefunLayer() 1147 self.assertEqual(defun_layer._call_fn_args, 1148 ['x', 'mask', 'a', 'training', 'b']) 1149 1150 def test_sequential_model(self): 1151 model = sequential.Sequential( 1152 [layers.Dense(10, input_shape=(10,)), 1153 layers.Dense(5)]) 1154 self.assertLen(model.layers, 2) 1155 self.assertLen(model.weights, 4) 1156 1157 # Make sure a subclass model also works when it is called 'Sequential'. 1158 class Sequential(training_lib.Model): 1159 1160 def __init__(self): 1161 super(Sequential, self).__init__() 1162 self.dense_layers = [layers.Dense(10), layers.Dense(5)] 1163 1164 def call(self, inputs): 1165 x = inputs 1166 for d in self.dense_layers: 1167 x = d(x) 1168 return x 1169 1170 s = Sequential() 1171 self.assertLen(s.layers, 2) 1172 self.assertLen(s.weights, 0) 1173 1174 s(input_layer.Input((10,))) 1175 self.assertLen(s.weights, 4) 1176 1177 1178@combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1179class NameScopingTest(keras_parameterized.TestCase): 1180 1181 def test_name_scope_layer(self): 1182 x = backend.placeholder(shape=(10, 10)) 1183 layer = layers.Dense(10, name='MyName') 1184 layer(x) 1185 self.assertEqual(layer.bias.name, 'MyName/bias:0') 1186 self.assertEqual(layer.kernel.name, 'MyName/kernel:0') 1187 1188 def test_name_scope_functional_api(self): 1189 inputs = input_layer.Input((3,)) 1190 layer = layers.Dense(10, name='MyName') 1191 _ = layer(inputs) 1192 self.assertEqual(layer.bias.name, 'MyName/bias:0') 1193 self.assertEqual(layer.kernel.name, 'MyName/kernel:0') 1194 1195 def test_name_scope_functional_api_nested(self): 1196 1197 class NestedLayer(base_layer.Layer): 1198 1199 def __init__(self, name='OuterName'): 1200 super(NestedLayer, self).__init__(name=name) 1201 self.dense = layers.Dense(10, name='InnerName') 1202 1203 def call(self, inputs): 1204 return self.dense(inputs) 1205 1206 inputs = input_layer.Input((3,)) 1207 layer = NestedLayer() 1208 _ = layer(inputs) 1209 self.assertEqual(layer.dense.bias.name, 'OuterName/InnerName/bias:0') 1210 self.assertEqual(layer.dense.kernel.name, 'OuterName/InnerName/kernel:0') 1211 1212 def test_name_scope_sublayer(self): 1213 1214 class NameScopeTracker(base_layer.Layer): 1215 1216 def call(self, inputs): 1217 self.active_name_scope = ops.get_name_scope() 1218 return inputs 1219 1220 x = backend.placeholder(shape=(10, 10)) 1221 sublayer = NameScopeTracker(name='Sublayer') 1222 layer = layers.Dense(10, activation=sublayer, name='MyName2') 1223 layer(x) 1224 self.assertEqual(layer.bias.name, 'MyName2/bias:0') 1225 self.assertEqual(layer.kernel.name, 'MyName2/kernel:0') 1226 self.assertEqual(sublayer.active_name_scope, 'MyName2/Sublayer') 1227 1228 def test_name_scope_tf_tensor(self): 1229 x = ops.convert_to_tensor_v2_with_dispatch(np.ones((10, 10))) 1230 layer = layers.Dense( 1231 10, activation=layers.ReLU(name='MyAct'), name='MyName3') 1232 layer(x) 1233 self.assertEqual(layer.bias.name, 'MyName3/bias:0') 1234 self.assertEqual(layer.kernel.name, 'MyName3/kernel:0') 1235 1236 1237@combinations.generate(combinations.keras_mode_combinations(mode=['eager'])) 1238class AutographControlFlowTest(keras_parameterized.TestCase): 1239 1240 def test_disabling_in_context_is_matched(self): 1241 1242 test_obj = self 1243 1244 class MyLayer(base_layer.Layer): 1245 1246 def call(self, inputs, training=None): 1247 with test_obj.assertRaisesRegex(TypeError, 'Tensor.*as.*bool'): 1248 if constant_op.constant(False): 1249 return inputs * 1. 1250 return inputs * 0. 1251 1252 @def_function.function(autograph=False) 1253 def test_fn(): 1254 return MyLayer()(constant_op.constant([[1., 2., 3.]])) 1255 1256 test_fn() 1257 1258 def test_if_training_pattern_output(self): 1259 1260 class MyLayer(base_layer.Layer): 1261 1262 def call(self, inputs, training=None): 1263 if training: 1264 return inputs * 1. 1265 return inputs * 0. 1266 1267 inputs = input_layer.Input((3,)) 1268 outputs = MyLayer()(inputs) 1269 model = training_lib.Model(inputs, outputs) 1270 model.compile( 1271 'sgd', 1272 'mse', 1273 run_eagerly=testing_utils.should_run_eagerly()) 1274 train_loss = model.train_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1275 self.assertEqual(train_loss, 0.) 1276 test_loss = model.test_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1277 self.assertEqual(test_loss, 1.) 1278 1279 def test_if_training_pattern_loss(self): 1280 1281 class MyLayer(base_layer.Layer): 1282 1283 def call(self, inputs, training=None): 1284 if training: 1285 loss = math_ops.reduce_sum(inputs) 1286 else: 1287 loss = 0. 1288 self.add_loss(loss) 1289 return inputs 1290 1291 inputs = input_layer.Input((3,)) 1292 outputs = MyLayer()(inputs) 1293 model = training_lib.Model(inputs, outputs) 1294 model.compile( 1295 'sgd', 1296 'mse', 1297 run_eagerly=testing_utils.should_run_eagerly()) 1298 train_loss = model.train_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1299 self.assertEqual(train_loss, 2 * 3) 1300 test_loss = model.test_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1301 self.assertEqual(test_loss, 0) 1302 1303 def test_if_training_pattern_metric(self): 1304 1305 class MyLayer(base_layer.Layer): 1306 1307 def call(self, inputs, training=None): 1308 if training: 1309 metric = math_ops.reduce_sum(inputs) 1310 else: 1311 metric = 0. 1312 self.add_metric(metric, name='my_metric', aggregation='mean') 1313 return inputs 1314 1315 inputs = input_layer.Input((3,)) 1316 outputs = MyLayer()(inputs) 1317 model = training_lib.Model(inputs, outputs) 1318 model.compile( 1319 'sgd', 1320 'mse', 1321 run_eagerly=testing_utils.should_run_eagerly()) 1322 for _ in range(3): 1323 _, train_metric = model.train_on_batch(np.ones((2, 3)), 1324 np.ones((2, 3))) 1325 1326 self.assertEqual(train_metric, 2 * 3) 1327 _, test_metric = model.test_on_batch(np.ones((2, 3)), 1328 np.ones((2, 3))) 1329 self.assertEqual(test_metric, 0) 1330 1331 def test_if_training_pattern_update(self): 1332 1333 class MyLayer(base_layer.Layer): 1334 1335 def build(self, input_shape): 1336 self.counter = self.add_weight( 1337 shape=(), trainable=False, initializer='zeros') 1338 1339 def call(self, inputs, training=None): 1340 if training: 1341 increment = 1. 1342 else: 1343 increment = 0. 1344 self.counter.assign_add(increment) 1345 return inputs 1346 1347 inputs = input_layer.Input((3,)) 1348 layer = MyLayer() 1349 outputs = layer(inputs) 1350 model = training_lib.Model(inputs, outputs) 1351 model.compile( 1352 'sgd', 1353 'mse', 1354 run_eagerly=testing_utils.should_run_eagerly()) 1355 model.train_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1356 self.assertEqual(backend.get_value(layer.counter), 1.) 1357 1358 def test_conditional_losses_in_call(self): 1359 1360 class MyLayer(base_layer.Layer): 1361 1362 def __init__(self): 1363 super(MyLayer, 1364 self).__init__(dynamic=testing_utils.should_run_eagerly()) 1365 1366 def call(self, inputs, training=None): 1367 if training: 1368 self.add_loss(math_ops.reduce_sum(inputs)) 1369 return inputs 1370 1371 def compute_output_shape(self, input_shape): 1372 return input_shape 1373 1374 inputs = input_layer.Input((3,)) 1375 layer = MyLayer() 1376 outputs = layer(inputs) 1377 model = training_lib.Model(inputs, outputs) 1378 model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) 1379 loss = model.train_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1380 self.assertEqual(loss, 2 * 3) 1381 1382 def test_conditional_callable_losses(self): 1383 model = sequential.Sequential([ 1384 layers.Dense( 1385 1, kernel_regularizer=regularizers.l2(1e-4), input_shape=(1,)) 1386 ]) 1387 model._run_eagerly = testing_utils.should_run_eagerly() 1388 1389 def assert_graph(t): 1390 if not context.executing_eagerly(): 1391 self.assertEqual(t.graph, ops.get_default_graph()) 1392 1393 @def_function.function 1394 def get_losses(t): 1395 if t < 0: 1396 return math_ops.reduce_sum(model.losses) * t 1397 else: 1398 return math_ops.reduce_sum(model.losses) 1399 1400 assert_graph(get_losses(constant_op.constant(2.))) 1401 assert_graph(get_losses(constant_op.constant(0.5))) 1402 1403 def test_conditional_metrics_in_call(self): 1404 1405 class MyLayer(base_layer.Layer): 1406 1407 def __init__(self): 1408 super(MyLayer, 1409 self).__init__(dynamic=testing_utils.should_run_eagerly()) 1410 1411 def call(self, inputs, training=None): 1412 if training: 1413 self.add_metric(math_ops.reduce_sum(inputs), 1414 name='sum', 1415 aggregation='mean') 1416 return inputs 1417 1418 def compute_output_shape(self, input_shape): 1419 return input_shape 1420 1421 inputs = input_layer.Input((3,)) 1422 layer = MyLayer() 1423 outputs = layer(inputs) 1424 model = training_lib.Model(inputs, outputs) 1425 model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) 1426 history = model.fit(np.ones((2, 3)), np.ones((2, 3))) 1427 self.assertEqual(history.history['sum'][-1], 2 * 3) 1428 1429 def test_conditional_activity_regularizer_in_call(self): 1430 1431 class TestModel(training_lib.Model): 1432 1433 def __init__(self): 1434 super(TestModel, self).__init__( 1435 name='test_model', dynamic=testing_utils.should_run_eagerly()) 1436 self.layer = layers.Dense(2, activity_regularizer='l2') 1437 1438 def call(self, x, training=None): 1439 if math_ops.greater(math_ops.reduce_sum(x), 0.0): 1440 return self.layer(x) 1441 else: 1442 return self.layer(x) 1443 1444 model = TestModel() 1445 model.compile( 1446 loss='mse', 1447 optimizer='sgd', 1448 run_eagerly=testing_utils.should_run_eagerly()) 1449 1450 x = np.ones(shape=(10, 1)) 1451 y = np.ones(shape=(10, 2)) 1452 1453 if testing_utils.should_run_eagerly(): 1454 model.fit(x, y, epochs=2, batch_size=5) 1455 else: 1456 with self.assertRaisesRegex(ValueError, 'ActivityRegularizer'): 1457 model.fit(x, y, epochs=2, batch_size=5) 1458 1459 def test_conditional_activity_regularizer_with_wrappers_in_call(self): 1460 1461 class TestModel(training_lib.Model): 1462 1463 def __init__(self): 1464 super(TestModel, self).__init__( 1465 name='test_model', dynamic=testing_utils.should_run_eagerly()) 1466 self.layer = layers.TimeDistributed( 1467 layers.Dense(2, activity_regularizer='l2'), input_shape=(3, 4)) 1468 1469 def call(self, x, training=None): 1470 if math_ops.greater(math_ops.reduce_sum(x), 0.0): 1471 return self.layer(x) 1472 else: 1473 return self.layer(x) 1474 1475 model = TestModel() 1476 model.compile( 1477 loss='mse', 1478 optimizer='sgd', 1479 run_eagerly=testing_utils.should_run_eagerly()) 1480 1481 x = np.ones(shape=(10, 3, 4)) 1482 y = np.ones(shape=(10, 3, 2)) 1483 1484 if testing_utils.should_run_eagerly(): 1485 model.fit(x, y, epochs=2, batch_size=5) 1486 else: 1487 with self.assertRaisesRegex(ValueError, 'ActivityRegularizer'): 1488 model.fit(x, y, epochs=2, batch_size=5) 1489 1490 1491class AddLayer(base_layer.Layer): 1492 """A layer which adds its input to a variable. 1493 1494 Useful for testing a layer with a variable 1495 """ 1496 1497 def build(self, _): 1498 self.v = self.add_weight('v', (), initializer='ones') 1499 self.built = True 1500 1501 def call(self, inputs): 1502 return inputs + self.v 1503 1504 1505class IdentityLayer(base_layer.Layer): 1506 """A layer that returns its input. 1507 1508 Useful for testing a layer without a variable. 1509 """ 1510 1511 def call(self, inputs): 1512 return inputs 1513 1514 1515@combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1516class DTypeTest(keras_parameterized.TestCase): 1517 1518 # This class only have tests relating to layer.dtype. Tests for dtype policies 1519 # are in mixed_precision/keras_test.py 1520 1521 # TODO(reedwm): Maybe have a separate test file for input casting tests. 1522 1523 def _const(self, dtype): 1524 return array_ops.constant(1, dtype=dtype) 1525 1526 @testing_utils.enable_v2_dtype_behavior 1527 def test_dtype_defaults_to_floatx(self): 1528 layer = AddLayer() 1529 self.assertEqual(layer.dtype, 'float32') 1530 layer(self._const('float64')) 1531 self.assertEqual(layer.dtype, 'float32') # dtype should not change 1532 1533 try: 1534 backend.set_floatx('float64') 1535 layer = AddLayer() 1536 self.assertEqual(layer.dtype, 'float64') 1537 finally: 1538 backend.set_floatx('float32') 1539 1540 @testing_utils.enable_v2_dtype_behavior 1541 def test_passing_dtype_to_constructor(self): 1542 layer = IdentityLayer(dtype='float64') 1543 layer(self._const('float32')) 1544 self.assertEqual(layer.dtype, 'float64') 1545 1546 layer = IdentityLayer(dtype='int32') 1547 layer(self._const('float32')) 1548 self.assertEqual(layer.dtype, 'int32') 1549 1550 layer = IdentityLayer(dtype=dtypes.float64) 1551 layer(self._const('float32')) 1552 self.assertEqual(layer.dtype, 'float64') 1553 1554 @testing_utils.enable_v2_dtype_behavior 1555 def input_cast_to_dtype(self): 1556 layer = AddLayer() 1557 1558 # Input should be cast to layer.dtype, so output should also be layer.dtype 1559 self.assertEqual(layer(self._const('float64')).dtype, 'float32') 1560 1561 layer = AddLayer(dtype='float64') 1562 self.assertEqual(layer(self._const('float32')).dtype, 'float64') 1563 1564 # Test inputs are not casted if layer.dtype is not floating-point 1565 layer = IdentityLayer(dtype='int32') 1566 self.assertEqual(layer(self._const('float64')).dtype, 'float64') 1567 1568 # Test inputs are not casted if the inputs are not floating-point 1569 layer = IdentityLayer(dtype='float32') 1570 self.assertEqual(layer(self._const('int32')).dtype, 'int32') 1571 1572 # Test Numpy arrays are casted 1573 layer = IdentityLayer(dtype='float64') 1574 self.assertEqual(layer(np.array(1, dtype='float32')).dtype, 'float64') 1575 1576 # Test Python floats are casted 1577 layer = IdentityLayer(dtype='float64') 1578 self.assertEqual(layer(1.).dtype, 'float64') 1579 1580 @testing_utils.enable_v2_dtype_behavior 1581 def multiple_inputs_cast_to_dtype(self): 1582 1583 class MultiIdentityLayer(base_layer.Layer): 1584 1585 def call(self, inputs): 1586 return [array_ops.identity(x) for x in inputs] 1587 1588 # Testing layer with default dtype of float32 1589 layer = MultiIdentityLayer() 1590 x, y = layer([self._const('float16'), self._const('float32')]) 1591 self.assertEqual(x.dtype, 'float32') 1592 self.assertEqual(y.dtype, 'float32') 1593 1594 # Test passing dtype to the constructor 1595 layer = MultiIdentityLayer(dtype='float64') 1596 x, y = layer([self._const('float16'), self._const('float32')]) 1597 self.assertEqual(x.dtype, 'float64') 1598 self.assertEqual(y.dtype, 'float64') 1599 1600 # Test several non-floating point types 1601 layer = MultiIdentityLayer(dtype='float64') 1602 x, y, z, w = layer([self._const('float16'), self._const('bool'), 1603 self._const('float64'), self._constant('complex64')]) 1604 self.assertEqual(x.dtype, 'float64') 1605 self.assertEqual(y.dtype, 'bool') 1606 self.assertEqual(z.dtype, 'float64') 1607 self.assertEqual(w.dtype, 'complex64') 1608 1609 @testing_utils.enable_v2_dtype_behavior 1610 def test_extra_args_and_kwargs_not_casted(self): 1611 1612 class IdentityLayerWithArgs(base_layer.Layer): 1613 1614 def call(self, inputs, *args, **kwargs): 1615 kwargs.pop('training', None) 1616 return nest.flatten([inputs, args, kwargs]) 1617 1618 layer = IdentityLayerWithArgs(dtype='float64') 1619 x, y, z = layer(self._const('float16'), self._const('float16'), 1620 kwarg=self._const('float16')) 1621 self.assertEqual(x.dtype, 'float64') 1622 self.assertEqual(y.dtype, 'float16') 1623 self.assertEqual(z.dtype, 'float16') 1624 1625 @testing_utils.enable_v2_dtype_behavior 1626 def test_layer_without_autocast(self): 1627 1628 class IdentityLayerWithoutAutocast(IdentityLayer): 1629 1630 def __init__(self, *args, **kwargs): 1631 kwargs['autocast'] = False 1632 super(IdentityLayerWithoutAutocast, self).__init__(*args, **kwargs) 1633 1634 layer = IdentityLayerWithoutAutocast(dtype='float64') 1635 self.assertEqual(layer(self._const('float32')).dtype, 'float32') 1636 1637 @testing_utils.enable_v2_dtype_behavior 1638 def test_compute_output_signature(self): 1639 1640 class IdentityLayerWithOutputShape(IdentityLayer): 1641 1642 def compute_output_shape(self, input_shape): 1643 return input_shape 1644 1645 layer = IdentityLayerWithOutputShape(dtype='float64') 1646 output_signature = layer.compute_output_signature( 1647 tensor_spec.TensorSpec(shape=(), dtype='float32')) 1648 self.assertEqual(output_signature.shape, ()) 1649 self.assertEqual(output_signature.dtype, 'float64') 1650 1651 @testing_utils.enable_v2_dtype_behavior 1652 def test_composite_tensors_input_casting(self): 1653 sparse = sparse_tensor.SparseTensor( 1654 indices=array_ops.constant([[0, 1], [2, 3]], dtype='int64'), 1655 values=array_ops.constant([0., 1.], dtype='float32'), 1656 dense_shape=array_ops.constant([4, 4], dtype='int64')) 1657 ragged = ragged_tensor.RaggedTensor.from_row_splits( 1658 values=array_ops.constant([1., 2., 3.], dtype='float32'), 1659 row_splits=array_ops.constant([0, 2, 2, 3], dtype='int64')) 1660 1661 layer = IdentityLayer(dtype='float16') 1662 1663 for x in sparse, ragged: 1664 self.assertEqual(x.dtype, 'float32') 1665 y = layer(x) 1666 self.assertEqual(y.dtype, 'float16') 1667 self.assertEqual(type(x), type(y)) 1668 1669 @testing_utils.enable_v2_dtype_behavior 1670 def test_passing_non_tensor(self): 1671 layer = IdentityLayer() 1672 x = object() 1673 y = layer(x) # Layer should not cast 'x', as it's not a tensor 1674 self.assertIs(x, y) 1675 1676 @testing_utils.disable_v2_dtype_behavior 1677 def test_v1_behavior(self): 1678 # Test dtype defaults to None and inferred from input 1679 layer = IdentityLayer() 1680 self.assertIsNone(layer.dtype) 1681 layer(self._const('float64')) 1682 self.assertEqual(layer.dtype, 'float64') 1683 1684 # Test layer does not cast to dtype 1685 self.assertEqual(layer(self._const('float32')).dtype, 'float32') 1686 1687 1688if __name__ == '__main__': 1689 ops.enable_eager_execution() 1690 test.main() 1691