1# Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for TensorFlow 2.0 layer behavior.""" 16 17import copy 18import os 19import sys 20import traceback 21 22import numpy as np 23 24from tensorflow.python.eager import context 25from tensorflow.python.eager import def_function 26from tensorflow.python.framework import composite_tensor 27from tensorflow.python.framework import constant_op 28from tensorflow.python.framework import dtypes 29from tensorflow.python.framework import errors_impl 30from tensorflow.python.framework import ops 31from tensorflow.python.framework import sparse_tensor 32from tensorflow.python.framework import tensor_spec 33from tensorflow.python.framework import type_spec 34from tensorflow.python.keras import backend 35from tensorflow.python.keras import combinations 36from tensorflow.python.keras import keras_parameterized 37from tensorflow.python.keras import layers 38from tensorflow.python.keras import regularizers 39from tensorflow.python.keras import testing_utils 40from tensorflow.python.keras.engine import base_layer 41from tensorflow.python.keras.engine import input_layer 42from tensorflow.python.keras.engine import sequential 43from tensorflow.python.keras.engine import training as training_lib 44from tensorflow.python.keras.legacy_tf_layers import core as legacy_core 45from tensorflow.python.keras.optimizer_v2 import rmsprop 46from tensorflow.python.keras.utils import control_flow_util 47from tensorflow.python.module import module 48from tensorflow.python.ops import array_ops 49from tensorflow.python.ops import math_ops 50from tensorflow.python.ops import state_ops 51from tensorflow.python.ops import summary_ops_v2 52from tensorflow.python.ops import tensor_array_ops 53from tensorflow.python.ops import variables 54from tensorflow.python.ops.ragged import ragged_tensor 55from tensorflow.python.platform import gfile 56from tensorflow.python.platform import test 57from tensorflow.python.summary import summary_iterator 58from tensorflow.python.util import nest 59 60 61class DynamicLayer(base_layer.Layer): 62 63 def __init__(self, dynamic=False, **kwargs): 64 super(DynamicLayer, self).__init__(dynamic=dynamic, **kwargs) 65 66 def call(self, inputs): 67 samples = tensor_array_ops.TensorArray( 68 dtype=dtypes.float32, size=array_ops.shape(inputs)[0]) 69 for idx, sample in enumerate(inputs): 70 samples = samples.write(idx, math_ops.square(sample)) 71 return samples.stack() 72 73 def compute_output_shape(self, input_shape): 74 return input_shape 75 76 77class InvalidLayer(base_layer.Layer): 78 79 def call(self, inputs): 80 raise ValueError('You did something wrong!') 81 82 83class BaseLayerTest(keras_parameterized.TestCase): 84 85 @combinations.generate(combinations.keras_mode_combinations()) 86 def test_layer_instrumentation(self): 87 layer = layers.Add() 88 self.assertTrue(layer._instrumented_keras_api) 89 self.assertTrue(layer._instrumented_keras_layer_class) 90 self.assertFalse(layer._instrumented_keras_model_class) 91 92 @combinations.generate(combinations.keras_model_type_combinations()) 93 def test_dynamic_layer(self): 94 model = testing_utils.get_model_from_layers([DynamicLayer(dynamic=True)], 95 input_shape=(3,)) 96 self.assertEqual(model.dynamic, True) 97 model.compile(rmsprop.RMSprop(0.001), loss='mse') 98 self.assertEqual(model.run_eagerly, True) 99 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 100 101 @combinations.generate(combinations.keras_model_type_combinations()) 102 def test_dynamic_layer_error(self): 103 # Functional Models hit the `dyanamic=True` error during construction. 104 # Subclass Models should just throw the original autograph error during 105 # execution. 106 raised_error = False 107 try: 108 model = testing_utils.get_model_from_layers([DynamicLayer()], 109 input_shape=(3,)) 110 model.compile(rmsprop.RMSprop(0.001), loss='mse') 111 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 112 except errors_impl.OperatorNotAllowedInGraphError as e: 113 if 'iterating over `tf.Tensor` is not allowed' in str(e): 114 raised_error = True 115 except TypeError as e: 116 if 'attempting to use Python control flow' in str(e): 117 raised_error = True 118 self.assertTrue(raised_error) 119 120 @combinations.generate(combinations.keras_model_type_combinations()) 121 def test_dynamic_layer_error_running_in_graph_mode(self): 122 with ops.get_default_graph().as_default(): 123 model = testing_utils.get_model_from_layers([DynamicLayer(dynamic=True)], 124 input_shape=(3,)) 125 self.assertEqual(model.dynamic, True) 126 # But then you cannot run the model since you're in a graph scope. 127 with self.assertRaisesRegex(ValueError, 128 'You must enable eager execution'): 129 model.compile(rmsprop.RMSprop(0.001), loss='mse') 130 131 def test_manual_compute_output_shape(self): 132 133 class BuildCounter(base_layer.Layer): 134 135 def __init__(self, *args, **kwargs): # pylint: disable=redefined-outer-name 136 super(BuildCounter, self).__init__(*args, **kwargs) 137 self.build_counter = 0 138 139 def build(self, input_shape): 140 self.build_counter += 1 141 self.build_shape = input_shape 142 143 def call(self, inputs): 144 return inputs 145 146 layer = BuildCounter(dtype=dtypes.float64) 147 output_shape = layer.compute_output_shape((None, 10)) 148 self.assertEqual(layer.build_counter, 1) 149 self.assertEqual(layer.build_shape.as_list(), [None, 10]) 150 self.assertEqual(output_shape.as_list(), [None, 10]) 151 output_signature = layer.compute_output_signature( 152 tensor_spec.TensorSpec(dtype=dtypes.float64, shape=[None, 10])) 153 self.assertEqual(layer.build_counter, 1) 154 self.assertEqual(layer.build_shape.as_list(), [None, 10]) 155 self.assertEqual(output_signature.dtype, dtypes.float64) 156 self.assertEqual(output_signature.shape.as_list(), [None, 10]) 157 layer(np.ones((5, 10))) 158 self.assertEqual(layer.build_counter, 1) 159 self.assertEqual(layer.build_shape.as_list(), [None, 10]) 160 161 def test_dynamic_layer_with_deferred_sequential_model(self): 162 model = sequential.Sequential([DynamicLayer(dynamic=True), layers.Dense(3)]) 163 self.assertEqual(model.dynamic, True) 164 model.compile(rmsprop.RMSprop(0.001), loss='mse') 165 self.assertEqual(model.run_eagerly, True) 166 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 167 168 def test_nested_dynamic_layers_in_eager_mode(self): 169 inputs = input_layer.Input((3,)) 170 outputs = DynamicLayer(dynamic=True)(inputs) 171 inner_model = training_lib.Model(inputs, outputs) 172 self.assertEqual(inner_model.dynamic, True) 173 174 inputs = input_layer.Input((3,)) 175 x = DynamicLayer(dynamic=True)(inputs) 176 outputs = inner_model(x) 177 178 model = training_lib.Model(inputs, outputs) 179 self.assertEqual(model.dynamic, True) 180 model.compile(rmsprop.RMSprop(0.001), loss='mse') 181 self.assertEqual(model.run_eagerly, True) 182 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 183 184 def test_dynamic_subclassed_model_no_shape_inference(self): 185 186 class MyModel(training_lib.Model): 187 188 def __init__(self): 189 super(MyModel, self).__init__(dynamic=True) 190 self.layer1 = layers.Dense(3) 191 self.layer2 = layers.Dense(3) 192 193 def call(self, inputs): 194 if math_ops.reduce_sum(inputs) > 0: 195 return self.layer1(inputs) 196 else: 197 return self.layer2(inputs) 198 199 model = MyModel() 200 self.assertEqual(model.dynamic, True) 201 model.compile(rmsprop.RMSprop(0.001), loss='mse') 202 self.assertEqual(model.run_eagerly, True) 203 model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) 204 self.assertEqual(model.outputs, None) 205 206 def test_dynamic_subclassed_model_with_shape_inference(self): 207 208 class MyModel(training_lib.Model): 209 210 def __init__(self): 211 super(MyModel, self).__init__(dynamic=True) 212 self.layer1 = layers.Dense(3) 213 self.layer2 = layers.Dense(3) 214 215 def call(self, inputs): 216 if math_ops.reduce_sum(inputs) > 0: 217 return self.layer1(inputs) 218 else: 219 return self.layer2(inputs) 220 221 def compute_output_shape(self, input_shape): 222 return tuple(input_shape[:-1].as_list()) + (3,) 223 224 model = MyModel() 225 self.assertEqual(model.dynamic, True) 226 model.compile(rmsprop.RMSprop(0.001), loss='mse') 227 x, y = np.random.random((2, 3)), np.random.random((2, 3)) 228 model.train_on_batch(x, y) 229 outputs = model(x) 230 self.assertEqual(outputs.shape.as_list(), [2, 3]) 231 232 def test_deepcopy(self): 233 bias_reg = lambda x: 1e-3 * math_ops.reduce_sum(x) 234 layer = layers.Conv2D(32, (3, 3), bias_regularizer=bias_reg) 235 # Call the Layer on data to generate regularize losses. 236 layer(array_ops.ones((1, 10, 10, 3))) 237 self.assertLen(layer.losses, 1) 238 new_layer = copy.deepcopy(layer) 239 self.assertEqual(new_layer.bias_regularizer, bias_reg) 240 self.assertEqual(layer.get_config(), new_layer.get_config()) 241 242 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 243 def test_invalid_forward_pass(self): 244 inputs = input_layer.Input((3,)) 245 with self.assertRaisesRegex(ValueError, 'You did something wrong!'): 246 _ = InvalidLayer()(inputs) 247 248 def test_no_legacy_model(self): 249 inputs = input_layer.Input((1,)) 250 legacy_dense_0 = legacy_core.Dense(1, name='legacy_dense_0') 251 legacy_dense_1 = legacy_core.Dense(1, name='legacy_dense_1') 252 253 layer = legacy_dense_0(inputs) 254 layer = layers.Dense(1)(layer) 255 layer = legacy_dense_1(layer) 256 257 expected_regex = (r'The following are legacy tf\.layers\.Layers:\n ' 258 '{}\n {}'.format(legacy_dense_0, legacy_dense_1)) 259 260 with self.assertRaisesRegex(TypeError, expected_regex): 261 _ = training_lib.Model(inputs=[inputs], outputs=[layer]) 262 263 model = training_lib.Model(inputs=[inputs], outputs=[inputs]) 264 with self.assertRaisesRegex(TypeError, expected_regex): 265 model._insert_layers([legacy_dense_0, legacy_dense_1]) 266 267 def test_no_legacy_sequential(self): 268 layer = [layers.Dense(1), legacy_core.Dense(1, name='legacy_dense_0')] 269 270 expected_regex = r'legacy tf\.layers\.Layers:\n {}'.format(layer[1]) 271 with self.assertRaisesRegex(TypeError, expected_regex): 272 _ = sequential.Sequential(layer) 273 274 with self.assertRaisesRegex(TypeError, expected_regex): 275 _ = sequential.Sequential([input_layer.Input(shape=(4,))] + layer) 276 277 model = sequential.Sequential() 278 with self.assertRaisesRegex(TypeError, expected_regex): 279 for l in layer: 280 model.add(l) 281 282 @combinations.generate( 283 combinations.times( 284 combinations.keras_model_type_combinations(), 285 combinations.combine(mode=['graph', 'eager']))) 286 def test_build_with_numpy_data(self): 287 model_layers = [ 288 layers.Dense(3, activation='relu', kernel_initializer='ones'), 289 layers.Dense(1, activation='sigmoid', kernel_initializer='ones') 290 ] 291 model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,)) 292 model(np.zeros((2, 4), dtype='float32')) 293 self.assertTrue(model.built) 294 295 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 296 def test_default_add_weight(self): 297 298 class TestLayer(base_layer.Layer): 299 300 def __init__(self): 301 super(TestLayer, self).__init__() 302 self.default_weight = self.add_weight() 303 self.weight_without_name = self.add_weight(shape=(3, 4)) 304 self.regularized_weight_without_name = self.add_weight( 305 shape=(3, 4), regularizer='l2') 306 307 layer = TestLayer() 308 self.assertEqual(layer.default_weight.shape.as_list(), []) 309 self.assertEqual(layer.weight_without_name.shape.as_list(), [3, 4]) 310 self.assertEqual(layer.default_weight.dtype.name, 'float32') 311 self.assertEqual(layer.weight_without_name.dtype.name, 'float32') 312 self.assertEqual(len(layer.losses), 1) 313 if not context.executing_eagerly(): 314 # Cannot access tensor.name in eager execution. 315 self.assertIn('Variable_2/Regularizer', layer.losses[0].name) 316 317 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 318 def test_add_weight_by_getter(self): 319 layer = base_layer.Layer() 320 variable = variables.Variable('abc') 321 added = layer.add_weight( 322 dtype=dtypes.string, getter=lambda *_, **__: variable) 323 self.assertIs(variable, added) 324 325 @combinations.generate(combinations.keras_mode_combinations(mode=['eager'])) 326 def test_learning_phase_freezing_for_layers(self): 327 328 class LearningPhaseLayer(base_layer.Layer): 329 330 def call(self, inputs): 331 return backend.in_train_phase(lambda: array_ops.ones_like(inputs), 332 lambda: array_ops.zeros_like(inputs)) 333 334 def get_learning_phase_value(): 335 model = sequential.Sequential([LearningPhaseLayer(input_shape=(1,))]) 336 model._run_eagerly = testing_utils.should_run_eagerly() 337 return np.sum(model(np.ones((1, 1)))) 338 339 self.assertEqual(get_learning_phase_value(), 0) 340 341 # Test scope. 342 with backend.learning_phase_scope(1): 343 self.assertEqual(get_learning_phase_value(), 1) 344 345 # The effects of the scope end after exiting it. 346 self.assertEqual(get_learning_phase_value(), 0) 347 348 # Test setting. 349 backend.set_learning_phase(1) 350 self.assertEqual(get_learning_phase_value(), 1) 351 backend.set_learning_phase(0) 352 self.assertEqual(get_learning_phase_value(), 0) 353 354 # Cannot be enabled with `run_eagerly=True`, see b/123904578 355 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 356 def test_layer_can_return_variable(self): 357 358 class ComputeSum(base_layer.Layer): 359 360 def __init__(self): 361 super(ComputeSum, self).__init__() 362 self.total = variables.Variable( 363 initial_value=array_ops.zeros((1, 1)), trainable=False) 364 if not context.executing_eagerly(): 365 backend.get_session().run(self.total.initializer) 366 367 def call(self, inputs): 368 self.total.assign_add(inputs) 369 return self.total 370 371 inputs = input_layer.Input(shape=(1,)) 372 model = training_lib.Model(inputs, ComputeSum()(inputs)) 373 model.predict(np.ones((1, 1))) 374 375 def _get_layer_with_training_arg(self): 376 377 class TrainingLayer(base_layer.Layer): 378 """A layer with a `training` argument in a defuned `call`.""" 379 380 @def_function.function 381 def call(self, inputs, training=None): 382 if training is None: 383 training = backend.learning_phase() 384 return control_flow_util.smart_cond( 385 training, lambda: array_ops.ones_like(inputs), 386 lambda: array_ops.zeros_like(inputs)) 387 388 return TrainingLayer() 389 390 # b/124459427: can't test with `run_eagerly=True` for now. 391 @combinations.generate( 392 combinations.times(combinations.keras_mode_combinations(), 393 combinations.keras_model_type_combinations())) 394 def test_training_arg_in_defun(self): 395 layer = self._get_layer_with_training_arg() 396 model = testing_utils.get_model_from_layers([layer], input_shape=(1,)) 397 model.compile(rmsprop.RMSprop(0.), 398 loss='mae') 399 history = model.fit(np.zeros((1, 1)), np.zeros((1, 1))) 400 self.assertEqual(history.history['loss'][0], 1.) 401 loss = model.evaluate(np.zeros((1, 1)), np.zeros((1, 1))) 402 self.assertEqual(loss, 0.) 403 404 # Test that the argument injection performed in `call` is not active 405 # when the argument is passed explicitly. 406 layer = self._get_layer_with_training_arg() 407 inputs = input_layer.Input(shape=(1,)) 408 # Pass `training` by name 409 outputs = layer(inputs, training=False) 410 model = training_lib.Model(inputs, outputs) 411 model.compile(rmsprop.RMSprop(0.), 412 loss='mae') 413 history = model.fit(np.zeros((1, 1)), np.zeros((1, 1))) 414 self.assertEqual(history.history['loss'][0], 0.) 415 416 @combinations.generate( 417 combinations.times(combinations.keras_mode_combinations(), 418 combinations.keras_model_type_combinations())) 419 def test_raw_variable_assignment(self): 420 421 class RawVariableLayer(base_layer.Layer): 422 423 def __init__(self, **kwargs): 424 super(RawVariableLayer, self).__init__(**kwargs) 425 # Test variables in nested structure. 426 self.var_list = [variables.Variable(1.), {'a': variables.Variable(2.)}] 427 428 def call(self, inputs): 429 return inputs * self.var_list[0] * self.var_list[1]['a'] 430 431 model = testing_utils.get_model_from_layers([RawVariableLayer()], 432 input_shape=(10,)) 433 model.compile( 434 'sgd', 435 'mse', 436 run_eagerly=testing_utils.should_run_eagerly()) 437 x, y = np.ones((10, 10)), np.ones((10, 10)) 438 # Checks that variables get initialized. 439 model.fit(x, y, batch_size=2, epochs=2) 440 441 @combinations.generate(combinations.combine(mode=['eager'])) 442 def test_composite_variable_assignment(self): 443 444 class Spec(type_spec.TypeSpec): 445 446 value_type = property(lambda self: CompositeVariable) 447 448 def _component_specs(self): 449 pass 450 451 def _serialize(self): 452 pass 453 454 def _to_components(self, value): 455 return value._variables 456 457 def _from_components(self, variable_list): 458 return CompositeVariable(variable_list) 459 460 class CompositeVariable(composite_tensor.CompositeTensor): 461 462 def __init__(self, variable_list): 463 self._variables = variable_list 464 465 @property 466 def _type_spec(self): 467 return Spec() 468 469 class CompositeVariableLayer(base_layer.Layer): 470 471 def __init__(self): 472 super().__init__() 473 self.composite_var = CompositeVariable( 474 [variables.Variable(1.), 475 variables.Variable(2.)]) 476 477 layer = CompositeVariableLayer() 478 self.assertLen(layer.weights, 2) 479 self.assertIsInstance(layer.weights[0], variables.Variable) 480 self.assertIsInstance(layer.weights[1], variables.Variable) 481 self.assertEqual(self.evaluate(layer.weights[0]), 1.) 482 self.assertEqual(self.evaluate(layer.weights[1]), 2.) 483 484 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 485 def test_layer_names(self): 486 inputs = input_layer.Input(shape=[2]) 487 add1 = inputs + inputs 488 add2 = layers.Add()([inputs, inputs]) 489 add3 = inputs + inputs 490 add4 = layers.Add()([inputs, inputs]) 491 model = training_lib.Model(inputs=[inputs], 492 outputs=[add1, add2, add3, add4]) 493 actual_names = [l.name for l in model.layers] 494 graph_names = [ 495 'input_1', 'tf_op_layer_add', 'add', 'tf_op_layer_add_2', 'add_1' 496 ] 497 eager_names = [ 498 'input_1', 'tf.__operators__.add', 'add', 'tf.__operators__.add_1', 499 'add_1' 500 ] 501 for actual, eager, graph in zip(actual_names, graph_names, eager_names): 502 self.assertIn(actual, {eager, graph}) 503 504 @combinations.generate(combinations.combine(mode=['eager'])) 505 def test_layer_names_after_loading(self): 506 backend.clear_session() 507 # Mimic loading a model that already contained add layers with 508 # name = 'add_1' and 'tf.__operators__.add' 509 layers.Add(name='add_1') 510 layers.Add(name='tf.__operators__.add') 511 512 inputs = input_layer.Input(shape=[2]) 513 add1 = inputs + inputs 514 add2 = layers.Add()([inputs, inputs]) 515 add3 = inputs + inputs 516 add4 = layers.Add()([inputs, inputs]) 517 model = training_lib.Model( 518 inputs=[inputs], outputs=[add1, add2, add3, add4]) 519 actual_names = [l.name for l in model.layers] 520 # The generated op layer names should have avoided layer names seen in 521 # the loaded model. (This avoiance should not apply to non-op-layers) 522 expected_names = [ 523 'input_1', 'tf.__operators__.add_1', 524 'add', 'tf.__operators__.add_2', 'add_1' 525 ] 526 self.assertAllEqual(actual_names, expected_names) 527 528 def test_add_trainable_weight_on_frozen_layer(self): 529 530 class TestLayer(base_layer.Layer): 531 532 def build(self, input_shape): 533 self.w = self.add_weight(shape=(), trainable=True) 534 535 def call(self, inputs): 536 return self.w * inputs 537 538 layer = TestLayer() 539 layer.trainable = False 540 layer.build(None) 541 layer.trainable = True 542 self.assertListEqual(layer.trainable_weights, [layer.w]) 543 544 @combinations.generate( 545 combinations.times(combinations.keras_mode_combinations(), 546 combinations.keras_model_type_combinations())) 547 def test_passing_initial_weights_values(self): 548 kernel_value = np.random.random((10, 2)) 549 layer_with_weights = layers.Dense(2, use_bias=False, weights=[kernel_value]) 550 551 model = testing_utils.get_model_from_layers([layer_with_weights], 552 input_shape=(10,)) 553 model.compile( 554 'sgd', 555 'mse', 556 run_eagerly=testing_utils.should_run_eagerly()) 557 inputs = np.random.random((3, 10)) 558 out = model.predict(inputs) 559 self.assertAllClose(model.layers[-1].get_weights()[0], kernel_value) 560 self.assertAllClose(out, np.dot(inputs, kernel_value)) 561 562 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 563 def test_set_weights_and_get_weights(self): 564 layer = layers.Dense(2) 565 layer.build((None, 10)) 566 kernel = np.random.random((10, 2)) 567 bias = np.random.random((2,)) 568 layer.set_weights([kernel, bias]) 569 weights = layer.get_weights() 570 self.assertEqual(len(weights), 2) 571 self.assertAllClose(weights[0], kernel) 572 self.assertAllClose(weights[1], bias) 573 with self.assertRaisesRegex(ValueError, 574 'but the layer was expecting 2 weights'): 575 layer.set_weights([1, 2, 3]) 576 with self.assertRaisesRegex(ValueError, 577 'not compatible with provided weight shape'): 578 layer.set_weights([kernel.T, bias]) 579 580 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 581 def test_set_weights_accepts_output_of_get_weights(self): 582 layer = layers.Layer() 583 layer.add_weight(name='scalar_float', shape=(), dtype=dtypes.float32) 584 layer.add_weight(name='scalar_string', shape=(), dtype=dtypes.string, 585 initializer=lambda *a, **k: 'abc') 586 layer.add_weight(name='vector_float', shape=(3,), dtype=dtypes.float32) 587 layer.add_weight(name='vector_string', shape=(2,), dtype=dtypes.string, 588 initializer=lambda *a, **k: 2 * ['abc']) 589 layer.set_weights(layer.get_weights()) 590 591 def test_get_config_error(self): 592 593 class MyLayer(base_layer.Layer): 594 595 def __init__(self, my_kwarg='default', **kwargs): 596 super(MyLayer, self).__init__(**kwargs) 597 self.my_kwarg = my_kwarg 598 599 # `__init__` includes kwargs but `get_config` is not overridden, so 600 # an error should be thrown: 601 with self.assertRaisesRegex(NotImplementedError, 'Layer MyLayer has'): 602 MyLayer('custom').get_config() 603 604 class MyLayerNew(base_layer.Layer): 605 606 def __init__(self, my_kwarg='default', **kwargs): 607 super(MyLayerNew, self).__init__(**kwargs) 608 self.my_kwarg = my_kwarg 609 610 def get_config(self): 611 config = super(MyLayerNew, self).get_config() 612 config['my_kwarg'] = self.my_kwarg 613 return config 614 615 # Test to make sure that error is not raised if the method call is 616 # from an overridden `get_config`: 617 self.assertEqual(MyLayerNew('custom').get_config()['my_kwarg'], 'custom') 618 619 class MyLayerNew2(base_layer.Layer): 620 621 def __init__(self, name='MyLayerName', dtype=None, **kwargs): # pylint:disable=redefined-outer-name 622 super(MyLayerNew2, self).__init__(name=name, dtype=dtype, **kwargs) 623 624 # Check that if the kwargs in `__init__` are base layer constructor 625 # arguments, no error is thrown: 626 self.assertEqual(MyLayerNew2(name='New').get_config()['name'], 'New') 627 628 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 629 def test_count_params(self): 630 dense = layers.Dense(16) 631 dense.build((None, 4)) 632 self.assertEqual(dense.count_params(), 16 * 4 + 16) 633 634 dense = layers.Dense(16) 635 with self.assertRaisesRegex(ValueError, 'call `count_params`'): 636 dense.count_params() 637 638 model = sequential.Sequential(layers.Dense(16)) 639 with self.assertRaisesRegex(ValueError, 'call `count_params`'): 640 model.count_params() 641 642 dense = layers.Dense(16, input_dim=4) 643 model = sequential.Sequential(dense) 644 self.assertEqual(model.count_params(), 16 * 4 + 16) 645 646 def test_super_not_called(self): 647 648 class CustomLayerNotCallingSuper(base_layer.Layer): 649 650 def __init__(self): 651 pass 652 653 layer = CustomLayerNotCallingSuper() 654 with self.assertRaisesRegex(RuntimeError, 'You must call `super()'): 655 layer(np.random.random((10, 2))) 656 657 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 658 def test_first_arg_not_called_inputs(self): 659 x, y = array_ops.ones((10, 1)), array_ops.ones((10, 1)) 660 661 class ArgLayer(base_layer.Layer): 662 663 def call(self, x, y): 664 return x + y 665 666 layer = ArgLayer() 667 out = self.evaluate(layer(x=x, y=y)) 668 self.assertAllClose(out, 2 * np.ones((10, 1))) 669 670 class KwargLayer(base_layer.Layer): 671 672 def call(self, x=None, y=None): 673 return x + y 674 675 layer = KwargLayer() 676 out = self.evaluate(layer(x=x, y=y)) 677 self.assertAllClose(out, 2 * np.ones((10, 1))) 678 679 with self.assertRaisesRegex(ValueError, 'must always be passed'): 680 layer(y=y) 681 682 class TFFunctionLayer(base_layer.Layer): 683 684 @def_function.function 685 def call(self, x, y=None): 686 if y is None: 687 return x 688 return x + y 689 690 layer = TFFunctionLayer() 691 out = self.evaluate(layer(x=x, y=y)) 692 self.assertAllClose(out, 2 * np.ones((10, 1))) 693 694 def test_build_input_shape(self): 695 696 class CustomLayer(base_layer.Layer): 697 698 def build(self, input_shape): 699 self.add_weight('w', shape=input_shape[1:]) 700 super(CustomLayer, self).build(input_shape) 701 702 layer = CustomLayer() 703 self.assertFalse(layer.built) 704 705 layer.build([None, 1, 2, 3]) 706 self.assertTrue(layer.built) 707 self.assertEqual([None, 1, 2, 3], layer._build_input_shape) 708 709 layer = CustomLayer() 710 layer(input_layer.Input((3,))) 711 self.assertTrue(layer.built) 712 self.assertEqual([None, 3], layer._build_input_shape.as_list()) 713 714 @combinations.generate(combinations.combine(mode=['eager'])) 715 def test_custom_layer_training_arg(self): 716 class CustomLayerNoTrainingArg(base_layer.Layer): 717 718 def __init__(self, nested_layer=None): 719 super(CustomLayerNoTrainingArg, self).__init__() 720 self._nested_layer = nested_layer or array_ops.identity 721 722 def call(self, inputs): 723 return self._nested_layer(inputs) 724 725 class CustomLayerDefaultTrainingMissing(base_layer.Layer): 726 727 def __init__(self, nested_layer=None): 728 super(CustomLayerDefaultTrainingMissing, self).__init__() 729 self._nested_layer = nested_layer or array_ops.identity 730 731 def call(self, inputs, training): 732 if training: 733 return self._nested_layer(inputs) 734 else: 735 return self._nested_layer(inputs) * 0.5 736 737 class CustomLayerDefaultTrainingNone(base_layer.Layer): 738 739 def __init__(self, nested_layer=None): 740 super(CustomLayerDefaultTrainingNone, self).__init__() 741 self._nested_layer = nested_layer or array_ops.identity 742 743 def call(self, inputs, training=None): 744 if training: 745 return self._nested_layer(inputs) 746 else: 747 return self._nested_layer(inputs) * 0.5 748 749 class CustomLayerDefaultTrainingFalse(base_layer.Layer): 750 751 def __init__(self, nested_layer=None): 752 super(CustomLayerDefaultTrainingFalse, self).__init__() 753 self._nested_layer = nested_layer or array_ops.identity 754 755 def call(self, inputs, training=False): 756 if training: 757 return self._nested_layer(inputs) 758 else: 759 return self._nested_layer(inputs) * 0.5 760 761 class CustomLayerDefaultTrainingTrue(base_layer.Layer): 762 763 def __init__(self, nested_layer=None): 764 super(CustomLayerDefaultTrainingTrue, self).__init__() 765 self._nested_layer = nested_layer or array_ops.identity 766 767 def call(self, inputs, training=True): 768 if training: 769 return self._nested_layer(inputs) 770 else: 771 return self._nested_layer(inputs) * 0.5 772 773 self._test_custom_layer_training_arg( 774 CustomLayerNoTrainingArg=CustomLayerNoTrainingArg, 775 CustomLayerDefaultTrainingMissing=CustomLayerDefaultTrainingMissing, 776 CustomLayerDefaultTrainingNone=CustomLayerDefaultTrainingNone, 777 CustomLayerDefaultTrainingFalse=CustomLayerDefaultTrainingFalse, 778 CustomLayerDefaultTrainingTrue=CustomLayerDefaultTrainingTrue) 779 780 @combinations.generate(combinations.combine(mode=['eager'])) 781 def test_custom_layer_training_arg_kwargonly(self): 782 class CustomLayerNoTrainingArg(base_layer.Layer): 783 784 def __init__(self, nested_layer=None): 785 super(CustomLayerNoTrainingArg, self).__init__() 786 self._nested_layer = nested_layer or array_ops.identity 787 788 def call(self, inputs): 789 return self._nested_layer(inputs) 790 791 class CustomLayerDefaultTrainingMissing(base_layer.Layer): 792 793 def __init__(self, nested_layer=None): 794 super(CustomLayerDefaultTrainingMissing, self).__init__() 795 self._nested_layer = nested_layer or array_ops.identity 796 797 def call(self, inputs, *, training): 798 if training: 799 return self._nested_layer(inputs) 800 else: 801 return self._nested_layer(inputs) * 0.5 802 803 class CustomLayerDefaultTrainingNone(base_layer.Layer): 804 805 def __init__(self, nested_layer=None): 806 super(CustomLayerDefaultTrainingNone, self).__init__() 807 self._nested_layer = nested_layer or array_ops.identity 808 809 def call(self, inputs, *, training=None): 810 if training: 811 return self._nested_layer(inputs) 812 else: 813 return self._nested_layer(inputs) * 0.5 814 815 class CustomLayerDefaultTrainingFalse(base_layer.Layer): 816 817 def __init__(self, nested_layer=None): 818 super(CustomLayerDefaultTrainingFalse, self).__init__() 819 self._nested_layer = nested_layer or array_ops.identity 820 821 def call(self, inputs, *, training=False): 822 if training: 823 return self._nested_layer(inputs) 824 else: 825 return self._nested_layer(inputs) * 0.5 826 827 class CustomLayerDefaultTrainingTrue(base_layer.Layer): 828 829 def __init__(self, nested_layer=None): 830 super(CustomLayerDefaultTrainingTrue, self).__init__() 831 self._nested_layer = nested_layer or array_ops.identity 832 833 def call(self, inputs, *, training=True): 834 if training: 835 return self._nested_layer(inputs) 836 else: 837 return self._nested_layer(inputs) * 0.5 838 839 self._test_custom_layer_training_arg( 840 CustomLayerNoTrainingArg=CustomLayerNoTrainingArg, 841 CustomLayerDefaultTrainingMissing=CustomLayerDefaultTrainingMissing, 842 CustomLayerDefaultTrainingNone=CustomLayerDefaultTrainingNone, 843 CustomLayerDefaultTrainingFalse=CustomLayerDefaultTrainingFalse, 844 CustomLayerDefaultTrainingTrue=CustomLayerDefaultTrainingTrue) 845 846 def _test_custom_layer_training_arg(self, 847 # pylint: disable=invalid-name 848 CustomLayerNoTrainingArg, 849 CustomLayerDefaultTrainingMissing, 850 CustomLayerDefaultTrainingNone, 851 CustomLayerDefaultTrainingFalse, 852 CustomLayerDefaultTrainingTrue, 853 # pylint: enable=invalid-name 854 ): 855 x = array_ops.ones(shape=(1, 1)) 856 857 # If the layer signature doesn't specify a default training arg, 858 # run it in inference mode when to training arg is passed 859 # to __call__ 860 layer = CustomLayerDefaultTrainingMissing() 861 self.assertAllEqual(layer(x), x * 0.5) 862 self.assertAllEqual(layer(x, training=False), x * 0.5) 863 self.assertAllEqual(layer(x, training=True), x) 864 865 # If the layer signature specifies `False` as the default training arg, 866 # run it in inference mode when no training arg is passed 867 # to __call__ 868 layer = CustomLayerDefaultTrainingFalse() 869 self.assertAllEqual(layer(x), x * 0.5) 870 self.assertAllEqual(layer(x, training=False), x * 0.5) 871 self.assertAllEqual(layer(x, training=True), x) 872 873 # If the layer signature specifies `True` as the default training arg, 874 # explicitly run it in training mode when no training arg is passed 875 # to __call__ 876 layer = CustomLayerDefaultTrainingTrue() 877 self.assertAllEqual(layer(x), x) 878 self.assertAllEqual(layer(x, training=False), x * 0.5) 879 self.assertAllEqual(layer(x, training=True), x) 880 881 # Outer layers/models should set the training context implicitly for all 882 # nested layers, respecting whatever mode the outer layer was run with. 883 layer = CustomLayerDefaultTrainingTrue(CustomLayerDefaultTrainingFalse()) 884 # No outer value passed: use local defaults 885 self.assertAllEqual(layer(x), x) # Use outer default True 886 # Outer value passed: override local defaults 887 self.assertAllEqual(layer(x, training=False), x * 0.25) 888 self.assertAllEqual(layer(x, training=True), x) 889 890 layer = CustomLayerDefaultTrainingFalse(CustomLayerDefaultTrainingTrue()) 891 # No outer value passed: use local defaults 892 self.assertAllEqual(layer(x), x * 0.25) # Use outer default False 893 # Outer value passed: override local defaults 894 self.assertAllEqual(layer(x, training=False), x * 0.25) 895 self.assertAllEqual(layer(x, training=True), x) 896 897 # If the outer layer `call` doesn't take a training argument at all, 898 # it'll set the nested scope as None when no training arg is passed in. 899 # If a training arg is passed in it won't use it directly in `call`, but 900 # it will set the nested training mode. 901 layer = CustomLayerNoTrainingArg(CustomLayerDefaultTrainingTrue()) 902 self.assertAllEqual(layer(x), x) # Use local default True 903 self.assertAllEqual(layer(x, training=False), x * 0.5) 904 self.assertAllEqual(layer(x, training=True), x) 905 906 layer = CustomLayerDefaultTrainingNone(CustomLayerDefaultTrainingTrue()) 907 self.assertAllEqual(layer(x), x * 0.5) # Nested use local default True 908 self.assertAllEqual(layer(x, training=False), x * 0.25) 909 self.assertAllEqual(layer(x, training=True), x) 910 911 def test_activity_regularizer_string(self): 912 913 class MyLayer(base_layer.Layer): 914 pass 915 916 layer = MyLayer(activity_regularizer='l2') 917 self.assertIsInstance(layer.activity_regularizer, regularizers.L2) 918 919 def test_tf_module_tracking(self): 920 921 class MyModule(module.Module): 922 923 def __init__(self): 924 super(MyModule, self).__init__() 925 self.v1 = variables.Variable(1., trainable=True, name='v1') 926 self.v2 = variables.Variable(2., trainable=False, name='v2') 927 928 def __call__(self, x): 929 return x * self.v1 * self.v2 930 931 class MyLayer(base_layer.Layer): 932 933 def __init__(self, **kwargs): 934 super(MyLayer, self).__init__(self, **kwargs) 935 self.my_modules = {} 936 self.my_modules['a'] = MyModule() 937 938 def call(self, x): 939 return self.my_modules['a'](x) 940 941 layer = MyLayer() 942 self.assertLen(layer.variables, 2) 943 self.assertLen(layer.trainable_variables, 1) 944 self.assertLen(layer.non_trainable_variables, 1) 945 946 layer.trainable = False 947 self.assertLen(layer.variables, 2) 948 self.assertLen(layer.trainable_variables, 0) 949 self.assertLen(layer.non_trainable_variables, 2) 950 951 class MyModel(training_lib.Model): 952 953 def __init__(self): 954 super(MyModel, self).__init__() 955 self.my_modules = [] 956 self.my_modules.append(MyModule()) 957 958 def call(self, x): 959 return self.my_modules[0](x) 960 961 model = MyModel() 962 self.assertLen(model.variables, 2) 963 self.assertLen(model.trainable_variables, 1) 964 self.assertLen(model.non_trainable_variables, 1) 965 966 model.trainable = False 967 self.assertLen(model.variables, 2) 968 self.assertLen(model.trainable_variables, 0) 969 self.assertLen(model.non_trainable_variables, 2) 970 971 972class SymbolicSupportTest(keras_parameterized.TestCase): 973 974 def test_using_symbolic_tensors_with_tf_ops(self): 975 # Single-input. 976 x = input_layer.Input((3,)) 977 math_ops.square(x) 978 979 # Multi-inputs. 980 x1, x2 = input_layer.Input((3,)), input_layer.Input((3,)) 981 array_ops.concat([x1, x2], axis=1) 982 983 # Mixing Keras symbolic tensors and graph tensors from the same graph works. 984 with backend.get_graph().as_default(): 985 x1 = input_layer.Input((3,)) 986 x2 = input_layer.Input((3,)) 987 math_ops.matmul(x1, x2) 988 989 # Creating same op type (matmul) multiple times in the Keras graph works. 990 x1 = input_layer.Input((3,)) 991 x2 = input_layer.Input((3,)) 992 math_ops.matmul(x1, x2) 993 994 def test_mixing_eager_and_graph_tensors(self): 995 with ops.Graph().as_default(): 996 x1 = array_ops.ones((3, 3)) 997 x2 = array_ops.ones((3, 3)) 998 with self.assertRaisesRegex(TypeError, 'Graph tensors'): 999 math_ops.matmul(x1, x2) 1000 1001 def test_mixing_numpy_arrays_and_graph_tensors(self): 1002 with ops.Graph().as_default(): 1003 x1 = array_ops.ones((3, 3)) 1004 x2 = np.ones((3, 3), dtype='float32') 1005 with self.assertRaisesRegex(TypeError, 'Graph tensors'): 1006 math_ops.matmul(x1, x2) 1007 1008 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1009 def test_mixing_keras_symbolic_tensors_and_eager_tensors(self): 1010 x1 = input_layer.Input((3,)) 1011 x2 = array_ops.ones((3, 3)) 1012 y = math_ops.matmul(x1, x2) 1013 1014 fn = backend.function(inputs=[x1], outputs=[y]) 1015 x_val = np.random.random((3, 3)) 1016 y_val = np.ones((3, 3)) 1017 self.assertAllClose(fn([x_val])[0], 1018 np.matmul(x_val, y_val), 1019 atol=1e-5) 1020 1021 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1022 def test_mixing_keras_symbolic_tensors_and_numpy_arrays(self): 1023 x1 = input_layer.Input((3,)) 1024 x2 = np.ones((3, 3), dtype='float32') 1025 y = math_ops.matmul(x1, x2) 1026 1027 fn = backend.function(inputs=[x1], outputs=[y]) 1028 x_val = np.random.random((3, 3)) 1029 y_val = np.ones((3, 3)) 1030 self.assertAllClose(fn([x_val])[0], 1031 np.matmul(x_val, y_val), 1032 atol=1e-5) 1033 1034 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1035 def test_reraising_exception(self): 1036 # When layer is not dynamic, we have some pattern matching during exception 1037 # handling to detect when the user is trying to use python control flow. 1038 # When an exception is thrown but the pattern doesn't match, we want to 1039 # preserve the originating stack trace. An early implementation of this 1040 # logic lost the stack trace. We test the correct behavior here. 1041 1042 class TypeErrorLayer(base_layer.Layer): 1043 1044 def call(self, inputs): 1045 def easily_identifiable_name(): 1046 raise TypeError('Non-matching TypeError message.') 1047 easily_identifiable_name() 1048 1049 inputs = input_layer.Input((3,)) 1050 1051 try: 1052 _ = TypeErrorLayer()(inputs) 1053 except TypeError as e: 1054 if hasattr(e, 'ag_error_metadata'): 1055 self.assertIn('easily_identifiable_name', str(e)) 1056 # See ErrorMetadataBase in autograph/pyct/errors.py 1057 function_name = e.ag_error_metadata.translated_stack[-1].function_name 1058 else: 1059 tb = traceback.extract_tb(sys.exc_info()[2]) 1060 last_entry = tb[-1] 1061 function_name = last_entry[2] 1062 self.assertEqual(function_name, 'easily_identifiable_name') 1063 1064 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1065 def test_summaries_in_tf_function(self): 1066 if not context.executing_eagerly(): 1067 return 1068 1069 class MyLayer(base_layer.Layer): 1070 1071 def call(self, inputs): 1072 summary_ops_v2.scalar('mean', math_ops.reduce_mean(inputs)) 1073 return inputs 1074 1075 tmp_dir = self.get_temp_dir() 1076 writer = summary_ops_v2.create_file_writer_v2(tmp_dir) 1077 with writer.as_default(step=1), summary_ops_v2.record_if(True): 1078 my_layer = MyLayer() 1079 x = array_ops.ones((10, 10)) 1080 1081 def my_fn(x): 1082 return my_layer(x) 1083 1084 _ = my_fn(x) 1085 1086 event_file = gfile.Glob(os.path.join(tmp_dir, 'events*')) 1087 self.assertLen(event_file, 1) 1088 event_file = event_file[0] 1089 tags = set() 1090 for e in summary_iterator.summary_iterator(event_file): 1091 for val in e.summary.value: 1092 tags.add(val.tag) 1093 self.assertEqual(set(['my_layer/mean']), tags) 1094 1095 @combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1096 def test_error_when_passing_non_tensor(self): 1097 # layers that have an `input_spec` will raise an error when called on 1098 # non-tensors. This covers all built-in layers. 1099 layer = layers.Dense(3) 1100 x = object() 1101 with self.assertRaisesRegex(TypeError, r'should be tensors'): 1102 layer(x) 1103 1104 1105@combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1106class NestedTrackingTest(test.TestCase): 1107 1108 def test_nested_layer_variable_tracking(self): 1109 # Test that variables from nested sublayers are 1110 # being tracked by subclassed layers. 1111 1112 class MyLayer(base_layer.Layer): 1113 1114 def __init__(self): 1115 super(MyLayer, self).__init__() 1116 self.dense1 = layers.Dense(1) 1117 self.dense2 = layers.BatchNormalization() 1118 1119 def build(self, input_shape): 1120 self.v1 = self.add_weight('v1', shape=input_shape[1:].as_list()) 1121 self.v2 = variables.Variable( 1122 name='v2', 1123 initial_value=np.zeros(input_shape[1:].as_list(), dtype='float32'), 1124 trainable=False) 1125 1126 def call(self, inputs): 1127 x = self.dense1(inputs) + self.dense2(inputs) 1128 return x + self.v1 + self.v2 1129 1130 layer = MyLayer() 1131 inputs = input_layer.Input((1,)) 1132 _ = layer(inputs) 1133 1134 self.assertEqual(len(layer.weights), 8) 1135 self.assertEqual(len(layer.trainable_weights), 5) 1136 self.assertEqual(len(layer.non_trainable_weights), 3) 1137 1138 layer.dense1.trainable = False 1139 self.assertEqual(len(layer.weights), 8) 1140 self.assertEqual(len(layer.trainable_weights), 3) 1141 self.assertEqual(len(layer.non_trainable_weights), 5) 1142 1143 layer.trainable = False 1144 self.assertEqual(len(layer.weights), 8) 1145 self.assertEqual(len(layer.trainable_weights), 0) 1146 self.assertEqual(len(layer.non_trainable_weights), 8) 1147 self.assertEqual( 1148 {id(v) for v in [layer.dense1, layer.dense2, layer.v1, layer.v2]}, 1149 {id(v) for _, v in layer._checkpoint_dependencies}) 1150 1151 def test_nested_layer_updates_losses_tracking(self): 1152 # Test that updates and losses from nested sublayers are 1153 # being tracked by subclassed layers. 1154 1155 class UpdateAndLossLayer(base_layer.Layer): 1156 1157 def build(self, _): 1158 self.v1 = self.add_weight('v1', shape=()) 1159 1160 def call(self, inputs): 1161 self.add_loss(math_ops.reduce_sum(inputs)) 1162 self.add_update(state_ops.assign_add(self.v1, 1)) 1163 return inputs + 1 1164 1165 class MyLayer(base_layer.Layer): 1166 1167 def build(self, _): 1168 self.v1 = self.add_weight('v1', shape=()) 1169 1170 def __init__(self): 1171 super(MyLayer, self).__init__() 1172 self.ul1 = UpdateAndLossLayer() 1173 self.ul2 = UpdateAndLossLayer() 1174 1175 def call(self, inputs): 1176 self.add_loss(math_ops.reduce_sum(inputs)) 1177 self.add_update(state_ops.assign_add(self.v1, 1)) 1178 x = self.ul1(inputs) 1179 return self.ul2(x) 1180 1181 layer = MyLayer() 1182 1183 if context.executing_eagerly(): 1184 inputs = array_ops.ones((3, 1)) 1185 _ = layer(inputs) 1186 self.assertEqual(len(layer.losses), 3) 1187 self.assertLen(layer.get_losses_for(None), 3) 1188 else: 1189 inputs = input_layer.Input((1,)) 1190 _ = layer(inputs) 1191 self.assertEqual(len(layer.losses), 3) 1192 self.assertEqual(len(layer.updates), 3) 1193 self.assertLen(layer.get_losses_for(None), 3) 1194 1195 def test_attribute_reassignment(self): 1196 l = base_layer.Layer() 1197 l.a = base_layer.Layer() 1198 l.a = [] 1199 l.a = variables.Variable(1.) 1200 l.a = base_layer.Layer() 1201 last_assignment = base_layer.Layer() 1202 l.a = last_assignment 1203 l.b = variables.Variable(1.) 1204 del l.b 1205 l.c = base_layer.Layer() 1206 del l.c 1207 l.d = last_assignment 1208 del l.d 1209 sublayers = list(l._flatten_layers(include_self=False, recursive=False)) 1210 self.assertEqual([last_assignment], sublayers) 1211 self.assertEqual([], l.trainable_weights) 1212 self.assertEqual([], l.non_trainable_weights) 1213 self.assertEqual([], l.weights) 1214 del l.a 1215 self.assertEqual([], l._self_tracked_trackables) 1216 1217 def test_layer_class_not_tracked_as_sublayer(self): 1218 # See https://github.com/tensorflow/tensorflow/issues/27431 for details. 1219 1220 class LayerWithClassAttribute(base_layer.Layer): 1221 1222 def __init__(self): 1223 super(LayerWithClassAttribute, self).__init__() 1224 self.layer_fn = layers.Dense 1225 1226 layer = LayerWithClassAttribute() 1227 self.assertEmpty(layer.variables) 1228 self.assertEmpty(layer.submodules) 1229 1230 def test_layer_call_fn_args(self): 1231 1232 class NonDefunLayer(base_layer.Layer): 1233 1234 def call(self, inputs, a, mask, b=None, training=None): 1235 return inputs 1236 1237 class DefunLayer(base_layer.Layer): 1238 1239 @def_function.function 1240 def call(self, x, mask, a, training=None, b=None): 1241 return x 1242 1243 nondefun_layer = NonDefunLayer() 1244 self.assertEqual(nondefun_layer._call_fn_args, 1245 ['inputs', 'a', 'mask', 'b', 'training']) 1246 defun_layer = DefunLayer() 1247 self.assertEqual(defun_layer._call_fn_args, 1248 ['x', 'mask', 'a', 'training', 'b']) 1249 1250 def test_sequential_model(self): 1251 model = sequential.Sequential( 1252 [layers.Dense(10, input_shape=(10,)), 1253 layers.Dense(5)]) 1254 self.assertLen(model.layers, 2) 1255 self.assertLen(model.weights, 4) 1256 1257 # Make sure a subclass model also works when it is called 'Sequential'. 1258 class Sequential(training_lib.Model): 1259 1260 def __init__(self): 1261 super(Sequential, self).__init__() 1262 self.dense_layers = [layers.Dense(10), layers.Dense(5)] 1263 1264 def call(self, inputs): 1265 x = inputs 1266 for d in self.dense_layers: 1267 x = d(x) 1268 return x 1269 1270 s = Sequential() 1271 self.assertLen(s.layers, 2) 1272 self.assertLen(s.weights, 0) 1273 1274 s(input_layer.Input((10,))) 1275 self.assertLen(s.weights, 4) 1276 1277 1278@combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1279class NameScopingTest(keras_parameterized.TestCase): 1280 1281 def test_name_scope_layer(self): 1282 x = backend.placeholder(shape=(10, 10)) 1283 layer = layers.Dense(10, name='MyName') 1284 layer(x) 1285 self.assertEqual(layer.bias.name, 'MyName/bias:0') 1286 self.assertEqual(layer.kernel.name, 'MyName/kernel:0') 1287 1288 def test_name_scope_functional_api(self): 1289 inputs = input_layer.Input((3,)) 1290 layer = layers.Dense(10, name='MyName') 1291 _ = layer(inputs) 1292 self.assertEqual(layer.bias.name, 'MyName/bias:0') 1293 self.assertEqual(layer.kernel.name, 'MyName/kernel:0') 1294 1295 def test_name_scope_functional_api_nested(self): 1296 1297 class NestedLayer(base_layer.Layer): 1298 1299 def __init__(self, name='OuterName'): 1300 super(NestedLayer, self).__init__(name=name) 1301 self.dense = layers.Dense(10, name='InnerName') 1302 1303 def call(self, inputs): 1304 return self.dense(inputs) 1305 1306 inputs = input_layer.Input((3,)) 1307 layer = NestedLayer() 1308 _ = layer(inputs) 1309 self.assertEqual(layer.dense.bias.name, 'OuterName/InnerName/bias:0') 1310 self.assertEqual(layer.dense.kernel.name, 'OuterName/InnerName/kernel:0') 1311 1312 def test_name_scope_sublayer(self): 1313 1314 class NameScopeTracker(base_layer.Layer): 1315 1316 def call(self, inputs): 1317 self.active_name_scope = ops.get_name_scope() 1318 return inputs 1319 1320 x = backend.placeholder(shape=(10, 10)) 1321 sublayer = NameScopeTracker(name='Sublayer') 1322 layer = layers.Dense(10, activation=sublayer, name='MyName2') 1323 layer(x) 1324 self.assertEqual(layer.bias.name, 'MyName2/bias:0') 1325 self.assertEqual(layer.kernel.name, 'MyName2/kernel:0') 1326 self.assertEqual(sublayer.active_name_scope, 'MyName2/Sublayer') 1327 1328 def test_name_scope_tf_tensor(self): 1329 x = ops.convert_to_tensor_v2_with_dispatch(np.ones((10, 10))) 1330 layer = layers.Dense( 1331 10, activation=layers.ReLU(name='MyAct'), name='MyName3') 1332 layer(x) 1333 self.assertEqual(layer.bias.name, 'MyName3/bias:0') 1334 self.assertEqual(layer.kernel.name, 'MyName3/kernel:0') 1335 1336 1337@combinations.generate(combinations.keras_mode_combinations(mode=['eager'])) 1338class AutographControlFlowTest(keras_parameterized.TestCase): 1339 1340 def test_disabling_in_context_is_matched(self): 1341 1342 test_obj = self 1343 1344 class MyLayer(base_layer.Layer): 1345 1346 def call(self, inputs, training=None): 1347 with test_obj.assertRaisesRegex(TypeError, 'Tensor.*as.*bool'): 1348 if constant_op.constant(False): 1349 return inputs * 1. 1350 return inputs * 0. 1351 1352 @def_function.function(autograph=False) 1353 def test_fn(): 1354 return MyLayer()(constant_op.constant([[1., 2., 3.]])) 1355 1356 test_fn() 1357 1358 def test_if_training_pattern_output(self): 1359 1360 class MyLayer(base_layer.Layer): 1361 1362 def call(self, inputs, training=None): 1363 if training: 1364 return inputs * 1. 1365 return inputs * 0. 1366 1367 inputs = input_layer.Input((3,)) 1368 outputs = MyLayer()(inputs) 1369 model = training_lib.Model(inputs, outputs) 1370 model.compile( 1371 'sgd', 1372 'mse', 1373 run_eagerly=testing_utils.should_run_eagerly()) 1374 train_loss = model.train_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1375 self.assertEqual(train_loss, 0.) 1376 test_loss = model.test_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1377 self.assertEqual(test_loss, 1.) 1378 1379 def test_if_training_pattern_loss(self): 1380 1381 class MyLayer(base_layer.Layer): 1382 1383 def call(self, inputs, training=None): 1384 if training: 1385 loss = math_ops.reduce_sum(inputs) 1386 else: 1387 loss = 0. 1388 self.add_loss(loss) 1389 return inputs 1390 1391 inputs = input_layer.Input((3,)) 1392 outputs = MyLayer()(inputs) 1393 model = training_lib.Model(inputs, outputs) 1394 model.compile( 1395 'sgd', 1396 'mse', 1397 run_eagerly=testing_utils.should_run_eagerly()) 1398 train_loss = model.train_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1399 self.assertEqual(train_loss, 2 * 3) 1400 test_loss = model.test_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1401 self.assertEqual(test_loss, 0) 1402 1403 def test_if_training_pattern_metric(self): 1404 1405 class MyLayer(base_layer.Layer): 1406 1407 def call(self, inputs, training=None): 1408 if training: 1409 metric = math_ops.reduce_sum(inputs) 1410 else: 1411 metric = 0. 1412 self.add_metric(metric, name='my_metric', aggregation='mean') 1413 return inputs 1414 1415 inputs = input_layer.Input((3,)) 1416 outputs = MyLayer()(inputs) 1417 model = training_lib.Model(inputs, outputs) 1418 model.compile( 1419 'sgd', 1420 'mse', 1421 run_eagerly=testing_utils.should_run_eagerly()) 1422 for _ in range(3): 1423 _, train_metric = model.train_on_batch(np.ones((2, 3)), 1424 np.ones((2, 3))) 1425 1426 self.assertEqual(train_metric, 2 * 3) 1427 _, test_metric = model.test_on_batch(np.ones((2, 3)), 1428 np.ones((2, 3))) 1429 self.assertEqual(test_metric, 0) 1430 1431 def test_if_training_pattern_update(self): 1432 1433 class MyLayer(base_layer.Layer): 1434 1435 def build(self, input_shape): 1436 self.counter = self.add_weight( 1437 shape=(), trainable=False, initializer='zeros') 1438 1439 def call(self, inputs, training=None): 1440 if training: 1441 increment = 1. 1442 else: 1443 increment = 0. 1444 self.counter.assign_add(increment) 1445 return inputs 1446 1447 inputs = input_layer.Input((3,)) 1448 layer = MyLayer() 1449 outputs = layer(inputs) 1450 model = training_lib.Model(inputs, outputs) 1451 model.compile( 1452 'sgd', 1453 'mse', 1454 run_eagerly=testing_utils.should_run_eagerly()) 1455 model.train_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1456 self.assertEqual(backend.get_value(layer.counter), 1.) 1457 1458 def test_conditional_losses_in_call(self): 1459 1460 class MyLayer(base_layer.Layer): 1461 1462 def __init__(self): 1463 super(MyLayer, 1464 self).__init__(dynamic=testing_utils.should_run_eagerly()) 1465 1466 def call(self, inputs, training=None): 1467 if training: 1468 self.add_loss(math_ops.reduce_sum(inputs)) 1469 return inputs 1470 1471 def compute_output_shape(self, input_shape): 1472 return input_shape 1473 1474 inputs = input_layer.Input((3,)) 1475 layer = MyLayer() 1476 outputs = layer(inputs) 1477 model = training_lib.Model(inputs, outputs) 1478 model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) 1479 loss = model.train_on_batch(np.ones((2, 3)), np.ones((2, 3))) 1480 self.assertEqual(loss, 2 * 3) 1481 1482 def test_conditional_callable_losses(self): 1483 model = sequential.Sequential([ 1484 layers.Dense( 1485 1, kernel_regularizer=regularizers.l2(1e-4), input_shape=(1,)) 1486 ]) 1487 model._run_eagerly = testing_utils.should_run_eagerly() 1488 1489 def assert_graph(t): 1490 if not context.executing_eagerly(): 1491 self.assertEqual(t.graph, ops.get_default_graph()) 1492 1493 @def_function.function 1494 def get_losses(t): 1495 if t < 0: 1496 return math_ops.reduce_sum(model.losses) * t 1497 else: 1498 return math_ops.reduce_sum(model.losses) 1499 1500 assert_graph(get_losses(constant_op.constant(2.))) 1501 assert_graph(get_losses(constant_op.constant(0.5))) 1502 1503 def test_conditional_metrics_in_call(self): 1504 1505 class MyLayer(base_layer.Layer): 1506 1507 def __init__(self): 1508 super(MyLayer, 1509 self).__init__(dynamic=testing_utils.should_run_eagerly()) 1510 1511 def call(self, inputs, training=None): 1512 if training: 1513 self.add_metric(math_ops.reduce_sum(inputs), 1514 name='sum', 1515 aggregation='mean') 1516 return inputs 1517 1518 def compute_output_shape(self, input_shape): 1519 return input_shape 1520 1521 inputs = input_layer.Input((3,)) 1522 layer = MyLayer() 1523 outputs = layer(inputs) 1524 model = training_lib.Model(inputs, outputs) 1525 model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) 1526 history = model.fit(np.ones((2, 3)), np.ones((2, 3))) 1527 self.assertEqual(history.history['sum'][-1], 2 * 3) 1528 1529 def test_conditional_activity_regularizer_in_call(self): 1530 1531 class TestModel(training_lib.Model): 1532 1533 def __init__(self): 1534 super(TestModel, self).__init__( 1535 name='test_model', dynamic=testing_utils.should_run_eagerly()) 1536 self.layer = layers.Dense(2, activity_regularizer='l2') 1537 1538 def call(self, x, training=None): 1539 if math_ops.greater(math_ops.reduce_sum(x), 0.0): 1540 return self.layer(x) 1541 else: 1542 return self.layer(x) 1543 1544 model = TestModel() 1545 model.compile( 1546 loss='mse', 1547 optimizer='sgd', 1548 run_eagerly=testing_utils.should_run_eagerly()) 1549 1550 x = np.ones(shape=(10, 1)) 1551 y = np.ones(shape=(10, 2)) 1552 1553 if testing_utils.should_run_eagerly(): 1554 model.fit(x, y, epochs=2, batch_size=5) 1555 else: 1556 with self.assertRaisesRegex(ValueError, 'ActivityRegularizer'): 1557 model.fit(x, y, epochs=2, batch_size=5) 1558 1559 def test_conditional_activity_regularizer_with_wrappers_in_call(self): 1560 1561 class TestModel(training_lib.Model): 1562 1563 def __init__(self): 1564 super(TestModel, self).__init__( 1565 name='test_model', dynamic=testing_utils.should_run_eagerly()) 1566 self.layer = layers.TimeDistributed( 1567 layers.Dense(2, activity_regularizer='l2'), input_shape=(3, 4)) 1568 1569 def call(self, x, training=None): 1570 if math_ops.greater(math_ops.reduce_sum(x), 0.0): 1571 return self.layer(x) 1572 else: 1573 return self.layer(x) 1574 1575 model = TestModel() 1576 model.compile( 1577 loss='mse', 1578 optimizer='sgd', 1579 run_eagerly=testing_utils.should_run_eagerly()) 1580 1581 x = np.ones(shape=(10, 3, 4)) 1582 y = np.ones(shape=(10, 3, 2)) 1583 1584 if testing_utils.should_run_eagerly(): 1585 model.fit(x, y, epochs=2, batch_size=5) 1586 else: 1587 with self.assertRaisesRegex(ValueError, 'ActivityRegularizer'): 1588 model.fit(x, y, epochs=2, batch_size=5) 1589 1590 1591class AddLayer(base_layer.Layer): 1592 """A layer which adds its input to a variable. 1593 1594 Useful for testing a layer with a variable 1595 """ 1596 1597 def build(self, _): 1598 self.v = self.add_weight('v', (), initializer='ones') 1599 self.built = True 1600 1601 def call(self, inputs): 1602 return inputs + self.v 1603 1604 1605class IdentityLayer(base_layer.Layer): 1606 """A layer that returns its input. 1607 1608 Useful for testing a layer without a variable. 1609 """ 1610 1611 def call(self, inputs): 1612 return inputs 1613 1614 1615@combinations.generate(combinations.combine(mode=['graph', 'eager'])) 1616class DTypeTest(keras_parameterized.TestCase): 1617 1618 def _const(self, dtype): 1619 return array_ops.constant(1, dtype=dtype) 1620 1621 @testing_utils.enable_v2_dtype_behavior 1622 def test_dtype_defaults_to_floatx(self): 1623 layer = AddLayer() 1624 self.assertEqual(layer.dtype, 'float32') 1625 layer(self._const('float64')) 1626 self.assertEqual(layer.dtype, 'float32') # dtype should not change 1627 1628 try: 1629 backend.set_floatx('float64') 1630 layer = AddLayer() 1631 self.assertEqual(layer.dtype, 'float64') 1632 finally: 1633 backend.set_floatx('float32') 1634 1635 @testing_utils.enable_v2_dtype_behavior 1636 def test_passing_dtype_to_constructor(self): 1637 layer = IdentityLayer(dtype='float64') 1638 layer(self._const('float32')) 1639 self.assertEqual(layer.dtype, 'float64') 1640 1641 layer = IdentityLayer(dtype='int32') 1642 layer(self._const('float32')) 1643 self.assertEqual(layer.dtype, 'int32') 1644 1645 layer = IdentityLayer(dtype=dtypes.float64) 1646 layer(self._const('float32')) 1647 self.assertEqual(layer.dtype, 'float64') 1648 1649 @testing_utils.enable_v2_dtype_behavior 1650 def input_cast_to_dtype(self): 1651 layer = AddLayer() 1652 1653 # Input should be cast to layer.dtype, so output should also be layer.dtype 1654 self.assertEqual(layer(self._const('float64')).dtype, 'float32') 1655 1656 layer = AddLayer(dtype='float64') 1657 self.assertEqual(layer(self._const('float32')).dtype, 'float64') 1658 1659 # Test inputs are not casted if layer.dtype is not floating-point 1660 layer = IdentityLayer(dtype='int32') 1661 self.assertEqual(layer(self._const('float64')).dtype, 'float64') 1662 1663 # Test inputs are not casted if the inputs are not floating-point 1664 layer = IdentityLayer(dtype='float32') 1665 self.assertEqual(layer(self._const('int32')).dtype, 'int32') 1666 1667 # Test Numpy arrays are casted 1668 layer = IdentityLayer(dtype='float64') 1669 self.assertEqual(layer(np.array(1, dtype='float32')).dtype, 'float64') 1670 1671 # Test Python floats are casted 1672 layer = IdentityLayer(dtype='float64') 1673 self.assertEqual(layer(1.).dtype, 'float64') 1674 1675 @testing_utils.enable_v2_dtype_behavior 1676 def multiple_inputs_cast_to_dtype(self): 1677 1678 class MultiIdentityLayer(base_layer.Layer): 1679 1680 def call(self, inputs): 1681 return [array_ops.identity(x) for x in inputs] 1682 1683 # Testing layer with default dtype of float32 1684 layer = MultiIdentityLayer() 1685 x, y = layer([self._const('float16'), self._const('float32')]) 1686 self.assertEqual(x.dtype, 'float32') 1687 self.assertEqual(y.dtype, 'float32') 1688 1689 # Test passing dtype to the constructor 1690 layer = MultiIdentityLayer(dtype='float64') 1691 x, y = layer([self._const('float16'), self._const('float32')]) 1692 self.assertEqual(x.dtype, 'float64') 1693 self.assertEqual(y.dtype, 'float64') 1694 1695 # Test several non-floating point types 1696 layer = MultiIdentityLayer(dtype='float64') 1697 x, y, z, w = layer([self._const('float16'), self._const('bool'), 1698 self._const('float64'), self._constant('complex64')]) 1699 self.assertEqual(x.dtype, 'float64') 1700 self.assertEqual(y.dtype, 'bool') 1701 self.assertEqual(z.dtype, 'float64') 1702 self.assertEqual(w.dtype, 'complex64') 1703 1704 @testing_utils.enable_v2_dtype_behavior 1705 def test_extra_args_and_kwargs_not_casted(self): 1706 1707 class IdentityLayerWithArgs(base_layer.Layer): 1708 1709 def call(self, inputs, *args, **kwargs): 1710 kwargs.pop('training', None) 1711 return nest.flatten([inputs, args, kwargs]) 1712 1713 layer = IdentityLayerWithArgs(dtype='float64') 1714 x, y, z = layer(self._const('float16'), self._const('float16'), 1715 kwarg=self._const('float16')) 1716 self.assertEqual(x.dtype, 'float64') 1717 self.assertEqual(y.dtype, 'float16') 1718 self.assertEqual(z.dtype, 'float16') 1719 1720 @testing_utils.enable_v2_dtype_behavior 1721 def test_layer_without_autocast(self): 1722 1723 class IdentityLayerWithoutAutocast(IdentityLayer): 1724 1725 def __init__(self, *args, **kwargs): 1726 kwargs['autocast'] = False 1727 super(IdentityLayerWithoutAutocast, self).__init__(*args, **kwargs) 1728 1729 layer = IdentityLayerWithoutAutocast(dtype='float64') 1730 self.assertEqual(layer(self._const('float32')).dtype, 'float32') 1731 1732 @testing_utils.enable_v2_dtype_behavior 1733 def test_compute_output_signature(self): 1734 1735 class IdentityLayerWithOutputShape(IdentityLayer): 1736 1737 def compute_output_shape(self, input_shape): 1738 return input_shape 1739 1740 layer = IdentityLayerWithOutputShape(dtype='float64') 1741 output_signature = layer.compute_output_signature( 1742 tensor_spec.TensorSpec(shape=(), dtype='float32')) 1743 self.assertEqual(output_signature.shape, ()) 1744 self.assertEqual(output_signature.dtype, 'float64') 1745 1746 @testing_utils.enable_v2_dtype_behavior 1747 def test_composite_tensors_input_casting(self): 1748 sparse = sparse_tensor.SparseTensor( 1749 indices=array_ops.constant([[0, 1], [2, 3]], dtype='int64'), 1750 values=array_ops.constant([0., 1.], dtype='float32'), 1751 dense_shape=array_ops.constant([4, 4], dtype='int64')) 1752 ragged = ragged_tensor.RaggedTensor.from_row_splits( 1753 values=array_ops.constant([1., 2., 3.], dtype='float32'), 1754 row_splits=array_ops.constant([0, 2, 2, 3], dtype='int64')) 1755 1756 layer = IdentityLayer(dtype='float16') 1757 1758 for x in sparse, ragged: 1759 self.assertEqual(x.dtype, 'float32') 1760 y = layer(x) 1761 self.assertEqual(y.dtype, 'float16') 1762 self.assertEqual(type(x), type(y)) 1763 1764 @testing_utils.enable_v2_dtype_behavior 1765 def test_passing_non_tensor(self): 1766 layer = IdentityLayer() 1767 x = object() 1768 y = layer(x) # Layer should not cast 'x', as it's not a tensor 1769 self.assertIs(x, y) 1770 1771 @testing_utils.disable_v2_dtype_behavior 1772 def test_v1_behavior(self): 1773 # Test dtype defaults to None and inferred from input 1774 layer = IdentityLayer() 1775 self.assertIsNone(layer.dtype) 1776 layer(self._const('float64')) 1777 self.assertEqual(layer.dtype, 'float64') 1778 1779 # Test layer does not cast to dtype 1780 self.assertEqual(layer(self._const('float32')).dtype, 'float32') 1781 1782 1783if __name__ == '__main__': 1784 ops.enable_eager_execution() 1785 test.main() 1786