1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15# pylint: disable=protected-access 16"""Home of the `Sequential` model.""" 17 18import copy 19import warnings 20 21from tensorflow.python import tf2 22from tensorflow.python.framework import ops 23from tensorflow.python.framework import tensor_util 24from tensorflow.python.keras import layers as layer_module 25from tensorflow.python.keras.engine import base_layer 26from tensorflow.python.keras.engine import functional 27from tensorflow.python.keras.engine import input_layer 28from tensorflow.python.keras.engine import training_utils 29from tensorflow.python.keras.saving.saved_model import model_serialization 30from tensorflow.python.keras.utils import generic_utils 31from tensorflow.python.keras.utils import layer_utils 32from tensorflow.python.keras.utils import tf_inspect 33from tensorflow.python.keras.utils import tf_utils 34from tensorflow.python.module import module 35from tensorflow.python.ops.numpy_ops import np_arrays 36from tensorflow.python.platform import tf_logging as logging 37from tensorflow.python.trackable import base as trackable 38from tensorflow.python.util import nest 39from tensorflow.python.util.tf_export import keras_export 40 41 42SINGLE_LAYER_OUTPUT_ERROR_MSG = ('All layers in a Sequential model should have ' 43 'a single output tensor. For multi-output ' 44 'layers, use the functional API.') 45 46 47@keras_export('keras.Sequential', 'keras.models.Sequential') 48class Sequential(functional.Functional): 49 """`Sequential` groups a linear stack of layers into a `tf.keras.Model`. 50 51 `Sequential` provides training and inference features on this model. 52 53 Examples: 54 55 >>> # Optionally, the first layer can receive an `input_shape` argument: 56 >>> model = tf.keras.Sequential() 57 >>> model.add(tf.keras.layers.Dense(8, input_shape=(16,))) 58 >>> # Afterwards, we do automatic shape inference: 59 >>> model.add(tf.keras.layers.Dense(4)) 60 61 >>> # This is identical to the following: 62 >>> model = tf.keras.Sequential() 63 >>> model.add(tf.keras.Input(shape=(16,))) 64 >>> model.add(tf.keras.layers.Dense(8)) 65 66 >>> # Note that you can also omit the `input_shape` argument. 67 >>> # In that case the model doesn't have any weights until the first call 68 >>> # to a training/evaluation method (since it isn't yet built): 69 >>> model = tf.keras.Sequential() 70 >>> model.add(tf.keras.layers.Dense(8)) 71 >>> model.add(tf.keras.layers.Dense(4)) 72 >>> # model.weights not created yet 73 74 >>> # Whereas if you specify the input shape, the model gets built 75 >>> # continuously as you are adding layers: 76 >>> model = tf.keras.Sequential() 77 >>> model.add(tf.keras.layers.Dense(8, input_shape=(16,))) 78 >>> model.add(tf.keras.layers.Dense(4)) 79 >>> len(model.weights) 80 4 81 82 >>> # When using the delayed-build pattern (no input shape specified), you can 83 >>> # choose to manually build your model by calling 84 >>> # `build(batch_input_shape)`: 85 >>> model = tf.keras.Sequential() 86 >>> model.add(tf.keras.layers.Dense(8)) 87 >>> model.add(tf.keras.layers.Dense(4)) 88 >>> model.build((None, 16)) 89 >>> len(model.weights) 90 4 91 92 ```python 93 # Note that when using the delayed-build pattern (no input shape specified), 94 # the model gets built the first time you call `fit`, `eval`, or `predict`, 95 # or the first time you call the model on some input data. 96 model = tf.keras.Sequential() 97 model.add(tf.keras.layers.Dense(8)) 98 model.add(tf.keras.layers.Dense(1)) 99 model.compile(optimizer='sgd', loss='mse') 100 # This builds the model for the first time: 101 model.fit(x, y, batch_size=32, epochs=10) 102 ``` 103 """ 104 105 @trackable.no_automatic_dependency_tracking 106 def __init__(self, layers=None, name=None): 107 """Creates a `Sequential` model instance. 108 109 Args: 110 layers: Optional list of layers to add to the model. 111 name: Optional name for the model. 112 """ 113 # Skip the init in FunctionalModel since model doesn't have input/output yet 114 super(functional.Functional, self).__init__( # pylint: disable=bad-super-call 115 name=name, autocast=False) 116 self.supports_masking = True 117 self._compute_output_and_mask_jointly = True 118 self._auto_track_sub_layers = False 119 self._inferred_input_shape = None 120 self._has_explicit_input_shape = False 121 self._input_dtype = None 122 self._layer_call_argspecs = {} 123 self._created_nodes = set() 124 # Flag that indicate whether the sequential network topology has been 125 # created. It is false when there isn't any layer, or the layers doesn't 126 # have input shape. 127 self._graph_initialized = False 128 129 # Unfortunately some Sequential models using custom layers or FeatureColumn 130 # layers have multiple inputs. This is fundamentally incompatible with 131 # most of the Sequential API, and we have to disable a number of features 132 # for such models. 133 self._use_legacy_deferred_behavior = False 134 135 # Add to the model any layers passed to the constructor. 136 if layers: 137 if not isinstance(layers, (list, tuple)): 138 layers = [layers] 139 for layer in layers: 140 self.add(layer) 141 142 @property 143 def layers(self): 144 # Historically, `sequential.layers` only returns layers that were added 145 # via `add`, and omits the auto-generated `InputLayer` that comes at the 146 # bottom of the stack. 147 # `Trackable` manages the `_layers` attributes and does filtering 148 # over it. 149 layers = super(Sequential, self).layers 150 if layers and isinstance(layers[0], input_layer.InputLayer): 151 return layers[1:] 152 return layers[:] 153 154 @trackable.no_automatic_dependency_tracking 155 def add(self, layer): 156 """Adds a layer instance on top of the layer stack. 157 158 Args: 159 layer: layer instance. 160 161 Raises: 162 TypeError: If `layer` is not a layer instance. 163 ValueError: In case the `layer` argument does not 164 know its input shape. 165 ValueError: In case the `layer` argument has 166 multiple output tensors, or is already connected 167 somewhere else (forbidden in `Sequential` models). 168 """ 169 # If we are passed a Keras tensor created by keras.Input(), we can extract 170 # the input layer from its keras history and use that without any loss of 171 # generality. 172 if hasattr(layer, '_keras_history'): 173 origin_layer = layer._keras_history[0] 174 if isinstance(origin_layer, input_layer.InputLayer): 175 layer = origin_layer 176 logging.warning( 177 'Please add `keras.layers.InputLayer` instead of `keras.Input` to ' 178 'Sequential model. `keras.Input` is intended to be used by ' 179 'Functional model.') 180 181 if isinstance(layer, module.Module): 182 if not isinstance(layer, base_layer.Layer): 183 layer = functional.ModuleWrapper(layer) 184 else: 185 raise TypeError('The added layer must be ' 186 'an instance of class Layer. ' 187 'Found: ' + str(layer)) 188 189 tf_utils.assert_no_legacy_layers([layer]) 190 if not self._is_layer_name_unique(layer): 191 raise ValueError('All layers added to a Sequential model ' 192 'should have unique names. Name "%s" is already the name' 193 ' of a layer in this model. Update the `name` argument ' 194 'to pass a unique name.' % (layer.name,)) 195 196 self.built = False 197 set_inputs = False 198 self._maybe_create_attribute('_self_tracked_trackables', []) 199 if not self._self_tracked_trackables: 200 if isinstance(layer, input_layer.InputLayer): 201 # Case where the user passes an Input or InputLayer layer via `add`. 202 set_inputs = True 203 else: 204 batch_shape, dtype = training_utils.get_input_shape_and_dtype(layer) 205 if batch_shape: 206 # Instantiate an input layer. 207 x = input_layer.Input( 208 batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input') 209 # This will build the current layer 210 # and create the node connecting the current layer 211 # to the input layer we just created. 212 layer(x) 213 set_inputs = True 214 215 if set_inputs: 216 outputs = nest.flatten(layer._inbound_nodes[-1].outputs) 217 if len(outputs) != 1: 218 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG) 219 self.outputs = outputs 220 self.inputs = layer_utils.get_source_inputs(self.outputs[0]) 221 self.built = True 222 self._has_explicit_input_shape = True 223 224 elif self.outputs: 225 # If the model is being built continuously on top of an input layer: 226 # refresh its output. 227 output_tensor = layer(self.outputs[0]) 228 if len(nest.flatten(output_tensor)) != 1: 229 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG) 230 self.outputs = [output_tensor] 231 self.built = True 232 233 if set_inputs or self._graph_initialized: 234 self._init_graph_network(self.inputs, self.outputs) 235 self._graph_initialized = True 236 else: 237 self._self_tracked_trackables.append(layer) 238 self._handle_deferred_layer_dependencies([layer]) 239 240 self._layer_call_argspecs[layer] = tf_inspect.getfullargspec(layer.call) 241 242 @trackable.no_automatic_dependency_tracking 243 def pop(self): 244 """Removes the last layer in the model. 245 246 Raises: 247 TypeError: if there are no layers in the model. 248 """ 249 if not self.layers: 250 raise TypeError('There are no layers in the model.') 251 252 layer = self._self_tracked_trackables.pop() 253 self._layer_call_argspecs.pop(layer) 254 if not self.layers: 255 self.outputs = None 256 self.inputs = None 257 self.built = False 258 self._inferred_input_shape = None 259 self._has_explicit_input_shape = False 260 self._graph_initialized = False 261 elif self._graph_initialized: 262 self.layers[-1]._outbound_nodes = [] 263 self.outputs = [self.layers[-1].output] 264 self._init_graph_network(self.inputs, self.outputs) 265 self.built = True 266 267 @trackable.no_automatic_dependency_tracking 268 def _build_graph_network_for_inferred_shape(self, 269 input_shape, 270 input_dtype=None): 271 if input_shape is None or not self.layers: 272 return 273 if not tf2.enabled() or not ops.executing_eagerly_outside_functions(): 274 # This behavior is disabled in V1 or when eager execution is disabled. 275 return 276 if (not self._has_explicit_input_shape and 277 not self._use_legacy_deferred_behavior): 278 # Determine whether the input shape is novel, i.e. whether the model 279 # should be rebuilt. 280 input_shape = tuple(input_shape) 281 if self._inferred_input_shape is None: 282 new_shape = input_shape 283 else: 284 new_shape = relax_input_shape(self._inferred_input_shape, input_shape) 285 if (new_shape is not None and new_shape != self._inferred_input_shape): 286 # A novel shape has been received: we need to rebuild the model. 287 # In case we are inside a graph function, we step out of it. 288 with ops.init_scope(): 289 inputs = input_layer.Input( 290 batch_shape=new_shape, 291 dtype=input_dtype, 292 name=self.layers[0].name + '_input') 293 layer_input = inputs 294 created_nodes = set() 295 for layer in self.layers: 296 # Clear nodes previously created via this method. This prevents 297 # node accumulation and ensures that e.g. `layer.output` is 298 # always connected to `model.inputs` 299 # (this is important e.g. for the feature extraction use case). 300 # We don't just do `layer._inbound_nodes = []` in order 301 # not to break shared layers added to Sequential models (which is 302 # technically illegal as per the `add()` docstring, 303 # but wasn't previously disabled). 304 clear_previously_created_nodes(layer, self._created_nodes) 305 try: 306 # Create Functional API connection by calling the current layer 307 layer_output = layer(layer_input) 308 except: # pylint:disable=bare-except 309 # Functional API calls may fail for a number of reasons: 310 # 1) The layer may be buggy. In this case it will be easier for 311 # the user to debug if we fail on the first call on concrete data, 312 # instead of our own call on a symbolic input. 313 # 2) The layer is dynamic (graph-incompatible) and hasn't 314 # overridden `compute_output_shape`. In this case, it is 315 # impossible to build a graph network. 316 # 3) The layer is otherwise incompatible with the Functional API 317 # (e.g. this is the case for some probabilistic layers that rely 318 # on hacks and that do not return tensors). 319 # In all these cases, we should avoid creating a graph network 320 # (or we simply can't). 321 self._use_legacy_deferred_behavior = True 322 return 323 if len(nest.flatten(layer_output)) != 1: 324 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG) 325 # Keep track of nodes just created above 326 track_nodes_created_by_last_call(layer, created_nodes) 327 layer_input = layer_output 328 outputs = layer_output 329 self._created_nodes = created_nodes 330 try: 331 # Initialize a graph Network. This call will never fail for 332 # a stack of valid Keras layers. 333 # However some users have layers that are fundamentally incompatible 334 # with the Functional API, which do not return tensors. In this 335 # case, we fall back to the legacy deferred behavior. 336 # TODO(fchollet): consider raising here, as we should not be 337 # supporting such layers. 338 self._init_graph_network(inputs, outputs) 339 self._graph_initialized = True 340 except: # pylint:disable=bare-except 341 self._use_legacy_deferred_behavior = True 342 self._inferred_input_shape = new_shape 343 344 @generic_utils.default 345 def build(self, input_shape=None): 346 if self._graph_initialized: 347 self._init_graph_network(self.inputs, self.outputs) 348 else: 349 if input_shape is None: 350 raise ValueError('You must provide an `input_shape` argument.') 351 self._build_graph_network_for_inferred_shape(input_shape) 352 if not self.built: 353 input_shape = tuple(input_shape) 354 self._build_input_shape = input_shape 355 super(Sequential, self).build(input_shape) 356 self.built = True 357 358 def call(self, inputs, training=None, mask=None): # pylint: disable=redefined-outer-name 359 # If applicable, update the static input shape of the model. 360 if not self._has_explicit_input_shape: 361 if not tensor_util.is_tf_type(inputs) and not isinstance( 362 inputs, np_arrays.ndarray): 363 # This is a Sequential with mutiple inputs. This is technically an 364 # invalid use case of Sequential, but we tolerate it for backwards 365 # compatibility. 366 self._use_legacy_deferred_behavior = True 367 self._build_input_shape = nest.map_structure(_get_shape_tuple, inputs) 368 if tf2.enabled(): 369 logging.warning('Layers in a Sequential model should only have a ' 370 'single input tensor, but we receive a %s input: %s' 371 '\nConsider rewriting this model with the Functional ' 372 'API.' % (type(inputs), inputs)) 373 else: 374 self._build_graph_network_for_inferred_shape(inputs.shape, inputs.dtype) 375 376 if self._graph_initialized: 377 if not self.built: 378 self._init_graph_network(self.inputs, self.outputs) 379 return super(Sequential, self).call(inputs, training=training, mask=mask) 380 381 outputs = inputs # handle the corner case where self.layers is empty 382 for layer in self.layers: 383 # During each iteration, `inputs` are the inputs to `layer`, and `outputs` 384 # are the outputs of `layer` applied to `inputs`. At the end of each 385 # iteration `inputs` is set to `outputs` to prepare for the next layer. 386 kwargs = {} 387 argspec = self._layer_call_argspecs[layer].args 388 if 'mask' in argspec: 389 kwargs['mask'] = mask 390 if 'training' in argspec: 391 kwargs['training'] = training 392 393 outputs = layer(inputs, **kwargs) 394 395 if len(nest.flatten(outputs)) != 1: 396 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG) 397 # `outputs` will be the inputs to the next layer. 398 inputs = outputs 399 mask = getattr(outputs, '_keras_mask', None) 400 return outputs 401 402 def compute_output_shape(self, input_shape): 403 shape = input_shape 404 for layer in self.layers: 405 shape = layer.compute_output_shape(shape) 406 return shape 407 408 def compute_mask(self, inputs, mask): 409 # TODO(omalleyt): b/123540974 This function is not really safe to call 410 # by itself because it will duplicate any updates and losses in graph 411 # mode by `call`ing the Layers again. 412 outputs = self.call(inputs, mask=mask) # pylint: disable=unexpected-keyword-arg 413 return getattr(outputs, '_keras_mask', None) 414 415 def predict_proba(self, x, batch_size=32, verbose=0): 416 """Generates class probability predictions for the input samples. 417 418 The input samples are processed batch by batch. 419 420 Args: 421 x: input data, as a Numpy array or list of Numpy arrays 422 (if the model has multiple inputs). 423 batch_size: integer. 424 verbose: verbosity mode, 0 or 1. 425 426 Returns: 427 A Numpy array of probability predictions. 428 """ 429 warnings.warn('`model.predict_proba()` is deprecated and ' 430 'will be removed after 2021-01-01. ' 431 'Please use `model.predict()` instead.') 432 preds = self.predict(x, batch_size, verbose) 433 if preds.min() < 0. or preds.max() > 1.: 434 logging.warning('Network returning invalid probability values. ' 435 'The last layer might not normalize predictions ' 436 'into probabilities ' 437 '(like softmax or sigmoid would).') 438 return preds 439 440 def predict_classes(self, x, batch_size=32, verbose=0): 441 """Generate class predictions for the input samples. 442 443 The input samples are processed batch by batch. 444 445 Args: 446 x: input data, as a Numpy array or list of Numpy arrays 447 (if the model has multiple inputs). 448 batch_size: integer. 449 verbose: verbosity mode, 0 or 1. 450 451 Returns: 452 A numpy array of class predictions. 453 """ 454 warnings.warn('`model.predict_classes()` is deprecated and ' 455 'will be removed after 2021-01-01. ' 456 'Please use instead:' 457 '* `np.argmax(model.predict(x), axis=-1)`, ' 458 ' if your model does multi-class classification ' 459 ' (e.g. if it uses a `softmax` last-layer activation).' 460 '* `(model.predict(x) > 0.5).astype("int32")`, ' 461 ' if your model does binary classification ' 462 ' (e.g. if it uses a `sigmoid` last-layer activation).') 463 proba = self.predict(x, batch_size=batch_size, verbose=verbose) 464 if proba.shape[-1] > 1: 465 return proba.argmax(axis=-1) 466 else: 467 return (proba > 0.5).astype('int32') 468 469 def get_config(self): 470 layer_configs = [] 471 for layer in super(Sequential, self).layers: 472 # `super().layers` include the InputLayer if available (it is filtered out 473 # of `self.layers`). Note that `self._self_tracked_trackables` is managed 474 # by the tracking infrastructure and should not be used. 475 layer_configs.append(generic_utils.serialize_keras_object(layer)) 476 config = { 477 'name': self.name, 478 'layers': copy.deepcopy(layer_configs) 479 } 480 if not self._is_graph_network and self._build_input_shape is not None: 481 config['build_input_shape'] = self._build_input_shape 482 return config 483 484 @classmethod 485 def from_config(cls, config, custom_objects=None): 486 if 'name' in config: 487 name = config['name'] 488 build_input_shape = config.get('build_input_shape') 489 layer_configs = config['layers'] 490 else: 491 name = None 492 build_input_shape = None 493 layer_configs = config 494 model = cls(name=name) 495 for layer_config in layer_configs: 496 layer = layer_module.deserialize(layer_config, 497 custom_objects=custom_objects) 498 model.add(layer) 499 if (not model.inputs and build_input_shape and 500 isinstance(build_input_shape, (tuple, list))): 501 model.build(build_input_shape) 502 return model 503 504 @property 505 def input_spec(self): 506 if hasattr(self, '_manual_input_spec'): 507 return self._manual_input_spec 508 if self.layers and hasattr(self.layers[0], 'input_spec'): 509 return self.layers[0].input_spec 510 return None 511 512 @input_spec.setter 513 def input_spec(self, value): 514 self._manual_input_spec = value 515 516 @property 517 def _trackable_saved_model_saver(self): 518 return model_serialization.SequentialSavedModelSaver(self) 519 520 def _is_layer_name_unique(self, layer): 521 for ref_layer in self.layers: 522 if layer.name == ref_layer.name and ref_layer is not layer: 523 return False 524 return True 525 526 def _assert_weights_created(self): 527 if self._graph_initialized: 528 return 529 # When the graph has not been initialized, use the Model's implementation to 530 # to check if the weights has been created. 531 super(functional.Functional, self)._assert_weights_created() # pylint: disable=bad-super-call 532 533 534def _get_shape_tuple(t): 535 if hasattr(t, 'shape'): 536 shape = t.shape 537 if isinstance(shape, tuple): 538 return shape 539 if shape.rank is not None: 540 return tuple(shape.as_list()) 541 return None 542 return None 543 544 545def relax_input_shape(shape_1, shape_2): 546 if shape_1 is None or shape_2 is None: 547 return None 548 if len(shape_1) != len(shape_2): 549 return None 550 return tuple(None if d1 != d2 else d1 for d1, d2 in zip(shape_1, shape_2)) 551 552 553def clear_previously_created_nodes(layer, created_nodes): 554 """Remove nodes from `created_nodes` from the layer's inbound_nodes.""" 555 for node in layer._inbound_nodes: 556 prev_layers = node.inbound_layers 557 for prev_layer in nest.flatten(prev_layers): 558 prev_layer._outbound_nodes = [ 559 n for n in prev_layer._outbound_nodes 560 if n not in created_nodes] 561 layer._inbound_nodes = [ 562 n for n in layer._inbound_nodes if n not in created_nodes] 563 564 565def track_nodes_created_by_last_call(layer, created_nodes): 566 """Adds to `created_nodes` the nodes created by the last call to `layer`.""" 567 if not layer._inbound_nodes: 568 return 569 created_nodes.add(layer._inbound_nodes[-1]) 570 prev_layers = layer._inbound_nodes[-1].inbound_layers 571 for prev_layer in nest.flatten(prev_layers): 572 if prev_layer._outbound_nodes: 573 created_nodes.add(prev_layer._outbound_nodes[-1]) 574