• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Tests for training routines."""
16
17from absl.testing import parameterized
18import numpy as np
19
20from tensorflow.python import keras
21from tensorflow.python.data.ops import dataset_ops
22from tensorflow.python.eager import context
23from tensorflow.python.framework import ops
24from tensorflow.python.keras import keras_parameterized
25from tensorflow.python.keras import metrics as metrics_module
26from tensorflow.python.keras import testing_utils
27from tensorflow.python.keras.optimizer_v2 import rmsprop
28from tensorflow.python.ops import array_ops
29from tensorflow.python.platform import test
30
31
32class TrainingTest(keras_parameterized.TestCase):
33
34  @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
35  def test_dynamic_model_has_trainable_weights(self):
36    if not context.executing_eagerly():
37      # Only test Eager modes, as Graph mode is not relevant for dynamic models.
38      return
39
40    class DynamicModel(keras.Model):
41
42      def __init__(self):
43        super(DynamicModel, self).__init__(dynamic=True)
44        self.dense = keras.layers.Dense(
45            1, kernel_initializer='zeros', bias_initializer='ones')
46
47      def call(self, inputs):
48        return self.dense(inputs)
49
50    model = DynamicModel()
51    model.compile(
52        'rmsprop', 'mae',
53        run_eagerly=True)
54    hist = model.fit(np.zeros((1, 1)), np.zeros((1, 1)))
55    self.assertEqual(hist.history['loss'][-1], 1)
56    self.assertEqual(len(model.trainable_weights), 2)
57    loss = model.train_on_batch(np.zeros((1, 1)), np.zeros((1, 1)))
58    # The loss must have been updated if the trainable weights are taken into
59    # account during tracking.
60    self.assertLess(loss, 1)
61
62  @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
63  @keras_parameterized.run_all_keras_modes
64  def test_model_methods_with_eager_tensors_multi_io(self):
65    if not context.executing_eagerly():
66      # Only test V2 Function and V2 Eager modes, as V1 Graph mode with
67      # symbolic tensors has different requirements.
68      return
69
70    input_a = keras.layers.Input(shape=(3,), name='input_a')
71    input_b = keras.layers.Input(shape=(3,), name='input_b')
72
73    dense = keras.layers.Dense(4, name='dense')
74    dropout = keras.layers.Dropout(0.5, name='dropout')
75
76    model = testing_utils.get_multi_io_model(
77        [input_a, dense], [input_b, dense, dropout])
78
79    optimizer = rmsprop.RMSprop(learning_rate=0.001)
80    loss = 'mse'
81    loss_weights = [1., 0.5]
82    metrics = ['mae', metrics_module.CategoricalAccuracy()]
83    model.compile(
84        optimizer,
85        loss,
86        metrics=metrics,
87        loss_weights=loss_weights,
88        run_eagerly=testing_utils.should_run_eagerly(),
89        sample_weight_mode=None)
90
91    input_a = array_ops.zeros(shape=(10, 3))
92    input_b = array_ops.zeros(shape=(10, 3))
93    target_a = array_ops.zeros(shape=(10, 4))
94    target_b = array_ops.zeros(shape=(10, 4))
95
96    model.fit(
97        [input_a, input_b], [target_a, target_b],
98        epochs=1,
99        batch_size=5,
100        verbose=0)
101    # Test: no shuffle.
102    model.fit(
103        [input_a, input_b], [target_a, target_b],
104        epochs=1,
105        batch_size=5,
106        verbose=0,
107        shuffle=False)
108    # Test: validation data.
109    model.fit([input_a, input_b], [target_a, target_b],
110              epochs=1, batch_size=2, verbose=0,
111              validation_data=([input_a, input_b], [target_a, target_b]))
112    model.train_on_batch([input_a, input_b], [target_a, target_b])
113    model.predict([input_a, input_b], batch_size=5)
114    model.evaluate([input_a, input_b], [target_a, target_b],
115                   batch_size=2, verbose=0)
116    model.test_on_batch([input_a, input_b], [target_a, target_b])
117
118    # Test: mix np and tensors.
119    input_b = np.zeros(shape=(10, 3)).astype('float32')
120    target_b = np.zeros(shape=(10, 4)).astype('float32')
121    model.fit(
122        [input_a, input_b], [target_a, target_b],
123        epochs=1,
124        batch_size=5,
125        verbose=0)
126    model.fit([input_a, input_b], [target_a, target_b],
127              epochs=1, batch_size=2, verbose=0,
128              validation_data=([input_a, input_b], [target_a, target_b]))
129    model.fit(
130        [input_a, input_b], [target_a, target_b],
131        epochs=1,
132        batch_size=5,
133        verbose=0,
134        shuffle=False)
135    model.train_on_batch([input_a, input_b], [target_a, target_b])
136    model.predict([input_a, input_b], batch_size=5)
137    model.evaluate([input_a, input_b], [target_a, target_b],
138                   batch_size=2, verbose=0)
139    model.test_on_batch([input_a, input_b], [target_a, target_b])
140
141  @keras_parameterized.run_with_all_model_types
142  @keras_parameterized.run_all_keras_modes
143  def test_model_methods_with_eager_tensors_single_io(self):
144    if not context.executing_eagerly():
145      # Only test V2 Function and V2 Eager modes, as V1 Graph mode with
146      # symbolic tensors has different requirements.
147      return
148
149    model = testing_utils.get_small_mlp(10, 4, 3)
150
151    optimizer = rmsprop.RMSprop(learning_rate=0.001)
152    loss = 'mse'
153    metrics = ['mae', metrics_module.CategoricalAccuracy()]
154    model.compile(
155        optimizer,
156        loss,
157        metrics=metrics,
158        run_eagerly=testing_utils.should_run_eagerly())
159
160    inputs = array_ops.zeros(shape=(10, 3))
161    targets = array_ops.zeros(shape=(10, 4))
162
163    model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0)
164    model.fit(inputs, targets, epochs=1, batch_size=3, verbose=0, shuffle=False)
165    model.fit(inputs, targets, epochs=1, batch_size=4, verbose=0,
166              validation_data=(inputs, targets))
167    model.evaluate(inputs, targets, batch_size=2, verbose=0)
168    model.predict(inputs, batch_size=2)
169    model.train_on_batch(inputs, targets)
170    model.test_on_batch(inputs, targets)
171
172  @keras_parameterized.run_with_all_model_types
173  def test_model_fit_and_validation_with_missing_arg_errors(self):
174    model = testing_utils.get_small_mlp(10, 4, 3)
175    model.compile(optimizer=rmsprop.RMSprop(learning_rate=0.001),
176                  loss='mse',
177                  run_eagerly=True)
178
179    x = array_ops.zeros(shape=(10, 3))
180    y = array_ops.zeros(shape=(10, 4))
181    dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat(10).batch(5)
182    validation_dataset = dataset_ops.Dataset.from_tensor_slices(
183        (x, y)).repeat().batch(5)  # Infinite dataset.
184
185    model.fit(dataset, epochs=1, verbose=0)
186
187    # Step argument is required for infinite datasets.
188    with self.assertRaises(ValueError):
189      model.fit(dataset, steps_per_epoch=2, epochs=1, verbose=0,
190                validation_data=validation_dataset)
191    with self.assertRaises(ValueError):
192      model.fit(dataset, steps_per_epoch=2, epochs=1, verbose=0,
193                validation_data=validation_dataset)
194
195  # TODO(b/120931266): Enable test on subclassed models after bug causing an
196  # extra dimension to be added to predict outputs is fixed.
197  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
198  def test_generator_methods(self):
199    model = testing_utils.get_small_mlp(10, 4, 3)
200    optimizer = rmsprop.RMSprop(learning_rate=0.001)
201    model.compile(
202        optimizer,
203        loss='mse',
204        metrics=['mae', metrics_module.CategoricalAccuracy()],
205        run_eagerly=True)
206
207    x = np.random.random((10, 3))
208    y = np.random.random((10, 4))
209
210    def numpy_iterator():
211      while True:
212        yield x, y
213
214    model.fit_generator(numpy_iterator(), steps_per_epoch=3, epochs=1)
215    model.evaluate_generator(numpy_iterator(), steps=3)
216
217    def inference_numpy_iterator():
218      while True:
219        yield x
220
221    out = model.predict_generator(inference_numpy_iterator(), steps=3)
222    self.assertEqual(out.shape, (30, 4))
223
224
225class CorrectnessTest(keras_parameterized.TestCase):
226
227  @keras_parameterized.run_with_all_model_types
228  @keras_parameterized.run_all_keras_modes
229  @parameterized.named_parameters([
230      ('', dict()),
231      ('_clipvalue_inf', {'clipvalue': 999999}),
232      ('_clipnorm_inf', {'clipnorm': 999999}),
233  ])
234  def test_loss_correctness(self, optimizer_kwargs):
235    # Test that training loss is the same in eager and graph
236    # (by comparing it to a reference value in a deterministic case)
237    layers = [
238        keras.layers.Dense(3, activation='relu',
239                           kernel_initializer='ones'),
240        keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')]
241    model = testing_utils.get_model_from_layers(layers, input_shape=(4,))
242    model.compile(
243        loss='sparse_categorical_crossentropy',
244        optimizer=rmsprop.RMSprop(learning_rate=0.001, **optimizer_kwargs),
245        run_eagerly=testing_utils.should_run_eagerly())
246    x = np.ones((100, 4))
247    np.random.seed(123)
248    y = np.random.randint(0, 1, size=(100, 1))
249    history = model.fit(x, y, epochs=1, batch_size=10)
250    self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
251
252  @keras_parameterized.run_with_all_model_types
253  @keras_parameterized.run_all_keras_modes
254  def test_loss_correctness_clipvalue_zero(self):
255    # Test that training loss is the same in eager and graph
256    # (by comparing it to a reference value in a deterministic case)
257    # And confirm that setting clipvalue to zero stops all training
258    layers = [
259        keras.layers.Dense(3, activation='relu',
260                           kernel_initializer='ones'),
261        keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')]
262    model = testing_utils.get_model_from_layers(layers, input_shape=(4,))
263    model.compile(
264        loss='sparse_categorical_crossentropy',
265        optimizer=rmsprop.RMSprop(learning_rate=0.001, clipvalue=0.0),
266        run_eagerly=testing_utils.should_run_eagerly())
267    x = np.ones((100, 4))
268    np.random.seed(123)
269    y = np.random.randint(0, 1, size=(100, 1))
270    history = model.fit(x, y, epochs=3, batch_size=10)
271    self.assertAlmostEqual(history.history['loss'][-3], 0.6931, 4)
272    self.assertAlmostEqual(history.history['loss'][-2], 0.6931, 4)
273    self.assertAlmostEqual(history.history['loss'][-1], 0.6931, 4)
274
275  @keras_parameterized.run_with_all_model_types
276  @keras_parameterized.run_all_keras_modes
277  def test_loss_correctness_with_iterator(self):
278    # Test that training loss is the same in eager and graph
279    # (by comparing it to a reference value in a deterministic case)
280    layers = [
281        keras.layers.Dense(3, activation='relu',
282                           kernel_initializer='ones'),
283        keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')]
284    model = testing_utils.get_model_from_layers(layers, input_shape=(4,))
285    model.compile(
286        loss='sparse_categorical_crossentropy',
287        optimizer=rmsprop.RMSprop(learning_rate=0.001),
288        run_eagerly=testing_utils.should_run_eagerly())
289    x = np.ones((100, 4), dtype=np.float32)
290    np.random.seed(123)
291    y = np.random.randint(0, 1, size=(100, 1))
292    dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
293    dataset = dataset.repeat(100)
294    dataset = dataset.batch(10)
295    history = model.fit(dataset, epochs=1, steps_per_epoch=10)
296    self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
297
298  @parameterized.named_parameters([
299      ('_None', None, 0., 4.),
300      ('_False', False, 4., 4.),
301      ('_True', True, 0., 0.),
302  ])
303  def test_nested_model_learning_phase(self, training,
304                                       expected_training_loss,
305                                       expected_validation_loss):
306    """Tests that learning phase is correctly set in an intermediate layer."""
307
308    def _make_unregularized_model():
309      inputs = keras.Input((4,))
310      # Zero out activations when `training=True`.
311      x = keras.layers.Dropout(1. - 1. / (1 << 24))(inputs)
312      x = keras.layers.Dense(
313          10,
314          activation='relu',
315          trainable=False,
316          bias_initializer='zeros',
317          kernel_initializer='ones')(
318              x)  # Just sum together all the activations.
319      outputs = keras.layers.Dense(3)(x)
320      return keras.Model(inputs, outputs)
321
322    def _regularize_model(unregularized_model):
323      # Regularize the most recent activations of a post-dropout layer.
324      sample_activations = unregularized_model.get_layer(
325          index=-2).get_output_at(-1)
326      regularization_loss = keras.backend.mean(sample_activations)
327      unregularized_model.add_loss(regularization_loss)
328      unregularized_model.add_metric(
329          regularization_loss, aggregation='mean', name='regularization_loss')
330      inputs = keras.Input(unregularized_model.inputs[0].shape[1:])
331      logits = unregularized_model(inputs, training=training)
332      outputs = keras.activations.softmax(logits)
333      model = keras.Model(inputs, outputs)
334      return model
335
336    # Make and compile models.
337    model = _regularize_model(_make_unregularized_model())
338    model.compile('sgd', 'sparse_categorical_crossentropy')
339    # Prepare fake data.
340    x = np.ones((20, 4)).astype(np.float32)
341    y = np.random.randint(0, 3, size=(20,)).astype(np.int64)
342    dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2)
343    results = model.evaluate(dataset)
344    evaluation_results = dict(zip(model.metrics_names, results))
345    # Rate of dropout depends on the learning phase.
346    self.assertEqual(evaluation_results['regularization_loss'],
347                     expected_validation_loss)
348    history = model.fit(dataset, epochs=2, validation_data=dataset).history
349    self.assertAllEqual(history['regularization_loss'],
350                        [expected_training_loss] * 2)
351    self.assertAllEqual(history['val_regularization_loss'],
352                        [expected_validation_loss] * 2)
353
354
355if __name__ == '__main__':
356  ops.enable_eager_execution()
357  test.main()
358