• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 # ==============================================================================
15 """Tests for Keras backend."""
16 from __future__ import absolute_import
17 from __future__ import division
18 from __future__ import print_function
19 
20 from absl.testing import parameterized
21 import numpy as np
22 import scipy.sparse
23 
24 from tensorflow.core.protobuf import config_pb2
25 from tensorflow.python import keras
26 from tensorflow.python.eager import context
27 from tensorflow.python.framework import config
28 from tensorflow.python.framework import errors_impl
29 from tensorflow.python.framework import ops
30 from tensorflow.python.framework import sparse_tensor
31 from tensorflow.python.framework import test_util
32 from tensorflow.python.ops import array_ops
33 from tensorflow.python.ops import nn
34 from tensorflow.python.ops import variables
35 from tensorflow.python.platform import test
36 from tensorflow.python.util import tf_inspect
37 
38 
39 def compare_single_input_op_to_numpy(keras_op,
40                                      np_op,
41                                      input_shape,
42                                      dtype='float32',
43                                      negative_values=True,
44                                      keras_args=None,
45                                      keras_kwargs=None,
46                                      np_args=None,
47                                      np_kwargs=None):
48   keras_args = keras_args or []
49   keras_kwargs = keras_kwargs or {}
50   np_args = np_args or []
51   np_kwargs = np_kwargs or {}
52   inputs = 2. * np.random.random(input_shape)
53   if negative_values:
54     inputs -= 1.
55   keras_output = keras_op(keras.backend.variable(inputs, dtype=dtype),
56                           *keras_args, **keras_kwargs)
57   keras_output = keras.backend.eval(keras_output)
58   np_output = np_op(inputs.astype(dtype), *np_args, **np_kwargs)
59   try:
60     np.testing.assert_allclose(keras_output, np_output, atol=1e-4)
61   except AssertionError:
62     raise AssertionError('Test for op `' + str(keras_op.__name__) + '` failed; '
63                          'Expected ' + str(np_output) + ' but got ' +
64                          str(keras_output))
65 
66 
67 def compare_two_inputs_op_to_numpy(keras_op,
68                                    np_op,
69                                    input_shape_a,
70                                    input_shape_b,
71                                    dtype='float32',
72                                    keras_args=None,
73                                    keras_kwargs=None,
74                                    np_args=None,
75                                    np_kwargs=None):
76   keras_args = keras_args or []
77   keras_kwargs = keras_kwargs or {}
78   np_args = np_args or []
79   np_kwargs = np_kwargs or {}
80   input_a = np.random.random(input_shape_a)
81   input_b = np.random.random(input_shape_b)
82   keras_output = keras_op(keras.backend.variable(input_a, dtype=dtype),
83                           keras.backend.variable(input_b, dtype=dtype),
84                           *keras_args, **keras_kwargs)
85   keras_output = keras.backend.eval(keras_output)
86   np_output = np_op(input_a.astype(dtype), input_b.astype(dtype),
87                     *np_args, **np_kwargs)
88   try:
89     np.testing.assert_allclose(keras_output, np_output, atol=1e-4)
90   except AssertionError:
91     raise AssertionError('Test for op `' + str(keras_op.__name__) + '` failed; '
92                          'Expected ' + str(np_output) + ' but got ' +
93                          str(keras_output))
94 
95 
96 class BackendResetTest(test.TestCase, parameterized.TestCase):
97 
98   @test_util.run_all_in_graph_and_eager_modes
99   def test_new_config(self):
100     # User defined jit setting
101     config.set_optimizer_jit(False)
102     sess = keras.backend.get_session()
103     default_config = context.context().config
104     self.assertEqual(
105         sess._config.graph_options.optimizer_options.global_jit_level,
106         default_config.graph_options.optimizer_options.global_jit_level)
107     keras.backend.clear_session()
108 
109     # New session has the same jit setting
110     sess = keras.backend.get_session()
111     default_config = context.context().config
112     self.assertEqual(
113         sess._config.graph_options.optimizer_options.global_jit_level,
114         default_config.graph_options.optimizer_options.global_jit_level)
115     keras.backend.clear_session()
116 
117     # Change respected
118     config.set_optimizer_jit(True)
119     sess = keras.backend.get_session()
120     default_config = context.context().config
121     self.assertEqual(
122         sess._config.graph_options.optimizer_options.global_jit_level,
123         default_config.graph_options.optimizer_options.global_jit_level)
124     keras.backend.clear_session()
125 
126   # We can't use the normal parameterized decorator because the test session
127   # will block graph clearing.
128   @parameterized.named_parameters(('_v1', context.graph_mode),
129                                   ('_v2', context.eager_mode))
130   def test_new_graph(self, test_context):
131     with test_context():
132       g_old = keras.backend.get_graph()
133       keras.backend.clear_session()
134       g = keras.backend.get_graph()
135 
136       assert g_old is not g
137 
138 
139 @test_util.run_all_in_graph_and_eager_modes
140 class BackendUtilsTest(test.TestCase):
141 
142   def test_backend(self):
143     self.assertEqual(keras.backend.backend(), 'tensorflow')
144 
145   def test_get_reset_uids(self):
146     self.assertEqual(keras.backend.get_uid('foo'), 1)
147     self.assertEqual(keras.backend.get_uid('foo'), 2)
148 
149     keras.backend.reset_uids()
150     self.assertEqual(keras.backend.get_uid('foo'), 1)
151 
152   def test_learning_phase(self):
153     with self.cached_session() as sess:
154       with self.assertRaises(ValueError):
155         keras.backend.set_learning_phase(2)
156 
157       # Test running with a learning-phase-consuming layer
158       with keras.backend.learning_phase_scope(0):
159         x = keras.Input((3,))
160         y = keras.layers.BatchNormalization()(x)
161         if not context.executing_eagerly():
162           self.evaluate(variables.global_variables_initializer())
163           sess.run(y, feed_dict={x: np.random.random((2, 3))})
164 
165   def test_learning_phase_name(self):
166     with ops.name_scope('test_scope'):
167       # Test that outer name scopes do not affect the learning phase's name.
168       lp = keras.backend.symbolic_learning_phase()
169     self.assertEqual(lp.name, 'keras_learning_phase:0')
170 
171   def test_learning_phase_scope(self):
172     initial_learning_phase = keras.backend.learning_phase()
173     with keras.backend.learning_phase_scope(1):
174       self.assertEqual(keras.backend.learning_phase(), 1)
175     self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
176     with keras.backend.learning_phase_scope(0):
177       self.assertEqual(keras.backend.learning_phase(), 0)
178     self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
179     with self.assertRaises(ValueError):
180       with keras.backend.learning_phase_scope(None):
181         pass
182     self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
183 
184     new_learning_phase = 0
185     keras.backend.set_learning_phase(new_learning_phase)
186     self.assertEqual(keras.backend.learning_phase(), new_learning_phase)
187     with keras.backend.learning_phase_scope(1):
188       self.assertEqual(keras.backend.learning_phase(), 1)
189     self.assertEqual(keras.backend.learning_phase(), new_learning_phase)
190 
191   def test_learning_phase_scope_in_graph(self):
192     initial_learning_phase_outside_graph = keras.backend.learning_phase()
193     with keras.backend.get_graph().as_default():
194       initial_learning_phase_in_graph = keras.backend.learning_phase()
195 
196     self.assertEqual(keras.backend.learning_phase(),
197                      initial_learning_phase_outside_graph)
198     with keras.backend.learning_phase_scope(1):
199       self.assertEqual(keras.backend.learning_phase(), 1)
200     self.assertEqual(keras.backend.learning_phase(),
201                      initial_learning_phase_outside_graph)
202 
203     with keras.backend.get_graph().as_default():
204       self.assertIs(keras.backend.learning_phase(),
205                     initial_learning_phase_in_graph)
206 
207     self.assertEqual(keras.backend.learning_phase(),
208                      initial_learning_phase_outside_graph)
209 
210   def test_int_shape(self):
211     x = keras.backend.ones(shape=(3, 4))
212     self.assertEqual(keras.backend.int_shape(x), (3, 4))
213 
214     if not context.executing_eagerly():
215       x = keras.backend.placeholder(shape=(None, 4))
216       self.assertEqual(keras.backend.int_shape(x), (None, 4))
217 
218   def test_in_train_phase(self):
219     y1 = keras.backend.variable(1)
220     y2 = keras.backend.variable(2)
221     if context.executing_eagerly():
222       with keras.backend.learning_phase_scope(0):
223         y_val_test = keras.backend.in_train_phase(y1, y2).numpy()
224       with keras.backend.learning_phase_scope(1):
225         y_val_train = keras.backend.in_train_phase(y1, y2).numpy()
226     else:
227       y = keras.backend.in_train_phase(y1, y2)
228       f = keras.backend.function([keras.backend.learning_phase()], [y])
229       y_val_test = f([0])[0]
230       y_val_train = f([1])[0]
231     self.assertAllClose(y_val_test, 2)
232     self.assertAllClose(y_val_train, 1)
233 
234   def test_is_keras_tensor(self):
235     x = keras.backend.variable(1)
236     self.assertEqual(keras.backend.is_keras_tensor(x), False)
237     x = keras.Input(shape=(1,))
238     self.assertEqual(keras.backend.is_keras_tensor(x), True)
239     x = keras.Input(shape=(None,), ragged=True)
240     self.assertEqual(keras.backend.is_keras_tensor(x), True)
241     x = keras.Input(shape=(None, None), sparse=True)
242     self.assertEqual(keras.backend.is_keras_tensor(x), True)
243     with self.assertRaises(ValueError):
244       keras.backend.is_keras_tensor(0)
245 
246   def test_stop_gradient(self):
247     x = keras.backend.variable(1)
248     y = keras.backend.stop_gradient(x)
249     if not context.executing_eagerly():
250       self.assertEqual(y.op.name[:12], 'StopGradient')
251 
252     xs = [keras.backend.variable(1) for _ in range(3)]
253     ys = keras.backend.stop_gradient(xs)
254     if not context.executing_eagerly():
255       for y in ys:
256         self.assertEqual(y.op.name[:12], 'StopGradient')
257 
258   def test_placeholder(self):
259     x = keras.backend.placeholder(shape=(3, 4))
260     self.assertEqual(x.shape.as_list(), [3, 4])
261     x = keras.backend.placeholder(shape=(3, 4), sparse=True)
262     self.assertEqual(x.shape.as_list(), [3, 4])
263 
264   def test_is_placeholder(self):
265     x = keras.backend.placeholder(shape=(1,))
266     self.assertEqual(keras.backend.is_placeholder(x), True)
267     x = keras.backend.variable(1)
268     self.assertEqual(keras.backend.is_placeholder(x), False)
269 
270   def test_print_tensor(self):
271     # Unfortunately it seems impossible to use `mock` (or any other method)
272     # to capture stdout when used inside a graph or graph function, thus
273     # we cannot test correctness.
274     # The message gets correctly printed in practice.
275     x = keras.backend.placeholder(shape=())
276     y = keras.backend.print_tensor(x, 'eager=%s' % context.executing_eagerly())
277     f = keras.backend.function(x, y)
278     f(0)
279 
280   def test_cast_to_floatx(self):
281     x = keras.backend.variable(1, dtype='float64')
282     x = keras.backend.cast_to_floatx(x)
283     self.assertEqual(x.dtype.name, 'float32')
284     x = keras.backend.cast_to_floatx(2)
285     self.assertEqual(x.dtype.name, 'float32')
286 
287 
288 @test_util.run_all_in_graph_and_eager_modes
289 class BackendVariableTest(test.TestCase):
290 
291   def test_zeros(self):
292     x = keras.backend.zeros((3, 4))
293     val = keras.backend.eval(x)
294     self.assertAllClose(val, np.zeros((3, 4)))
295 
296   def test_ones(self):
297     x = keras.backend.ones((3, 4))
298     val = keras.backend.eval(x)
299     self.assertAllClose(val, np.ones((3, 4)))
300 
301   def test_eye(self):
302     x = keras.backend.eye(4)
303     val = keras.backend.eval(x)
304     self.assertAllClose(val, np.eye(4))
305 
306   def test_zeros_like(self):
307     x = keras.backend.zeros((3, 4))
308     y = keras.backend.zeros_like(x)
309     val = keras.backend.eval(y)
310     self.assertAllClose(val, np.zeros((3, 4)))
311 
312   def test_ones_like(self):
313     x = keras.backend.zeros((3, 4))
314     y = keras.backend.ones_like(x)
315     val = keras.backend.eval(y)
316     self.assertAllClose(val, np.ones((3, 4)))
317 
318   def test_random_uniform_variable(self):
319     x = keras.backend.random_uniform_variable((30, 20), low=1, high=2, seed=0)
320     val = keras.backend.eval(x)
321     self.assertAllClose(val.mean(), 1.5, atol=1e-1)
322     self.assertAllClose(val.max(), 2., atol=1e-1)
323     self.assertAllClose(val.min(), 1., atol=1e-1)
324 
325   def test_random_normal_variable(self):
326     x = keras.backend.random_normal_variable((30, 20), 1., 0.5, seed=0)
327     val = keras.backend.eval(x)
328     self.assertAllClose(val.mean(), 1., atol=1e-1)
329     self.assertAllClose(val.std(), 0.5, atol=1e-1)
330 
331   def test_count_params(self):
332     x = keras.backend.zeros((4, 5))
333     val = keras.backend.count_params(x)
334     self.assertAllClose(val, 20)
335 
336   def test_constant(self):
337     ref_val = np.random.random((3, 4)).astype('float32')
338     x = keras.backend.constant(ref_val)
339     val = keras.backend.eval(x)
340     self.assertAllClose(val, ref_val)
341 
342   def test_sparse_variable(self):
343     val = scipy.sparse.eye(10)
344     x = keras.backend.variable(val)
345     self.assertTrue(isinstance(x, sparse_tensor.SparseTensor))
346 
347     y = keras.backend.to_dense(x)
348     self.assertFalse(keras.backend.is_sparse(y))
349 
350 
351 @test_util.run_all_in_graph_and_eager_modes
352 class BackendLinearAlgebraTest(test.TestCase, parameterized.TestCase):
353 
354   def test_dot(self):
355     x = keras.backend.ones(shape=(2, 3))
356     y = keras.backend.ones(shape=(3, 4))
357     xy = keras.backend.dot(x, y)
358     self.assertEqual(xy.shape.as_list(), [2, 4])
359 
360     x = keras.backend.ones(shape=(32, 28, 3))
361     y = keras.backend.ones(shape=(3, 4))
362     xy = keras.backend.dot(x, y)
363     self.assertEqual(xy.shape.as_list(), [32, 28, 4])
364 
365   @parameterized.parameters(
366       [(2, 3, 4, 5), (2, 5, 6, 7), (2, 3, 4, 6, 7), (3, 1)],
367       [(2, 20, 1), (2, 30, 20), (2, 1, 30), (1, 2)],
368       [(4, 2, 3), (4, 5, 3), (4, 2, 5), (2, 2)],
369       [(4, 2), (4, 2, 3), (4, 3), (1, 1)],
370       [(4, 2), (4, 2, 3), (4, 3), 1],
371       [(4, 2, 3), (4, 3), (4, 2), (2, 1)],
372   )
373   def test_batch_dot(self, x_shape, y_shape, output_shape, axes):
374     x_val = np.random.random(x_shape)
375     y_val = np.random.random(y_shape)
376     x = keras.backend.variable(x_val)
377     y = keras.backend.variable(y_val)
378     xy = keras.backend.batch_dot(x, y, axes=axes)
379     self.assertEqual(tuple(xy.shape.as_list()), output_shape)
380     xy_val = keras.backend.eval(xy)
381     ref_val = self._reference_batch_dot(x_val, y_val, axes)
382     self.assertAllClose(xy_val, ref_val, atol=1e-5)
383 
384   def _reference_batch_dot(self, x, y, axes):
385     if isinstance(axes, int):
386       axes = [axes, axes]
387     elif isinstance(axes, tuple):
388       axes = list(axes)
389     if axes is None:
390       if y.ndim == 2:
391         axes = [x.ndim - 1, y.ndim - 1]
392       else:
393         axes = [x.ndim - 1, y.ndim - 2]
394     if axes[0] < 0:
395       axes[0] += x.ndim
396     if axes[1] < 0:
397       axes[1] += y.ndim
398     result = []
399     axes = [axes[0] - 1, axes[1] - 1]
400     for xi, yi in zip(x, y):
401       result.append(np.tensordot(xi, yi, axes))
402     result = np.array(result)
403     if result.ndim == 1:
404       result = np.expand_dims(result, -1)
405     return result
406 
407   def test_reduction_ops(self):
408     ops_to_test = [
409         (keras.backend.max, np.max),
410         (keras.backend.min, np.min),
411         (keras.backend.sum, np.sum),
412         (keras.backend.prod, np.prod),
413         (keras.backend.var, np.var),
414         (keras.backend.std, np.std),
415         (keras.backend.mean, np.mean),
416         (keras.backend.argmin, np.argmin),
417         (keras.backend.argmax, np.argmax),
418     ]
419     for keras_op, np_op in ops_to_test:
420       compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
421                                        keras_kwargs={'axis': 1},
422                                        np_kwargs={'axis': 1})
423       compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
424                                        keras_kwargs={'axis': -1},
425                                        np_kwargs={'axis': -1})
426       if 'keepdims' in tf_inspect.getargspec(keras_op).args:
427         compare_single_input_op_to_numpy(keras_op, np_op,
428                                          input_shape=(4, 7, 5),
429                                          keras_kwargs={'axis': 1,
430                                                        'keepdims': True},
431                                          np_kwargs={'axis': 1,
432                                                     'keepdims': True})
433 
434   def test_elementwise_ops(self):
435     ops_to_test = [
436         (keras.backend.square, np.square),
437         (keras.backend.abs, np.abs),
438         (keras.backend.round, np.round),
439         (keras.backend.sign, np.sign),
440         (keras.backend.sin, np.sin),
441         (keras.backend.cos, np.cos),
442         (keras.backend.exp, np.exp),
443     ]
444     for keras_op, np_op in ops_to_test:
445       compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7))
446 
447     ops_to_test = [
448         (keras.backend.sqrt, np.sqrt),
449         (keras.backend.log, np.log),
450     ]
451     for keras_op, np_op in ops_to_test:
452       compare_single_input_op_to_numpy(keras_op, np_op,
453                                        input_shape=(4, 7),
454                                        negative_values=False)
455 
456     compare_single_input_op_to_numpy(
457         keras.backend.clip, np.clip,
458         input_shape=(6, 4),
459         keras_kwargs={'min_value': 0.1, 'max_value': 2.4},
460         np_kwargs={'a_min': 0.1, 'a_max': 1.4})
461 
462     compare_single_input_op_to_numpy(
463         keras.backend.pow, np.power,
464         input_shape=(6, 4),
465         keras_args=[3],
466         np_args=[3])
467 
468   def test_two_tensor_ops(self):
469     ops_to_test = [
470         (keras.backend.equal, np.equal),
471         (keras.backend.not_equal, np.not_equal),
472         (keras.backend.greater, np.greater),
473         (keras.backend.greater_equal, np.greater_equal),
474         (keras.backend.less, np.less),
475         (keras.backend.less_equal, np.less_equal),
476         (keras.backend.maximum, np.maximum),
477         (keras.backend.minimum, np.minimum),
478     ]
479     for keras_op, np_op in ops_to_test:
480       compare_two_inputs_op_to_numpy(keras_op, np_op,
481                                      input_shape_a=(4, 7),
482                                      input_shape_b=(4, 7))
483 
484   def test_relu(self):
485     x = ops.convert_to_tensor([[-4, 0], [2, 7]], 'float32')
486 
487     # standard relu
488     relu_op = keras.backend.relu(x)
489     self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
490 
491     # alpha (leaky relu used)
492     relu_op = keras.backend.relu(x, alpha=0.5)
493     if not context.executing_eagerly():
494       self.assertTrue('LeakyRelu' in relu_op.name)
495     self.assertAllClose(keras.backend.eval(relu_op), [[-2, 0], [2, 7]])
496 
497     # max_value < some elements
498     relu_op = keras.backend.relu(x, max_value=5)
499     self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 5]])
500 
501     # nn.relu6 used
502     relu_op = keras.backend.relu(x, max_value=6)
503     if not context.executing_eagerly():
504       self.assertTrue('Relu6' in relu_op.name)  # uses tf.nn.relu6
505     self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 6]])
506 
507     # max value > 6
508     relu_op = keras.backend.relu(x, max_value=10)
509     self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
510 
511     # max value is float
512     relu_op = keras.backend.relu(x, max_value=4.3)
513     self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 4.3]])
514 
515     # max value == 0
516     relu_op = keras.backend.relu(x, max_value=0)
517     self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 0]])
518 
519     # alpha and max_value
520     relu_op = keras.backend.relu(x, alpha=0.25, max_value=3)
521     self.assertAllClose(keras.backend.eval(relu_op), [[-1, 0], [2, 3]])
522 
523     # threshold
524     relu_op = keras.backend.relu(x, threshold=3)
525     self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 7]])
526 
527     # threshold is float
528     relu_op = keras.backend.relu(x, threshold=1.5)
529     self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
530 
531     # threshold is negative
532     relu_op = keras.backend.relu(x, threshold=-5)
533     self.assertAllClose(keras.backend.eval(relu_op), [[-4, 0], [2, 7]])
534 
535     # threshold and max_value
536     relu_op = keras.backend.relu(x, threshold=3, max_value=5)
537     self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 5]])
538 
539     # threshold and alpha
540     relu_op = keras.backend.relu(x, alpha=0.25, threshold=4)
541     self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 7]])
542 
543     # threshold, alpha, and max_value
544     relu_op = keras.backend.relu(x, alpha=0.25, threshold=4, max_value=5)
545     self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 5]])
546 
547     # Test case for GitHub issue 35430, with integer dtype
548     x = keras.Input(shape=(), name='x', dtype='int64')
549     y = keras.layers.ReLU(max_value=100, dtype='int64')(x)
550 
551 
552 @test_util.run_all_in_graph_and_eager_modes
553 class BackendShapeOpsTest(test.TestCase):
554 
555   def test_reshape(self):
556     compare_single_input_op_to_numpy(keras.backend.reshape, np.reshape,
557                                      input_shape=(4, 7),
558                                      keras_args=[(2, 14)],
559                                      np_args=[(2, 14)])
560 
561   def test_concatenate(self):
562     a = keras.backend.variable(np.ones((1, 2, 3)))
563     b = keras.backend.variable(np.ones((1, 2, 2)))
564     y = keras.backend.concatenate([a, b], axis=-1)
565     self.assertEqual(y.shape.as_list(), [1, 2, 5])
566 
567   def test_permute_dimensions(self):
568     compare_single_input_op_to_numpy(keras.backend.permute_dimensions,
569                                      np.transpose,
570                                      input_shape=(4, 7),
571                                      keras_args=[(1, 0)],
572                                      np_args=[(1, 0)])
573 
574   def test_resize_images(self):
575     height_factor = 2
576     width_factor = 2
577     data_format = 'channels_last'
578     x = keras.backend.variable(np.ones((1, 2, 2, 3)))
579     y = keras.backend.resize_images(x,
580                                     height_factor,
581                                     width_factor,
582                                     data_format)
583     self.assertEqual(y.shape.as_list(), [1, 4, 4, 3])
584 
585     data_format = 'channels_first'
586     x = keras.backend.variable(np.ones((1, 3, 2, 2)))
587     y = keras.backend.resize_images(x,
588                                     height_factor,
589                                     width_factor,
590                                     data_format)
591     self.assertEqual(y.shape.as_list(), [1, 3, 4, 4])
592 
593     # Invalid use:
594     with self.assertRaises(ValueError):
595       keras.backend.resize_images(x,
596                                   height_factor,
597                                   width_factor,
598                                   data_format='unknown')
599 
600   def test_resize_volumes(self):
601     height_factor = 2
602     width_factor = 2
603     depth_factor = 2
604     data_format = 'channels_last'
605     x = keras.backend.variable(np.ones((1, 2, 2, 2, 3)))
606     y = keras.backend.resize_volumes(x,
607                                      depth_factor,
608                                      height_factor,
609                                      width_factor,
610                                      data_format)
611     self.assertEqual(y.shape.as_list(), [1, 4, 4, 4, 3])
612 
613     data_format = 'channels_first'
614     x = keras.backend.variable(np.ones((1, 3, 2, 2, 2)))
615     y = keras.backend.resize_volumes(x,
616                                      depth_factor,
617                                      height_factor,
618                                      width_factor,
619                                      data_format)
620     self.assertEqual(y.shape.as_list(), [1, 3, 4, 4, 4])
621 
622     # Invalid use:
623     with self.assertRaises(ValueError):
624       keras.backend.resize_volumes(x,
625                                    depth_factor,
626                                    height_factor,
627                                    width_factor,
628                                    data_format='unknown')
629 
630   def test_repeat_elements(self):
631     x = keras.backend.variable(np.ones((1, 3, 2)))
632     y = keras.backend.repeat_elements(x, 3, axis=1)
633     self.assertEqual(y.shape.as_list(), [1, 9, 2])
634 
635     # Use with a dynamic axis:
636     if not context.executing_eagerly():
637       x = keras.backend.placeholder(shape=(2, None, 2))
638       y = keras.backend.repeat_elements(x, 3, axis=1)
639       self.assertEqual(y.shape.as_list(), [2, None, 2])
640 
641   def test_repeat(self):
642     x = keras.backend.variable(np.ones((1, 3)))
643     y = keras.backend.repeat(x, 2)
644     self.assertEqual(y.shape.as_list(), [1, 2, 3])
645 
646   def test_flatten(self):
647     compare_single_input_op_to_numpy(keras.backend.flatten,
648                                      np.reshape,
649                                      input_shape=(4, 7, 6),
650                                      np_args=[(4 * 7 * 6,)])
651 
652   def test_batch_flatten(self):
653     compare_single_input_op_to_numpy(keras.backend.batch_flatten,
654                                      np.reshape,
655                                      input_shape=(4, 7, 6),
656                                      np_args=[(4, 7 * 6)])
657 
658   def test_temporal_padding(self):
659 
660     def ref_op(x, padding):
661       shape = list(x.shape)
662       shape[1] += padding[0] + padding[1]
663       y = np.zeros(tuple(shape))
664       y[:, padding[0]:-padding[1], :] = x
665       return y
666 
667     compare_single_input_op_to_numpy(keras.backend.temporal_padding,
668                                      ref_op,
669                                      input_shape=(4, 7, 6),
670                                      keras_args=[(2, 3)],
671                                      np_args=[(2, 3)])
672 
673   def test_spatial_2d_padding(self):
674 
675     def ref_op(x, padding, data_format='channels_last'):
676       shape = list(x.shape)
677       if data_format == 'channels_last':
678         shape[1] += padding[0][0] + padding[0][1]
679         shape[2] += padding[1][0] + padding[1][1]
680         y = np.zeros(tuple(shape))
681         y[:, padding[0][0]:-padding[0][1], padding[1][0]:-padding[1][1], :] = x
682       else:
683         shape[2] += padding[0][0] + padding[0][1]
684         shape[3] += padding[1][0] + padding[1][1]
685         y = np.zeros(tuple(shape))
686         y[:, :, padding[0][0]:-padding[0][1], padding[1][0]:-padding[1][1]] = x
687       return y
688 
689     compare_single_input_op_to_numpy(
690         keras.backend.spatial_2d_padding,
691         ref_op,
692         input_shape=(2, 3, 2, 3),
693         keras_args=[((2, 3), (1, 2))],
694         keras_kwargs={'data_format': 'channels_last'},
695         np_args=[((2, 3), (1, 2))],
696         np_kwargs={'data_format': 'channels_last'})
697     compare_single_input_op_to_numpy(
698         keras.backend.spatial_2d_padding,
699         ref_op,
700         input_shape=(2, 3, 2, 3),
701         keras_args=[((2, 3), (1, 2))],
702         keras_kwargs={'data_format': 'channels_first'},
703         np_args=[((2, 3), (1, 2))],
704         np_kwargs={'data_format': 'channels_first'})
705 
706   def test_spatial_3d_padding(self):
707 
708     def ref_op(x, padding, data_format='channels_last'):
709       shape = list(x.shape)
710       if data_format == 'channels_last':
711         shape[1] += padding[0][0] + padding[0][1]
712         shape[2] += padding[1][0] + padding[1][1]
713         shape[3] += padding[2][0] + padding[2][1]
714         y = np.zeros(tuple(shape))
715         y[:,
716           padding[0][0]:-padding[0][1],
717           padding[1][0]:-padding[1][1],
718           padding[2][0]:-padding[2][1],
719           :] = x
720       else:
721         shape[2] += padding[0][0] + padding[0][1]
722         shape[3] += padding[1][0] + padding[1][1]
723         shape[4] += padding[2][0] + padding[2][1]
724         y = np.zeros(tuple(shape))
725         y[:, :,
726           padding[0][0]:-padding[0][1],
727           padding[1][0]:-padding[1][1],
728           padding[2][0]:-padding[2][1]] = x
729       return y
730 
731     compare_single_input_op_to_numpy(
732         keras.backend.spatial_3d_padding,
733         ref_op,
734         input_shape=(2, 3, 2, 3, 2),
735         keras_args=[((2, 3), (1, 2), (2, 3))],
736         keras_kwargs={'data_format': 'channels_last'},
737         np_args=[((2, 3), (1, 2), (2, 3))],
738         np_kwargs={'data_format': 'channels_last'})
739     compare_single_input_op_to_numpy(
740         keras.backend.spatial_3d_padding,
741         ref_op,
742         input_shape=(2, 3, 2, 3, 2),
743         keras_args=[((2, 3), (1, 2), (2, 3))],
744         keras_kwargs={'data_format': 'channels_first'},
745         np_args=[((2, 3), (1, 2), (2, 3))],
746         np_kwargs={'data_format': 'channels_first'})
747 
748 
749 @test_util.run_all_in_graph_and_eager_modes
750 class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
751 
752   def test_bias_add(self):
753     keras_op = keras.backend.bias_add
754     np_op = np.add
755     compare_two_inputs_op_to_numpy(keras_op, np_op,
756                                    input_shape_a=(4, 7),
757                                    input_shape_b=(7,))
758     compare_two_inputs_op_to_numpy(keras_op, np_op,
759                                    input_shape_a=(4, 3, 7),
760                                    input_shape_b=(7,))
761     compare_two_inputs_op_to_numpy(keras_op, np_op,
762                                    input_shape_a=(4, 3, 5, 7),
763                                    input_shape_b=(7,))
764     compare_two_inputs_op_to_numpy(keras_op, np_op,
765                                    input_shape_a=(4, 3, 5, 2, 7),
766                                    input_shape_b=(7,))
767 
768     with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
769       x = keras.backend.variable((3, 4))
770       b = keras.backend.variable((3, 4))
771       keras.backend.bias_add(x, b)
772     with self.assertRaises(ValueError):
773       x = keras.backend.variable((3, 4))
774       b = keras.backend.variable((4,))
775       keras.backend.bias_add(x, b, data_format='unknown')
776 
777   def test_bias_add_channels_first(self):
778 
779     def keras_op(x, b):
780       return keras.backend.bias_add(x, b, data_format='channels_first')
781 
782     def np_op(x, b):
783       if x.ndim == 3:
784         b = b.reshape((1, b.shape[0], 1))
785       if x.ndim == 4:
786         b = b.reshape((1, b.shape[0], 1, 1))
787       return x + b
788 
789     compare_two_inputs_op_to_numpy(keras_op, np_op,
790                                    input_shape_a=(4, 3, 7),
791                                    input_shape_b=(3,))
792     compare_two_inputs_op_to_numpy(keras_op, np_op,
793                                    input_shape_a=(4, 3, 5, 7),
794                                    input_shape_b=(3,))
795 
796   def test_pool2d(self):
797     val = np.random.random((10, 3, 10, 10))
798     x = keras.backend.variable(val)
799     y = keras.backend.pool2d(x, (2, 2), strides=(1, 1),
800                              padding='valid', data_format='channels_first',
801                              pool_mode='max')
802     self.assertEqual(y.shape.as_list(), [10, 3, 9, 9])
803 
804     y = keras.backend.pool2d(x, (2, 2), strides=(1, 1),
805                              padding='valid', data_format='channels_first',
806                              pool_mode='avg')
807     self.assertEqual(y.shape.as_list(), [10, 3, 9, 9])
808 
809     val = np.random.random((10, 10, 10, 3))
810     x = keras.backend.variable(val)
811     y = keras.backend.pool2d(x, (2, 2), strides=(1, 1),
812                              padding='valid', data_format='channels_last')
813     self.assertEqual(y.shape.as_list(), [10, 9, 9, 3])
814 
815     val = np.random.random((10, 10, 10, 3))
816     x = keras.backend.variable(val)
817     y = keras.backend.pool2d(x, (2, 2), strides=(1, 1),
818                              padding='same', data_format='channels_last')
819     self.assertEqual(y.shape.as_list(), [10, 10, 10, 3])
820 
821     val = np.random.random((10, 10, 10, 3))
822     x = keras.backend.variable(val)
823     y = keras.backend.pool2d(x, (2, 2), strides=(2, 2),
824                              padding='same', data_format='channels_last')
825     self.assertEqual(y.shape.as_list(), [10, 5, 5, 3])
826 
827     with self.assertRaises(ValueError):
828       y = keras.backend.pool2d(x, (2, 2), strides=(2, 2),
829                                padding='other', data_format='channels_last')
830     with self.assertRaises(ValueError):
831       y = keras.backend.pool2d(x, (2, 2), strides=(2, 2),
832                                data_format='other')
833     with self.assertRaises(ValueError):
834       y = keras.backend.pool2d(x, (2, 2, 2), strides=(2, 2))
835     with self.assertRaises(ValueError):
836       y = keras.backend.pool2d(x, (2, 2), strides=(2, 2, 2))
837     with self.assertRaises(ValueError):
838       y = keras.backend.pool2d(x, (2, 2), strides=(2, 2), pool_mode='other')
839 
840   def test_pool3d(self):
841     if test.is_built_with_rocm():
842       self.skipTest('Pooling with 3D tensors is not supported in ROCm')
843     val = np.random.random((10, 3, 10, 10, 10))
844     x = keras.backend.variable(val)
845     y = keras.backend.pool3d(x, (2, 2, 2), strides=(1, 1, 1),
846                              padding='valid', data_format='channels_first',
847                              pool_mode='max')
848     self.assertEqual(y.shape.as_list(), [10, 3, 9, 9, 9])
849 
850     y = keras.backend.pool3d(x, (2, 2, 2), strides=(1, 1, 1),
851                              padding='valid', data_format='channels_first',
852                              pool_mode='avg')
853     self.assertEqual(y.shape.as_list(), [10, 3, 9, 9, 9])
854 
855     val = np.random.random((10, 10, 10, 10, 3))
856     x = keras.backend.variable(val)
857     y = keras.backend.pool3d(x, (2, 2, 2), strides=(1, 1, 1),
858                              padding='valid', data_format='channels_last')
859     self.assertEqual(y.shape.as_list(), [10, 9, 9, 9, 3])
860 
861     val = np.random.random((10, 10, 10, 10, 3))
862     x = keras.backend.variable(val)
863     y = keras.backend.pool3d(x, (2, 2, 2), strides=(1, 1, 1),
864                              padding='same', data_format='channels_last')
865     self.assertEqual(y.shape.as_list(), [10, 10, 10, 10, 3])
866 
867     val = np.random.random((10, 10, 10, 10, 3))
868     x = keras.backend.variable(val)
869     y = keras.backend.pool3d(x, (2, 2, 2), strides=(2, 2, 2),
870                              padding='same', data_format='channels_last')
871     self.assertEqual(y.shape.as_list(), [10, 5, 5, 5, 3])
872 
873   def test_conv1d(self):
874     val = np.random.random((10, 4, 10))
875     x = keras.backend.variable(val)
876     kernel_val = np.random.random((3, 4, 5))
877     k = keras.backend.variable(kernel_val)
878     y = keras.backend.conv1d(x, k, strides=(1,),
879                              padding='valid', data_format='channels_first')
880     self.assertEqual(y.shape.as_list(), [10, 5, 8])
881 
882     val = np.random.random((10, 10, 4))
883     x = keras.backend.variable(val)
884     y = keras.backend.conv1d(x, k, strides=(1,),
885                              padding='valid', data_format='channels_last')
886     self.assertEqual(y.shape.as_list(), [10, 8, 5])
887 
888     val = np.random.random((10, 10, 4))
889     x = keras.backend.variable(val)
890     y = keras.backend.conv1d(x, k, strides=(1,),
891                              padding='same', data_format='channels_last')
892     self.assertEqual(y.shape.as_list(), [10, 10, 5])
893 
894     val = np.random.random((10, 10, 4))
895     x = keras.backend.variable(val)
896     y = keras.backend.conv1d(x, k, strides=(2,),
897                              padding='same', data_format='channels_last')
898     self.assertEqual(y.shape.as_list(), [10, 5, 5])
899 
900   def test_local_conv_channels_dim(self):
901     filters = 3
902     batch_size = 2
903 
904     for input_shape in [(3, 5), (2, 3, 5), (2, 5, 3, 4)]:
905       channels_in = input_shape[0]
906       input_spatial_shape = input_shape[1:]
907       dim = len(input_spatial_shape)
908 
909       inputs = np.random.normal(0, 1, (batch_size,) + input_shape)
910       inputs_cf = keras.backend.variable(inputs)
911 
912       for kernel_size in [1, 2]:
913         for stride in [1, 2]:
914           kernel_sizes = (kernel_size,) * dim
915           strides = (stride,) * dim
916 
917           output_shape = tuple([(i - kernel_size + stride) // stride
918                                 for i in input_spatial_shape])
919 
920           kernel_shape = (np.prod(output_shape),
921                           np.prod(kernel_sizes) * channels_in,
922                           filters)
923 
924           kernel = np.random.normal(
925               0,
926               1,
927               output_shape + (channels_in, np.prod(kernel_sizes), filters)
928           )
929 
930           kernel_cf = np.reshape(kernel, kernel_shape)
931           kernel_cf = keras.backend.variable(kernel_cf)
932 
933           conv_cf = keras.backend.local_conv(inputs_cf,
934                                              kernel_cf,
935                                              kernel_sizes,
936                                              strides,
937                                              output_shape,
938                                              'channels_first')
939 
940           inputs_cl = np.transpose(inputs, [0, 2] + list(range(3, dim + 2)) +
941                                    [1])
942           inputs_cl = keras.backend.variable(inputs_cl)
943 
944           kernel_cl = np.reshape(
945               np.transpose(kernel, list(range(dim)) + [dim + 1, dim, dim + 2]),
946               kernel_shape
947           )
948           kernel_cl = keras.backend.variable(kernel_cl)
949 
950           conv_cl = keras.backend.local_conv(inputs_cl,
951                                              kernel_cl,
952                                              kernel_sizes,
953                                              strides,
954                                              output_shape,
955                                              'channels_last')
956 
957           conv_cf = keras.backend.eval(conv_cf)
958           conv_cl = keras.backend.eval(conv_cl)
959 
960           self.assertAllCloseAccordingToType(
961               conv_cf,
962               np.transpose(conv_cl,
963                            [0, dim + 1] + list(range(1, dim + 1))),
964               atol=1e-5
965           )
966 
967   @parameterized.named_parameters(
968       ('local_conv1d', (5, 6), (3,), (1,), (3,)),
969       ('local_conv2d', (4, 5, 6), (3, 3), (1, 1), (2, 3)))
970   def test_local_conv_1d_and_2d(self,
971                                 input_shape,
972                                 kernel_sizes,
973                                 strides,
974                                 output_shape):
975     filters = 3
976     batch_size = 2
977 
978     inputs = np.random.normal(0, 1, (batch_size,) + input_shape)
979     inputs = keras.backend.variable(inputs)
980 
981     kernel = np.random.normal(0, 1, (np.prod(output_shape),
982                                      np.prod(kernel_sizes) * input_shape[-1],
983                                      filters))
984     kernel = keras.backend.variable(kernel)
985 
986     local_conv = keras.backend.local_conv(inputs,
987                                           kernel,
988                                           kernel_sizes,
989                                           strides,
990                                           output_shape,
991                                           'channels_last')
992     if len(output_shape) == 1:
993       local_conv_dim = keras.backend.local_conv1d(inputs,
994                                                   kernel,
995                                                   kernel_sizes,
996                                                   strides,
997                                                   'channels_last')
998     else:
999       local_conv_dim = keras.backend.local_conv2d(inputs,
1000                                                   kernel,
1001                                                   kernel_sizes,
1002                                                   strides,
1003                                                   output_shape,
1004                                                   'channels_last')
1005 
1006     local_conv = keras.backend.eval(local_conv)
1007     local_conv_dim = keras.backend.eval(local_conv_dim)
1008 
1009     self.assertAllCloseAccordingToType(local_conv, local_conv_dim)
1010 
1011   def test_conv2d(self):
1012     kernel_val = np.random.random((3, 3, 4, 5))
1013     k = keras.backend.variable(kernel_val)
1014 
1015     # Test channels_first
1016     val = np.random.random((10, 4, 10, 10))
1017     x = keras.backend.variable(val)
1018     y = keras.backend.conv2d(x, k,
1019                              padding='valid', data_format='channels_first')
1020     self.assertEqual(y.shape.as_list(), [10, 5, 8, 8])
1021 
1022     # Test channels_last
1023     val = np.random.random((10, 10, 10, 4))
1024     x = keras.backend.variable(val)
1025     y = keras.backend.conv2d(x, k, strides=(1, 1),
1026                              padding='valid', data_format='channels_last')
1027     self.assertEqual(y.shape.as_list(), [10, 8, 8, 5])
1028 
1029     # Test same padding
1030     val = np.random.random((10, 10, 10, 4))
1031     x = keras.backend.variable(val)
1032     y = keras.backend.conv2d(x, k,
1033                              padding='same', data_format='channels_last')
1034     self.assertEqual(y.shape.as_list(), [10, 10, 10, 5])
1035 
1036     # Test dilation_rate
1037     val = np.random.random((10, 10, 10, 4))
1038     x = keras.backend.variable(val)
1039     y = keras.backend.conv2d(x, k, dilation_rate=(2, 2),
1040                              padding='same', data_format='channels_last')
1041     self.assertEqual(y.shape.as_list(), [10, 10, 10, 5])
1042 
1043     # Test strides
1044     val = np.random.random((10, 10, 10, 4))
1045     x = keras.backend.variable(val)
1046     y = keras.backend.conv2d(x, k, strides=(2, 2),
1047                              padding='same', data_format='channels_last')
1048     self.assertEqual(y.shape.as_list(), [10, 5, 5, 5])
1049 
1050     # Test invalid arguments
1051     with self.assertRaises(ValueError):
1052       y = keras.backend.conv2d(x, k, (2, 2),
1053                                padding='other', data_format='channels_last')
1054     with self.assertRaises(ValueError):
1055       y = keras.backend.conv2d(x, k, (2, 2),
1056                                data_format='other')
1057     with self.assertRaises(ValueError):
1058       y = keras.backend.conv2d(x, k, (2, 2, 2))
1059 
1060   def test_conv2d_transpose(self):
1061     input_size = (7, 8)
1062     kernel_size = (3, 3)
1063     input_depth = 6
1064     filters = 6
1065     batch_size = 2
1066 
1067     kernel_val = np.random.random(kernel_size + (input_depth, filters))
1068     k = keras.backend.variable(kernel_val)
1069 
1070     # Test channels_first
1071     input_val = np.random.random((batch_size, input_depth) + input_size)
1072     x = keras.backend.variable(input_val)
1073     y = keras.backend.conv2d_transpose(x, k, (batch_size, filters) + input_size,
1074                                        padding='same',
1075                                        data_format='channels_first')
1076     self.assertEqual(
1077         tuple(y.shape.as_list()), (batch_size, filters) + input_size)
1078 
1079     # Test channels_last
1080     input_val = np.random.random((batch_size,) + input_size + (input_depth,))
1081     x = keras.backend.variable(input_val)
1082     y = keras.backend.conv2d_transpose(
1083         x, k, (batch_size,) + input_size + (filters,),
1084         padding='same',
1085         data_format='channels_last')
1086     self.assertEqual(
1087         tuple(y.shape.as_list()), (batch_size,) + input_size + (filters,))
1088 
1089     # Test dilation_rate
1090     y = keras.backend.conv2d_transpose(
1091         x, k, (batch_size,) + input_size + (filters,),
1092         padding='same',
1093         data_format='channels_last',
1094         dilation_rate=(2, 2))
1095     self.assertEqual(
1096         tuple(y.shape.as_list()), (batch_size,) + input_size + (filters,))
1097 
1098     # Test batch size of None in output_shape
1099     y = keras.backend.conv2d_transpose(x, k, (None,) + input_size + (filters,),
1100                                        padding='same',
1101                                        data_format='channels_last')
1102     self.assertEqual(
1103         tuple(y.shape.as_list()), (batch_size,) + input_size + (filters,))
1104 
1105     # Test invalid values
1106     with self.assertRaises(ValueError):
1107       y = keras.backend.conv2d_transpose(x, k, (2, 2, 8, 9),
1108                                          padding='other',
1109                                          data_format='channels_last')
1110     with self.assertRaises(ValueError):
1111       y = keras.backend.conv2d_transpose(x, k, (2, 2, 8, 9),
1112                                          data_format='other')
1113 
1114   def test_separable_conv2d(self):
1115     val = np.random.random((10, 4, 10, 10))
1116     x = keras.backend.variable(val)
1117     depthwise_kernel_val = np.random.random((3, 3, 4, 1))
1118     pointwise_kernel_val = np.random.random((1, 1, 4, 5))
1119     dk = keras.backend.variable(depthwise_kernel_val)
1120     pk = keras.backend.variable(pointwise_kernel_val)
1121     y = keras.backend.separable_conv2d(
1122         x, dk, pk, padding='valid', data_format='channels_first')
1123     self.assertEqual(y.shape.as_list(), [10, 5, 8, 8])
1124 
1125     val = np.random.random((10, 10, 10, 4))
1126     x = keras.backend.variable(val)
1127     y = keras.backend.separable_conv2d(
1128         x, dk, pk, strides=(1, 1), padding='valid', data_format='channels_last')
1129     self.assertEqual(y.shape.as_list(), [10, 8, 8, 5])
1130 
1131     val = np.random.random((10, 10, 10, 4))
1132     x = keras.backend.variable(val)
1133     y = keras.backend.separable_conv2d(
1134         x, dk, pk, strides=(1, 1), padding='same', data_format='channels_last')
1135     self.assertEqual(y.shape.as_list(), [10, 10, 10, 5])
1136 
1137     val = np.random.random((10, 10, 10, 4))
1138     x = keras.backend.variable(val)
1139     y = keras.backend.separable_conv2d(
1140         x, dk, pk, strides=(2, 2), padding='same', data_format='channels_last')
1141     self.assertEqual(y.shape.as_list(), [10, 5, 5, 5])
1142     with self.assertRaises(ValueError):
1143       y = keras.backend.separable_conv2d(
1144           x, dk, pk, (2, 2), padding='other', data_format='channels_last')
1145     with self.assertRaises(ValueError):
1146       y = keras.backend.separable_conv2d(
1147           x, dk, pk, (2, 2), data_format='other')
1148     with self.assertRaises(ValueError):
1149       y = keras.backend.separable_conv2d(x, dk, pk, (2, 2, 2))
1150 
1151   def test_conv3d(self):
1152     val = np.random.random((10, 4, 10, 10, 10))
1153     x = keras.backend.variable(val)
1154     kernel_val = np.random.random((3, 3, 3, 4, 5))
1155     k = keras.backend.variable(kernel_val)
1156     y = keras.backend.conv3d(x, k,
1157                              padding='valid', data_format='channels_first')
1158     self.assertEqual(y.shape.as_list(), [10, 5, 8, 8, 8])
1159 
1160     val = np.random.random((10, 10, 10, 10, 4))
1161     x = keras.backend.variable(val)
1162     y = keras.backend.conv3d(x, k, strides=(1, 1, 1),
1163                              padding='valid', data_format='channels_last')
1164     self.assertEqual(y.shape.as_list(), [10, 8, 8, 8, 5])
1165 
1166     val = np.random.random((10, 10, 10, 10, 4))
1167     x = keras.backend.variable(val)
1168     y = keras.backend.conv3d(x, k, strides=(1, 1, 1),
1169                              padding='same', data_format='channels_last')
1170     self.assertEqual(y.shape.as_list(), [10, 10, 10, 10, 5])
1171 
1172     val = np.random.random((10, 10, 10, 10, 4))
1173     x = keras.backend.variable(val)
1174     y = keras.backend.conv3d(x, k, strides=(2, 2, 2),
1175                              padding='same', data_format='channels_last')
1176     self.assertEqual(y.shape.as_list(), [10, 5, 5, 5, 5])
1177     with self.assertRaises(ValueError):
1178       y = keras.backend.conv3d(x, k, (2, 2, 2),
1179                                padding='other', data_format='channels_last')
1180     with self.assertRaises(ValueError):
1181       y = keras.backend.conv3d(x, k, (2, 2, 2),
1182                                data_format='other')
1183     with self.assertRaises(ValueError):
1184       y = keras.backend.conv3d(x, k, (2, 2))
1185 
1186   def test_rnn(self):
1187     # implement a simple RNN
1188     num_samples = 4
1189     input_dim = 5
1190     output_dim = 3
1191     timesteps = 6
1192 
1193     input_val = np.random.random(
1194         (num_samples, timesteps, input_dim)).astype(np.float32)
1195     init_state_val = np.random.random(
1196         (num_samples, output_dim)).astype(np.float32)
1197     w_i_val = np.random.random((input_dim, output_dim)).astype(np.float32)
1198     w_o_val = np.random.random((output_dim, output_dim)).astype(np.float32)
1199     np_mask = np.random.randint(2, size=(num_samples, timesteps))
1200 
1201     def rnn_step_fn():
1202       w_i = keras.backend.variable(w_i_val)
1203       w_o = keras.backend.variable(w_o_val)
1204 
1205       def step_function(x, states):
1206         assert len(states) == 1
1207         prev_output = states[0]
1208         output = keras.backend.dot(x, w_i) + keras.backend.dot(prev_output, w_o)
1209         return output, [output]
1210 
1211       return step_function
1212 
1213     # test default setup
1214     last_output_list = [[], [], [], [], [], []]
1215     outputs_list = [[], [], [], [], [], []]
1216     state_list = [[], [], [], [], [], []]
1217 
1218     rnn_fn = rnn_step_fn()
1219     inputs = keras.backend.variable(input_val)
1220     initial_states = [keras.backend.variable(init_state_val)]
1221     mask = keras.backend.variable(np_mask)
1222 
1223     kwargs_list = [
1224         {'go_backwards': False, 'mask': None},
1225         {'go_backwards': False, 'mask': None, 'unroll': True},
1226         {'go_backwards': True, 'mask': None},
1227         {'go_backwards': True, 'mask': None, 'unroll': True},
1228         {'go_backwards': False, 'mask': mask},
1229         {'go_backwards': False, 'mask': mask, 'unroll': True},
1230     ]
1231     for i, kwargs in enumerate(kwargs_list):
1232       last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
1233                                                            initial_states,
1234                                                            **kwargs)
1235       # check static shape inference
1236       self.assertEqual(last_output.shape.as_list(), [num_samples, output_dim])
1237       self.assertEqual(outputs.shape.as_list(),
1238                        [num_samples, timesteps, output_dim])
1239       for state in new_states:
1240         self.assertEqual(state.shape.as_list(), [num_samples, output_dim])
1241 
1242       last_output_list[i].append(keras.backend.eval(last_output))
1243       outputs_list[i].append(keras.backend.eval(outputs))
1244       self.assertLen(new_states, 1)
1245       state_list[i].append(keras.backend.eval(new_states[0]))
1246 
1247       def assert_list_pairwise(z_list, atol=1e-05):
1248         for (z1, z2) in zip(z_list[1:], z_list[:-1]):
1249           self.assertAllClose(z1, z2, atol=atol)
1250 
1251       assert_list_pairwise(last_output_list[0], atol=1e-04)
1252       assert_list_pairwise(outputs_list[0], atol=1e-04)
1253       assert_list_pairwise(state_list[0], atol=1e-04)
1254       assert_list_pairwise(last_output_list[2], atol=1e-04)
1255       assert_list_pairwise(outputs_list[2], atol=1e-04)
1256       assert_list_pairwise(state_list[2], atol=1e-04)
1257 
1258       for l, u_l in zip(last_output_list[0], last_output_list[1]):
1259         self.assertAllClose(l, u_l, atol=1e-04)
1260 
1261       for o, u_o in zip(outputs_list[0], outputs_list[1]):
1262         self.assertAllClose(o, u_o, atol=1e-04)
1263 
1264       for s, u_s in zip(state_list[0], state_list[1]):
1265         self.assertAllClose(s, u_s, atol=1e-04)
1266 
1267       for b_l, b_u_l in zip(last_output_list[2], last_output_list[3]):
1268         self.assertAllClose(b_l, b_u_l, atol=1e-04)
1269 
1270       for b_o, b_u_o in zip(outputs_list[2], outputs_list[3]):
1271         self.assertAllClose(b_o, b_u_o, atol=1e-04)
1272 
1273       for b_s, b_u_s in zip(state_list[2], state_list[3]):
1274         self.assertAllClose(b_s, b_u_s, atol=1e-04)
1275 
1276   def test_rnn_additional_states(self):
1277     # implement a simple RNN
1278     num_samples = 4
1279     input_dim = 5
1280     output_dim = 3
1281     timesteps = 6
1282 
1283     input_val = np.random.random(
1284         (num_samples, timesteps, input_dim)).astype(np.float32)
1285     init_state_val = np.random.random(
1286         (num_samples, output_dim)).astype(np.float32)
1287     w_i_val = np.random.random((input_dim, output_dim)).astype(np.float32)
1288     w_o_val = np.random.random((output_dim, output_dim)).astype(np.float32)
1289     np_mask = np.random.randint(2, size=(num_samples, timesteps))
1290 
1291     def rnn_step_fn():
1292       w_i = keras.backend.variable(w_i_val)
1293       w_o = keras.backend.variable(w_o_val)
1294 
1295       def step_function(x, states):
1296         assert len(states) == 2
1297         prev_output = states[0]
1298         output = keras.backend.dot(x, w_i) + keras.backend.dot(prev_output, w_o)
1299         return output, [output,
1300                         keras.backend.concatenate([output, output], axis=-1)]
1301 
1302       return step_function
1303 
1304     # test default setup
1305     last_output_list = [[], [], [], [], [], []]
1306     outputs_list = [[], [], [], [], [], []]
1307     state_list = [[], [], [], [], [], []]
1308     additional_state_list = [[], [], [], [], [], []]
1309 
1310     rnn_fn = rnn_step_fn()
1311     inputs = keras.backend.variable(input_val)
1312     initial_states = [
1313         keras.backend.variable(init_state_val),
1314         ops.convert_to_tensor(
1315             np.concatenate([init_state_val, init_state_val], axis=-1))
1316     ]
1317     mask = keras.backend.variable(np_mask)
1318 
1319     kwargs_list = [
1320         {'go_backwards': False, 'mask': None},
1321         {'go_backwards': False, 'mask': None, 'unroll': True},
1322         {'go_backwards': True, 'mask': None},
1323         {'go_backwards': True, 'mask': None, 'unroll': True},
1324         {'go_backwards': False, 'mask': mask},
1325         {'go_backwards': False, 'mask': mask, 'unroll': True},
1326     ]
1327     for i, kwargs in enumerate(kwargs_list):
1328       last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
1329                                                            initial_states,
1330                                                            **kwargs)
1331       # check static shape inference
1332       self.assertEqual(last_output.shape.as_list(), [num_samples, output_dim])
1333       self.assertEqual(outputs.shape.as_list(),
1334                        [num_samples, timesteps, output_dim])
1335       # for state in new_states:
1336       #   self.assertEqual(state.shape.as_list(),
1337       #                     [num_samples, output_dim])
1338       self.assertEqual(new_states[0].shape.as_list(), [num_samples, output_dim])
1339       self.assertEqual(new_states[1].shape.as_list(),
1340                        [num_samples, 2 * output_dim])
1341 
1342       last_output_list[i].append(keras.backend.eval(last_output))
1343       outputs_list[i].append(keras.backend.eval(outputs))
1344       self.assertLen(new_states, 2)
1345       state_list[i].append(keras.backend.eval(new_states[0]))
1346       additional_state_list[i].append(keras.backend.eval(new_states[1]))
1347 
1348       def assert_list_pairwise(z_list, atol=1e-05):
1349         for (z1, z2) in zip(z_list[1:], z_list[:-1]):
1350           self.assertAllClose(z1, z2, atol=atol)
1351 
1352       assert_list_pairwise(last_output_list[0], atol=1e-04)
1353       assert_list_pairwise(outputs_list[0], atol=1e-04)
1354       assert_list_pairwise(state_list[0], atol=1e-04)
1355       assert_list_pairwise(additional_state_list[0], atol=1e-04)
1356       assert_list_pairwise(last_output_list[2], atol=1e-04)
1357       assert_list_pairwise(outputs_list[2], atol=1e-04)
1358       assert_list_pairwise(state_list[2], atol=1e-04)
1359       assert_list_pairwise(additional_state_list[2], atol=1e-04)
1360 
1361       for l, u_l in zip(last_output_list[0], last_output_list[1]):
1362         self.assertAllClose(l, u_l, atol=1e-04)
1363 
1364       for o, u_o in zip(outputs_list[0], outputs_list[1]):
1365         self.assertAllClose(o, u_o, atol=1e-04)
1366 
1367       for s, u_s in zip(state_list[0], state_list[1]):
1368         self.assertAllClose(s, u_s, atol=1e-04)
1369 
1370       for s, u_s in zip(additional_state_list[0], additional_state_list[1]):
1371         self.assertAllClose(s, u_s, atol=1e-04)
1372 
1373       for b_l, b_u_l in zip(last_output_list[2], last_output_list[3]):
1374         self.assertAllClose(b_l, b_u_l, atol=1e-04)
1375 
1376       for b_o, b_u_o in zip(outputs_list[2], outputs_list[3]):
1377         self.assertAllClose(b_o, b_u_o, atol=1e-04)
1378 
1379       for b_s, b_u_s in zip(state_list[2], state_list[3]):
1380         self.assertAllClose(b_s, b_u_s, atol=1e-04)
1381 
1382       for s, u_s in zip(additional_state_list[2], additional_state_list[3]):
1383         self.assertAllClose(s, u_s, atol=1e-04)
1384 
1385   def test_rnn_output_and_state_masking_independent(self):
1386     num_samples = 2
1387     num_timesteps = 4
1388     state_and_io_size = 2
1389     mask_last_num_timesteps = 2  # for second sample only
1390 
1391     # a step function that just outputs inputs,
1392     # but increments states +1 per timestep
1393     def step_function(inputs, states):
1394       return inputs, [s + 1 for s in states]
1395 
1396     inputs_vals = np.random.random((num_samples, num_timesteps,
1397                                     state_and_io_size))
1398     initial_state_vals = np.random.random((num_samples, state_and_io_size))
1399     # masking of two last timesteps for second sample only
1400     mask_vals = np.ones((num_samples, num_timesteps))
1401     mask_vals[1, -mask_last_num_timesteps:] = 0
1402 
1403     # outputs expected to be same as inputs for the first sample
1404     expected_outputs = inputs_vals.copy()
1405     # but for the second sample all outputs in masked region should be the same
1406     # as last output before masked region
1407     expected_outputs[1, -mask_last_num_timesteps:] = \
1408         expected_outputs[1, -(mask_last_num_timesteps + 1)]
1409 
1410     expected_last_state = initial_state_vals.copy()
1411     # first state should be incremented for every timestep (no masking)
1412     expected_last_state[0] += num_timesteps
1413     # second state should not be incremented for last two timesteps
1414     expected_last_state[1] += (num_timesteps - mask_last_num_timesteps)
1415 
1416     # verify same expected output for `unroll=true/false`
1417     inputs = keras.backend.variable(inputs_vals)
1418     initial_states = [keras.backend.variable(initial_state_vals)]
1419     mask = keras.backend.variable(mask_vals)
1420     for unroll in [True, False]:
1421       _, outputs, last_states = keras.backend.rnn(
1422           step_function,
1423           inputs,
1424           initial_states,
1425           mask=mask,
1426           unroll=unroll,
1427           input_length=num_timesteps if unroll else None)
1428 
1429       self.assertAllClose(keras.backend.eval(outputs), expected_outputs)
1430       self.assertAllClose(
1431           keras.backend.eval(last_states[0]), expected_last_state)
1432 
1433   def test_rnn_output_num_dim_larger_than_2_masking(self):
1434     num_samples = 3
1435     num_timesteps = 4
1436     num_features = 5
1437 
1438     def step_function(inputs, states):
1439       outputs = keras.backend.tile(keras.backend.expand_dims(inputs), [1, 1, 2])
1440       return outputs, [keras.backend.identity(s) for s in states]
1441       # Note: cannot just return states (which can be a problem) ->
1442       # tensorflow/python/ops/resource_variable_ops.py", line 824, in set_shape
1443       # NotImplementedError: ResourceVariable does not implement set_shape()
1444 
1445     inputs_vals = np.random.random((num_samples, num_timesteps, num_features))
1446     initial_state_vals = np.random.random((num_samples, 6))
1447     mask_vals = np.ones((num_samples, num_timesteps))
1448     mask_vals[-1, -1] = 0  # final timestep masked for last sample
1449 
1450     expected_outputs = np.repeat(inputs_vals[..., None], repeats=2, axis=-1)
1451     # for the last sample, the final timestep (in masked region) should be the
1452     # same as the second to final output (before masked region)
1453     expected_outputs[-1, -1] = expected_outputs[-1, -2]
1454 
1455     inputs = keras.backend.variable(inputs_vals)
1456     initial_states = [keras.backend.variable(initial_state_vals)]
1457     mask = keras.backend.variable(mask_vals)
1458     for unroll in [True, False]:
1459       _, outputs, _ = keras.backend.rnn(
1460           step_function,
1461           inputs,
1462           initial_states,
1463           mask=mask,
1464           unroll=unroll,
1465           input_length=num_timesteps if unroll else None)
1466 
1467       self.assertAllClose(keras.backend.eval(outputs), expected_outputs)
1468 
1469   def test_rnn_state_num_dim_larger_than_2_masking(self):
1470     num_samples = 3
1471     num_timesteps = 4
1472 
1473     def step_function(inputs, states):
1474       return inputs, [s + 1 for s in states]
1475 
1476     inputs_vals = np.random.random((num_samples, num_timesteps, 5))
1477     initial_state_vals = np.random.random((num_samples, 6, 7))
1478     mask_vals = np.ones((num_samples, num_timesteps))
1479     mask_vals[0, -2:] = 0  # final two timesteps masked for first sample
1480 
1481     expected_last_state = initial_state_vals.copy()
1482     expected_last_state[0] += (num_timesteps - 2)
1483     expected_last_state[1:] += num_timesteps
1484 
1485     inputs = keras.backend.variable(inputs_vals)
1486     initial_states = [keras.backend.variable(initial_state_vals)]
1487     mask = keras.backend.variable(mask_vals)
1488     for unroll in [True, False]:
1489       _, _, last_states = keras.backend.rnn(
1490           step_function,
1491           inputs,
1492           initial_states,
1493           mask=mask,
1494           unroll=unroll,
1495           input_length=num_timesteps if unroll else None)
1496 
1497       self.assertAllClose(
1498           keras.backend.eval(last_states[0]), expected_last_state)
1499 
1500   def test_batch_normalization(self):
1501     g_val = np.random.random((3,))
1502     b_val = np.random.random((3,))
1503     gamma = keras.backend.variable(g_val)
1504     beta = keras.backend.variable(b_val)
1505 
1506     # 3D NHC case
1507     val = np.random.random((10, 5, 3))
1508     x = keras.backend.variable(val)
1509     mean, var = nn.moments(x, (0, 1), None, None, False)
1510     normed = keras.backend.batch_normalization(
1511         x, mean, var, beta, gamma, axis=-1, epsilon=1e-3)
1512     self.assertEqual(normed.shape.as_list(), [10, 5, 3])
1513 
1514     # 4D NHWC case
1515     val = np.random.random((10, 5, 5, 3))
1516     x = keras.backend.variable(val)
1517     mean, var = nn.moments(x, (0, 1, 2), None, None, False)
1518     normed = keras.backend.batch_normalization(
1519         x, mean, var, beta, gamma, axis=-1, epsilon=1e-3)
1520     self.assertEqual(normed.shape.as_list(), [10, 5, 5, 3])
1521 
1522     # 4D NCHW case
1523     if not context.executing_eagerly():
1524       # Eager CPU kernel for NCHW does not exist.
1525       val = np.random.random((10, 3, 5, 5))
1526       x = keras.backend.variable(val)
1527       mean, var = nn.moments(x, (0, 2, 3), None, None, False)
1528       normed = keras.backend.batch_normalization(
1529           x, mean, var, beta, gamma, axis=1, epsilon=1e-3)
1530       self.assertEqual(normed.shape.as_list(), [10, 3, 5, 5])
1531 
1532   def test_normalize_batch_in_training(self):
1533     val = np.random.random((10, 3, 10, 10))
1534     x = keras.backend.variable(val)
1535     reduction_axes = (0, 2, 3)
1536 
1537     g_val = np.random.random((3,))
1538     b_val = np.random.random((3,))
1539     gamma = keras.backend.variable(g_val)
1540     beta = keras.backend.variable(b_val)
1541     normed, mean, var = keras.backend.normalize_batch_in_training(
1542         x, gamma, beta, reduction_axes, epsilon=1e-3)
1543     self.assertEqual(normed.shape.as_list(), [10, 3, 10, 10])
1544     self.assertEqual(mean.shape.as_list(), [
1545         3,
1546     ])
1547     self.assertEqual(var.shape.as_list(), [
1548         3,
1549     ])
1550 
1551     # case: gamma=None
1552     gamma = None
1553     normed, mean, var = keras.backend.normalize_batch_in_training(
1554         x, gamma, beta, reduction_axes, epsilon=1e-3)
1555     self.assertEqual(normed.shape.as_list(), [10, 3, 10, 10])
1556     self.assertEqual(mean.shape.as_list(), [
1557         3,
1558     ])
1559     self.assertEqual(var.shape.as_list(), [
1560         3,
1561     ])
1562 
1563     # case: beta=None
1564     beta = None
1565     normed, mean, var = keras.backend.normalize_batch_in_training(
1566         x, gamma, beta, reduction_axes, epsilon=1e-3)
1567     self.assertEqual(normed.shape.as_list(), [10, 3, 10, 10])
1568     self.assertEqual(mean.shape.as_list(), [
1569         3,
1570     ])
1571     self.assertEqual(var.shape.as_list(), [
1572         3,
1573     ])
1574 
1575   def test_dropout(self):
1576     inputs = array_ops.ones((200, 200))
1577     outputs = keras.backend.dropout(inputs, 0.2)
1578     outputs_val = keras.backend.eval(outputs)
1579     self.assertEqual(np.min(outputs_val), 0)
1580     self.assertAllClose(np.count_nonzero(outputs_val), 32000, atol=1000)
1581     # Test noise shape
1582     outputs = keras.backend.dropout(inputs, 0.2, noise_shape=(200, 1))
1583     outputs_val = keras.backend.eval(outputs)
1584     self.assertAllClose(outputs_val[2, :], outputs_val[3, :], atol=1e-5)
1585 
1586 
1587 class BackendCrossEntropyLossesTest(test.TestCase):
1588 
1589   @test_util.run_in_graph_and_eager_modes
1590   def test_binary_crossentropy_with_sigmoid(self):
1591     t = keras.backend.constant([[0, 1, 0]])
1592     logits = keras.backend.constant([[8., 1., 1.]])
1593     p = keras.backend.sigmoid(logits)
1594     p = array_ops.identity(array_ops.identity(p))
1595     result = self.evaluate(keras.backend.binary_crossentropy(t, p))
1596     self.assertArrayNear(result[0], [8., 0.313, 1.313], 1e-3)
1597 
1598   @test_util.run_in_graph_and_eager_modes
1599   def test_categorical_crossentropy_loss(self):
1600     t = keras.backend.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
1601 
1602     p = keras.backend.constant([[.9, .05, .05], [.05, .89, .06],
1603                                 [.05, .01, .94]])
1604     result = keras.backend.categorical_crossentropy(t, p)
1605     self.assertArrayNear(self.evaluate(result), [.105, .116, .062], 1e-3)
1606 
1607     p = keras.backend.constant([[.9, .05, .05], [.05, .89, .01],
1608                                 [.05, .06, .94]])
1609     result = keras.backend.categorical_crossentropy(t, p, axis=0)
1610     self.assertArrayNear(self.evaluate(result), [.105, .116, .062], 1e-3)
1611 
1612     p = keras.backend.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
1613     result = keras.backend.categorical_crossentropy(t, p, from_logits=True),
1614     self.assertArrayNear(self.evaluate(result)[0], [.002, 0, .17], 1e-3)
1615 
1616     p = keras.backend.constant([[8., 0., 2.], [1., 9., 3.], [1., 1., 5.]])
1617     result = keras.backend.categorical_crossentropy(
1618         t, p, from_logits=True, axis=0),
1619     self.assertArrayNear(self.evaluate(result)[0], [.002, 0, .17], 1e-3)
1620 
1621   @test_util.run_in_graph_and_eager_modes
1622   def test_categorical_crossentropy_loss_with_unknown_rank_tensor(self):
1623     t = keras.backend.placeholder()
1624     p = keras.backend.placeholder()
1625     o = keras.backend.categorical_crossentropy(t, p)
1626 
1627     t_val = ops.convert_to_tensor([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]])
1628     p_val = ops.convert_to_tensor([[.9, .05, .05], [.05, .89, .06],
1629                                    [.05, .01, .94]])
1630     f = keras.backend.function([t, p], o)
1631 
1632     result = f([t_val, p_val])
1633     self.assertArrayNear(result, [.105, .116, .062], 1e-3)
1634 
1635     # With axis set
1636     o = keras.backend.categorical_crossentropy(t, p, axis=0)
1637     f = keras.backend.function([t, p], o)
1638 
1639     result = f([t_val, p_val])
1640     self.assertArrayNear(result, [.105, .065, .111], 1e-3)
1641 
1642     # from logits
1643     p_val = ops.convert_to_tensor([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
1644     o = keras.backend.categorical_crossentropy(t, p, from_logits=True)
1645     f = keras.backend.function([t, p], o)
1646 
1647     result = f([t_val, p_val])
1648     self.assertArrayNear(result, [.002, 0, .17], 1e-3)
1649 
1650     # from logits and axis set
1651     o = keras.backend.categorical_crossentropy(t, p, from_logits=True, axis=0)
1652     f = keras.backend.function([t, p], o)
1653 
1654     result = f([t_val, p_val])
1655     self.assertArrayNear(result, [.002, .003, .036], 1e-3)
1656 
1657   @test_util.run_in_graph_and_eager_modes
1658   def test_categorical_crossentropy_with_softmax(self):
1659     t = keras.backend.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
1660     logits = keras.backend.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
1661     p = keras.backend.softmax(logits)
1662     p = array_ops.identity(array_ops.identity(p))
1663     result = self.evaluate(keras.backend.categorical_crossentropy(t, p))
1664     self.assertArrayNear(result, [0.002, 0.0005, 0.17], 1e-3)
1665 
1666   @test_util.run_in_graph_and_eager_modes
1667   def test_sparse_categorical_crossentropy_loss(self):
1668     t = keras.backend.constant([0, 1, 2])
1669 
1670     p = keras.backend.constant([[.9, .05, .05], [.05, .89, .06],
1671                                 [.05, .01, .94]])
1672     result = keras.backend.sparse_categorical_crossentropy(t, p)
1673     self.assertArrayNear(self.evaluate(result), [.105, .116, .062], 1e-3)
1674 
1675     p = keras.backend.constant([[.9, .05, .05], [.05, .89, .01],
1676                                 [.05, .06, .94]])
1677     result = keras.backend.sparse_categorical_crossentropy(t, p, axis=0)
1678     self.assertArrayNear(self.evaluate(result), [.105, .116, .062], 1e-3)
1679 
1680     p = keras.backend.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
1681     result = keras.backend.sparse_categorical_crossentropy(
1682         t, p, from_logits=True),
1683     self.assertArrayNear(self.evaluate(result)[0], [.002, 0, .17], 1e-3)
1684 
1685     p = keras.backend.constant([[8., 0., 2.], [1., 9., 3.], [1., 1., 5.]])
1686     result = keras.backend.sparse_categorical_crossentropy(
1687         t, p, from_logits=True, axis=0),
1688     self.assertArrayNear(self.evaluate(result)[0], [.002, 0, .17], 1e-3)
1689 
1690   @test_util.run_in_graph_and_eager_modes
1691   def test_sparse_categorical_crossentropy_loss_with_unknown_rank_tensor(self):
1692     t = keras.backend.placeholder()
1693     p = keras.backend.placeholder()
1694     o = keras.backend.sparse_categorical_crossentropy(t, p)
1695 
1696     t_val = ops.convert_to_tensor([0, 1, 2])
1697     p_val = ops.convert_to_tensor([[.9, .05, .05], [.05, .89, .06],
1698                                    [.05, .01, .94]])
1699     f = keras.backend.function([t, p], o)
1700 
1701     result = f([t_val, p_val])
1702     self.assertArrayNear(result, [.105, .116, .062], 1e-3)
1703 
1704     # With axis set
1705     with self.assertRaisesRegex(
1706         ValueError,
1707         'Cannot compute sparse categorical crossentropy with `axis=0`'):
1708       o = keras.backend.sparse_categorical_crossentropy(t, p, axis=0)
1709       f = keras.backend.function([t, p], o)
1710 
1711       _ = f([t_val, p_val])
1712 
1713     # from logits
1714     p_val = ops.convert_to_tensor([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
1715     o = keras.backend.sparse_categorical_crossentropy(t, p, from_logits=True)
1716     f = keras.backend.function([t, p], o)
1717 
1718     result = f([t_val, p_val])
1719     self.assertArrayNear(result, [.002, 0, .17], 1e-3)
1720 
1721     # from logits and axis set
1722     with self.assertRaisesRegex(
1723         ValueError,
1724         'Cannot compute sparse categorical crossentropy with `axis=0`'):
1725       o = keras.backend.sparse_categorical_crossentropy(
1726           t, p, from_logits=True, axis=0)
1727       f = keras.backend.function([t, p], o)
1728 
1729       _ = f([t_val, p_val])
1730 
1731   @test_util.run_in_graph_and_eager_modes
1732   def test_sparse_categorical_crossentropy_with_softmax(self):
1733     t = keras.backend.constant([0, 1, 2])
1734     logits = keras.backend.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
1735     p = keras.backend.softmax(logits)
1736     p = array_ops.identity(array_ops.identity(p))
1737     result = self.evaluate(keras.backend.sparse_categorical_crossentropy(t, p))
1738     self.assertArrayNear(result, [0.002, 0.0005, 0.17], 1e-3)
1739 
1740 
1741 @test_util.run_all_in_graph_and_eager_modes
1742 @test_util.with_control_flow_v2
1743 class TestCTC(test.TestCase):
1744 
1745   def test_ctc_decode(self):
1746     depth = 6
1747     seq_len_0 = 5
1748     input_prob_matrix_0 = np.asarray(
1749         [[0.30999, 0.309938, 0.0679938, 0.0673362, 0.0708352, 0.173908],
1750          [0.215136, 0.439699, 0.0370931, 0.0393967, 0.0381581, 0.230517],
1751          [0.199959, 0.489485, 0.0233221, 0.0251417, 0.0233289, 0.238763],
1752          [0.279611, 0.452966, 0.0204795, 0.0209126, 0.0194803, 0.20655],
1753          [0.51286, 0.288951, 0.0243026, 0.0220788, 0.0219297, 0.129878],
1754          # Random entry added in at time=5
1755          [0.155251, 0.164444, 0.173517, 0.176138, 0.169979, 0.160671]],
1756         dtype=np.float32)
1757 
1758     # len max_time_steps array of batch_size x depth matrices
1759     inputs = ([input_prob_matrix_0[t, :][np.newaxis, :]
1760                for t in range(seq_len_0)] +  # Pad to max_time_steps = 8
1761               2 * [np.zeros((1, depth), dtype=np.float32)])
1762 
1763     inputs = keras.backend.variable(np.asarray(inputs).transpose((1, 0, 2)))
1764 
1765     # batch_size length vector of sequence_lengths
1766     input_length = keras.backend.variable(
1767         np.array([seq_len_0], dtype=np.int32))
1768     # batch_size length vector of negative log probabilities
1769     log_prob_truth = np.array([
1770         -3.5821197,  # output beam 0
1771         -3.777835    # output beam 1
1772     ], np.float32)[np.newaxis, :]
1773 
1774     decode_truth = [np.array([1, 0]), np.array([0, 1, 0])]
1775     beam_width = 2
1776     top_paths = 2
1777 
1778     decode_pred_tf, log_prob_pred_tf = keras.backend.ctc_decode(
1779         inputs,
1780         input_length,
1781         greedy=False,
1782         beam_width=beam_width,
1783         top_paths=top_paths)
1784 
1785     self.assertEqual(len(decode_pred_tf), top_paths)
1786     log_prob_pred = keras.backend.eval(log_prob_pred_tf)
1787     for i in range(top_paths):
1788       self.assertTrue(
1789           np.alltrue(
1790               decode_truth[i] == keras.backend.eval(decode_pred_tf[i])))
1791     self.assertAllClose(log_prob_truth, log_prob_pred)
1792 
1793   def test_ctc_batch_cost(self):
1794     with self.cached_session():
1795       label_lens = np.expand_dims(np.asarray([5, 4]), 1)
1796       input_lens = np.expand_dims(np.asarray([5, 5]), 1)  # number of timesteps
1797       loss_log_probs = [3.34211, 5.42262]
1798 
1799       # dimensions are batch x time x categories
1800       labels = np.asarray([[0, 1, 2, 1, 0], [0, 1, 1, 0, -1]])
1801       inputs = np.asarray(
1802           [[[0.633766, 0.221185, 0.0917319, 0.0129757, 0.0142857, 0.0260553],
1803             [0.111121, 0.588392, 0.278779, 0.0055756, 0.00569609, 0.010436],
1804             [0.0357786, 0.633813, 0.321418, 0.00249248, 0.00272882, 0.0037688],
1805             [0.0663296, 0.643849, 0.280111, 0.00283995, 0.0035545, 0.00331533],
1806             [0.458235, 0.396634, 0.123377, 0.00648837, 0.00903441, 0.00623107]],
1807            [[0.30176, 0.28562, 0.0831517, 0.0862751, 0.0816851, 0.161508],
1808             [0.24082, 0.397533, 0.0557226, 0.0546814, 0.0557528, 0.19549],
1809             [0.230246, 0.450868, 0.0389607, 0.038309, 0.0391602, 0.202456],
1810             [0.280884, 0.429522, 0.0326593, 0.0339046, 0.0326856, 0.190345],
1811             [0.423286, 0.315517, 0.0338439, 0.0393744, 0.0339315, 0.154046]]],
1812           dtype=np.float32)
1813 
1814       labels = keras.backend.variable(labels, dtype='int32')
1815       inputs = keras.backend.variable(inputs, dtype='float32')
1816       input_lens = keras.backend.variable(input_lens, dtype='int32')
1817       label_lens = keras.backend.variable(label_lens, dtype='int32')
1818       res = keras.backend.eval(
1819           keras.backend.ctc_batch_cost(labels, inputs, input_lens, label_lens))
1820       self.assertAllClose(res[:, 0], loss_log_probs, atol=1e-05)
1821 
1822       # test when batch_size = 1, that is, one sample only
1823       ref = [3.34211]
1824       input_lens = np.expand_dims(np.asarray([5]), 1)
1825       label_lens = np.expand_dims(np.asarray([5]), 1)
1826 
1827       labels = np.asarray([[0, 1, 2, 1, 0]])
1828       inputs = np.asarray(
1829           [[[0.633766, 0.221185, 0.0917319, 0.0129757, 0.0142857, 0.0260553], [
1830               0.111121, 0.588392, 0.278779, 0.0055756, 0.00569609, 0.010436
1831           ], [0.0357786, 0.633813, 0.321418, 0.00249248, 0.00272882, 0.0037688],
1832             [0.0663296, 0.643849, 0.280111, 0.00283995, 0.0035545, 0.00331533],
1833             [0.458235, 0.396634, 0.123377, 0.00648837, 0.00903441, 0.00623107]]
1834           ],
1835           dtype=np.float32)
1836 
1837       k_labels = keras.backend.variable(labels, dtype='int32')
1838       k_inputs = keras.backend.variable(inputs, dtype='float32')
1839       k_input_lens = keras.backend.variable(input_lens, dtype='int32')
1840       k_label_lens = keras.backend.variable(label_lens, dtype='int32')
1841       res = keras.backend.eval(
1842           keras.backend.ctc_batch_cost(k_labels, k_inputs, k_input_lens,
1843                                        k_label_lens))
1844       self.assertAllClose(res[:, 0], ref, atol=1e-05)
1845 
1846 
1847 @test_util.run_all_in_graph_and_eager_modes
1848 class TestRandomOps(test.TestCase):
1849 
1850   def test_random_normal(self):
1851     np.random.seed(123)
1852     x = keras.backend.random_normal((500, 500))
1853     val = keras.backend.eval(x)
1854     self.assertAllClose(np.mean(val), 0., atol=0.01)
1855     self.assertAllClose(np.std(val), 1., atol=0.01)
1856 
1857   def test_random_uniform(self):
1858     np.random.seed(123)
1859     x = keras.backend.random_uniform((500, 500))
1860     val = keras.backend.eval(x)
1861     self.assertAllClose(np.mean(val), 0.5, atol=0.01)
1862     self.assertAllClose(np.max(val), 1., atol=0.01)
1863     self.assertAllClose(np.min(val), 0., atol=0.01)
1864 
1865   def test_random_binomial(self):
1866     np.random.seed(123)
1867     x = keras.backend.random_binomial((500, 500), p=0.5)
1868     self.assertAllClose(np.mean(keras.backend.eval(x)), 0.5, atol=0.01)
1869 
1870   def test_truncated_normal(self):
1871     np.random.seed(123)
1872     x = keras.backend.truncated_normal((500, 500), mean=0.0, stddev=1.0)
1873     x = keras.backend.truncated_normal((1000, 1000), mean=0.0, stddev=1.0)
1874     y = keras.backend.eval(x)
1875     self.assertAllClose(np.mean(y), 0., atol=0.01)
1876     self.assertAllClose(np.std(y), 0.88, atol=0.01)
1877     self.assertAllClose(np.max(y), 2., atol=0.01)
1878     self.assertAllClose(np.min(y), -2., atol=0.01)
1879 
1880 
1881 @test_util.run_all_in_graph_and_eager_modes
1882 class FunctionTest(test.TestCase):
1883 
1884   def test_function_basics(self):
1885     x1 = keras.backend.placeholder(shape=(), dtype='float32')
1886     x2 = keras.backend.placeholder(shape=(), dtype='int32')
1887     v = keras.backend.variable(10.)
1888 
1889     y1 = x1 + keras.backend.cast(x2, 'float32') + v
1890     y2 = x1 * keras.backend.cast(x2, 'float32')
1891 
1892     with ops.control_dependencies([y1]):
1893       u = keras.backend.update(v, x1)
1894 
1895     f = keras.backend.function([x1, x2], [y1, y2], updates=[u])
1896     output_values = f([2, 3])
1897     self.assertEqual(output_values, [15., 6.])
1898     self.assertEqual(keras.backend.eval(v), 2.)
1899 
1900   def test_function_dict_outputs(self):
1901     x_ph = keras.backend.placeholder(shape=(), name='x')
1902     y_ph = keras.backend.placeholder(shape=(), name='y')
1903     outputs = {'x*y': y_ph * x_ph, 'x*x': x_ph * x_ph}
1904 
1905     f = keras.backend.function(inputs=[x_ph, y_ph], outputs=outputs)
1906     x, y = 2., 5.
1907     results = f([x, y])
1908 
1909     self.assertEqual(results['x*y'], 10.)
1910     self.assertEqual(results['x*x'], 4)
1911 
1912   def test_function_dict_inputs(self):
1913     placeholders = {
1914         'x': keras.backend.placeholder(shape=()),
1915         'y': keras.backend.placeholder(shape=())
1916     }
1917     outputs = [placeholders['x'] * placeholders['y']]
1918 
1919     f = keras.backend.function(inputs=placeholders, outputs=outputs)
1920     results = f({'x': 2., 'y': 3.})
1921     self.assertEqual(results[0], 6.)
1922 
1923   def test_function_single_input_output(self):
1924     x_ph = keras.backend.placeholder(shape=(), name='x')
1925     output = x_ph * x_ph
1926     f = keras.backend.function(x_ph, output)
1927     result = f(2.)
1928     self.assertEqual(result, 4.)
1929 
1930   def test_tuple_updates(self):
1931     x_ph = keras.backend.placeholder(ndim=2)
1932     v = keras.backend.variable(np.ones((4, 2)))
1933     output = x_ph ** 2 + v
1934     new_v = v + x_ph
1935     f = keras.backend.function(x_ph, output, updates=[(v, new_v)])
1936     input_val = np.random.random((4, 2))
1937     result = f(input_val)
1938     self.assertAllClose(result, input_val ** 2 + 1)
1939     self.assertAllClose(keras.backend.get_value(v), np.ones((4, 2)) + input_val)
1940 
1941 
1942 class BackendGraphTests(test.TestCase):
1943 
1944   @test_util.run_in_graph_and_eager_modes
1945   def test_function_placeholder_with_default(self):
1946     with keras.backend.get_graph().as_default():
1947       x1 = array_ops.placeholder_with_default(
1948           np.array(2., dtype='float32'), shape=())
1949       x2 = array_ops.placeholder_with_default(
1950           np.array(3, dtype='int32'), shape=())
1951     y1 = x1 + keras.backend.cast(x2, 'float32')
1952     y2 = x1 * keras.backend.cast(x2, 'float32')
1953     f = keras.backend.function([x1, x2], [y1, y2])
1954     output_values = f([4, 5])
1955     self.assertEqual(output_values, [9., 20.])
1956     output_values = f([None, None])
1957     self.assertEqual(output_values, [5., 6.])
1958 
1959   @test_util.run_deprecated_v1
1960   def test_function_tf_feed_symbols(self):
1961     # Test Keras backend functions with TF tensor inputs.
1962     with self.cached_session():
1963       # Test feeding a resource variable to `function`.
1964       x1 = keras.backend.placeholder(shape=())
1965       x2 = keras.backend.placeholder(shape=())
1966       lr = keras.backend.learning_phase()  # Include a placeholder_with_default.
1967 
1968       y1 = keras.backend.variable(10.)
1969       y2 = 3
1970 
1971       f = keras.backend.function(
1972           inputs=[x1, x2, lr],
1973           outputs=[x1 + 1, keras.backend.in_train_phase(x2 + 2, x2 - 1)])
1974       outs = f([y1, y2, None])  # Use default learning_phase value.
1975       self.assertEqual(outs, [11., 2.])
1976       outs = f([y1, y2, 1])  # Set learning phase value.
1977       self.assertEqual(outs, [11., 5.])
1978 
1979       # Test triggering a callable refresh by changing the input.
1980       y3 = keras.backend.constant(20.)  # Test with tensor
1981       outs = f([y3, y2, None])
1982       self.assertEqual(outs, [21., 2.])
1983 
1984       y4 = 4  # Test with non-symbol
1985       outs = f([y4, y2, None])
1986       self.assertEqual(outs, [5., 2.])
1987 
1988       # Test with a different dtype
1989       y5 = keras.backend.constant(10., dtype='float64')
1990       outs = f([y5, y2, None])
1991       self.assertEqual(outs, [11., 2.])
1992 
1993   @test_util.run_deprecated_v1
1994   def test_function_tf_fetches(self):
1995     # Additional operations can be passed to tf.compat.v1.Session().run() via
1996     # its `fetches` arguments. In contrast to `updates` argument of
1997     # keras.backend.function() these do not have control dependency on `outputs`
1998     # so they can run in parallel. Also they should not contribute to output of
1999     # keras.backend.function().
2000     with self.cached_session():
2001       x = keras.backend.variable(0.)
2002       y = keras.backend.variable(0.)
2003       x_placeholder = keras.backend.placeholder(shape=())
2004       y_placeholder = keras.backend.placeholder(shape=())
2005 
2006       f = keras.backend.function(
2007           inputs=[x_placeholder, y_placeholder],
2008           outputs=[x_placeholder + y_placeholder],
2009           updates=[(x, x_placeholder + 1.)],
2010           fetches=[keras.backend.update(y, 5.)])
2011       output = f([10., 20.])
2012       self.assertEqual(output, [30.])
2013       self.assertEqual(keras.backend.get_session().run(fetches=[x, y]),
2014                        [11., 5.])
2015 
2016   @test_util.run_deprecated_v1
2017   def test_function_tf_feed_dict(self):
2018     # Additional substitutions can be passed to `tf.compat.v1.Session().run()`
2019     # via its `feed_dict` arguments. Note that the feed_dict is passed once in
2020     # the constructor but we can modify the values in the dictionary. Through
2021     # this feed_dict we can provide additional substitutions besides Keras
2022     # inputs.
2023     with self.cached_session():
2024       x = keras.backend.variable(0.)
2025       y = keras.backend.variable(0.)
2026       x_placeholder = keras.backend.placeholder(shape=())
2027       y_placeholder = keras.backend.placeholder(shape=())
2028 
2029       feed_dict = {y_placeholder: 3.}
2030       fetches = [keras.backend.update(y, y_placeholder * 10.)]
2031       f = keras.backend.function(
2032           inputs=[x_placeholder],
2033           outputs=[x_placeholder + 1.],
2034           updates=[(x, x_placeholder + 10.)],
2035           feed_dict=feed_dict,
2036           fetches=fetches)
2037       output = f([10.])
2038       self.assertEqual(output, [11.])
2039       self.assertEqual(keras.backend.get_session().run(fetches=[x, y]),
2040                        [20., 30.])
2041 
2042       # updated value in feed_dict will be modified within the K.function()
2043       feed_dict[y_placeholder] = 4.
2044       output = f([20.])
2045       self.assertEqual(output, [21.])
2046       self.assertEqual(keras.backend.get_session().run(fetches=[x, y]),
2047                        [30., 40.])
2048 
2049   @test_util.run_deprecated_v1
2050   def test_function_tf_run_options_with_run_metadata(self):
2051     with self.cached_session():
2052       x_placeholder = keras.backend.placeholder(shape=())
2053       y_placeholder = keras.backend.placeholder(shape=())
2054 
2055       run_options = config_pb2.RunOptions(output_partition_graphs=True)
2056       run_metadata = config_pb2.RunMetadata()
2057       # enable run_options.
2058       f = keras.backend.function(
2059           inputs=[x_placeholder, y_placeholder],
2060           outputs=[x_placeholder + y_placeholder],
2061           options=run_options,
2062           run_metadata=run_metadata)
2063       output = f([10., 20.])
2064       self.assertEqual(output, [30.])
2065       self.assertGreater(len(run_metadata.partition_graphs), 0)
2066       # disable run_options.
2067       f1 = keras.backend.function(
2068           inputs=[x_placeholder, y_placeholder],
2069           outputs=[x_placeholder + y_placeholder],
2070           run_metadata=run_metadata)
2071       output1 = f1([10., 20.])
2072       self.assertEqual(output1, [30.])
2073       self.assertEqual(len(run_metadata.partition_graphs), 0)
2074 
2075   @test_util.run_deprecated_v1
2076   def test_function_fetch_callbacks(self):
2077 
2078     class CallbackStub(object):
2079 
2080       def __init__(self):
2081         self.times_called = 0
2082         self.callback_result = 0
2083 
2084       def _fetch_callback(self, result):
2085         self.times_called += 1
2086         self.callback_result = result
2087 
2088     with self.cached_session():
2089       callback = CallbackStub()
2090       x_placeholder = keras.backend.placeholder(shape=())
2091       y_placeholder = keras.backend.placeholder(shape=())
2092 
2093       callback_op = x_placeholder * y_placeholder
2094 
2095       f = keras.backend.function(
2096           inputs=[x_placeholder, y_placeholder],
2097           outputs=[x_placeholder + y_placeholder])
2098       f.fetches.append(callback_op)
2099       f.fetch_callbacks[callback_op] = callback._fetch_callback
2100 
2101       _ = f([10., 20.])
2102 
2103       self.assertEqual(callback.times_called, 1)
2104       self.assertEqual(callback.callback_result, 200)
2105 
2106   def test_get_session_different_graphs(self):
2107     with ops.Graph().as_default():
2108       x = keras.backend.constant(1)
2109       session = keras.backend.get_session()
2110       self.assertIs(session, keras.backend.get_session((x,)))
2111       self.assertIs(session, keras.backend.get_session())
2112     with ops.Graph().as_default():
2113       self.assertIs(session, keras.backend.get_session((x,)))
2114       self.assertIsNot(session, keras.backend.get_session())
2115 
2116 
2117 @test_util.run_all_in_graph_and_eager_modes
2118 class ControlOpsTests(test.TestCase):
2119 
2120   def test_function_switch_basics(self):
2121     x = array_ops.constant(2.0)
2122     y = array_ops.constant(3.0)
2123 
2124     def xpowy():
2125       return keras.backend.pow(x, y)
2126 
2127     def ypowx():
2128       return keras.backend.pow(y, x)
2129 
2130     tensor = keras.backend.switch(keras.backend.less(x, y), xpowy, ypowx)
2131     self.assertEqual(keras.backend.eval(tensor), [8.0])
2132 
2133     tensor = keras.backend.switch(keras.backend.greater(x, y), xpowy, ypowx)
2134     self.assertEqual(keras.backend.eval(tensor), [9.0])
2135 
2136   def test_unequal_rank(self):
2137     x = ops.convert_to_tensor(np.array([[1, 2, 3], [4, 5, 6]]), dtype='float32')
2138     y = ops.convert_to_tensor(np.array([1, 2, 3]), dtype='float32')
2139 
2140     def true_func():
2141       return x
2142 
2143     def false_func():
2144       return y
2145 
2146     with self.assertRaisesRegexp(ValueError,
2147                                  'Rank of `condition` should be less than'):
2148       keras.backend.switch(keras.backend.equal(x, x), false_func, true_func)
2149 
2150 
2151 if __name__ == '__main__':
2152   test.main()
2153