1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for tensorflow.ops.math_ops.matmul.""" 16 17import operator 18 19import numpy as np 20 21from tensorflow.python import tf2 22from tensorflow.python.framework import constant_op 23from tensorflow.python.framework import dtypes 24from tensorflow.python.framework import ops 25from tensorflow.python.framework import test_util 26from tensorflow.python.ops import array_ops 27from tensorflow.python.ops import gradient_checker_v2 28from tensorflow.python.ops import math_ops 29from tensorflow.python.ops import random_ops 30from tensorflow.python.ops import variables 31from tensorflow.python.platform import test as test_lib 32 33# TODO(yangzihao): Currently matmul autotuning is disabled by default. Use 34# os.environ["TF_MATMUL_AUTOTUNE_ENABLE"] = "1" to enable it. 35 36 37@test_util.with_eager_op_as_function 38class MatMulMixedType(test_lib.TestCase): 39 """Simple test for tf.matmul where Tout is different from T.""" 40 41 def testBatchMatMulV3OutputType(self): 42 # TODO(shivaniagrawal): uint8 is not supported for mixed matmul type in XLA. 43 for (a_dtype, b_dtype) in [(np.int8, np.int8), (np.uint8, np.uint8)]: 44 a = np.array([[1, 2], [3, 4]], dtype=a_dtype) 45 b = np.array([[1, 2], [3, 4]], dtype=b_dtype) 46 c = math_ops.batch_mat_mul_v3(a, b, adj_y=True, Tout=np.int32) 47 self.assertAllEqual((2, 2), c.shape) 48 self.assertAllEqual([[5, 11], [11, 25]], c) 49 50 def testBatchMatMulV3MixedPrec(self): 51 # TODO(shivaniagrawal): uint8 is not supported for mixed matmul type in XLA. 52 np_bf16 = dtypes.bfloat16.as_numpy_dtype 53 a = np.array([[1, 2], [3, 4]], dtype=np.int8) 54 b = np.array([[1, 2], [3, 4]], dtype=np_bf16) 55 c = math_ops.batch_mat_mul_v3(a, b, adj_y=True, Tout=np_bf16) 56 self.assertAllEqual((2, 2), c.shape) 57 self.assertAllEqual([[5, 11], [11, 25]], c) 58 59 60@test_util.with_eager_op_as_function 61class MatVecTest(test_lib.TestCase): 62 """Simple test for matvec, which is sugar on top of matmul.""" 63 64 def testTwoByTwoCase(self): 65 a = np.array([[1, 2], [3, 4]]) 66 b = np.array([5, 6]) 67 c = math_ops.matvec(a, b) 68 self.assertAllEqual((2,), c.shape) 69 self.assertAllEqual([5 + 2 * 6, 3 * 5 + 4 * 6], c) 70 71 def testEmpty(self): 72 full = np.array([[1., 2.], [3., 4.], [5., 6.]]) 73 empty = np.empty([3, 0]) 74 self.assertShapeEqual( 75 np.matmul(full.T, empty), math_ops.matmul(full, empty, adjoint_a=True)) 76 self.assertShapeEqual( 77 np.matmul(empty.T, full), math_ops.matmul(empty, full, adjoint_a=True)) 78 79 80def _AddTest(test, op_name, testcase_name, fn): 81 test_name = "_".join(["test", op_name, testcase_name]) 82 if hasattr(test, test_name): 83 raise RuntimeError("Test %s defined more than once" % test_name) 84 setattr(test, test_name, test_util.deprecated_graph_mode_only(fn)) 85 86 87def _GetTransposedMatrices(x, x_name, kwargs): 88 if kwargs["transpose_" + x_name] is True: 89 return x.T 90 elif kwargs["adjoint_" + x_name] is True: 91 return np.conj(x.T) 92 else: 93 return x 94 95 96@test_util.with_eager_op_as_function 97class MatMulTest(test_lib.TestCase): 98 pass # Filled in below 99 100 101def _GetMatMulTest(a_np_, b_np_, use_static_shape_, **kwargs_): 102 103 @test_util.run_without_tensor_float_32("Tests matmul") 104 def Test(self): 105 np_val = np.matrix(a_np_) * np.matrix(b_np_) 106 107 use_gpu = True 108 if a_np_.dtype is np.float16 and ( 109 not test_util.GpuSupportsHalfMatMulAndConv()): 110 use_gpu = False 111 print("Built without fp16 matmul support for Cuda, running test on CPU.") 112 113 # Transpose and possibly conjugate a_np_ and b_np_ according to the 114 # attributes such that tf.matmul(effective_a_np, effective_b_np, **kwargs) 115 # results in a valid matrix multiplication and produces the same result as 116 # np.matrix(a_np_) * np.matrix(b_np_) 117 effective_a_np = _GetTransposedMatrices(a_np_, "a", kwargs_) 118 effective_b_np = _GetTransposedMatrices(b_np_, "b", kwargs_) 119 with self.cached_session() as sess, test_util.device(use_gpu): 120 if use_static_shape_: 121 a = constant_op.constant(effective_a_np) 122 b = constant_op.constant(effective_b_np) 123 res = math_ops.matmul(a, b, **kwargs_) 124 tf_val = self.evaluate(res) 125 else: 126 a = array_ops.placeholder(a_np_.dtype) 127 b = array_ops.placeholder(b_np_.dtype) 128 res = math_ops.matmul(a, b, **kwargs_) 129 tf_val = sess.run(res, feed_dict={a: effective_a_np, b: effective_b_np}) 130 131 self.assertAllCloseAccordingToType( 132 tf_val, 133 np_val, 134 float_rtol=3e-5, 135 float_atol=3e-5, 136 half_rtol=0.2, 137 half_atol=0.2) 138 139 return Test 140 141 142@test_util.with_eager_op_as_function 143class MatMulGradientTest(test_lib.TestCase): 144 pass # Will be filled in below. 145 146 147def _GetMatMulGradientTest(a_np_, b_np_, use_static_shape_, **kwargs_): 148 149 @test_util.run_without_tensor_float_32("Tests matmul") 150 def Test(self): 151 if not use_static_shape_ or a_np_.dtype in (np.int32, np.int64, np.float16): 152 self.skipTest("Skipping infeasible gradient test.") 153 154 # Transpose and possibly conjugate a_np_ and b_np_ according to the 155 # attributes such that tf.matmul(effective_a_np, effective_b_np, **kwargs) 156 # results in a valid matrix multiplication and produces the same result as 157 # np.matrix(a_np_) * np.matrix(b_np_) 158 effective_a_np = _GetTransposedMatrices(a_np_, "a", kwargs_) 159 effective_b_np = _GetTransposedMatrices(b_np_, "b", kwargs_) 160 161 epsilon = np.finfo(a_np_.dtype).eps 162 delta = epsilon**(1.0 / 3.0) 163 tol = 20 * delta 164 with self.session(): 165 theoretical, numerical = gradient_checker_v2.compute_gradient( 166 lambda x: math_ops.matmul(x, effective_b_np, **kwargs_), 167 [effective_a_np], 168 delta=delta) 169 self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol) 170 171 theoretical, numerical = gradient_checker_v2.compute_gradient( 172 lambda x: math_ops.matmul(effective_a_np, x, **kwargs_), 173 [effective_b_np], 174 delta=delta) 175 self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol) 176 177 return Test 178 179 180@test_util.with_eager_op_as_function 181class MatMulStatsTest(test_lib.TestCase): 182 183 @test_util.run_v1_only("Test requires a Graph and NodeDef inspection") 184 def testSimpleStatistics(self): 185 a = variables.Variable(random_ops.random_normal([25, 16])) 186 b = variables.Variable(random_ops.random_normal([16, 9])) 187 math_ops.matmul(a, b) 188 g = ops.get_default_graph() 189 for op in g.get_operations(): 190 flops = ops.get_stats_for_node_def(g, op.node_def, "flops").value 191 if op.name == "MatMul": 192 self.assertEqual(7200, flops) 193 194 @test_util.run_v1_only("Test requires a Graph and NodeDef inspection") 195 def testTransposedStatistics(self): 196 a = variables.Variable(random_ops.random_normal([16, 25])) 197 b = variables.Variable(random_ops.random_normal([16, 9])) 198 math_ops.matmul(a, b, transpose_a=True) 199 g = ops.get_default_graph() 200 for op in g.get_operations(): 201 flops = ops.get_stats_for_node_def(g, op.node_def, "flops").value 202 if op.name == "MatMul": 203 self.assertEqual(7200, flops) 204 205 206try: 207 # @ operator supported since python 3.5. 208 infix_matmul = operator.matmul 209except AttributeError: 210 211 # For earlier versions of python, emulate regular behavior. 212 # Useful to build and test for 3.5+ on earlier versions. 213 def infix_matmul(x, y): # pylint: disable=invalid-name 214 try: 215 r = type(x).__matmul__(x, y) 216 except AttributeError: 217 r = NotImplemented 218 if r is NotImplemented and type(x) is not type(y): 219 try: 220 r = type(y).__rmatmul__(y, x) 221 except AttributeError: 222 r = NotImplemented 223 if r is NotImplemented: 224 raise TypeError("unsupported operand type(s) for @: '{}' and '{}'" 225 .format(type(x).__name__, type(y).__name__)) 226 return r 227 228 229@test_util.with_eager_op_as_function 230class MatMulInfixOperatorTest(test_lib.TestCase): 231 232 def testMismatchedShape(self): 233 with self.assertRaisesRegex( 234 Exception, (r"(In\[0\] and In\[1\] has different ndims|In\[0\] " 235 r"ndims must be >= 2|Shape must be rank 2 but is rank 1)")): 236 infix_matmul( 237 ops.convert_to_tensor([10.0, 20.0, 30.0]), 238 ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]])) 239 240 def testMismatchedDimensions(self): 241 with self.assertRaisesRegex( 242 Exception, r"(Matrix size-incompatible: In\[0\]: .* In\[1\]|" 243 r"Dimensions must be equal)"): 244 infix_matmul( 245 ops.convert_to_tensor([[10.0, 20.0, 30.0]]), 246 ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]])) 247 248 @test_util.run_v1_only("Tensor.op is generally not applicable in TF 2") 249 def testInfixMatmulIsTfMatmul(self): 250 a = ops.convert_to_tensor([[10.0, 20.0, 30.0]]) 251 b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]]) 252 c = infix_matmul(a, b) 253 self.assertEqual(c.op.type, "MatMul") 254 255 def testInfixMatmulDoesDotProduct(self): 256 a = ops.convert_to_tensor([[10.0, 20.0, 30.0]]) 257 b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]]) 258 c = infix_matmul(a, b) 259 d = math_ops.matmul(a, b) 260 self.assertAllEqual(c, d) 261 262 263if __name__ == "__main__": 264 sizes = [1, 3, 5] 265 trans_options = [[False, False], [True, False], [False, True]] 266 dtypes_to_test = [ 267 np.int32, np.int64, np.float16, np.float32, np.float64, np.complex64, 268 np.complex128 269 ] 270 # TF2 does not support placeholders under eager so we skip it 271 for use_static_shape in set([True, tf2.enabled()]): 272 for dtype in dtypes_to_test: 273 for m in sizes: 274 for n in sizes: 275 for k in sizes: 276 # Construct compatible random matrices a_np of size [m, k] and b_np 277 # of size [k, n]. 278 a_np = np.random.normal(-5, 5, m * k).astype(dtype).reshape([m, k]) 279 if dtype in (np.complex64, np.complex128): 280 a_np.imag = np.random.normal(-5, 5, 281 m * k).astype(dtype).reshape([m, k]) 282 b_np = np.random.normal(-5, 5, k * n).astype(dtype).reshape([k, n]) 283 if dtype in (np.complex64, np.complex128): 284 b_np.imag = np.random.normal(-5, 5, 285 k * n).astype(dtype).reshape([k, n]) 286 for adjoint_a, transpose_a in trans_options: 287 for adjoint_b, transpose_b in trans_options: 288 name = "%s_%s_%s_%s_%s_%s_%s_%s_%s" % ( 289 use_static_shape, dtype.__name__, m, n, k, adjoint_a, 290 transpose_a, adjoint_b, transpose_b) 291 _AddTest(MatMulTest, "MatMulTest", name, 292 _GetMatMulTest( 293 a_np, 294 b_np, 295 use_static_shape, 296 adjoint_a=adjoint_a, 297 transpose_a=transpose_a, 298 adjoint_b=adjoint_b, 299 transpose_b=transpose_b)) 300 _AddTest(MatMulGradientTest, "MatMulGradientTest", name, 301 _GetMatMulGradientTest( 302 a_np, 303 b_np, 304 use_static_shape, 305 adjoint_a=adjoint_a, 306 transpose_a=transpose_a, 307 adjoint_b=adjoint_b, 308 transpose_b=transpose_b)) 309 310 test_lib.main() 311