1# Copyright 2019 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================ 15 16import numpy as np 17import pytest 18 19import mindspore.context as context 20import mindspore.nn as nn 21from mindspore import Tensor 22from mindspore.common.parameter import ParameterTuple 23from mindspore.ops import composite as C 24from mindspore.ops import operations as P 25from mindspore.ops.composite import GradOperation 26from mindspore.ops.operations import _inner_ops as inner 27 28class BiasAdd(nn.Cell): 29 def __init__(self): 30 super(BiasAdd, self).__init__() 31 self.ba = P.BiasAdd() 32 33 def construct(self, x, b): 34 return self.ba(x, b) 35 36 37@pytest.mark.level0 38@pytest.mark.platform_x86_gpu_training 39@pytest.mark.env_onecard 40def test_biasadd(): 41 x = Tensor(np.array([[0.1, 0.2, 0.3, 0.4], 42 [0.5, 0.6, 0.7, 0.8], 43 [0.9, 1.0, 1.1, 1.2]]).astype(np.float32)) 44 b = Tensor(np.array([0.1, 0.2, 0.3, 0.4]).astype(np.float32)) 45 expect = np.array([[0.2, 0.4, 0.6, 0.8], 46 [0.6, 0.8, 1.0, 1.2], 47 [1.0, 1.2, 1.4, 1.6]]) 48 error = np.ones(shape=[3, 4]) * 1.0e-6 49 50 context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") 51 ba = BiasAdd() 52 result = ba(x, b) 53 diff = result.asnumpy() - expect 54 assert np.all(diff < error) 55 assert np.all(-diff < error) 56 57 context.set_context(mode=context.GRAPH_MODE, device_target="GPU") 58 ba = BiasAdd() 59 result = ba(x, b) 60 diff = result.asnumpy() - expect 61 assert np.all(diff < error) 62 assert np.all(-diff < error) 63 64 65class GradData(nn.Cell): 66 def __init__(self, network): 67 super(GradData, self).__init__() 68 self.grad = GradOperation(get_all=True, sens_param=True) 69 self.network = network 70 71 def construct(self, inputs, output_grad): 72 return self.grad(self.network)(inputs, output_grad) 73 74 75class GradWeight(nn.Cell): 76 def __init__(self, network): 77 super(GradWeight, self).__init__() 78 self.network = network 79 self.weights = ParameterTuple(network.trainable_params()) 80 self.grad = C.GradOperation(get_by_list=True, 81 sens_param=True) 82 83 def construct(self, x, output_grad): 84 weights = self.weights 85 grads = self.grad(self.network, weights)(x, output_grad) 86 return grads 87 88 89class DenseNet(nn.Cell): 90 def __init__(self): 91 super(DenseNet, self).__init__() 92 w = np.array([[0.1, 0.8, 0.1, 0.1], 93 [1, 1, 1, 1]]).astype(np.float32) 94 b = np.array([0.3, 0.6]).astype(np.float32) 95 self.dense = nn.Dense(4, 2, weight_init=Tensor(w), bias_init=Tensor(b)) 96 97 def construct(self, x): 98 return self.dense(x) 99 100 101@pytest.mark.level0 102@pytest.mark.platform_x86_gpu_training 103@pytest.mark.env_onecard 104def test_dx(): 105 x = np.array([[0.1, 0.2, 0.3, 0.4], 106 [0.1, 0.2, 0.3, 0.4], 107 [0.1, 0.2, 0.3, 0.4]]).astype(np.float32) 108 dy = np.array([[1, 1], 109 [1, 1], 110 [1, 1]]).astype(np.float32) 111 dx_expect = np.array([[1.1, 1.8, 1.1, 1.1], 112 [1.1, 1.8, 1.1, 1.1], 113 [1.1, 1.8, 1.1, 1.1]]).astype(np.float32) 114 error = np.ones(shape=[3, 4]) * 1.0e-6 115 116 context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") 117 net = GradData(DenseNet()) 118 dx = net(Tensor(x), Tensor(dy)) 119 diff = dx[0].asnumpy() - dx_expect 120 assert np.all(diff < error) 121 assert np.all(-diff < error) 122 123 context.set_context(mode=context.GRAPH_MODE, device_target="GPU") 124 net = GradData(DenseNet()) 125 dx = net(Tensor(x), Tensor(dy)) 126 diff = dx[0].asnumpy() - dx_expect 127 assert np.all(diff < error) 128 assert np.all(-diff < error) 129 130 131@pytest.mark.level0 132@pytest.mark.platform_x86_gpu_training 133@pytest.mark.env_onecard 134def test_dx_ND(): 135 x = np.array([[[0.1, 0.2, 0.3, 0.4], 136 [0.1, 0.2, 0.3, 0.4], 137 [0.1, 0.2, 0.3, 0.4]], 138 [[0.1, 0.2, 0.3, 0.4], 139 [0.1, 0.2, 0.3, 0.4], 140 [0.1, 0.2, 0.3, 0.4]] 141 ]).astype(np.float32) 142 dy = np.array([[[1, 1], 143 [1, 1], 144 [1, 1]], 145 [[1, 1], 146 [1, 1], 147 [1, 1]]]).astype(np.float32) 148 dx_expect = np.array([[[1.1, 1.8, 1.1, 1.1], 149 [1.1, 1.8, 1.1, 1.1], 150 [1.1, 1.8, 1.1, 1.1]], 151 [[1.1, 1.8, 1.1, 1.1], 152 [1.1, 1.8, 1.1, 1.1], 153 [1.1, 1.8, 1.1, 1.1]] 154 ]).astype(np.float32) 155 error = np.ones(shape=[2, 3, 4]) * 1.0e-6 156 157 context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") 158 net = GradData(DenseNet()) 159 dx = net(Tensor(x), Tensor(dy)) 160 diff = dx[0].asnumpy() - dx_expect 161 assert np.all(diff < error) 162 assert np.all(-diff < error) 163 164 context.set_context(mode=context.GRAPH_MODE, device_target="GPU") 165 net = GradData(DenseNet()) 166 dx = net(Tensor(x), Tensor(dy)) 167 diff = dx[0].asnumpy() - dx_expect 168 assert np.all(diff < error) 169 assert np.all(-diff < error) 170 171 172@pytest.mark.level0 173@pytest.mark.platform_x86_gpu_training 174@pytest.mark.env_onecard 175def test_dw(): 176 x = np.array([[0.1, 0.2, 0.3, 0.4], 177 [0.1, 0.2, 0.3, 0.4], 178 [0.1, 0.2, 0.3, 0.4]]).astype(np.float32) 179 dy = np.array([[1, 1], 180 [1, 1], 181 [1, 1]]).astype(np.float32) 182 dw_expect = np.array([[0.3, 0.6, 0.9, 1.2], 183 [0.3, 0.6, 0.9, 1.2]]).astype(np.float32) 184 dw_error = np.ones(shape=[2, 4]) * 1.0e-6 185 db_expect = np.array([3, 3]).astype(np.float32) 186 db_error = np.ones(shape=[2]) * 1.0e-6 187 188 context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") 189 net = GradWeight(DenseNet()) 190 dw, db = net(Tensor(x), Tensor(dy)) 191 diff = dw.asnumpy() - dw_expect 192 assert np.all(diff < dw_error) 193 assert np.all(-diff < dw_error) 194 diff = db.asnumpy() - db_expect 195 assert np.all(diff < db_error) 196 assert np.all(-diff < db_error) 197 198 context.set_context(mode=context.GRAPH_MODE, device_target="GPU") 199 net = GradWeight(DenseNet()) 200 dw, db = net(Tensor(x), Tensor(dy)) 201 diff = dw.asnumpy() - dw_expect 202 assert np.all(diff < dw_error) 203 assert np.all(-diff < dw_error) 204 diff = db.asnumpy() - db_expect 205 assert np.all(diff < db_error) 206 assert np.all(-diff < db_error) 207 208 209@pytest.mark.level0 210@pytest.mark.platform_x86_gpu_training 211@pytest.mark.env_onecard 212def test_dw_ND(): 213 x = np.array([[[0.1, 0.2, 0.3, 0.4], 214 [0.1, 0.2, 0.3, 0.4], 215 [0.1, 0.2, 0.3, 0.4]], 216 [[0.1, 0.2, 0.3, 0.4], 217 [0.1, 0.2, 0.3, 0.4], 218 [0.1, 0.2, 0.3, 0.4]]]).astype(np.float32) 219 dy = np.array([[[1, 1], 220 [1, 1], 221 [1, 1]], 222 [[1, 1], 223 [1, 1], 224 [1, 1]]]).astype(np.float32) 225 dw_expect = 2 * np.array([[0.3, 0.6, 0.9, 1.2], 226 [0.3, 0.6, 0.9, 1.2]]).astype(np.float32) 227 dw_error = np.ones(shape=[2, 4]) * 1.0e-6 228 db_expect = 2 * np.array([3, 3]).astype(np.float32) 229 db_error = np.ones(shape=[2]) * 1.0e-6 230 231 context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") 232 net = GradWeight(DenseNet()) 233 dw, db = net(Tensor(x), Tensor(dy)) 234 diff = dw.asnumpy() - dw_expect 235 assert np.all(diff < dw_error) 236 assert np.all(-diff < dw_error) 237 diff = db.asnumpy() - db_expect 238 assert np.all(diff < db_error) 239 assert np.all(-diff < db_error) 240 241 context.set_context(mode=context.GRAPH_MODE, device_target="GPU") 242 net = GradWeight(DenseNet()) 243 dw, db = net(Tensor(x), Tensor(dy)) 244 diff = dw.asnumpy() - dw_expect 245 assert np.all(diff < dw_error) 246 assert np.all(-diff < dw_error) 247 diff = db.asnumpy() - db_expect 248 assert np.all(diff < db_error) 249 assert np.all(-diff < db_error) 250 251 252class Grad(nn.Cell): 253 def __init__(self, network): 254 super(Grad, self).__init__() 255 self.grad = GradOperation(get_all=True, sens_param=True) 256 self.network = network 257 258 def construct(self, input_, bias, dy): 259 return self.grad(self.network)(input_, bias, dy) 260 261 262@pytest.mark.level0 263@pytest.mark.platform_x86_gpu_training 264@pytest.mark.env_onecard 265def test_biasadd_3d(): 266 x = Tensor(np.array([[[1, 2, 3, 4, 5, 6, 7, 8], 267 [9, 10, 11, 12, 13, 14, 15, 16], 268 [17, 18, 19, 20, 21, 22, 23, 24], 269 [25, 26, 27, 28, 29, 30, 31, 32]], 270 271 [[33, 34, 35, 36, 37, 38, 39, 40], 272 [41, 42, 43, 44, 45, 46, 47, 48], 273 [49, 50, 51, 52, 53, 54, 55, 56], 274 [57, 58, 59, 60, 61, 62, 63, 64]], 275 276 [[65, 66, 67, 68, 69, 70, 71, 72], 277 [73, 74, 75, 76, 77, 78, 79, 80], 278 [81, 82, 83, 84, 85, 86, 87, 88], 279 [89, 90, 91, 92, 93, 94, 95, 96]]]).astype(np.float32)) 280 b = Tensor(np.array([1, 2, 3, 4]).astype(np.float32)) 281 dy = Tensor(np.array([[[1, 2, 3, 4, 5, 6, 7, 8], 282 [9, 10, 11, 12, 13, 14, 15, 16], 283 [17, 18, 19, 20, 21, 22, 23, 24], 284 [25, 26, 27, 28, 29, 30, 31, 32]], 285 286 [[33, 34, 35, 36, 37, 38, 39, 40], 287 [41, 42, 43, 44, 45, 46, 47, 48], 288 [49, 50, 51, 52, 53, 54, 55, 56], 289 [57, 58, 59, 60, 61, 62, 63, 64]], 290 291 [[65, 66, 67, 68, 69, 70, 71, 72], 292 [73, 74, 75, 76, 77, 78, 79, 80], 293 [81, 82, 83, 84, 85, 86, 87, 88], 294 [89, 90, 91, 92, 93, 94, 95, 96]]]).astype(np.float32)) 295 296 expect = np.array([[[2, 3, 4, 5, 6, 7, 8, 9], 297 [11, 12, 13, 14, 15, 16, 17, 18], 298 [20, 21, 22, 23, 24, 25, 26, 27], 299 [29, 30, 31, 32, 33, 34, 35, 36]], 300 301 [[34, 35, 36, 37, 38, 39, 40, 41], 302 [43, 44, 45, 46, 47, 48, 49, 50], 303 [52, 53, 54, 55, 56, 57, 58, 59], 304 [61, 62, 63, 64, 65, 66, 67, 68]], 305 306 [[66, 67, 68, 69, 70, 71, 72, 73], 307 [75, 76, 77, 78, 79, 80, 81, 82], 308 [84, 85, 86, 87, 88, 89, 90, 91], 309 [93, 94, 95, 96, 97, 98, 99, 100]]]) 310 311 error = np.ones(shape=[3, 4, 8]) * 1.0e-6 312 context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") 313 net = BiasAdd() 314 net.set_grad() 315 result = net(x, b) 316 diff = result.asnumpy() - expect 317 assert np.all(diff < error) 318 assert np.all(-diff < error) 319 320 net = Grad(net) 321 _, result = net(x, b, dy) 322 expect = np.array([876., 1068., 1260., 1452.]) 323 diff = result.asnumpy() - expect 324 error = np.ones(shape=[4]) * 1.0e-6 325 assert np.all(diff < error) 326 assert np.all(-diff < error) 327 328 329@pytest.mark.level0 330@pytest.mark.platform_x86_gpu_training 331@pytest.mark.env_onecard 332def test_biasadd_4d(): 333 x = Tensor(np.array([[[[1, 2, 3, 4], 334 [5, 6, 7, 8], 335 [9, 10, 11, 12], 336 [13, 14, 15, 16]], 337 338 [[17, 18, 19, 20], 339 [21, 22, 23, 24], 340 [25, 26, 27, 28], 341 [29, 30, 31, 32]], 342 343 [[33, 34, 35, 36], 344 [37, 38, 39, 40], 345 [41, 42, 43, 44], 346 [45, 46, 47, 48]]], 347 348 [[[49, 50, 51, 52], 349 [53, 54, 55, 56], 350 [57, 58, 59, 60], 351 [61, 62, 63, 64]], 352 353 [[65, 66, 67, 68], 354 [69, 70, 71, 72], 355 [73, 74, 75, 76], 356 [77, 78, 79, 80]], 357 358 [[81, 82, 83, 84], 359 [85, 86, 87, 88], 360 [89, 90, 91, 92], 361 [93, 94, 95, 96]]]]).astype(np.float32)) 362 b = Tensor(np.array([1, 2, 3]).astype(np.float32)) 363 dy = Tensor(np.array([[[[1, 2, 3, 4], 364 [5, 6, 7, 8], 365 [9, 10, 11, 12], 366 [13, 14, 15, 16]], 367 368 [[17, 18, 19, 20], 369 [21, 22, 23, 24], 370 [25, 26, 27, 28], 371 [29, 30, 31, 32]], 372 373 [[33, 34, 35, 36], 374 [37, 38, 39, 40], 375 [41, 42, 43, 44], 376 [45, 46, 47, 48]]], 377 378 [[[49, 50, 51, 52], 379 [53, 54, 55, 56], 380 [57, 58, 59, 60], 381 [61, 62, 63, 64]], 382 383 [[65, 66, 67, 68], 384 [69, 70, 71, 72], 385 [73, 74, 75, 76], 386 [77, 78, 79, 80]], 387 388 [[81, 82, 83, 84], 389 [85, 86, 87, 88], 390 [89, 90, 91, 92], 391 [93, 94, 95, 96]]]]).astype(np.float32)) 392 393 expect = np.array([[[[2, 3, 4, 5], 394 [6, 7, 8, 9], 395 [10, 11, 12, 13], 396 [14, 15, 16, 17]], 397 398 [[19, 20, 21, 22], 399 [23, 24, 25, 26], 400 [27, 28, 29, 30], 401 [31, 32, 33, 34]], 402 403 [[36, 37, 38, 39], 404 [40, 41, 42, 43], 405 [44, 45, 46, 47], 406 [48, 49, 50, 51]]], 407 408 [[[50, 51, 52, 53], 409 [54, 55, 56, 57], 410 [58, 59, 60, 61], 411 [62, 63, 64, 65]], 412 413 [[67, 68, 69, 70], 414 [71, 72, 73, 74], 415 [75, 76, 77, 78], 416 [79, 80, 81, 82]], 417 418 [[84, 85, 86, 87], 419 [88, 89, 90, 91], 420 [92, 93, 94, 95], 421 [96, 97, 98, 99]]]]) 422 error = np.ones(shape=[2, 3, 4, 4]) * 1.0e-6 423 424 context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") 425 ba = BiasAdd() 426 result = ba(x, b) 427 diff = result.asnumpy() - expect 428 assert np.all(diff < error) 429 assert np.all(-diff < error) 430 431 context.set_context(mode=context.GRAPH_MODE, device_target="GPU") 432 net = BiasAdd() 433 result = net(x, b) 434 diff = result.asnumpy() - expect 435 assert np.all(diff < error) 436 assert np.all(-diff < error) 437 438 net = Grad(net) 439 _, result = net(x, b, dy) 440 expect = np.array([1040., 1552., 2064.]) 441 diff = result.asnumpy() - expect 442 error = np.ones(shape=[3]) * 1.0e-6 443 assert np.all(diff < error) 444 assert np.all(-diff < error) 445 446 447class BiasAddDynamic(nn.Cell): 448 def __init__(self): 449 super(BiasAddDynamic, self).__init__() 450 self.ba = P.BiasAdd() 451 self.test_dynamic = inner.GpuConvertToDynamicShape() 452 453 def construct(self, x, b): 454 x = self.test_dynamic(x) 455 output = self.ba(x, b) 456 return output 457 458@pytest.mark.level0 459@pytest.mark.platform_x86_gpu_training 460@pytest.mark.env_onecard 461def test_bias_add_dynamic_two_inputs(): 462 context.set_context(mode=context.GRAPH_MODE, device_target="GPU") 463 net = BiasAddDynamic() 464 465 x_1 = Tensor(np.array([[0.1, 0.2, 0.3, 0.4], 466 [0.5, 0.6, 0.7, 0.8], 467 [0.9, 1.0, 1.1, 1.2]]).astype(np.float32)) 468 b_1 = Tensor(np.array([0.1, 0.2, 0.3, 0.4]).astype(np.float32)) 469 expect_1 = np.array([[0.2, 0.4, 0.6, 0.8], 470 [0.6, 0.8, 1.0, 1.2], 471 [1.0, 1.2, 1.4, 1.6]]) 472 error_1 = np.ones(shape=[3, 4]) * 1.0e-6 473 result_1 = net(x_1, b_1) 474 diff_1 = result_1.asnumpy() - expect_1 475 assert np.all(diff_1 < error_1) 476 assert np.all(-diff_1 < error_1) 477 478 x_2 = Tensor(np.array([[[1, 2, 3, 4, 5, 6, 7, 8], 479 [9, 10, 11, 12, 13, 14, 15, 16], 480 [17, 18, 19, 20, 21, 22, 23, 24], 481 [25, 26, 27, 28, 29, 30, 31, 32]], 482 [[33, 34, 35, 36, 37, 38, 39, 40], 483 [41, 42, 43, 44, 45, 46, 47, 48], 484 [49, 50, 51, 52, 53, 54, 55, 56], 485 [57, 58, 59, 60, 61, 62, 63, 64]], 486 [[65, 66, 67, 68, 69, 70, 71, 72], 487 [73, 74, 75, 76, 77, 78, 79, 80], 488 [81, 82, 83, 84, 85, 86, 87, 88], 489 [89, 90, 91, 92, 93, 94, 95, 96]]]).astype(np.float32)) 490 b_2 = Tensor(np.array([1, 2, 3, 4]).astype(np.float32)) 491 expect_2 = np.array([[[2, 3, 4, 5, 6, 7, 8, 9], 492 [11, 12, 13, 14, 15, 16, 17, 18], 493 [20, 21, 22, 23, 24, 25, 26, 27], 494 [29, 30, 31, 32, 33, 34, 35, 36]], 495 [[34, 35, 36, 37, 38, 39, 40, 41], 496 [43, 44, 45, 46, 47, 48, 49, 50], 497 [52, 53, 54, 55, 56, 57, 58, 59], 498 [61, 62, 63, 64, 65, 66, 67, 68]], 499 [[66, 67, 68, 69, 70, 71, 72, 73], 500 [75, 76, 77, 78, 79, 80, 81, 82], 501 [84, 85, 86, 87, 88, 89, 90, 91], 502 [93, 94, 95, 96, 97, 98, 99, 100]]]) 503 error_2 = np.ones(shape=[3, 4, 8]) * 1.0e-6 504 result_2 = net(x_2, b_2) 505 diff_2 = result_2.asnumpy() - expect_2 506 assert np.all(diff_2 < error_2) 507 assert np.all(-diff_2 < error_2) 508