1# Copyright 2020 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import numpy as np 16 17import mindspore as ms 18from mindspore import context, Tensor, Parameter 19from mindspore.common.api import _cell_graph_executor 20from mindspore.nn import Cell, TrainOneStepCell, Momentum 21from mindspore.ops import operations as P 22 23 24class Net(Cell): 25 def __init__(self, mul_weight, batch_matmul_weight, transpose_b=False, strategy1=None, strategy2=None): 26 super().__init__() 27 self.mul = P.Mul().shard(strategy1) 28 self.batch_matmul = P.BatchMatMul(transpose_b=transpose_b).shard(strategy2) 29 self.mul_weight = Parameter(mul_weight, "w1") 30 self.batch_matmul_weight = Parameter(batch_matmul_weight, "w2") 31 32 def construct(self, x, b): 33 out = self.mul(x, self.mul_weight) 34 out = self.batch_matmul(out, self.batch_matmul_weight) 35 return out 36 37 38_x = Tensor(np.ones([128, 64, 32]), dtype=ms.float32) 39_w1 = Tensor(np.ones([128, 64, 32]), dtype=ms.float32) 40_w2 = Tensor(np.ones([128, 32, 32]), dtype=ms.float32) 41_b = Tensor(np.ones([128, 64, 16]), dtype=ms.float32) 42 43 44def compile_net(net): 45 optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) 46 train_net = TrainOneStepCell(net, optimizer) 47 train_net.set_auto_parallel() 48 train_net.set_train() 49 _cell_graph_executor.compile(train_net, _x, _b) 50 context.reset_auto_parallel_context() 51 52 53def test_batch_matmul_data_parallel(): 54 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 55 strategy1 = ((16, 1, 1), (16, 1, 1)) 56 strategy2 = ((16, 1, 1), (16, 1, 1)) 57 net = Net(_w1, _w2, False, strategy1, strategy2) 58 compile_net(net) 59 60 61def test_batch_matmul_model_parallel(): 62 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 63 strategy1 = ((1, 1, 1), (1, 1, 1)) 64 strategy2 = ((1, 1, 1), (1, 1, 16)) 65 net = Net(_w1, _w2, False, strategy1, strategy2) 66 compile_net(net) 67 68 69def test_batch_matmul_hybrid_parallel(): 70 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 71 strategy1 = ((2, 2, 2), (2, 2, 2)) 72 strategy2 = ((2, 2, 2), (2, 2, 2)) 73 net = Net(_w1, _w2, False, strategy1, strategy2) 74 compile_net(net) 75 76 77def test_batch_matmul_auto_parallel(): 78 context.set_auto_parallel_context(parallel_mode="auto_parallel", device_num=16, global_rank=0) 79 net = Net(_w1, _w2, False) 80 compile_net(net) 81 82 83def test_batch_matmul_repeat_calc(): 84 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 85 strategy1 = ((2, 2, 4), (2, 2, 4)) 86 strategy2 = ((1, 2, 2), (1, 2, 2)) 87 net = Net(_w1, _w2, False, strategy1, strategy2) 88 compile_net(net) 89 90 91def test_batch_matmul_transpose_b(): 92 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 93 strategy1 = ((2, 2, 4), (2, 2, 4)) 94 strategy2 = ((1, 2, 2), (1, 2, 2)) 95 net = Net(_w1, _w2, True, strategy1, strategy2) 96 compile_net(net) 97