1# Copyright 2020 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================ 15import numpy as np 16 17import mindspore as ms 18from mindspore import context, Tensor, Parameter 19from mindspore.common.api import _cell_graph_executor 20from mindspore.nn import Cell 21from mindspore.ops import operations as P 22 23 24class Net(Cell): 25 def __init__(self, mul_weight, strategy1=None, strategy2=None): 26 super().__init__() 27 self.mul = P.Mul().shard(strategy1) 28 self.neg = P.Neg().shard(strategy2) 29 self.mul_weight = Parameter(mul_weight, "w1") 30 31 def construct(self, x, b): 32 out = self.mul(x, self.mul_weight) 33 out = self.neg(out) 34 return out, b 35 36 37_x = Tensor(np.ones([128, 64, 32]), dtype=ms.float32) 38_w1 = Tensor(np.ones([128, 64, 32]), dtype=ms.float32) 39_b = Tensor(np.ones([128, 64, 32]), dtype=ms.float32) 40 41 42def compile_net(net): 43 net.set_auto_parallel() 44 net.set_train() 45 _cell_graph_executor.compile(net, _x, _b) 46 context.reset_auto_parallel_context() 47 48 49def test_forward_graph_data_parallel(): 50 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 51 strategy1 = ((16, 1, 1), (16, 1, 1)) 52 strategy2 = ((16, 1, 1),) 53 net = Net(_w1, strategy1, strategy2) 54 compile_net(net) 55 56 57def test_forward_graph_model_parallel(): 58 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 59 strategy1 = ((1, 1, 16), (1, 1, 16)) 60 strategy2 = ((1, 1, 16),) 61 net = Net(_w1, strategy1, strategy2) 62 compile_net(net) 63 64 65def test_forward_graph_hybrid_parallel(): 66 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 67 strategy1 = ((2, 2, 4), (2, 2, 4)) 68 strategy2 = ((2, 2, 4),) 69 net = Net(_w1, strategy1, strategy2) 70 compile_net(net) 71 72 73def test_forward_graph_auto_parallel(): 74 context.set_auto_parallel_context(parallel_mode="auto_parallel", device_num=16, global_rank=0) 75 net = Net(_w1) 76 compile_net(net) 77 78 79def test_forward_graph_repeat_calc(): 80 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 81 strategy1 = ((2, 2, 4), (2, 2, 4)) 82 strategy2 = ((1, 2, 2),) 83 net = Net(_w1, strategy1, strategy2) 84 compile_net(net) 85