1# Copyright 2020 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import numpy as np 16import pytest 17import mindspore as ms 18from mindspore import context, Tensor, Parameter 19from mindspore.common.api import _cell_graph_executor 20from mindspore.nn import Cell, TrainOneStepCell, Momentum 21from mindspore.ops import operations as P 22 23 24class Net(Cell): 25 def __init__(self, mul_weight, strategy1=None, strategy2=None): 26 super().__init__() 27 self.mul = P.Mul().shard(strategy1) 28 self.mul2 = P.Mul().shard(strategy2) 29 self.mul_weight = Parameter(mul_weight, "w1") 30 31 def construct(self, x, b): 32 out = self.mul(x, self.mul_weight) 33 out = self.mul2(out, self.mul_weight) 34 return out 35 36 37class Net2(Cell): 38 def __init__(self, mul_weight, strategy1=None, strategy2=None): 39 super().__init__() 40 self.mul = P.Mul().shard(strategy1) 41 self.mul2 = P.Mul().shard(strategy2) 42 self.mul_weight = Parameter(mul_weight, "w1") 43 44 def construct(self, x, b): 45 out = self.mul(x, self.mul_weight) 46 out = self.mul2(x, out) 47 return out 48 49 50class Net3(Cell): 51 def __init__(self, mul_weight, strategy1=None, strategy2=None): 52 super().__init__() 53 self.mul = P.MatMul().shard(strategy1) 54 self.mul2 = P.MatMul().shard(strategy2) 55 self.mul_weight = Parameter(mul_weight, "w1") 56 57 def construct(self, x, b): 58 out = self.mul(x, self.mul_weight) 59 out = self.mul2(out, self.mul_weight) 60 return out 61 62 63_x = Tensor(np.ones([16, 16]), dtype=ms.float32) 64_w = Tensor(np.ones([16, 16]), dtype=ms.float32) 65_b = Tensor(np.ones([16, 16]), dtype=ms.float32) 66 67 68def compile_net(net): 69 optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) 70 train_net = TrainOneStepCell(net, optimizer) 71 train_net.set_auto_parallel() 72 train_net.set_train() 73 _cell_graph_executor.compile(train_net, _x, _b) 74 context.reset_auto_parallel_context() 75 76 77def test_parameter_same_split(): 78 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 79 strategy1 = ((16, 1), (16, 1)) 80 strategy2 = ((16, 1), (16, 1)) 81 net = Net(_w, strategy1, strategy2) 82 compile_net(net) 83 84 85def test_parameter_different_split(): 86 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 87 strategy1 = ((16, 1), (16, 1)) 88 strategy2 = ((4, 4), (4, 4)) 89 net = Net(_w, strategy1, strategy2) 90 with pytest.raises(RuntimeError): 91 compile_net(net) 92 93 94def test_input_same_split(): 95 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 96 strategy1 = ((16, 1), (16, 1)) 97 strategy2 = ((16, 1), (16, 1)) 98 net = Net(_w, strategy1, strategy2) 99 compile_net(net) 100 101 102def test_input_different_split(): 103 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 104 strategy1 = ((16, 1), (16, 1)) 105 strategy2 = ((4, 4), (4, 4)) 106 net = Net2(_w, strategy1, strategy2) 107 with pytest.raises(RuntimeError): 108 compile_net(net) 109 110 111def test_parameter_different_group(): 112 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 113 strategy1 = ((1, 2), (2, 1)) 114 strategy2 = ((8, 2), (2, 1)) 115 net = Net3(_w, strategy1, strategy2) 116 with pytest.raises(RuntimeError): 117 compile_net(net) 118