1# Copyright 2020 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================ 15import numpy as np 16import pytest 17 18import mindspore as ms 19from mindspore import context, Tensor, Parameter 20from mindspore.common.api import _cell_graph_executor 21from mindspore.nn import Cell, TrainOneStepCell, Momentum 22from mindspore.ops import operations as P 23 24 25class Net(Cell): 26 def __init__(self, weight, w2, begin, end, strategy1=None, strategy2=None, is_parameter=True): 27 super().__init__() 28 self.mul = P.Mul().shard(strategy1) 29 self.slice = P.Slice().shard(strategy2) 30 if is_parameter: 31 self.weight = Parameter(weight, "w1") 32 else: 33 self.weight = weight 34 self.mul2 = P.Mul() 35 self.weight2 = Parameter(w2, "w2") 36 self.begin = begin 37 self.end = end 38 39 def construct(self, x, b): 40 out = self.slice(self.weight, self.begin, self.end) 41 out = self.mul(x, out) 42 out = self.mul2(out, self.weight2) 43 return out 44 45 46class Net2(Cell): 47 def __init__(self, weight2, begin, end, strategy1=None, strategy2=None): 48 super().__init__() 49 self.mul = P.Mul().shard(strategy1) 50 self.slice = P.Slice().shard(strategy2) 51 self.weight2 = Parameter(weight2, "w2") 52 self.begin = begin 53 self.end = end 54 55 def construct(self, x, b): 56 out = self.mul(x, self.weight2) 57 out = self.slice(out, self.begin, self.end) 58 return out 59 60 61_x = Tensor(np.ones([128, 64, 1]), dtype=ms.float32) 62_w1 = Tensor(np.ones([256, 64, 32]), dtype=ms.float32) 63_w2 = Tensor(np.ones([128, 64, 1]), dtype=ms.float32) 64_b = Tensor(np.ones([128, 64, 32]), dtype=ms.float32) 65 66 67def compile_net(net): 68 optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) 69 train_net = TrainOneStepCell(net, optimizer) 70 train_net.set_auto_parallel() 71 train_net.set_train() 72 _cell_graph_executor.compile(train_net, _x, _b) 73 context.reset_auto_parallel_context() 74 75 76def test_slice_no_fully_fetch_split_error(): 77 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=8, global_rank=0) 78 strategy1 = ((2, 2, 2), (2, 2, 2)) 79 strategy2 = ((2, 2, 2),) 80 net = Net(_w1, _w2, (0, 0, 0), (128, 64, 32), strategy1, strategy2, is_parameter=True) 81 with pytest.raises(RuntimeError): 82 compile_net(net) 83 84def test_slice_parameter(): 85 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=8, global_rank=0) 86 strategy1 = ((1, 4, 1), (1, 4, 2)) 87 strategy2 = ((1, 4, 2),) 88 net = Net(_w1, _w2, (0, 0, 0), (128, 64, 32), strategy1, strategy2) 89 compile_net(net) 90 91 92def test_slice_tensor(): 93 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=8, global_rank=0) 94 strategy1 = ((1, 4, 1), (1, 4, 2)) 95 strategy2 = ((1, 4, 2),) 96 net = Net(_w1, _w2, (0, 0, 0), (128, 64, 32), strategy1, strategy2, is_parameter=False) 97 compile_net(net) 98 99 100def test_slice_parameter_no_full_split(): 101 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=8, global_rank=0) 102 strategy1 = ((1, 4, 1), (1, 4, 2)) 103 strategy2 = ((1, 2, 2),) 104 net = Net(_w1, _w2, (0, 0, 0), (128, 64, 32), strategy1, strategy2, is_parameter=True) 105 compile_net(net) 106 107 108def test_slice_output(): 109 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=8, global_rank=0) 110 strategy1 = ((1, 8, 1), (1, 8, 1)) 111 strategy2 = ((1, 8, 1),) 112 net = Net2(_w2, (0, 0, 0), (64, 64, 1), strategy1, strategy2) 113 compile_net(net) 114 115 116def test_stridedslice_output_no_full_split(): 117 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=8, global_rank=0) 118 strategy1 = ((1, 8, 1), (1, 8, 1)) 119 strategy2 = ((1, 4, 1),) 120 net = Net2(_w2, (0, 0, 0), (64, 64, 1), strategy1, strategy2) 121 compile_net(net) 122 123 124def test_stridedslice_no_strategy(): 125 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=8, global_rank=0) 126 strategy1 = ((1, 8, 1), (1, 8, 1)) 127 strategy2 = None 128 net = Net2(_w2, (0, 0, 0), (128, 64, 1), strategy1, strategy2) 129 compile_net(net) 130 131 132def test_slice_auto_parallel(): 133 context.set_auto_parallel_context(parallel_mode="auto_parallel", device_num=8, global_rank=0) 134 net = Net2(_w2, (0, 0, 0), (32, 64, 1)) 135 compile_net(net) 136