1# Copyright 2020 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import numpy as np 16 17import mindspore as ms 18from mindspore import context, Tensor, Parameter 19from mindspore.common.api import _cell_graph_executor 20from mindspore.nn import Cell, TrainOneStepCell, Momentum 21from mindspore.ops import operations as P 22 23 24class Net(Cell): 25 def __init__(self, mul_weight, strategy1=None, strategy2=None): 26 super().__init__() 27 self.mul = P.Mul().shard(strategy1) 28 self.dropout_do_mask = P.DropoutDoMask().shard(strategy2) 29 self.dropout_gen_mask = P.DropoutGenMask() 30 self.get_shape = P.Shape() 31 self.cast = P.Cast() 32 self.mul_weight = Parameter(mul_weight, "w1") 33 self.keep_prob = Tensor(0.9) 34 35 def construct(self, x, b): 36 out = self.mul(x, self.mul_weight) 37 shape = self.get_shape(out) 38 dtype = P.DType()(out) 39 keep_prob = self.cast(self.keep_prob, dtype) 40 mask = self.dropout_gen_mask(shape, keep_prob) 41 out = self.dropout_do_mask(out, mask, keep_prob) 42 return out 43 44 45_x = Tensor(np.ones([128, 64]), dtype=ms.float32) 46_w1 = Tensor(np.ones([128, 64]), dtype=ms.float32) 47_b = Tensor(np.ones([128, 64]), dtype=ms.float32) 48 49 50def compile_net(net): 51 optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) 52 train_net = TrainOneStepCell(net, optimizer) 53 train_net.set_auto_parallel() 54 train_net.set_train() 55 _cell_graph_executor.compile(train_net, _x, _b) 56 context.reset_auto_parallel_context() 57 58 59def test_dropout_do_mask_data_parallel(): 60 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 61 strategy1 = ((16, 1), (16, 1)) 62 strategy2 = ((16, 1),) 63 net = Net(_w1, strategy1, strategy2) 64 compile_net(net) 65 66 67def test_dropout_do_mask_model_parallel(): 68 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 69 strategy1 = ((1, 16), (1, 16)) 70 strategy2 = ((1, 16),) 71 net = Net(_w1, strategy1, strategy2) 72 compile_net(net) 73 74 75def test_dropout_do_mask_hybrid_parallel(): 76 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 77 strategy1 = ((4, 4), (4, 4)) 78 strategy2 = ((4, 4),) 79 net = Net(_w1, strategy1, strategy2) 80 compile_net(net) 81 82 83def test_dropout_do_mask_auto_parallel(): 84 context.set_auto_parallel_context(parallel_mode="auto_parallel", device_num=16, global_rank=0) 85 net = Net(_w1) 86 compile_net(net) 87 88 89def test_dropout_do_mask_repeat_calc(): 90 context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0) 91 strategy1 = ((4, 4), (4, 4)) 92 strategy2 = ((2, 4),) 93 net = Net(_w1, strategy1, strategy2) 94 compile_net(net) 95