• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14import numpy as np
15
16import mindspore as ms
17import mindspore.nn as nn
18from mindspore import Tensor
19from mindspore import context
20from mindspore.common.api import _cell_graph_executor
21from mindspore.ops import composite as C
22from mindspore.ops import operations as P
23from tests.ut.python.ops.test_math_ops import VirtualLoss
24
25
26grad_all = C.GradOperation(get_all=True)
27
28
29class AddRelu(nn.Cell):
30    def __init__(self, strategy0=None, strategy1=None):
31        super(AddRelu, self).__init__()
32        self.add = P.Add().shard(strategy=strategy0)
33        self.relu = P.ReLU().shard(strategy=strategy1)
34
35    def construct(self, x, z):
36        out = self.add(x, z)
37        return self.relu(out)
38
39
40class NetWithLoss(nn.Cell):
41    def __init__(self, network):
42        super(NetWithLoss, self).__init__()
43        self.loss = VirtualLoss()
44        self.network = network
45
46    def construct(self, x, z):
47        predict = self.network(x, z)
48        return self.loss(predict)
49
50
51class Grad(nn.Cell):
52    def __init__(self, network):
53        super(Grad, self).__init__()
54        self.network = network
55
56    def construct(self, x, y):
57        return grad_all(self.network)(x, y)
58
59
60def compile_net(net, x, y):
61    net.set_auto_parallel()
62    net.set_train()
63    _cell_graph_executor.compile(net, x, y)
64
65
66def test_add_relu_stride_slice():
67    context.set_auto_parallel_context(device_num=8, global_rank=7)
68    context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
69
70    strategy0 = ((1, 1), (1, 1))
71    strategy1 = ((8, 1),)
72    net = Grad(NetWithLoss(AddRelu(strategy0, strategy1)))
73
74    x = Tensor(np.ones([128, 32]), dtype=ms.float32)
75    y = Tensor(np.ones([128, 32]), dtype=ms.float32)
76    compile_net(net, x, y)
77
78
79def test_add_relu_all_gather():
80    context.set_auto_parallel_context(device_num=8, global_rank=7)
81    context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
82
83    strategy0 = ((8, 1), (8, 1))
84    strategy1 = ((1, 1),)
85    net = Grad(NetWithLoss(AddRelu(strategy0, strategy1)))
86
87    x = Tensor(np.ones([128, 32]), dtype=ms.float32)
88    y = Tensor(np.ones([128, 32]), dtype=ms.float32)
89    compile_net(net, x, y)
90