• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15import pytest
16import numpy as np
17
18import mindspore as ms
19import mindspore.nn as nn
20from mindspore.common.api import _cell_graph_executor
21from mindspore.ops import operations as P
22from mindspore.ops import composite as C
23from mindspore import Tensor, context
24from mindspore.nn import TrainOneStepCell, Adam
25from tests.ut.python.ops.test_math_ops import VirtualLoss
26
27grad_all = C.GradOperation(get_all=True)
28
29
30@pytest.fixture(name="test_context")
31def _test_context():
32    context.set_context(enable_sparse=True)
33    yield
34    context.set_context(enable_sparse=False)
35    context.reset_auto_parallel_context()
36
37
38class GradWrap(nn.Cell):
39    def __init__(self, network):
40        super(GradWrap, self).__init__()
41        self.network = network
42
43    def construct(self, x, y, z):
44        return grad_all(self.network)(x, y, z)
45
46
47class NetWithLoss(nn.Cell):
48    def __init__(self, network):
49        super(NetWithLoss, self).__init__()
50        self.loss = VirtualLoss()
51        self.network = network
52
53    def construct(self, x, y, z):
54        predict = self.network(x, y, z)
55        return self.loss(predict)
56
57
58class Net(nn.Cell):
59    def __init__(self, shape, field_size=10, slice_mode=nn.EmbeddingLookup.BATCH_SLICE, target="Device",
60                 operator='SUM'):
61        super().__init__()
62        self.embedding = nn.MultiFieldEmbeddingLookup(vocab_size=32, embedding_size=64, target=target,
63                                                      field_size=field_size, slice_mode=slice_mode, operator=operator)
64        self.reshape = P.Reshape()
65        self.batch_size = shape[0]
66
67    def construct(self, x, y, z):
68        out = self.embedding(x, y, z)
69        out = self.reshape(out, (self.batch_size, -1))
70        return out
71
72
73def compile_net(net, shape):
74    x = Tensor(np.ones(shape), dtype=ms.int32)
75    y = Tensor(np.ones(shape), dtype=ms.float32)
76    z = Tensor(np.ones(shape), dtype=ms.int32)
77    optimizer = Adam(net.trainable_params(), learning_rate=0.1)
78    train_net = TrainOneStepCell(net, optimizer)
79    train_net.set_auto_parallel()
80    train_net.set_train()
81    _cell_graph_executor.compile(train_net, x, y, z)
82    context.reset_auto_parallel_context()
83
84
85def test_embeddinglookup_batch_parallel_sum(test_context):
86    context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode="semi_auto_parallel")
87    shape = [64, 64]
88    net = NetWithLoss(Net(shape, field_size=10, target='DEVICE'))
89    compile_net(net, shape)
90
91
92def test_embeddinglookup_row_parallel_sum(test_context):
93    context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode="semi_auto_parallel")
94    shape = [64, 64]
95    net = NetWithLoss(Net(shape, field_size=9, slice_mode=nn.EmbeddingLookup.TABLE_ROW_SLICE, target='DEVICE'))
96    compile_net(net, shape)
97
98
99def test_embeddinglookup_column_parallel_sum(test_context):
100    context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode="semi_auto_parallel")
101    shape = [64, 64]
102    net = NetWithLoss(Net(shape, field_size=10, slice_mode=nn.EmbeddingLookup.TABLE_COLUMN_SLICE, target='DEVICE'))
103    compile_net(net, shape)
104
105
106def test_embeddinglookup_batch_parallel_mean(test_context):
107    context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode="semi_auto_parallel")
108    shape = [64, 64]
109    net = NetWithLoss(Net(shape, field_size=1, target='DEVICE', operator='MEAN'))
110    compile_net(net, shape)
111
112
113def test_embeddinglookup_column_parallel_mean(test_context):
114    context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode="semi_auto_parallel")
115    shape = [64, 64]
116    net = NetWithLoss(Net(shape, target='DEVICE', slice_mode=nn.EmbeddingLookup.TABLE_COLUMN_SLICE, operator='MEAN'))
117    compile_net(net, shape)
118
119
120def test_embeddinglookup_row_parallel_mean(test_context):
121    context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode="semi_auto_parallel")
122    shape = [64, 64]
123    net = NetWithLoss(Net(shape, target='DEVICE', slice_mode=nn.EmbeddingLookup.TABLE_ROW_SLICE, operator='MEAN'))
124    compile_net(net, shape)
125
126
127def test_embeddinglookup_batch_parallel_max(test_context):
128    context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode="semi_auto_parallel")
129    shape = [64, 64]
130    net = NetWithLoss(Net(shape, target='DEVICE', operator='MAX'))
131    compile_net(net, shape)
132
133
134def test_embeddinglookup_column_parallel_max(test_context):
135    context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode="semi_auto_parallel")
136    shape = [64, 64]
137    net = NetWithLoss(Net(shape, target='DEVICE', slice_mode=nn.EmbeddingLookup.TABLE_COLUMN_SLICE, operator='MAX'))
138    compile_net(net, shape)
139
140
141def test_embeddinglookup_row_parallel_max(test_context):
142    context.set_auto_parallel_context(device_num=8, global_rank=0, parallel_mode="semi_auto_parallel")
143    shape = [64, 64]
144    net = NetWithLoss(Net(shape, target='DEVICE', slice_mode=nn.EmbeddingLookup.TABLE_ROW_SLICE, operator='MAX'))
145    compile_net(net, shape)
146