• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2021 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15import numpy as np
16import pytest
17from mindspore import nn, context
18from mindspore import ops as P
19from mindspore.train import DatasetHelper, connect_network_with_dataset
20import mindspore.dataset as ds
21context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
22
23def _exec_preprocess(network, is_train, dataset, dataset_sink_mode, sink_size=-1, epoch_num=1, dataset_helper=None):
24    if dataset_sink_mode and not is_train:
25        dataset.__loop_size__ = 1
26
27    if dataset_helper is None:
28        dataset_helper = DatasetHelper(dataset, dataset_sink_mode, sink_size, epoch_num)
29
30    if dataset_sink_mode:
31        network = connect_network_with_dataset(network, dataset_helper)
32
33    network.set_train(is_train)
34
35    return dataset_helper, network
36
37
38def _eval_dataset_sink_process(network, valid_dataset):
39    dataset_helper, eval_network = _exec_preprocess(network, is_train=False, dataset=valid_dataset,
40                                                    dataset_sink_mode=True)
41    for inputs1, inputs2 in zip(dataset_helper, valid_dataset.create_dict_iterator()):
42        outputs = eval_network(*inputs1)
43        for elem1, (_, elem2) in zip(outputs, inputs2.items()):
44            assert elem1.shape == elem2.shape
45
46def dataset_generator():
47    for i in range(1, 10):
48        yield (
49            np.ones((32, i), dtype=np.float32), np.zeros((32, i, i, 3), dtype=np.int32),
50            np.ones((32,), dtype=np.float32),
51            np.ones((32, i, 8), dtype=np.float32), np.ones((32, 8, 8), dtype=np.float32))
52
53class Net(nn.Cell):
54    def __init__(self):
55        super(Net, self).__init__()
56        self.relu = P.ReLU()
57
58    def construct(self, x1, x2, x3, x4, x5):
59        x1 = self.relu(x1)
60        x1 = self.relu(x1)
61
62        x2 = self.relu(x2)
63
64        x3 = self.relu(x3)
65        x3 = self.relu(x3)
66
67        x4 = self.relu(x4)
68
69        x5 = self.relu(x5)
70        return x1, x2, x3, x4, x5
71
72@pytest.mark.level0
73@pytest.mark.platform_arm_ascend_training
74@pytest.mark.platform_x86_ascend_training
75@pytest.mark.env_onecard
76def test_getnext_dynamic_pipeline():
77    network = Net()
78    dataset = ds.GeneratorDataset(dataset_generator, ["data1", "data2", "data3", "data4", "data5"])
79    dataset.set_dynamic_columns(columns={"data1": [32, None], "data2": [32, None, None, 3],
80                                         "data3": [32], "data4": [32, None, 8], "data5": [32, 8, 8]})
81    _eval_dataset_sink_process(network, dataset)
82