• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""
16Testing RandomOrder op in DE
17"""
18import numpy as np
19import mindspore.dataset as ds
20import mindspore.dataset.transforms.py_transforms as py_transforms
21import mindspore.dataset.vision.py_transforms as py_vision
22from mindspore import log as logger
23from util import visualize_list, config_get_set_seed, \
24    config_get_set_num_parallel_workers, save_and_check_md5
25
26GENERATE_GOLDEN = False
27
28DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
29SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
30
31
32def test_random_order_op(plot=False):
33    """
34    Test RandomOrder in python transformations
35    """
36    logger.info("test_random_order_op")
37    # define map operations
38    transforms_list = [py_vision.CenterCrop(64), py_vision.RandomRotation(30)]
39    transforms1 = [
40        py_vision.Decode(),
41        py_transforms.RandomOrder(transforms_list),
42        py_vision.ToTensor()
43    ]
44    transform1 = py_transforms.Compose(transforms1)
45
46    transforms2 = [
47        py_vision.Decode(),
48        py_vision.ToTensor()
49    ]
50    transform2 = py_transforms.Compose(transforms2)
51
52    #  First dataset
53    data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
54    data1 = data1.map(operations=transform1, input_columns=["image"])
55    #  Second dataset
56    data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
57    data2 = data2.map(operations=transform2, input_columns=["image"])
58
59    image_order = []
60    image_original = []
61    for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True),
62                            data2.create_dict_iterator(num_epochs=1, output_numpy=True)):
63        image1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
64        image2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
65        image_order.append(image1)
66        image_original.append(image2)
67    if plot:
68        visualize_list(image_original, image_order)
69
70
71def test_random_order_md5():
72    """
73    Test RandomOrder op with md5 check
74    """
75    logger.info("test_random_order_md5")
76    original_seed = config_get_set_seed(8)
77    original_num_parallel_workers = config_get_set_num_parallel_workers(1)
78    # define map operations
79    transforms_list = [py_vision.RandomCrop(64), py_vision.RandomRotation(30)]
80    transforms = [
81        py_vision.Decode(),
82        py_transforms.RandomOrder(transforms_list),
83        py_vision.ToTensor()
84    ]
85    transform = py_transforms.Compose(transforms)
86
87    #  Generate dataset
88    data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
89    data = data.map(operations=transform, input_columns=["image"])
90
91    # check results with md5 comparison
92    filename = "random_order_01_result.npz"
93    save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
94
95    # Restore configuration
96    ds.config.set_seed(original_seed)
97    ds.config.set_num_parallel_workers((original_num_parallel_workers))
98
99
100if __name__ == '__main__':
101    test_random_order_op(plot=True)
102    test_random_order_md5()
103