1# Copyright 2020 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15""" 16Testing RandomApply op in DE 17""" 18import numpy as np 19import mindspore.dataset as ds 20import mindspore.dataset.transforms.py_transforms as py_transforms 21import mindspore.dataset.vision.py_transforms as py_vision 22from mindspore import log as logger 23from util import visualize_list, config_get_set_seed, \ 24 config_get_set_num_parallel_workers, save_and_check_md5 25 26GENERATE_GOLDEN = False 27 28DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] 29SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" 30 31 32def test_random_apply_op(plot=False): 33 """ 34 Test RandomApply in python transformations 35 """ 36 logger.info("test_random_apply_op") 37 # define map operations 38 transforms_list = [py_vision.CenterCrop(64), py_vision.RandomRotation(30)] 39 transforms1 = [ 40 py_vision.Decode(), 41 py_transforms.RandomApply(transforms_list, prob=0.6), 42 py_vision.ToTensor() 43 ] 44 transform1 = py_transforms.Compose(transforms1) 45 46 transforms2 = [ 47 py_vision.Decode(), 48 py_vision.ToTensor() 49 ] 50 transform2 = py_transforms.Compose(transforms2) 51 52 # First dataset 53 data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) 54 data1 = data1.map(operations=transform1, input_columns=["image"]) 55 # Second dataset 56 data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) 57 data2 = data2.map(operations=transform2, input_columns=["image"]) 58 59 image_apply = [] 60 image_original = [] 61 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), 62 data2.create_dict_iterator(num_epochs=1, output_numpy=True)): 63 image1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8) 64 image2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) 65 image_apply.append(image1) 66 image_original.append(image2) 67 if plot: 68 visualize_list(image_original, image_apply) 69 70 71def test_random_apply_md5(): 72 """ 73 Test RandomApply op with md5 check 74 """ 75 logger.info("test_random_apply_md5") 76 original_seed = config_get_set_seed(10) 77 original_num_parallel_workers = config_get_set_num_parallel_workers(1) 78 # define map operations 79 transforms_list = [py_vision.CenterCrop(64), py_vision.RandomRotation(30)] 80 transforms = [ 81 py_vision.Decode(), 82 # Note: using default value "prob=0.5" 83 py_transforms.RandomApply(transforms_list), 84 py_vision.ToTensor() 85 ] 86 transform = py_transforms.Compose(transforms) 87 88 # Generate dataset 89 data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) 90 data = data.map(operations=transform, input_columns=["image"]) 91 92 # check results with md5 comparison 93 filename = "random_apply_01_result.npz" 94 save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) 95 96 # Restore configuration 97 ds.config.set_seed(original_seed) 98 ds.config.set_num_parallel_workers((original_num_parallel_workers)) 99 100 101def test_random_apply_exception_random_crop_badinput(): 102 """ 103 Test RandomApply: test invalid input for one of the transform functions, 104 expected to raise error 105 """ 106 logger.info("test_random_apply_exception_random_crop_badinput") 107 original_seed = config_get_set_seed(200) 108 original_num_parallel_workers = config_get_set_num_parallel_workers(1) 109 # define map operations 110 transforms_list = [py_vision.Resize([32, 32]), 111 py_vision.RandomCrop(100), # crop size > image size 112 py_vision.RandomRotation(30)] 113 transforms = [ 114 py_vision.Decode(), 115 py_transforms.RandomApply(transforms_list, prob=0.6), 116 py_vision.ToTensor() 117 ] 118 transform = py_transforms.Compose(transforms) 119 # Generate dataset 120 data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) 121 data = data.map(operations=transform, input_columns=["image"]) 122 try: 123 _ = data.create_dict_iterator(num_epochs=1).__next__() 124 except RuntimeError as e: 125 logger.info("Got an exception in DE: {}".format(str(e))) 126 assert "Crop size" in str(e) 127 # Restore configuration 128 ds.config.set_seed(original_seed) 129 ds.config.set_num_parallel_workers(original_num_parallel_workers) 130 131 132if __name__ == '__main__': 133 test_random_apply_op(plot=True) 134 test_random_apply_md5() 135 test_random_apply_exception_random_crop_badinput() 136