1# Copyright 2019 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15import mindspore.dataset as ds 16 17DATA_DIR = ["./data.data"] 18SCHEMA_DIR = "./schema.json" 19 20 21def test_case_0(): 22 """ 23 Test PyFunc 24 """ 25 print("Test 1-1 PyFunc : lambda x : x + x") 26 27 col = "col0" 28 29 # apply dataset operations 30 ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) 31 32 ds1 = ds1.map(operations=(lambda x: x + x), input_columns=col, output_columns="out") 33 34 print("************** Output Tensor *****************") 35 for data in ds1.create_dict_iterator(num_epochs=1, output_numpy=True): # each data is a dictionary 36 # in this example, each dictionary has keys "image" and "label" 37 print(data["out"]) 38 print("************** Output Tensor *****************") 39 40 41def test_case_1(): 42 """ 43 Test PyFunc 44 """ 45 print("Test 1-n PyFunc : (lambda x : (x , x + x)) ") 46 47 col = "col0" 48 49 # apply dataset operations 50 ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) 51 52 ds1 = ds1.map(operations=(lambda x: (x, x + x)), input_columns=col, output_columns=["out0", "out1"]) 53 54 print("************** Output Tensor *****************") 55 for data in ds1.create_dict_iterator(num_epochs=1, output_numpy=True): # each data is a dictionary 56 # in this example, each dictionary has keys "image" and "label" 57 print("out0") 58 print(data["out0"]) 59 print("out1") 60 print(data["out1"]) 61 print("************** Output Tensor *****************") 62 63 64def test_case_2(): 65 """ 66 Test PyFunc 67 """ 68 print("Test n-1 PyFunc : (lambda x, y : x + y) ") 69 70 col = ["col0", "col1"] 71 72 # apply dataset operations 73 ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) 74 75 ds1 = ds1.map(operations=(lambda x, y: x + y), input_columns=col, output_columns="out") 76 77 print("************** Output Tensor *****************") 78 for data in ds1.create_dict_iterator(num_epochs=1, output_numpy=True): # each data is a dictionary 79 # in this example, each dictionary has keys "image" and "label" 80 print(data["out"]) 81 82 print("************** Output Tensor *****************") 83 84 85def test_case_3(): 86 """ 87 Test PyFunc 88 """ 89 print("Test n-m PyFunc : (lambda x, y : (x , x + 1, x + y)") 90 91 col = ["col0", "col1"] 92 93 # apply dataset operations 94 ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) 95 96 ds1 = ds1.map(operations=(lambda x, y: (x, x + y, x + x + y)), input_columns=col, 97 output_columns=["out0", "out1", "out2"]) 98 99 print("************** Output Tensor *****************") 100 for data in ds1.create_dict_iterator(num_epochs=1, output_numpy=True): # each data is a dictionary 101 # in this example, each dictionary has keys "image" and "label" 102 print("out0") 103 print(data["out0"]) 104 print("out1") 105 print(data["out1"]) 106 print("out2") 107 print(data["out2"]) 108 print("************** Output Tensor *****************") 109 110 111def test_case_4(): 112 """ 113 Test PyFunc 114 """ 115 print("Test Parallel n-m PyFunc : (lambda x, y : (x , x + 1, x + y)") 116 117 col = ["col0", "col1"] 118 119 # apply dataset operations 120 ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) 121 122 ds1 = ds1.map(operations=(lambda x, y: (x, x + y, x + x + y)), input_columns=col, 123 output_columns=["out0", "out1", "out2"], num_parallel_workers=4) 124 125 print("************** Output Tensor *****************") 126 for data in ds1.create_dict_iterator(num_epochs=1, output_numpy=True): # each data is a dictionary 127 # in this example, each dictionary has keys "image" and "label" 128 print("out0") 129 print(data["out0"]) 130 print("out1") 131 print(data["out1"]) 132 print("out2") 133 print(data["out2"]) 134 print("************** Output Tensor *****************") 135 136 137if __name__ == "__main__": 138 test_case_0() 139 # test_case_1() 140 # test_case_2() 141 # test_case_3() 142 # test_case_4() 143