1 /** 2 * Copyright 2020-2021 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "minddata/dataset/engine/tree_adapter.h" 18 #include "common/common.h" 19 #include "minddata/dataset/core/tensor_row.h" 20 #include "minddata/dataset/include/dataset/datasets.h" 21 #include "minddata/dataset/include/dataset/transforms.h" 22 23 // IR non-leaf nodes 24 #include "minddata/dataset/engine/ir/datasetops/batch_node.h" 25 #include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" 26 #include "minddata/dataset/engine/ir/datasetops/concat_node.h" 27 #include "minddata/dataset/engine/ir/datasetops/map_node.h" 28 #include "minddata/dataset/engine/ir/datasetops/project_node.h" 29 #include "minddata/dataset/engine/ir/datasetops/rename_node.h" 30 #include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" 31 #include "minddata/dataset/engine/ir/datasetops/skip_node.h" 32 #include "minddata/dataset/engine/ir/datasetops/zip_node.h" 33 34 35 using namespace mindspore::dataset; 36 using mindspore::dataset::Tensor; 37 38 class MindDataTestTreeAdapter : public UT::DatasetOpTesting { 39 protected: 40 }; 41 42 TEST_F(MindDataTestTreeAdapter, TestSimpleTreeAdapter) { 43 MS_LOG(INFO) << "Doing MindDataTestTreeAdapter-TestSimpleTreeAdapter."; 44 45 // Create a Mnist Dataset 46 std::string folder_path = datasets_root_path_ + "/testMnistData/"; 47 std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 4)); 48 EXPECT_NE(ds, nullptr); 49 50 ds = ds->Batch(2); 51 EXPECT_NE(ds, nullptr); 52 53 mindspore::dataset::TreeAdapter tree_adapter; 54 55 Status rc = tree_adapter.Compile(ds->IRNode(), 1); 56 57 EXPECT_TRUE(rc.IsOk()); 58 59 const std::unordered_map<std::string, int32_t> map = {{"label", 1}, {"image", 0}}; 60 EXPECT_EQ(tree_adapter.GetColumnNameMap(), map); 61 62 std::vector<size_t> row_sizes = {2, 2, 0}; 63 64 TensorRow row; 65 for (size_t sz : row_sizes) { 66 rc = tree_adapter.GetNext(&row); 67 EXPECT_TRUE(rc.IsOk()); 68 EXPECT_EQ(row.size(), sz); 69 } 70 71 rc = tree_adapter.GetNext(&row); 72 EXPECT_TRUE(rc.IsError()); 73 const std::string err_msg = rc.ToString(); 74 EXPECT_TRUE(err_msg.find("EOF buffer encountered.") != err_msg.npos); 75 } 76 77 TEST_F(MindDataTestTreeAdapter, TestTreeAdapterWithRepeat) { 78 MS_LOG(INFO) << "Doing MindDataTestTreeAdapter-TestTreeAdapterWithRepeat."; 79 80 // Create a Mnist Dataset 81 std::string folder_path = datasets_root_path_ + "/testMnistData/"; 82 std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 3)); 83 EXPECT_NE(ds, nullptr); 84 85 ds = ds->Batch(2, false); 86 EXPECT_NE(ds, nullptr); 87 88 mindspore::dataset::TreeAdapter tree_adapter; 89 90 Status rc = tree_adapter.Compile(ds->IRNode(), 2); 91 EXPECT_TRUE(rc.IsOk()); 92 93 const std::unordered_map<std::string, int32_t> map = tree_adapter.GetColumnNameMap(); 94 EXPECT_EQ(tree_adapter.GetColumnNameMap(), map); 95 96 std::vector<size_t> row_sizes = {2, 2, 0, 2, 2, 0}; 97 98 TensorRow row; 99 for (size_t sz : row_sizes) { 100 rc = tree_adapter.GetNext(&row); 101 EXPECT_TRUE(rc.IsOk()); 102 EXPECT_EQ(row.size(), sz); 103 } 104 rc = tree_adapter.GetNext(&row); 105 const std::string err_msg = rc.ToString(); 106 EXPECT_TRUE(err_msg.find("EOF buffer encountered.") != err_msg.npos); 107 } 108 109 TEST_F(MindDataTestTreeAdapter, TestProjectMapTreeAdapter) { 110 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestProjectMap."; 111 112 // Create an ImageFolder Dataset 113 std::string folder_path = datasets_root_path_ + "/testPK/data/"; 114 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, 2)); 115 EXPECT_NE(ds, nullptr); 116 117 // Create objects for the tensor ops 118 std::shared_ptr<TensorTransform> one_hot = std::make_shared<transforms::OneHot>(10); 119 EXPECT_NE(one_hot, nullptr); 120 121 // Create a Map operation, this will automatically add a project after map 122 ds = ds->Map({one_hot}, {"label"}, {"label"}, {"label"}); 123 EXPECT_NE(ds, nullptr); 124 125 mindspore::dataset::TreeAdapter tree_adapter; 126 127 Status rc = tree_adapter.Compile(ds->IRNode(), 2); 128 129 EXPECT_TRUE(rc.IsOk()); 130 131 const std::unordered_map<std::string, int32_t> map = {{"label", 0}}; 132 EXPECT_EQ(tree_adapter.GetColumnNameMap(), map); 133 134 std::vector<size_t> row_sizes = {1, 1, 0, 1, 1, 0}; 135 TensorRow row; 136 137 for (size_t sz : row_sizes) { 138 rc = tree_adapter.GetNext(&row); 139 EXPECT_TRUE(rc.IsOk()); 140 EXPECT_EQ(row.size(), sz); 141 } 142 rc = tree_adapter.GetNext(&row); 143 const std::string err_msg = rc.ToString(); 144 EXPECT_TRUE(err_msg.find("EOF buffer encountered.") != err_msg.npos); 145 } 146