1 /** 2 * Copyright 2019 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #include <string> 17 #include "minddata/dataset/util/circular_pool.h" 18 #include "minddata/dataset/core/client.h" 19 #include "minddata/dataset/engine/execution_tree.h" 20 #include "minddata/dataset/engine/datasetops/shuffle_op.h" 21 #include "minddata/dataset/engine/datasetops/source/tf_reader_op.h" 22 #include "minddata/dataset/engine/jagged_connector.h" 23 #include "common/common.h" 24 #include "gtest/gtest.h" 25 #include "utils/log_adapter.h" 26 27 using namespace mindspore::dataset; 28 using mindspore::LogStream; 29 using mindspore::ExceptionType::NoExceptionType; 30 using mindspore::MsLogLevel::INFO; 31 32 class MindDataTestExecutionTree : public UT::DatasetOpTesting { 33 public: 34 }; 35 36 // Construct some tree nodes and play with them 37 TEST_F(MindDataTestExecutionTree, TestExecutionTree1) { 38 MS_LOG(INFO) << "Doing MindDataTestExecutionTree1."; 39 40 // Start with an empty execution tree 41 auto my_tree = std::make_shared<ExecutionTree>(); 42 ASSERT_NE(my_tree, nullptr); 43 uint32_t shuffle_size = 32; 44 uint32_t connector_size = 8; 45 46 std::shared_ptr<ShuffleOp> leaf_op1 = std::make_shared<ShuffleOp>(shuffle_size, 0, connector_size, false); 47 ASSERT_NE(leaf_op1, nullptr); 48 my_tree->AssociateNode(leaf_op1); 49 shuffle_size = 16; 50 std::shared_ptr<ShuffleOp> leaf_op2 = std::make_shared<ShuffleOp>(shuffle_size, 0, connector_size, false); 51 ASSERT_NE(leaf_op2, nullptr); 52 my_tree->AssociateNode(leaf_op2); 53 shuffle_size = 8; 54 std::shared_ptr<ShuffleOp> parent_op = std::make_shared<ShuffleOp>(shuffle_size, 0, connector_size, false); 55 ASSERT_NE(parent_op, nullptr); 56 my_tree->AssociateNode(parent_op); 57 58 // It's up to you if you want to use move semantic or not here. 59 // By using move, we transfer ownership of the child to the parent. 60 // If you do not use move, 61 // we get a reference count bump to the pointer and you have 62 // your own pointer to it, plus the parent has a copy of the pointer. 63 parent_op->AddChild(std::move(leaf_op1)); 64 parent_op->AddChild(std::move(leaf_op2)); 65 shuffle_size = 4; 66 std::shared_ptr<DatasetOp> root_op = std::make_shared<ShuffleOp>(shuffle_size, 0, connector_size, false); 67 my_tree->AssignRoot(root_op); 68 root_op->AddChild(parent_op); 69 ASSERT_NE(root_op, nullptr); 70 // Testing Iterator 71 MS_LOG(INFO) << "Testing Tree Iterator from root."; 72 for (auto itr = my_tree->begin(); itr != my_tree->end(); ++itr) { 73 itr->Print(std::cout, false); 74 } 75 MS_LOG(INFO) << "Finished testing Tree Iterator from root."; 76 MS_LOG(INFO) << "Testing Tree Iterator from parentOp."; 77 for (auto itr = my_tree->begin(parent_op); itr != my_tree->end(); ++itr) { 78 itr->Print(std::cout, false); 79 } 80 MS_LOG(INFO) << "Finished testing Tree Iterator from parentOp."; 81 82 // At this point, since move semantic was used, 83 // I don't have any operator access myself now. 84 // Ownership is fully transferred into the tree. 85 86 // explicitly drive tree destruction rather than 87 // wait for descoping (to examine in debugger 88 // just to see it work) 89 my_tree.reset(); 90 MS_LOG(INFO) << "Done."; 91 } 92 93 // Construct some tree nodes and play with them 94 TEST_F(MindDataTestExecutionTree, TestExecutionTree2) { 95 MS_LOG(INFO) << "Doing MindDataTestExecutionTree2."; 96 Status rc; 97 auto my_tree = std::make_shared<ExecutionTree>(); 98 99 std::string dataset_path = datasets_root_path_ + "/testDataset1/testDataset1.data"; 100 std::shared_ptr<ConfigManager> config_manager = GlobalContext::config_manager(); 101 auto op_connector_size = config_manager->op_connector_size(); 102 std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); 103 std::vector<std::string> columns_to_load = {}; 104 std::vector<std::string> files = {dataset_path}; 105 std::shared_ptr<TFReaderOp> my_tfreader_op = std::make_shared<TFReaderOp>( 106 1, 2, 0, files, std::move(schema), op_connector_size, columns_to_load, false, 1, 0, false); 107 rc = my_tfreader_op->Init(); 108 ASSERT_OK(rc); 109 rc = my_tree->AssociateNode(my_tfreader_op); 110 ASSERT_OK(rc); 111 rc = my_tree->AssignRoot(my_tfreader_op); 112 ASSERT_OK(rc); 113 114 // prepare the tree 115 rc = my_tree->Prepare(); 116 ASSERT_OK(rc); 117 118 // Launch the tree execution to kick off threads 119 // and start running the pipeline 120 MS_LOG(INFO) << "Launching my tree."; 121 rc = my_tree->Launch(); 122 ASSERT_OK(rc); 123 124 // Simulate a parse of data from our pipeline. 125 std::shared_ptr<DatasetOp> root_node = my_tree->root(); 126 127 // Start the loop of reading from our pipeline using iterator 128 MS_LOG(INFO) << "Testing DatasetIterator in testTree2."; 129 DatasetIterator di(my_tree); 130 TensorRow buffer; 131 rc = di.FetchNextTensorRow(&buffer); 132 EXPECT_TRUE(rc.IsOk()); 133 134 while (!buffer.empty()) { 135 rc = di.FetchNextTensorRow(&buffer); 136 EXPECT_TRUE(rc.IsOk()); 137 } 138 } 139 140 // Construct some tree nodes and play with them 141 TEST_F(MindDataTestExecutionTree, TestExecutionTree3) { MS_LOG(INFO) << "Doing MindDataTestExecutionTree3."; } 142