1 /**
2 * Copyright 2019 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include <string>
17 #include "minddata/dataset/util/circular_pool.h"
18 #include "minddata/dataset/core/client.h"
19 #include "minddata/dataset/engine/execution_tree.h"
20 #include "minddata/dataset/engine/datasetops/shuffle_op.h"
21 #include "minddata/dataset/engine/datasetops/source/tf_reader_op.h"
22 #include "minddata/dataset/engine/jagged_connector.h"
23 #include "common/common.h"
24 #include "gtest/gtest.h"
25 #include "utils/log_adapter.h"
26
27 using namespace mindspore::dataset;
28 using mindspore::LogStream;
29 using mindspore::ExceptionType::NoExceptionType;
30 using mindspore::MsLogLevel::INFO;
31
32 class MindDataTestExecutionTree : public UT::DatasetOpTesting {
33 public:
34 };
35
36 // Construct some tree nodes and play with them
TEST_F(MindDataTestExecutionTree,TestExecutionTree1)37 TEST_F(MindDataTestExecutionTree, TestExecutionTree1) {
38 MS_LOG(INFO) << "Doing MindDataTestExecutionTree1.";
39
40 // Start with an empty execution tree
41 auto my_tree = std::make_shared<ExecutionTree>();
42 ASSERT_NE(my_tree, nullptr);
43 uint32_t shuffle_size = 32;
44 uint32_t connector_size = 8;
45
46 std::shared_ptr<ShuffleOp> leaf_op1 = std::make_shared<ShuffleOp>(shuffle_size, 0, connector_size, false);
47 ASSERT_NE(leaf_op1, nullptr);
48 my_tree->AssociateNode(leaf_op1);
49 shuffle_size = 16;
50 std::shared_ptr<ShuffleOp> leaf_op2 = std::make_shared<ShuffleOp>(shuffle_size, 0, connector_size, false);
51 ASSERT_NE(leaf_op2, nullptr);
52 my_tree->AssociateNode(leaf_op2);
53 shuffle_size = 8;
54 std::shared_ptr<ShuffleOp> parent_op = std::make_shared<ShuffleOp>(shuffle_size, 0, connector_size, false);
55 ASSERT_NE(parent_op, nullptr);
56 my_tree->AssociateNode(parent_op);
57
58 // It's up to you if you want to use move semantic or not here.
59 // By using move, we transfer ownership of the child to the parent.
60 // If you do not use move,
61 // we get a reference count bump to the pointer and you have
62 // your own pointer to it, plus the parent has a copy of the pointer.
63 parent_op->AddChild(std::move(leaf_op1));
64 parent_op->AddChild(std::move(leaf_op2));
65 shuffle_size = 4;
66 std::shared_ptr<DatasetOp> root_op = std::make_shared<ShuffleOp>(shuffle_size, 0, connector_size, false);
67 my_tree->AssignRoot(root_op);
68 root_op->AddChild(parent_op);
69 ASSERT_NE(root_op, nullptr);
70 // Testing Iterator
71 MS_LOG(INFO) << "Testing Tree Iterator from root.";
72 for (auto itr = my_tree->begin(); itr != my_tree->end(); ++itr) {
73 itr->Print(std::cout, false);
74 }
75 MS_LOG(INFO) << "Finished testing Tree Iterator from root.";
76 MS_LOG(INFO) << "Testing Tree Iterator from parentOp.";
77 for (auto itr = my_tree->begin(parent_op); itr != my_tree->end(); ++itr) {
78 itr->Print(std::cout, false);
79 }
80 MS_LOG(INFO) << "Finished testing Tree Iterator from parentOp.";
81
82 // At this point, since move semantic was used,
83 // I don't have any operator access myself now.
84 // Ownership is fully transferred into the tree.
85
86 // explicitly drive tree destruction rather than
87 // wait for descoping (to examine in debugger
88 // just to see it work)
89 my_tree.reset();
90 MS_LOG(INFO) << "Done.";
91 }
92
93 // Construct some tree nodes and play with them
TEST_F(MindDataTestExecutionTree,TestExecutionTree2)94 TEST_F(MindDataTestExecutionTree, TestExecutionTree2) {
95 MS_LOG(INFO) << "Doing MindDataTestExecutionTree2.";
96 Status rc;
97 auto my_tree = std::make_shared<ExecutionTree>();
98
99 std::string dataset_path = datasets_root_path_ + "/testDataset1/testDataset1.data";
100 std::shared_ptr<ConfigManager> config_manager = GlobalContext::config_manager();
101 auto op_connector_size = config_manager->op_connector_size();
102 std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
103 std::vector<std::string> columns_to_load = {};
104 std::vector<std::string> files = {dataset_path};
105 std::shared_ptr<TFReaderOp> my_tfreader_op = std::make_shared<TFReaderOp>(
106 1, 2, 0, files, std::move(schema), op_connector_size, columns_to_load, false, 1, 0, false);
107 rc = my_tfreader_op->Init();
108 ASSERT_OK(rc);
109 rc = my_tree->AssociateNode(my_tfreader_op);
110 ASSERT_OK(rc);
111 rc = my_tree->AssignRoot(my_tfreader_op);
112 ASSERT_OK(rc);
113
114 // prepare the tree
115 rc = my_tree->Prepare();
116 ASSERT_OK(rc);
117
118 // Launch the tree execution to kick off threads
119 // and start running the pipeline
120 MS_LOG(INFO) << "Launching my tree.";
121 rc = my_tree->Launch();
122 ASSERT_OK(rc);
123
124 // Simulate a parse of data from our pipeline.
125 std::shared_ptr<DatasetOp> root_node = my_tree->root();
126
127 // Start the loop of reading from our pipeline using iterator
128 MS_LOG(INFO) << "Testing DatasetIterator in testTree2.";
129 DatasetIterator di(my_tree);
130 TensorRow buffer;
131 rc = di.FetchNextTensorRow(&buffer);
132 EXPECT_TRUE(rc.IsOk());
133
134 while (!buffer.empty()) {
135 rc = di.FetchNextTensorRow(&buffer);
136 EXPECT_TRUE(rc.IsOk());
137 }
138 }
139
140 // Construct some tree nodes and play with them
TEST_F(MindDataTestExecutionTree,TestExecutionTree3)141 TEST_F(MindDataTestExecutionTree, TestExecutionTree3) { MS_LOG(INFO) << "Doing MindDataTestExecutionTree3."; }
142