1 /**
2 * Copyright 2019-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common.h"
17 #include <algorithm>
18 #include <fstream>
19 #include <string>
20 #include <vector>
21 #include "minddata/dataset/core/client.h"
22 #include "minddata/dataset/core/config_manager.h"
23 #include "minddata/dataset/core/pybind_support.h"
24 #include "minddata/dataset/core/tensor.h"
25 #include "minddata/dataset/core/tensor_shape.h"
26 #include "minddata/dataset/engine/datasetops/batch_op.h"
27 #include "minddata/dataset/engine/datasetops/repeat_op.h"
28 #include "minddata/dataset/engine/datasetops/source/tf_reader_op.h"
29
30 namespace UT {
31 #ifdef __cplusplus
32 #if __cplusplus
33 extern "C" {
34 #endif
35 #endif
36
SetUp()37 void DatasetOpTesting::SetUp() {
38 std::string install_home = "data/dataset";
39 datasets_root_path_ = install_home;
40 mindrecord_root_path_ = "data/mindrecord";
41 }
42
ToTensorShapeVec(const std::vector<std::vector<int64_t>> & v)43 std::vector<mindspore::dataset::TensorShape> DatasetOpTesting::ToTensorShapeVec(
44 const std::vector<std::vector<int64_t>> &v) {
45 std::vector<mindspore::dataset::TensorShape> ret_v;
46 std::transform(v.begin(), v.end(), std::back_inserter(ret_v),
47 [](const auto &s) { return mindspore::dataset::TensorShape(s); });
48 return ret_v;
49 }
50
ToDETypes(const std::vector<mindspore::DataType> & t)51 std::vector<mindspore::dataset::DataType> DatasetOpTesting::ToDETypes(const std::vector<mindspore::DataType> &t) {
52 std::vector<mindspore::dataset::DataType> ret_t;
53 std::transform(t.begin(), t.end(), std::back_inserter(ret_t), [](const mindspore::DataType &t) {
54 return mindspore::dataset::MSTypeToDEType(static_cast<mindspore::TypeId>(t));
55 });
56 return ret_t;
57 }
58
59 // Function to read a file into an MSTensor
60 // Note: This provides the analogous support for DETensor's CreateFromFile.
ReadFileToTensor(const std::string & file)61 mindspore::MSTensor DatasetOpTesting::ReadFileToTensor(const std::string &file) {
62 if (file.empty()) {
63 MS_LOG(ERROR) << "Pointer file is nullptr; return an empty Tensor.";
64 return mindspore::MSTensor();
65 }
66 std::ifstream ifs(file);
67 if (!ifs.good()) {
68 MS_LOG(ERROR) << "File: " << file << " does not exist; return an empty Tensor.";
69 return mindspore::MSTensor();
70 }
71 if (!ifs.is_open()) {
72 MS_LOG(ERROR) << "File: " << file << " open failed; return an empty Tensor.";
73 return mindspore::MSTensor();
74 }
75
76 ifs.seekg(0, std::ios::end);
77 size_t size = ifs.tellg();
78 mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast<int64_t>(size)}, nullptr, size);
79
80 ifs.seekg(0, std::ios::beg);
81 ifs.read(reinterpret_cast<char *>(buf.MutableData()), size);
82 ifs.close();
83
84 return buf;
85 }
86
87 // Helper function to create a batch op
Batch(int32_t batch_size,bool drop,mindspore::dataset::PadInfo pad_map)88 std::shared_ptr<mindspore::dataset::BatchOp> DatasetOpTesting::Batch(int32_t batch_size, bool drop,
89 mindspore::dataset::PadInfo pad_map) {
90 std::shared_ptr<mindspore::dataset::ConfigManager> cfg = mindspore::dataset::GlobalContext::config_manager();
91 int32_t num_workers = cfg->num_parallel_workers();
92 int32_t op_connector_size = cfg->op_connector_size();
93 std::vector<std::string> output_columns = {};
94 std::vector<std::string> input_columns = {};
95 mindspore::dataset::py::function batch_size_func;
96 mindspore::dataset::py::function batch_map_func;
97 bool pad = false;
98 if (!pad_map.empty()) {
99 pad = true;
100 }
101 std::shared_ptr<mindspore::dataset::BatchOp> op =
102 std::make_shared<mindspore::dataset::BatchOp>(batch_size, drop, pad, op_connector_size, num_workers, input_columns,
103 output_columns, batch_size_func, batch_map_func, pad_map);
104 return op;
105 }
106
Repeat(int repeat_cnt)107 std::shared_ptr<mindspore::dataset::RepeatOp> DatasetOpTesting::Repeat(int repeat_cnt) {
108 std::shared_ptr<mindspore::dataset::RepeatOp> op = std::make_shared<mindspore::dataset::RepeatOp>(repeat_cnt);
109 return op;
110 }
111
TFReader(std::string file,int num_works)112 std::shared_ptr<mindspore::dataset::TFReaderOp> DatasetOpTesting::TFReader(std::string file, int num_works) {
113 std::shared_ptr<mindspore::dataset::ConfigManager> config_manager =
114 mindspore::dataset::GlobalContext::config_manager();
115 auto op_connector_size = config_manager->op_connector_size();
116 auto worker_connector_size = config_manager->worker_connector_size();
117 std::vector<std::string> columns_to_load = {};
118 std::vector<std::string> files = {file};
119 std::shared_ptr<mindspore::dataset::TFReaderOp> so = std::make_shared<mindspore::dataset::TFReaderOp>(
120 num_works, worker_connector_size, 0, files, std::make_unique<mindspore::dataset::DataSchema>(), op_connector_size,
121 columns_to_load, false, 1, 0, false, CompressionType::NONE, true);
122 (void)so->Init();
123 return so;
124 }
125
Build(std::vector<std::shared_ptr<mindspore::dataset::DatasetOp>> ops)126 std::shared_ptr<mindspore::dataset::ExecutionTree> DatasetOpTesting::Build(
127 std::vector<std::shared_ptr<mindspore::dataset::DatasetOp>> ops) {
128 std::shared_ptr<mindspore::dataset::ExecutionTree> tree = std::make_shared<mindspore::dataset::ExecutionTree>();
129 for (int i = 0; i < ops.size(); i++) {
130 tree->AssociateNode(ops[i]);
131 if (i > 0) {
132 ops[i]->AddChild(std::move(ops[i - 1]));
133 }
134 if (i == ops.size() - 1) {
135 tree->AssignRoot(ops[i]);
136 }
137 }
138 return tree;
139 }
140
141 #ifdef __cplusplus
142 #if __cplusplus
143 }
144 #endif
145 #endif
146 } // namespace UT
147
148 // Helper function to get the session id from SESSION_ID env variable
GetSessionFromEnv(uint32_t * session_id)149 Status GetSessionFromEnv(uint32_t *session_id) {
150 RETURN_UNEXPECTED_IF_NULL(session_id);
151 if (const char *session_env = std::getenv("SESSION_ID")) {
152 std::string session_id_str(session_env);
153 try {
154 *session_id = std::stoul(session_id_str);
155 } catch (const std::exception &e) {
156 std::string err_msg = "Invalid numeric value for session id in env var: " + session_id_str;
157 return Status(StatusCode::kMDSyntaxError, err_msg);
158 }
159 } else {
160 RETURN_STATUS_UNEXPECTED("Test case requires a session id to be provided via SESSION_ID environment variable.");
161 }
162 return Status::OK();
163 }
164
165 namespace mindspore {
166 namespace dataset {
Predicate1(MSTensorVec in)167 MSTensorVec Predicate1(MSTensorVec in) {
168 // Return true if input is equal to 3
169 uint64_t input_value;
170 TensorRow input = VecToRow(in);
171 (void)input.at(0)->GetItemAt(&input_value, {0});
172 bool result = (input_value == 3);
173
174 // Convert from boolean to TensorRow
175 TensorRow output;
176 std::shared_ptr<Tensor> out;
177 (void)Tensor::CreateEmpty(TensorShape({}), DataType(DataType::Type::DE_BOOL), &out);
178 (void)out->SetItemAt({}, result);
179 output.push_back(out);
180
181 return RowToVec(output);
182 }
183
Predicate2(MSTensorVec in)184 MSTensorVec Predicate2(MSTensorVec in) {
185 // Return true if label is more than 1
186 // The index of label in input is 1
187 uint64_t input_value;
188 TensorRow input = VecToRow(in);
189 (void)input.at(1)->GetItemAt(&input_value, {0});
190 bool result = (input_value > 1);
191
192 // Convert from boolean to TensorRow
193 TensorRow output;
194 std::shared_ptr<Tensor> out;
195 (void)Tensor::CreateEmpty(TensorShape({}), DataType(mindspore::dataset::DataType::Type::DE_BOOL), &out);
196 (void)out->SetItemAt({}, result);
197 output.push_back(out);
198
199 return RowToVec(output);
200 }
201
Predicate3(MSTensorVec in)202 MSTensorVec Predicate3(MSTensorVec in) {
203 // Return true if label is non-negative (don't filter!)
204 uint64_t input_value;
205 TensorRow input = VecToRow(in);
206 (void)input.at(0)->GetItemAt(&input_value, {0});
207 bool result = (input_value >= 0);
208
209 // Convert from boolean to TensorRow
210 TensorRow output;
211 std::shared_ptr<Tensor> out;
212 (void)Tensor::CreateEmpty(TensorShape({}), DataType(mindspore::dataset::DataType::Type::DE_BOOL), &out);
213 (void)out->SetItemAt({}, result);
214 output.push_back(out);
215
216 return RowToVec(output);
217 }
218
BGRToRGB(const cv::Mat & img)219 cv::Mat BGRToRGB(const cv::Mat &img) {
220 cv::Mat image(img.rows, img.cols, CV_8UC3);
221 for (int i = 0; i < img.rows; ++i) {
222 auto p1 = img.ptr<cv::Vec3b>(i);
223 auto p2 = image.ptr<cv::Vec3b>(i);
224 for (int j = 0; j < img.cols; ++j) {
225 p2[j][2] = p1[j][0];
226 p2[j][1] = p1[j][1];
227 p2[j][0] = p1[j][2];
228 }
229 }
230 return image;
231 }
232 } // namespace dataset
233 } // namespace mindspore
234