1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include <fstream>
17 #include <stack>
18 #include "minddata/dataset/engine/serdes.h"
19
20 #include "minddata/dataset/core/pybind_support.h"
21 #include "utils/file_utils.h"
22 #include "utils/utils.h"
23
24 namespace mindspore {
25 namespace dataset {
26
27 std::map<std::string, Status (*)(nlohmann::json json_obj, std::shared_ptr<TensorOperation> *operation)>
28 Serdes::func_ptr_ = Serdes::InitializeFuncPtr();
29
SaveToJSON(std::shared_ptr<DatasetNode> node,const std::string & filename,nlohmann::json * out_json)30 Status Serdes::SaveToJSON(std::shared_ptr<DatasetNode> node, const std::string &filename, nlohmann::json *out_json) {
31 RETURN_UNEXPECTED_IF_NULL(node);
32 RETURN_UNEXPECTED_IF_NULL(out_json);
33 // Dump attributes of current node to json string
34 nlohmann::json args;
35 RETURN_IF_NOT_OK(node->to_json(&args));
36 args["op_type"] = node->Name();
37
38 // If the current node isn't leaf node, visit all its children and get all attributes
39 std::vector<nlohmann::json> children_pipeline;
40 if (!node->IsLeaf()) {
41 for (auto child : node->Children()) {
42 nlohmann::json child_args;
43 RETURN_IF_NOT_OK(SaveToJSON(child, "", &child_args));
44 children_pipeline.push_back(child_args);
45 }
46 }
47 args["children"] = children_pipeline;
48
49 // Save json string into file if filename is given.
50 if (!filename.empty()) {
51 RETURN_IF_NOT_OK(SaveJSONToFile(args, filename));
52 }
53
54 *out_json = args;
55 return Status::OK();
56 }
57
SaveJSONToFile(nlohmann::json json_string,const std::string & file_name)58 Status Serdes::SaveJSONToFile(nlohmann::json json_string, const std::string &file_name) {
59 try {
60 std::optional<std::string> dir = "";
61 std::optional<std::string> local_file_name = "";
62 FileUtils::SplitDirAndFileName(file_name, &dir, &local_file_name);
63 if (!dir.has_value()) {
64 dir = ".";
65 }
66 auto realpath = FileUtils::GetRealPath(dir.value().data());
67 if (!realpath.has_value()) {
68 MS_LOG(ERROR) << "Get real path failed, path=" << file_name;
69 RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_name);
70 }
71
72 std::optional<std::string> whole_path = "";
73 FileUtils::ConcatDirAndFileName(&realpath, &local_file_name, &whole_path);
74
75 std::ofstream file(whole_path.value());
76 file << json_string;
77 file.close();
78
79 ChangeFileMode(whole_path.value(), S_IRUSR | S_IWUSR);
80 } catch (const std::exception &err) {
81 RETURN_STATUS_UNEXPECTED("Invalid data, failed to save json string into file: " + file_name);
82 }
83 return Status::OK();
84 }
85
Deserialize(const std::string & json_filepath,std::shared_ptr<DatasetNode> * ds)86 Status Serdes::Deserialize(const std::string &json_filepath, std::shared_ptr<DatasetNode> *ds) {
87 nlohmann::json json_obj;
88 CHECK_FAIL_RETURN_UNEXPECTED(json_filepath.size() != 0, "Json path is null");
89 std::ifstream json_in(json_filepath);
90 CHECK_FAIL_RETURN_UNEXPECTED(json_in, "Invalid file, failed to open json file: " + json_filepath);
91 try {
92 json_in >> json_obj;
93 } catch (const std::exception &e) {
94 return Status(StatusCode::kMDSyntaxError, "Invalid file, failed to parse json file: " + json_filepath);
95 }
96 RETURN_IF_NOT_OK(ConstructPipeline(json_obj, ds));
97 return Status::OK();
98 }
99
ConstructPipeline(nlohmann::json json_obj,std::shared_ptr<DatasetNode> * ds)100 Status Serdes::ConstructPipeline(nlohmann::json json_obj, std::shared_ptr<DatasetNode> *ds) {
101 CHECK_FAIL_RETURN_UNEXPECTED(json_obj.find("children") != json_obj.end(), "Failed to find children");
102 std::shared_ptr<DatasetNode> child_ds;
103
104 if (json_obj["children"].size() == 0) {
105 // If the JSON object has no child, then this node is a leaf node. Call create node to construct the corresponding
106 // leaf node
107 RETURN_IF_NOT_OK(CreateNode(nullptr, json_obj, ds));
108 } else if (json_obj["children"].size() == 1) {
109 // This node only has one child, construct the sub-tree under it first, and then call create node to construct the
110 // corresponding node
111 RETURN_IF_NOT_OK(ConstructPipeline(json_obj["children"][0], &child_ds));
112 RETURN_IF_NOT_OK(CreateNode(child_ds, json_obj, ds));
113 } else {
114 std::vector<std::shared_ptr<DatasetNode>> datasets;
115 for (auto child_json_obj : json_obj["children"]) {
116 RETURN_IF_NOT_OK(ConstructPipeline(child_json_obj, &child_ds));
117 datasets.push_back(child_ds);
118 }
119 if (json_obj["op_type"] == "Zip") {
120 CHECK_FAIL_RETURN_UNEXPECTED(datasets.size() > 1, "Should zip more than 1 dataset");
121 RETURN_IF_NOT_OK(ZipNode::from_json(datasets, ds));
122 } else if (json_obj["op_type"] == "Concat") {
123 CHECK_FAIL_RETURN_UNEXPECTED(datasets.size() > 1, "Should concat more than 1 dataset");
124 RETURN_IF_NOT_OK(ConcatNode::from_json(json_obj, datasets, ds));
125 } else {
126 return Status(StatusCode::kMDUnexpectedError,
127 "Invalid data, unsupported operation type: " + std::string(json_obj["op_type"]));
128 }
129 }
130 return Status::OK();
131 }
132
CreateNode(const std::shared_ptr<DatasetNode> & child_ds,nlohmann::json json_obj,std::shared_ptr<DatasetNode> * ds)133 Status Serdes::CreateNode(const std::shared_ptr<DatasetNode> &child_ds, nlohmann::json json_obj,
134 std::shared_ptr<DatasetNode> *ds) {
135 CHECK_FAIL_RETURN_UNEXPECTED(json_obj.find("op_type") != json_obj.end(), "Failed to find op_type in json.");
136 std::string op_type = json_obj["op_type"];
137 if (child_ds == nullptr) {
138 // if dataset doesn't have any child, then create a source dataset IR. e.g., ImageFolderNode, CocoNode
139 RETURN_IF_NOT_OK(CreateDatasetNode(json_obj, op_type, ds));
140 } else {
141 // if the dataset has at least one child, then create an operation dataset IR, e.g., BatchNode, MapNode
142 RETURN_IF_NOT_OK(CreateDatasetOperationNode(child_ds, json_obj, op_type, ds));
143 }
144 return Status::OK();
145 }
146
CreateDatasetNode(const nlohmann::json & json_obj,const std::string & op_type,std::shared_ptr<DatasetNode> * ds)147 Status Serdes::CreateDatasetNode(const nlohmann::json &json_obj, const std::string &op_type,
148 std::shared_ptr<DatasetNode> *ds) {
149 if (op_type == kAlbumNode) {
150 RETURN_IF_NOT_OK(AlbumNode::from_json(json_obj, ds));
151 } else if (op_type == kCelebANode) {
152 RETURN_IF_NOT_OK(CelebANode::from_json(json_obj, ds));
153 } else if (op_type == kCifar10Node) {
154 RETURN_IF_NOT_OK(Cifar10Node::from_json(json_obj, ds));
155 } else if (op_type == kCifar100Node) {
156 RETURN_IF_NOT_OK(Cifar100Node::from_json(json_obj, ds));
157 } else if (op_type == kCLUENode) {
158 RETURN_IF_NOT_OK(CLUENode::from_json(json_obj, ds));
159 } else if (op_type == kCocoNode) {
160 RETURN_IF_NOT_OK(CocoNode::from_json(json_obj, ds));
161 } else if (op_type == kCSVNode) {
162 RETURN_IF_NOT_OK(CSVNode::from_json(json_obj, ds));
163 } else if (op_type == kFlickrNode) {
164 RETURN_IF_NOT_OK(FlickrNode::from_json(json_obj, ds));
165 } else if (op_type == kImageFolderNode) {
166 RETURN_IF_NOT_OK(ImageFolderNode::from_json(json_obj, ds));
167 } else if (op_type == kManifestNode) {
168 RETURN_IF_NOT_OK(ManifestNode::from_json(json_obj, ds));
169 } else if (op_type == kMnistNode) {
170 RETURN_IF_NOT_OK(MnistNode::from_json(json_obj, ds));
171 } else if (op_type == kTextFileNode) {
172 RETURN_IF_NOT_OK(TextFileNode::from_json(json_obj, ds));
173 } else if (op_type == kTFRecordNode) {
174 RETURN_IF_NOT_OK(TFRecordNode::from_json(json_obj, ds));
175 } else if (op_type == kVOCNode) {
176 RETURN_IF_NOT_OK(VOCNode::from_json(json_obj, ds));
177 } else {
178 return Status(StatusCode::kMDUnexpectedError, "Invalid data, unsupported operation type: " + op_type);
179 }
180 return Status::OK();
181 }
182
CreateDatasetOperationNode(const std::shared_ptr<DatasetNode> & ds,const nlohmann::json & json_obj,const std::string & op_type,std::shared_ptr<DatasetNode> * result)183 Status Serdes::CreateDatasetOperationNode(const std::shared_ptr<DatasetNode> &ds, const nlohmann::json &json_obj,
184 const std::string &op_type, std::shared_ptr<DatasetNode> *result) {
185 if (op_type == kBatchNode) {
186 RETURN_IF_NOT_OK(BatchNode::from_json(json_obj, ds, result));
187 } else if (op_type == kMapNode) {
188 RETURN_IF_NOT_OK(MapNode::from_json(json_obj, ds, result));
189 } else if (op_type == kProjectNode) {
190 RETURN_IF_NOT_OK(ProjectNode::from_json(json_obj, ds, result));
191 } else if (op_type == kRenameNode) {
192 RETURN_IF_NOT_OK(RenameNode::from_json(json_obj, ds, result));
193 } else if (op_type == kRepeatNode) {
194 RETURN_IF_NOT_OK(RepeatNode::from_json(json_obj, ds, result));
195 } else if (op_type == kShuffleNode) {
196 RETURN_IF_NOT_OK(ShuffleNode::from_json(json_obj, ds, result));
197 } else if (op_type == kSkipNode) {
198 RETURN_IF_NOT_OK(SkipNode::from_json(json_obj, ds, result));
199 } else if (op_type == kTransferNode) {
200 RETURN_IF_NOT_OK(TransferNode::from_json(json_obj, ds, result));
201 } else if (op_type == kTakeNode) {
202 RETURN_IF_NOT_OK(TakeNode::from_json(json_obj, ds, result));
203 } else {
204 return Status(StatusCode::kMDUnexpectedError, "Invalid data, unsupported operation type: " + op_type);
205 }
206 return Status::OK();
207 }
208
ConstructSampler(nlohmann::json json_obj,std::shared_ptr<SamplerObj> * sampler)209 Status Serdes::ConstructSampler(nlohmann::json json_obj, std::shared_ptr<SamplerObj> *sampler) {
210 CHECK_FAIL_RETURN_UNEXPECTED(json_obj.find("num_samples") != json_obj.end(), "Failed to find num_samples");
211 CHECK_FAIL_RETURN_UNEXPECTED(json_obj.find("sampler_name") != json_obj.end(), "Failed to find sampler_name");
212 int64_t num_samples = json_obj["num_samples"];
213 std::string sampler_name = json_obj["sampler_name"];
214 if (sampler_name == "DistributedSampler") {
215 RETURN_IF_NOT_OK(DistributedSamplerObj::from_json(json_obj, num_samples, sampler));
216 } else if (sampler_name == "PKSampler") {
217 RETURN_IF_NOT_OK(PKSamplerObj::from_json(json_obj, num_samples, sampler));
218 } else if (sampler_name == "RandomSampler") {
219 RETURN_IF_NOT_OK(RandomSamplerObj::from_json(json_obj, num_samples, sampler));
220 } else if (sampler_name == "SequentialSampler") {
221 RETURN_IF_NOT_OK(SequentialSamplerObj::from_json(json_obj, num_samples, sampler));
222 } else if (sampler_name == "SubsetSampler") {
223 RETURN_IF_NOT_OK(SubsetSamplerObj::from_json(json_obj, num_samples, sampler));
224 } else if (sampler_name == "SubsetRandomSampler") {
225 RETURN_IF_NOT_OK(SubsetRandomSamplerObj::from_json(json_obj, num_samples, sampler));
226 } else if (sampler_name == "WeightedRandomSampler") {
227 RETURN_IF_NOT_OK(WeightedRandomSamplerObj::from_json(json_obj, num_samples, sampler));
228 } else {
229 return Status(StatusCode::kMDUnexpectedError, "Invalid data, unsupported sampler type: " + sampler_name);
230 }
231 return Status::OK();
232 }
233
ConstructTensorOps(nlohmann::json json_obj,std::vector<std::shared_ptr<TensorOperation>> * result)234 Status Serdes::ConstructTensorOps(nlohmann::json json_obj, std::vector<std::shared_ptr<TensorOperation>> *result) {
235 std::vector<std::shared_ptr<TensorOperation>> output;
236 for (nlohmann::json item : json_obj) {
237 if (item.find("python_module") != item.end()) {
238 if (Py_IsInitialized()) {
239 RETURN_IF_NOT_OK(PyFuncOp::from_json(item, result));
240 } else {
241 RETURN_STATUS_SYNTAX_ERROR("Python module is not initialized or Pyfunction is not supported on this platform.");
242 }
243 } else {
244 CHECK_FAIL_RETURN_UNEXPECTED(item.find("tensor_op_name") != item.end(), "Failed to find tensor_op_name");
245 CHECK_FAIL_RETURN_UNEXPECTED(item.find("tensor_op_params") != item.end(), "Failed to find tensor_op_params");
246 std::string op_name = item["tensor_op_name"];
247 nlohmann::json op_params = item["tensor_op_params"];
248 std::shared_ptr<TensorOperation> operation = nullptr;
249 CHECK_FAIL_RETURN_UNEXPECTED(func_ptr_.find(op_name) != func_ptr_.end(),
250 "Invalid data, unsupported operation: " + op_name);
251 RETURN_IF_NOT_OK(func_ptr_[op_name](op_params, &operation));
252 output.push_back(operation);
253 *result = output;
254 }
255 }
256 return Status::OK();
257 }
258
259 std::map<std::string, Status (*)(nlohmann::json json_obj, std::shared_ptr<TensorOperation> *operation)>
InitializeFuncPtr()260 Serdes::InitializeFuncPtr() {
261 std::map<std::string, Status (*)(nlohmann::json json_obj, std::shared_ptr<TensorOperation> * operation)> ops_ptr;
262 ops_ptr[vision::kAdjustGammaOperation] = &(vision::AdjustGammaOperation::from_json);
263 ops_ptr[vision::kAffineOperation] = &(vision::AffineOperation::from_json);
264 ops_ptr[vision::kAutoContrastOperation] = &(vision::AutoContrastOperation::from_json);
265 ops_ptr[vision::kBoundingBoxAugmentOperation] = &(vision::BoundingBoxAugmentOperation::from_json);
266 ops_ptr[vision::kCenterCropOperation] = &(vision::CenterCropOperation::from_json);
267 ops_ptr[vision::kCropOperation] = &(vision::CropOperation::from_json);
268 ops_ptr[vision::kCutMixBatchOperation] = &(vision::CutMixBatchOperation::from_json);
269 ops_ptr[vision::kCutOutOperation] = &(vision::CutOutOperation::from_json);
270 ops_ptr[vision::kDecodeOperation] = &(vision::DecodeOperation::from_json);
271 #ifdef ENABLE_ACL
272 ops_ptr[vision::kDvppCropJpegOperation] = &(vision::DvppCropJpegOperation::from_json);
273 ops_ptr[vision::kDvppDecodeResizeOperation] = &(vision::DvppDecodeResizeOperation::from_json);
274 ops_ptr[vision::kDvppDecodeResizeCropOperation] = &(vision::DvppDecodeResizeCropOperation::from_json);
275 ops_ptr[vision::kDvppNormalizeOperation] = &(vision::DvppNormalizeOperation::from_json);
276 ops_ptr[vision::kDvppResizeJpegOperation] = &(vision::DvppResizeJpegOperation::from_json);
277 #endif
278 ops_ptr[vision::kEqualizeOperation] = &(vision::EqualizeOperation::from_json);
279 ops_ptr[vision::kGaussianBlurOperation] = &(vision::GaussianBlurOperation::from_json);
280 ops_ptr[vision::kHorizontalFlipOperation] = &(vision::HorizontalFlipOperation::from_json);
281 ops_ptr[vision::kHwcToChwOperation] = &(vision::HwcToChwOperation::from_json);
282 ops_ptr[vision::kInvertOperation] = &(vision::InvertOperation::from_json);
283 ops_ptr[vision::kMixUpBatchOperation] = &(vision::MixUpBatchOperation::from_json);
284 ops_ptr[vision::kNormalizeOperation] = &(vision::NormalizeOperation::from_json);
285 ops_ptr[vision::kNormalizePadOperation] = &(vision::NormalizePadOperation::from_json);
286 ops_ptr[vision::kPadOperation] = &(vision::PadOperation::from_json);
287 ops_ptr[vision::kRandomAffineOperation] = &(vision::RandomAffineOperation::from_json);
288 ops_ptr[vision::kRandomColorOperation] = &(vision::RandomColorOperation::from_json);
289 ops_ptr[vision::kRandomColorAdjustOperation] = &(vision::RandomColorAdjustOperation::from_json);
290 ops_ptr[vision::kRandomCropDecodeResizeOperation] = &(vision::RandomCropDecodeResizeOperation::from_json);
291 ops_ptr[vision::kRandomCropOperation] = &(vision::RandomCropOperation::from_json);
292 ops_ptr[vision::kRandomCropWithBBoxOperation] = &(vision::RandomCropWithBBoxOperation::from_json);
293 ops_ptr[vision::kRandomHorizontalFlipOperation] = &(vision::RandomHorizontalFlipOperation::from_json);
294 ops_ptr[vision::kRandomHorizontalFlipWithBBoxOperation] = &(vision::RandomHorizontalFlipWithBBoxOperation::from_json);
295 ops_ptr[vision::kRandomPosterizeOperation] = &(vision::RandomPosterizeOperation::from_json);
296 ops_ptr[vision::kRandomResizeOperation] = &(vision::RandomResizeOperation::from_json);
297 ops_ptr[vision::kRandomResizeWithBBoxOperation] = &(vision::RandomResizeWithBBoxOperation::from_json);
298 ops_ptr[vision::kRandomResizedCropOperation] = &(vision::RandomResizedCropOperation::from_json);
299 ops_ptr[vision::kRandomResizedCropWithBBoxOperation] = &(vision::RandomResizedCropWithBBoxOperation::from_json);
300 ops_ptr[vision::kRandomRotationOperation] = &(vision::RandomRotationOperation::from_json);
301 ops_ptr[vision::kRandomSelectSubpolicyOperation] = &(vision::RandomSelectSubpolicyOperation::from_json);
302 ops_ptr[vision::kRandomSharpnessOperation] = &(vision::RandomSharpnessOperation::from_json);
303 ops_ptr[vision::kRandomSolarizeOperation] = &(vision::RandomSolarizeOperation::from_json);
304 ops_ptr[vision::kRandomVerticalFlipOperation] = &(vision::RandomVerticalFlipOperation::from_json);
305 ops_ptr[vision::kRandomVerticalFlipWithBBoxOperation] = &(vision::RandomVerticalFlipWithBBoxOperation::from_json);
306 ops_ptr[vision::kRandomSharpnessOperation] = &(vision::RandomSharpnessOperation::from_json);
307 ops_ptr[vision::kRandomSolarizeOperation] = &(vision::RandomSolarizeOperation::from_json);
308 ops_ptr[vision::kRescaleOperation] = &(vision::RescaleOperation::from_json);
309 ops_ptr[vision::kResizeOperation] = &(vision::ResizeOperation::from_json);
310 ops_ptr[vision::kResizePreserveAROperation] = &(vision::ResizePreserveAROperation::from_json);
311 ops_ptr[vision::kResizeWithBBoxOperation] = &(vision::ResizeWithBBoxOperation::from_json);
312 ops_ptr[vision::kRgbaToBgrOperation] = &(vision::RgbaToBgrOperation::from_json);
313 ops_ptr[vision::kRgbaToRgbOperation] = &(vision::RgbaToRgbOperation::from_json);
314 ops_ptr[vision::kRgbToBgrOperation] = &(vision::RgbToBgrOperation::from_json);
315 ops_ptr[vision::kRgbToGrayOperation] = &(vision::RgbToGrayOperation::from_json);
316 ops_ptr[vision::kRotateOperation] = &(vision::RotateOperation::from_json);
317 ops_ptr[vision::kSlicePatchesOperation] = &(vision::SlicePatchesOperation::from_json);
318 ops_ptr[vision::kSoftDvppDecodeRandomCropResizeJpegOperation] =
319 &(vision::SoftDvppDecodeRandomCropResizeJpegOperation::from_json);
320 ops_ptr[vision::kSoftDvppDecodeResizeJpegOperation] = &(vision::SoftDvppDecodeResizeJpegOperation::from_json);
321 ops_ptr[vision::kSwapRedBlueOperation] = &(vision::SwapRedBlueOperation::from_json);
322 ops_ptr[vision::kUniformAugOperation] = &(vision::UniformAugOperation::from_json);
323 ops_ptr[vision::kVerticalFlipOperation] = &(vision::VerticalFlipOperation::from_json);
324 ops_ptr[transforms::kFillOperation] = &(transforms::FillOperation::from_json);
325 ops_ptr[transforms::kOneHotOperation] = &(transforms::OneHotOperation::from_json);
326 ops_ptr[transforms::kTypeCastOperation] = &(transforms::TypeCastOperation::from_json);
327 ops_ptr[text::kToNumberOperation] = &(text::ToNumberOperation::from_json);
328 return ops_ptr;
329 }
330
ParseMindIRPreprocess(const std::string & dataset_json,const std::string & process_column,std::vector<std::shared_ptr<mindspore::dataset::Execute>> * data_graph)331 Status Serdes::ParseMindIRPreprocess(const std::string &dataset_json, const std::string &process_column,
332 std::vector<std::shared_ptr<mindspore::dataset::Execute>> *data_graph) {
333 CHECK_FAIL_RETURN_UNEXPECTED(!dataset_json.empty(), "Invalid data, no json data in dataset_json.");
334
335 nlohmann::json dataset_js;
336 try {
337 dataset_js = nlohmann::json::parse(dataset_json);
338 } catch (const std::exception &err) {
339 MS_LOG(ERROR) << "Invalid json content, failed to parse JSON data.";
340 RETURN_STATUS_UNEXPECTED("Invalid json content, failed to parse JSON data.");
341 }
342
343 // Note1: We have to consider if pipeline has multibranch, how to deal with this situation?
344 // op1 - map - |
345 // op2 - map - concat - map - ...
346 std::stack<nlohmann::json> reverse_traversal;
347 nlohmann::json dataset_nodes = dataset_js;
348 while (dataset_nodes != nullptr) {
349 reverse_traversal.push(dataset_nodes);
350 if (dataset_nodes["children"].size() > 1) {
351 MS_LOG(WARNING) << "Need to support dataset_node with more than one child.";
352 }
353 dataset_nodes = dataset_nodes["children"][0];
354 }
355
356 // Note2: We have to consider if the "image" column does not named with "image", how to select its map ops?
357 // In MindRecord, TFRecord, GeneratorDataset or RenameDataset, it seems that the column names are not fixed.
358 while (!reverse_traversal.empty()) {
359 nlohmann::json node = reverse_traversal.top();
360 reverse_traversal.pop();
361 if (node["op_type"] == "Map") {
362 std::vector<std::shared_ptr<TensorOperation>> tensor_ops;
363 RETURN_IF_NOT_OK(ConstructTensorOps(node["operations"], &tensor_ops));
364 if (node["input_columns"][0] == process_column) {
365 std::vector<std::string> op_names;
366 std::transform(tensor_ops.begin(), tensor_ops.end(), std::back_inserter(op_names),
367 [](const auto &op) { return op->Name(); });
368 MS_LOG(INFO) << "Find valid preprocess operations: " << op_names;
369 data_graph->push_back(std::make_shared<Execute>(tensor_ops));
370 }
371 }
372 }
373
374 if (!data_graph->size()) {
375 MS_LOG(WARNING) << "Can not find any valid preprocess operation.";
376 }
377 return Status::OK();
378 }
379
380 // In the current stage, there is a cyclic dependency between libmindspore.so and c_dataengine.so,
381 // we make a C function here and dlopen by libminspore.so to avoid linking explicitly,
382 // will be fix after decouling libminspore.so into multi submodules
383 extern "C" {
384 // ParseMindIRPreprocess_C has C-linkage specified, but returns user-defined type 'mindspore::Status'
385 // which is incompatible with C
ParseMindIRPreprocess_C(const std::string & dataset_json,const std::string & process_column,std::vector<std::shared_ptr<mindspore::dataset::Execute>> * data_graph,Status * s)386 void ParseMindIRPreprocess_C(const std::string &dataset_json, const std::string &process_column,
387 std::vector<std::shared_ptr<mindspore::dataset::Execute>> *data_graph, Status *s) {
388 Status ret = Serdes::ParseMindIRPreprocess(dataset_json, process_column, data_graph);
389 *s = Status(ret);
390 }
391 }
392
393 } // namespace dataset
394 } // namespace mindspore
395