1 /**
2 * Copyright 2020-2023 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <utility>
18 #include <vector>
19 #include "minddata/dataset/engine/consumers/python_tree_consumer.h"
20
21 namespace mindspore::dataset {
22 namespace consumers_util {
GetNextAsPythonList(TreeConsumer * consumer,const py::list * out)23 Status GetNextAsPythonList(TreeConsumer *consumer, const py::list *out) {
24 RETURN_UNEXPECTED_IF_NULL(out);
25 std::vector<TensorPtr> row;
26 {
27 py::gil_scoped_release gil_release;
28 RETURN_IF_NOT_OK(consumer->GetNextAsVector(&row));
29 }
30 for (auto el : row) {
31 (*out).append(el);
32 }
33 return Status::OK();
34 }
35
GetNextAsPythonDict(TreeConsumer * consumer,const py::dict * out)36 Status GetNextAsPythonDict(TreeConsumer *consumer, const py::dict *out) {
37 RETURN_UNEXPECTED_IF_NULL(out);
38 std::vector<std::pair<std::string, std::shared_ptr<Tensor>>> vec;
39 {
40 py::gil_scoped_release gil_release;
41 RETURN_IF_NOT_OK(consumer->GetNextAsOrderedPair(&vec));
42 }
43 // Generate Python dict, python dict maintains its insertion order
44 for (const auto &pair : vec) {
45 (*out)[common::SafeCStr(pair.first)] = pair.second;
46 }
47 return Status::OK();
48 }
49 } // namespace consumers_util
50
GetNextAsList(const py::list * out)51 Status PythonIteratorConsumer::GetNextAsList(const py::list *out) {
52 return consumers_util::GetNextAsPythonList(this, out);
53 }
54
GetNextAsDict(const py::dict * out)55 Status PythonIteratorConsumer::GetNextAsDict(const py::dict *out) {
56 return consumers_util::GetNextAsPythonDict(this, out);
57 }
58
GetNextAsList(const py::list * out)59 Status PythonPullBasedIteratorConsumer::GetNextAsList(const py::list *out) {
60 return consumers_util::GetNextAsPythonList(this, out);
61 }
62
GetNextAsDict(const py::dict * out)63 Status PythonPullBasedIteratorConsumer::GetNextAsDict(const py::dict *out) {
64 return consumers_util::GetNextAsPythonDict(this, out);
65 }
66
Start()67 Status PythonBuildVocabConsumer::Start() {
68 py::gil_scoped_release gil_release;
69 return BuildVocabConsumer::Start();
70 }
71
Save()72 Status PythonSaveToDisk::Save() {
73 py::gil_scoped_release gil_release;
74 return SaveToDisk::Save();
75 }
76
PythonSaveToDisk(const std::string & datasetPath,int32_t numFiles,const std::string & datasetType)77 PythonSaveToDisk::PythonSaveToDisk(const std::string &datasetPath, int32_t numFiles, const std::string &datasetType)
78 : SaveToDisk(datasetPath, numFiles, datasetType) {}
79
GetRow(TensorRow * const r)80 Status PythonTreeGetters::GetRow(TensorRow *const r) {
81 RETURN_UNEXPECTED_IF_NULL(r);
82 py::gil_scoped_release gil_release;
83 return TreeGetters::GetRow(r);
84 }
85
GetRow(const std::shared_ptr<TreeAdapter> & tree_adapter,TensorRow * r)86 Status PythonDatasetSizeGetter::GetRow(const std::shared_ptr<TreeAdapter> &tree_adapter, TensorRow *r) {
87 RETURN_UNEXPECTED_IF_NULL(tree_adapter);
88 RETURN_UNEXPECTED_IF_NULL(r);
89 py::gil_scoped_release gil_release;
90 return DatasetSizeGetter::GetRow(tree_adapter, r);
91 }
92 } // namespace mindspore::dataset
93