1 /** 2 * Copyright 2020-2024 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_MAP_NODE_H_ 18 #define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_MAP_NODE_H_ 19 20 #include <memory> 21 #include <string> 22 #include <vector> 23 24 #ifdef ENABLE_PYTHON 25 #include "minddata/dataset/api/python/python_mp.h" 26 #endif 27 #include "minddata/dataset/engine/ir/datasetops/dataset_node.h" 28 29 namespace mindspore { 30 namespace dataset { 31 class MapNode : public DatasetNode { 32 public: 33 /// \brief Constructor 34 MapNode(std::shared_ptr<DatasetNode> child, std::vector<std::shared_ptr<TensorOperation>> operations, 35 std::vector<std::string> input_columns = {}, std::vector<std::string> output_columns = {}, 36 const std::shared_ptr<DatasetCache> &cache = nullptr, std::vector<std::shared_ptr<DSCallback>> callbacks = {}, 37 ManualOffloadMode offload = ManualOffloadMode::kUnspecified, 38 std::shared_ptr<PythonMultiprocessingRuntime> python_mp = nullptr); 39 40 /// \brief Constructor used in InsertMap pass. 41 MapNode(std::vector<std::shared_ptr<TensorOperation>> operations, std::vector<std::string> input_columns, 42 std::vector<std::string> output_columns); 43 44 /// \brief Destructor 45 ~MapNode() override = default; 46 47 /// \brief Node name getter 48 /// \return Name of the current node Name()49 std::string Name() const override { return kMapNode; } 50 51 /// \brief Print the description 52 /// \param out - The output stream to write output to 53 void Print(std::ostream &out) const override; 54 55 /// \brief Copy the node to a new object 56 /// \return A shared pointer to the new copy 57 std::shared_ptr<DatasetNode> Copy() override; 58 59 /// \brief a base class override function to create the required runtime dataset op objects for this class 60 /// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create 61 /// \return Status Status::OK() if build successfully 62 Status Build(std::vector<std::shared_ptr<DatasetOp>> *const node_ops) override; 63 64 /// \brief Parameters validation 65 /// \return Status Status::OK() if all the parameters are valid 66 Status ValidateParams() override; 67 68 /// \brief Base-class override for accepting IRNodePass visitor 69 /// \param[in] p The node to visit 70 /// \param[out] modified Indicator if the node was modified 71 /// \return Status of the node visit 72 Status Accept(IRNodePass *const p, bool *const modified) override; 73 74 /// \brief Base-class override for accepting IRNodePass visitor 75 /// \param[in] p The node to visit 76 /// \param[out] modified Indicator if the node was modified 77 /// \return Status of the node visit 78 Status AcceptAfter(IRNodePass *const p, bool *const modified) override; 79 80 /// \brief clear all callbacks ClearCallbacks()81 void ClearCallbacks() { callbacks_.clear(); } 82 83 /// \brief getter to get all tensor operations 84 std::vector<std::shared_ptr<TensorOperation>> operations(); 85 86 /// \brief setter to set all tensor operations 87 void setOperations(const std::vector<std::shared_ptr<TensorOperation>> &operations); 88 89 /// \brief Getter functions 90 /// \brief Getter of tensor operations 91 /// \return Vector of operations the Map node will process TensorOperations()92 const auto &TensorOperations() const { return operations_; } InputColumns()93 const std::vector<std::string> &InputColumns() const { return input_columns_; } OutputColumns()94 const std::vector<std::string> &OutputColumns() const { return output_columns_; } Callbacks()95 const std::vector<std::shared_ptr<DSCallback>> &Callbacks() const { return callbacks_; } GetOffload()96 ManualOffloadMode GetOffload() const { return offload_; } 97 98 /// \brief setter to set offload flag of node 99 void SetOffload(ManualOffloadMode offload); 100 101 /// \brief Get the arguments of node 102 /// \param[out] out_json JSON string of all attributes 103 /// \return Status of the function 104 Status to_json(nlohmann::json *out_json) override; 105 106 #ifndef ENABLE_ANDROID 107 /// \brief Function for read dataset operation from json 108 /// \param[in] json_obj The JSON object to be deserialized 109 /// \param[in] ds dataset node constructed 110 /// \param[out] result Deserialized dataset after the operation 111 /// \return Status The status code returned 112 static Status from_json(nlohmann::json json_obj, std::shared_ptr<DatasetNode> ds, 113 std::shared_ptr<DatasetNode> *result); 114 #endif 115 116 /// \brief Base-class override for GetDatasetSize 117 /// \param[in] size_getter Shared pointer to DatasetSizeGetter 118 /// \param[in] estimate This is only supported by some of the ops and it's used to speed up the process of getting 119 /// dataset size at the expense of accuracy. 120 /// \param[out] dataset_size the size of the dataset 121 /// \return Status of the function 122 Status GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size_getter, bool estimate, 123 int64_t *dataset_size) override; 124 125 private: 126 std::vector<std::shared_ptr<TensorOperation>> operations_; 127 std::vector<std::string> input_columns_; 128 std::vector<std::string> output_columns_; 129 std::vector<std::shared_ptr<DSCallback>> callbacks_; 130 131 /// \brief ManualOffloadMode to indicate manual_offload status 132 ManualOffloadMode offload_; 133 134 std::shared_ptr<PythonMultiprocessingRuntime> python_mp_; 135 }; 136 } // namespace dataset 137 } // namespace mindspore 138 #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_MAP_NODE_H_ 139