1 /** 2 * Copyright 2022-2023 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef MINDSPORE_INCLUDE_API_MODEL_PARALLEL_RUNNER_H 17 #define MINDSPORE_INCLUDE_API_MODEL_PARALLEL_RUNNER_H 18 #include <vector> 19 #include <memory> 20 #include <utility> 21 #include <map> 22 #include <string> 23 #include "include/api/status.h" 24 #include "include/api/context.h" 25 namespace mindspore { 26 /// \brief The RunnerConfig class is used to store environment variables during execution 27 /// management. 28 class MS_API RunnerConfig { 29 public: 30 struct Data; 31 RunnerConfig(); 32 ~RunnerConfig(); 33 34 /// \brief Set the number of workers at runtime. Only valid for ModelParallelRunner. 35 /// 36 /// \param[in] workers_num the number of workers at runtime. 37 void SetWorkersNum(int32_t workers_num); 38 39 /// \brief Get the current operators parallel workers number setting. Only valid for ModelParallelRunner. 40 /// 41 /// \return The current operators parallel workers number setting. 42 int32_t GetWorkersNum() const; 43 44 /// \brief Set the context at runtime. Only valid for ModelParallelRunner. 45 /// 46 /// \param[in] context store environment variables at runtime. 47 void SetContext(const std::shared_ptr<Context> &context); 48 49 /// \brief Get the current context setting. Only valid for ModelParallelRunner. 50 /// 51 /// \return The current operators context setting. 52 std::shared_ptr<Context> GetContext() const; 53 54 /// \brief Set the config before runtime. Only valid for ModelParallelRunner. 55 /// 56 /// \param[in] section The category of the configuration parameter. 57 /// \param[in] config store environment variables before runtime. 58 inline void SetConfigInfo(const std::string §ion, const std::map<std::string, std::string> &config); 59 60 /// \brief Get the current config setting. Only valid for ModelParallelRunner. 61 /// 62 /// \return The current config setting. 63 inline std::map<std::string, std::map<std::string, std::string>> GetConfigInfo() const; 64 65 /// \brief Set the config path before runtime. Only valid for ModelParallelRunner. 66 /// 67 /// \param[in] config_path The path of the configuration parameter. 68 inline void SetConfigPath(const std::string &config_path); 69 70 /// \brief Get the current config path. Only valid for ModelParallelRunner. 71 /// 72 /// \return The current config path. 73 inline std::string GetConfigPath() const; 74 75 /// \brief Set the device id list at runtime, make the model run on different npu/gpu devices. Only valid for 76 /// ModelParallelRunner. 77 /// 78 /// \param[in] device_ids The device id list at runtime. 79 void SetDeviceIds(const std::vector<uint32_t> &device_ids); 80 81 /// \brief Get the device id list at runtime. Only valid for ModelParallelRunner. 82 /// 83 /// \return The device id list. 84 std::vector<uint32_t> GetDeviceIds() const; 85 86 private: 87 void SetConfigInfo(const std::vector<char> §ion, const std::map<std::vector<char>, std::vector<char>> &config); 88 std::map<std::vector<char>, std::map<std::vector<char>, std::vector<char>>> GetConfigInfoChar() const; 89 void SetConfigPath(const std::vector<char> &config_path); 90 std::vector<char> GetConfigPathChar() const; 91 std::shared_ptr<Data> data_ = nullptr; 92 }; 93 94 void RunnerConfig::SetConfigInfo(const std::string §ion, const std::map<std::string, std::string> &config) { 95 SetConfigInfo(StringToChar(section), MapStringToVectorChar(config)); 96 } 97 98 std::map<std::string, std::map<std::string, std::string>> RunnerConfig::GetConfigInfo() const { 99 return MapMapCharToString(GetConfigInfoChar()); 100 } 101 102 void RunnerConfig::SetConfigPath(const std::string &config_path) { SetConfigPath(StringToChar(config_path)); } 103 104 std::string RunnerConfig::GetConfigPath() const { return CharToString(GetConfigPathChar()); } 105 106 class ModelParallelRunnerImpl; 107 108 /// \brief The ModelParallelRunner class is used to define a MindSpore ModelParallelRunner, facilitating Model 109 /// management. 110 class MS_API ModelParallelRunner { 111 public: 112 ModelParallelRunner(); 113 ~ModelParallelRunner(); 114 115 /// \brief build a model parallel runner from model path so that it can run on a device. Supports importing the `ms` 116 /// model (exported by the `converter_lite` tool) and the `mindir` model (exported by MindSpore or exported by the 117 /// `converter_lite` tool). The support for the `ms` model will be removed in future iterations, and it is recommended 118 /// to use the `mindir` model for inference. When using the `ms` model for inference, please keep the suffix name of 119 /// the model as `.ms`, otherwise it will not be recognized. 120 /// 121 /// \param[in] model_path Define the model path. 122 /// \param[in] runner_config Define the config used to store options during model pool init. 123 /// 124 /// \return Status. 125 inline Status Init(const std::string &model_path, const std::shared_ptr<RunnerConfig> &runner_config = nullptr); 126 127 /// \brief build a model parallel runner from model buffer so that it can run on a device. This interface only 128 /// supports passing in `mindir` model file data. 129 /// 130 /// \param[in] model_data Define the buffer read from a model file. 131 /// \param[in] data_size Define bytes number of model buffer. 132 /// \param[in] runner_config Define the config used to store options during model pool init. 133 /// 134 /// \return Status. 135 Status Init(const void *model_data, const size_t data_size, 136 const std::shared_ptr<RunnerConfig> &runner_config = nullptr); 137 138 /// \brief Obtains all input tensors information of the model. 139 /// 140 /// \return The vector that includes all input tensors. 141 std::vector<MSTensor> GetInputs(); 142 143 /// \brief Obtains all output tensors information of the model. 144 /// 145 /// \return The vector that includes all output tensors. 146 std::vector<MSTensor> GetOutputs(); 147 148 /// \brief Inference ModelParallelRunner. 149 /// 150 /// \param[in] inputs A vector where model inputs are arranged in sequence. 151 /// \param[out] outputs Which is a pointer to a vector. The model outputs are filled in the container in sequence. 152 /// \param[in] before CallBack before predict. 153 /// \param[in] after CallBack after predict. 154 /// 155 /// \return Status. 156 Status Predict(const std::vector<MSTensor> &inputs, std::vector<MSTensor> *outputs, 157 const MSKernelCallBack &before = nullptr, const MSKernelCallBack &after = nullptr); 158 159 private: 160 Status Init(const std::vector<char> &model_path, const std::shared_ptr<RunnerConfig> &runner_config); 161 std::shared_ptr<ModelParallelRunnerImpl> model_parallel_runner_impl_ = nullptr; 162 }; 163 164 Status ModelParallelRunner::Init(const std::string &model_path, const std::shared_ptr<RunnerConfig> &runner_config) { 165 return Init(StringToChar(model_path), runner_config); 166 } 167 } // namespace mindspore 168 #endif // MINDSPORE_INCLUDE_API_MODEL_PARALLEL_RUNNER_H 169