1 /** 2 * Copyright 2020 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MINDSPORE_LITE_TOOLS_BENCHMARK_BENCHMARK_UNIFIED_API_H_ 18 #define MINDSPORE_LITE_TOOLS_BENCHMARK_BENCHMARK_UNIFIED_API_H_ 19 20 #include <signal.h> 21 #include <random> 22 #include <unordered_map> 23 #include <fstream> 24 #include <iostream> 25 #include <map> 26 #include <cmath> 27 #include <string> 28 #include <vector> 29 #include <memory> 30 #include <cfloat> 31 #include <utility> 32 #include <atomic> 33 #ifndef BENCHMARK_CLIP_JSON 34 #include <nlohmann/json.hpp> 35 #endif 36 #include "tools/benchmark/benchmark_base.h" 37 #include "tools/common/flag_parser.h" 38 #include "src/common/file_utils.h" 39 #include "src/common/utils.h" 40 #include "include/api/types.h" 41 #include "include/api/model.h" 42 #include "include/api/context.h" 43 #include "tools/common/opengl_util.h" 44 #ifdef PARALLEL_INFERENCE 45 #include "include/api/model_parallel_runner.h" 46 #endif 47 48 namespace mindspore::lite { 49 class MS_API BenchmarkUnifiedApi : public BenchmarkBase { 50 public: BenchmarkUnifiedApi(BenchmarkFlags * flags)51 explicit BenchmarkUnifiedApi(BenchmarkFlags *flags) : BenchmarkBase(flags) {} 52 53 virtual ~BenchmarkUnifiedApi(); 54 55 int RunBenchmark() override; 56 57 protected: 58 int CompareDataGetTotalBiasAndSize(const std::string &name, mindspore::MSTensor *tensor, float *total_bias, 59 int *total_size, float relative_tolerance = kRelativeTolerance, 60 float absolute_tolerance = kAbsoluteTolerance); 61 int CompareDataGetTotalCosineDistanceAndSize(const std::string &name, mindspore::MSTensor *tensor, 62 float *total_cosine_distance, int *total_size); 63 void InitContext(const std::shared_ptr<mindspore::Context> &context); 64 65 int CompileGraph(mindspore::ModelType model_type, const std::shared_ptr<Context> &context, 66 const std::string &model_name); 67 68 int GenerateGLTexture(std::map<std::string, GLuint> *inputGlTexture); 69 70 int LoadAndBindGLTexture(); 71 72 int ReadGLTextureFile(std::map<std::string, GLuint> *inputGlTexture); 73 74 int FillGLTextureToTensor(std::map<std::string, GLuint> *gl_texture, mindspore::MSTensor *tensor, std::string name, 75 void *data = nullptr); 76 77 // call GenerateRandomData to fill inputTensors 78 int LoadInput() override; 79 int GenerateInputData() override; 80 81 int ReadInputFile() override; 82 83 int InitMSContext(const std::shared_ptr<Context> &context); 84 85 int GetDataTypeByTensorName(const std::string &tensor_name) override; 86 87 int CompareOutput() override; 88 89 int CompareOutputByCosineDistance(float cosine_distance_threshold); 90 91 int InitTimeProfilingCallbackParameter() override; 92 93 int InitPerfProfilingCallbackParameter() override; 94 95 int InitDumpTensorDataCallbackParameter() override; 96 97 int InitPrintTensorDataCallbackParameter() override; 98 99 int PrintInputData(); 100 #ifdef PARALLEL_INFERENCE 101 int RunParallelBenchmark(std::shared_ptr<mindspore::Context> context); 102 int CompareOutputForModelPool(std::vector<mindspore::MSTensor> *outputs); 103 void ModelParallelRunnerWarmUp(int index); 104 void ModelParallelRunnerRun(int task_num, int parallel_idx); 105 int ParallelInference(std::shared_ptr<mindspore::Context> context); 106 int AddConfigInfo(const std::shared_ptr<RunnerConfig> &runner_config); 107 #endif 108 109 int PrintOutputData(); 110 111 template <typename T> ConverterToInt64Vector(const std::vector<T> & srcDims)112 std::vector<int64_t> ConverterToInt64Vector(const std::vector<T> &srcDims) { 113 std::vector<int64_t> dims; 114 for (auto shape : srcDims) { 115 dims.push_back(static_cast<int64_t>(shape)); 116 } 117 return dims; 118 } 119 120 int MarkPerformance(); 121 122 int MarkAccuracy(); 123 124 int GetBenchmarkResult(); 125 126 void UpdateDistributionName(const std::shared_ptr<mindspore::Context> &context, std::string *name); 127 128 private: 129 void UpdateConfigInfo(); 130 std::vector<std::vector<int64_t>> ParseGraphInputShapeMap(const std::vector<MSTensor> &inputs); 131 132 private: 133 mindspore::OpenGL::OpenGLRuntime gl_runtime_; 134 mindspore::Model ms_model_; 135 std::vector<mindspore::MSTensor> ms_inputs_for_api_; 136 std::vector<mindspore::MSTensor> ms_outputs_for_api_; 137 138 MSKernelCallBack ms_before_call_back_ = nullptr; 139 MSKernelCallBack ms_after_call_back_ = nullptr; 140 #ifdef PARALLEL_INFERENCE 141 std::vector<std::vector<int64_t>> resize_dims_; 142 std::vector<std::vector<void *>> all_inputs_data_; 143 std::vector<std::vector<mindspore::MSTensor>> all_outputs_; 144 std::atomic<bool> model_parallel_runner_ret_failed_{false}; 145 std::atomic<bool> runner_run_start_ = false; 146 mindspore::ModelParallelRunner model_runner_; 147 #endif 148 149 void InitMSContextForGPU(const std::shared_ptr<Context> &context, 150 std::vector<std::shared_ptr<DeviceInfoContext>> *device_list); 151 152 void InitMSContextForAscend(const std::shared_ptr<Context> &context, 153 std::vector<std::shared_ptr<DeviceInfoContext>> *device_list); 154 }; 155 156 } // namespace mindspore::lite 157 #endif // MINNIE_BENCHMARK_BENCHMARK_H_ 158