1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_ML_NN_RUNTIME_CALLBACKS_H 18 #define ANDROID_ML_NN_RUNTIME_CALLBACKS_H 19 20 #include <android-base/thread_annotations.h> 21 #include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h> 22 #include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h> 23 #include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h> 24 #include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h> 25 #include <hidl/MQDescriptor.h> 26 #include <hidl/Status.h> 27 #include <condition_variable> 28 #include <functional> 29 #include <mutex> 30 #include <thread> 31 32 /* 33 * The Callback classes are used internally by the NeuralNetworks runtime to 34 * synchronize between different threads. An asynchronous task is launched 35 * paired with a callback object. When a client thread requires the output being 36 * generated by the asynchronous task, the client thread can wait for the result 37 * and be blocked until it has completed. Any wait may safely be called 38 * concurrently, even on the same callback object. When the asynchronous task 39 * has finished its workload, it must immediately call "notify*". If the 40 * asynchronous task has failed to launch, the function that tried to launch the 41 * asynchronous task must immediately call "notify*". This "notify*" call 42 * awakens any client threads waiting on the callback object. 43 * 44 * These classes exist to enable synchronization across HIDL. When 45 * synchronization is only required in the same process, consider using 46 * std::future, std::mutex, std::condition_variable, or std::experimental::latch 47 * instead. 48 */ 49 50 namespace android::hardware::neuralnetworks::V1_2::implementation { 51 52 using V1_0::ErrorStatus; 53 54 /** 55 * The PreparedModelCallback class is used to receive the error status of 56 * preparing a model as well as the prepared model from a task executing 57 * asynchronously with respect to the runtime. If a calling thread calls wait 58 * or get* on a PreparedModelCallback object and the corresponding asynchronous 59 * task has not finished preparing the model, the calling thread will block 60 * until the asynchronous task has either called notify or notify_1_2. 61 * 62 * If the callback object is notified more than once, only the results of the 63 * first call to notify* are used, and the results from subsequent calls are 64 * discarded. 65 * 66 * This callback object is passed as an argument to IDevice::prepareModel*. 67 */ 68 class PreparedModelCallback : public IPreparedModelCallback { 69 public: 70 /** 71 * IPreparedModelCallback::notify marks the callback object with the return 72 * status of the asynchronous model preparation along with the prepared 73 * model, and allows all prior and future wait calls on the 74 * PreparedModelCallback object to proceed. 75 * 76 * Either IPreparedModelCallback::notify or 77 * IPreparedModelCallback::notify_1_2 must be called on a given 78 * PreparedModelCallback object. 79 * 80 * If the callback object is notified more than once, only the results of 81 * the first call to notify* are used, and the results from subsequent calls 82 * are discarded. 83 * 84 * @param status Error status returned from asynchronously preparing the 85 * model; will be: 86 * - NONE if the asynchronous preparation was successful 87 * - DEVICE_UNAVAILABLE if driver is offline or busy 88 * - GENERAL_FAILURE if there is an unspecified error 89 * - INVALID_ARGUMENT if the input model is invalid 90 * @param preparedModel Returned model that has been prepared for execution, 91 * nullptr if the model was unable to be prepared. 92 */ 93 Return<void> notify(ErrorStatus status, const sp<V1_0::IPreparedModel>& preparedModel) override; 94 95 /** 96 * IPreparedModelCallback::notify_1_2 marks the callback object with the 97 * return status of the asynchronous model preparation along with the 98 * prepared model, and allows all prior and future wait calls on the 99 * PreparedModelCallback object to proceed. 100 * 101 * Either IPreparedModelCallback::notify or 102 * IPreparedModelCallback::notify_1_2 must be called on a given 103 * PreparedModelCallback object. 104 * 105 * If the callback object is notified more than once, only the results of 106 * the first call to notify* are used, and the results from subsequent calls 107 * are discarded. 108 * 109 * @param status Error status returned from asynchronously preparing the 110 * model; will be: 111 * - NONE if the asynchronous preparation was successful 112 * - DEVICE_UNAVAILABLE if driver is offline or busy 113 * - GENERAL_FAILURE if there is an unspecified error 114 * - INVALID_ARGUMENT if the input model is invalid 115 * @param preparedModel Returned model that has been prepared for execution, 116 * nullptr if the model was unable to be prepared. 117 */ 118 Return<void> notify_1_2(ErrorStatus status, 119 const sp<V1_2::IPreparedModel>& preparedModel) override; 120 121 /** 122 * PreparedModelCallback::wait blocks until notify* has been called on the 123 * callback object. 124 */ 125 void wait() const; 126 127 /** 128 * Retrieves the error status returned from the asynchronous task launched 129 * by IDevice::prepareModel*. If IDevice::prepareModel* has not finished 130 * asynchronously preparing the model, this call will block until the 131 * asynchronous task notifies the object. 132 * 133 * @return status Error status returned from asynchronously preparing the 134 * model; will be: 135 * - NONE if the asynchronous preparation was successful 136 * - DEVICE_UNAVAILABLE if driver is offline or busy 137 * - GENERAL_FAILURE if there is an unspecified error 138 * - INVALID_ARGUMENT if the input model is invalid 139 */ 140 ErrorStatus getStatus() const; 141 142 /** 143 * Retrieves the model that has been prepared for execution from the 144 * asynchronous task launched by IDevice::prepareModel*. If 145 * IDevice::prepareModel* has not finished asynchronously preparing the 146 * model, this call will block until the asynchronous task notifies the 147 * object. 148 * 149 * @return preparedModel Returned model that has been prepared for 150 * execution, nullptr if the model was unable to be prepared. 151 */ 152 sp<V1_0::IPreparedModel> getPreparedModel() const; 153 154 private: 155 mutable std::mutex mMutex; 156 mutable std::condition_variable mCondition; 157 bool mNotified GUARDED_BY(mMutex) = false; 158 ErrorStatus mErrorStatus = ErrorStatus::GENERAL_FAILURE; 159 sp<V1_0::IPreparedModel> mPreparedModel; 160 }; 161 162 /** 163 * The ExecutionCallback class is used to receive the results of the execution 164 * from a task executing asynchronously with respect to the runtime. If a 165 * calling thread calls wait or get* on a ExecutionCallback object and the 166 * corresponding asynchronous task has not finished the execution, the calling 167 * thread will block until the asynchronous task has either called notify or 168 * notify_1_2. 169 * 170 * If the callback object is notified more than once, only the results of the 171 * first call to notify* are used, and the results from subsequent calls are 172 * discarded. 173 * 174 * This callback object is passed as an argument to IPreparedModel::execute*. 175 */ 176 class ExecutionCallback : public IExecutionCallback { 177 using ExecutionFinish = 178 std::function<ErrorStatus(ErrorStatus, const std::vector<OutputShape>&)>; 179 180 public: 181 /** 182 * IExecutionCallback::notify marks the callback object with the return 183 * status of the asynchronous execution that held this callback and enables 184 * all prior and future wait calls on the ExecutionCallback object to 185 * proceed. 186 * 187 * Either IExecutionCallback::notify or IExecutionCallback::notify_1_2 must 188 * be called on a given ExecutionCallback object. 189 * 190 * If the callback object is notified more than once, only the results of 191 * the first call to notify* are used, and the results from subsequent calls 192 * are discarded. 193 * 194 * @param status Error status returned from launching the asynchronous task 195 * (if the launch fails) or from the asynchronous task itself (if the 196 * launch succeeds). Must be: 197 * - NONE if the asynchronous execution was successful 198 * - DEVICE_UNAVAILABLE if driver is offline or busy 199 * - GENERAL_FAILURE if there is an unspecified error 200 * - OUTPUT_INSUFFICIENT_SIZE if provided output buffer is not large 201 * enough to store the resultant values 202 * - INVALID_ARGUMENT if the input request is invalid 203 */ 204 Return<void> notify(ErrorStatus status) override; 205 206 /** 207 * IExecutionCallback::notify_1_2 marks the callback object with the results 208 * (error status, dynamic output shapes, and timing information) of the 209 * asynchronous execution that held this callback and enables all prior and 210 * future wait calls on the ExecutionCallback object to proceed. 211 * 212 * Either IExecutionCallback::notify or IExecutionCallback::notify_1_2 must 213 * be called on a given ExecutionCallback object. 214 * 215 * If the callback object is notified more than once, only the results of 216 * the first call to notify* are used, and the results from subsequent calls 217 * are discarded. 218 * 219 * @param status Error status returned from launching the asynchronous task 220 * (if the launch fails) or from the asynchronous task itself (if the 221 * launch succeeds). Must be: 222 * - NONE if the asynchronous execution was successful 223 * - DEVICE_UNAVAILABLE if driver is offline or busy 224 * - GENERAL_FAILURE if the asynchronous task resulted in an unspecified 225 * error 226 * - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is 227 * not large enough to store the corresponding output 228 * - INVALID_ARGUMENT if one of the input arguments to prepareModel is 229 * invalid 230 * @param outputShapes A list of shape information of model output operands. 231 * The index into "outputShapes" corresponds to the index of the output 232 * operand in the Request outputs vector. outputShapes must be empty 233 * unless the status is either NONE or OUTPUT_INSUFFICIENT_SIZE. 234 * @param Timing Duration of execution. Unless MeasureTiming::YES was passed 235 * when launching the execution and status is NONE, all times must be 236 * reported as UINT64_MAX. A driver may choose to report any time as 237 * UINT64_MAX, indicating that particular measurement is not available. 238 */ 239 Return<void> notify_1_2(ErrorStatus status, const hidl_vec<OutputShape>& outputShapes, 240 const Timing& timing) override; 241 242 // An overload of the latest notify interface to hide the version from ExecutionBuilder. notify(ErrorStatus status,const hidl_vec<OutputShape> & outputShapes,const Timing & timing)243 Return<void> notify(ErrorStatus status, const hidl_vec<OutputShape>& outputShapes, 244 const Timing& timing) { 245 return notify_1_2(status, outputShapes, timing); 246 } 247 248 /** 249 * ExecutionCallback::wait blocks until notify* has been called on the 250 * callback object. 251 */ 252 void wait() const; 253 254 /** 255 * Retrieves the error status returned from the asynchronous task launched 256 * by either IPreparedModel::execute or IPreparedModel::execute_1_2. If 257 * IPreparedModel::execute or IPreparedModel::execute_1_2 has not finished 258 * asynchronously executing, this call will block until the asynchronous 259 * task notifies the object. 260 * 261 * @return status Error status returned from launching the asynchronous task 262 * (if the launch fails) or from the asynchronous task itself (if the 263 * launch succeeds). Must be: 264 * - NONE if the asynchronous execution was successful 265 * - DEVICE_UNAVAILABLE if driver is offline or busy 266 * - GENERAL_FAILURE if the asynchronous task resulted in an unspecified 267 * error 268 * - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is 269 * not large enough to store the corresponding output 270 * - INVALID_ARGUMENT if one of the input arguments to prepareModel is 271 * invalid 272 */ 273 ErrorStatus getStatus() const; 274 275 /** 276 * Retrieves the output shapes returned from the asynchronous task launched 277 * by IPreparedModel::execute_1_2. If IPreparedModel::execute_1_2 has not 278 * finished asynchronously executing, this call will block until the 279 * asynchronous task notifies the object. 280 * 281 * If the asynchronous task was launched by IPreparedModel::execute, an 282 * empty vector will be returned. 283 * 284 * @return outputShapes A list of shape information of model output 285 * operands. The index into "outputShapes" corresponds to the index of 286 * the output operand in the Request outputs vector. outputShapes must 287 * be empty unless the status is either NONE or 288 * OUTPUT_INSUFFICIENT_SIZE. outputShaps may be empty if the status is 289 * NONE and all model output operands are fully-specified at execution 290 * time. outputShapes must have the same number of elements as the 291 * number of model output operands if the status is 292 * OUTPUT_INSUFFICIENT_SIZE, or if the status is NONE and the model has 293 * at least one output operand that is not fully-specified. 294 */ 295 const std::vector<OutputShape>& getOutputShapes() const; 296 297 /** 298 * Retrieves the duration of execution of the asynchronous task launched by 299 * IPreparedModel::execute_1_2. If IPreparedModel::execute_1_2 has not 300 * finished asynchronously executing, this call will block until the 301 * asynchronous task notifies the object. 302 * 303 * If the asynchronous task was launched by IPreparedModel::execute, every 304 * time must be UINT64_MAX. 305 * 306 * @return timing Duration of the execution. Every time must be UINT64_MAX 307 * unless the status is NONE. 308 */ 309 Timing getTiming() const; 310 311 /** 312 * ExecutionCallback::bindThread binds a thread to the ExecutionCallback 313 * object. The bound thread is later joined by ExecutionCallback::wait or 314 * ExecutionCallback::get*. 315 * 316 * Once a thread is bound with ExecutionCallback::bindThread, the client 317 * code must ensure that ExecutionCallback::wait or ExecutionCallback::get* 318 * has been called before the ExecutionCallback object is destroyed. 319 * 320 * The bound thread must not call any ExecutionCallback method with the 321 * exception of ExecutionCallback::notify*, which it must call when the 322 * thread has finished its computation. 323 * 324 * ExecutionCallback::bindThread can be called at most once on a given 325 * callback object. 326 * 327 * @param asyncThread Thread to be bound to the callback object. The thread 328 * object must represent a thread of execution -- i.e., 329 * std::thread::joinable() must be true. 330 * @return bool True if successful, false if thread was not properly bound. 331 */ 332 bool bindThread(std::thread asyncThread); 333 334 /** 335 * ExecutionCallback::setOnFinish binds a callback to the ExecutionCallback 336 * object that will be executed during one of the ExecutionCallback::notify* 337 * calls but before any calls to wait or get* return. This provided callback 338 * is provided with both the ErrorStatus and the output shapes from 339 * ExecutionCallback::notify*. 340 * 341 * The bound function must not synchronize with or otherwise access the 342 * callback object it is bound to, as this could cause a deadlock. 343 * 344 * This call will not bind the provided callback if any of the following 345 * occur: 346 * (1) the provided callback is invalid (i.e., "(bool) finish" is false) 347 * (2) ExecutionCallback already contains a bound callback 348 * (3) ExecutionCallback has already been notified with results 349 * 350 * @param finish Callback to be executed when ExecutionCallback is notified 351 * with results. 352 */ 353 void setOnFinish(const ExecutionFinish& finish); 354 355 private: 356 /* 357 * ExecutionCallback::notifyInternal stores the results of the execution 358 * (status, output shapes, and timing information) in the ExecutionCallback 359 * object and invokes the bound callback function "mOnFinish" (if present) 360 * before any call to wait or get* return. It then enables all prior and 361 * future wait calls on the ExecutionCallback object to proceed. 362 */ 363 void notifyInternal(ErrorStatus errorStatus, const hidl_vec<OutputShape>& outputShapes, 364 const Timing& timing); 365 366 // members 367 mutable std::mutex mMutex; 368 mutable std::condition_variable mCondition; 369 mutable std::thread mThread GUARDED_BY(mMutex); 370 ExecutionFinish mOnFinish GUARDED_BY(mMutex); 371 bool mNotified GUARDED_BY(mMutex) = false; 372 ErrorStatus mErrorStatus = ErrorStatus::GENERAL_FAILURE; 373 std::vector<OutputShape> mOutputShapes = {}; 374 Timing mTiming = {}; 375 }; 376 377 } // namespace android::hardware::neuralnetworks::V1_2::implementation 378 379 namespace android::nn { 380 381 using ::android::hardware::neuralnetworks::V1_2::implementation::ExecutionCallback; 382 using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback; 383 384 } // namespace android::nn 385 386 #endif // ANDROID_ML_NN_RUNTIME_CALLBACKS_H 387