1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_FRAMEWORKS_ML_NN_COMMON_NNAPI_IBURST_H 18 #define ANDROID_FRAMEWORKS_ML_NN_COMMON_NNAPI_IBURST_H 19 20 #include <android-base/scopeguard.h> 21 22 #include <functional> 23 #include <memory> 24 #include <optional> 25 #include <utility> 26 #include <vector> 27 28 #include "nnapi/Types.h" 29 30 namespace android::nn { 31 32 /** 33 * IBurst represents a burst execution object. 34 * 35 * Burst executions are a sequence of executions of the same prepared model that occur in rapid 36 * succession, such as frames of a camera capture or successive audio samples. A burst object is 37 * used to control a set of burst executions, and to preserve resources between executions, enabling 38 * executions to have lower overhead. Burst objects enable some optimizations: 39 * (1) A burst object is created before a sequence of executions, and freed when the sequence has 40 * ended. Because of this, the lifetime of the burst object hints to a driver how long it should 41 * remain in a high performance state. 42 * (2) A burst object can preserve resources between executions. For example, a driver can map a 43 * memory object on the first execution and cache the mapping in the burst object for reuse in 44 * subsequent executions. Any cached resource can be released when the burst object is destroyed 45 * or when the NNAPI runtime notifies the burst object that the resource is no longer required. 46 * (3) A burst object may be used for at most one execution at a time. This enables any transient 47 * execution resources such as intermediate tensors to be allocated once when the burst object 48 * is created and freed when the burst object is destroyed. 49 * 50 * This interface is thread-safe, and any class that implements this interface must be thread-safe. 51 */ 52 class IBurst { 53 public: 54 using OptionalCacheHold = std::shared_ptr<const base::ScopeGuard<std::function<void()>>>; 55 56 /** 57 * Cache a memory object in the burst. 58 * 59 * This can enable multiple executions that reuse the same memory to be more efficient. 60 * 61 * @param memory The memory object to be cached as long as CacheHandle is held. 62 * @return An optional cache handle that will release the corresponding cahced object once the 63 * cache handle is released, or nullptr. 64 */ 65 virtual OptionalCacheHold cacheMemory(const SharedMemory& memory) const = 0; 66 67 /** 68 * Performs a synchronous execution on a prepared model. 69 * 70 * At most one execution may occur on a burst object at any given time. 71 * 72 * The execution is performed synchronously with respect to the caller. IBurst::execute must 73 * verify the inputs to the function are correct. If there is an error, IBurst::execute must 74 * immediately return {@link ErrorStatus::INVALID_ARGUMENT} as a ExecutionError. If the inputs 75 * to the function are valid and there is no error, IBurst::execute must perform the execution, 76 * and must not return until the execution is complete. 77 * 78 * The caller must not change the content of any data object referenced by request (described by 79 * the {@link DataLocation} of a {@link RequestArgument}) until IBurst::execute returns. 80 * IBurst::execute must not change the content of any of the data objects corresponding to 81 * request inputs. 82 * 83 * If the prepared model was prepared from a model wherein all tensor operands have fully 84 * specified dimensions, and the inputs to the function are valid, and at execution time every 85 * operation's input operands have legal values, then the execution should complete 86 * successfully. There must be no failure unless the device itself is in a bad state. 87 * 88 * @param request The input and output information on which the prepared model is to be 89 * executed. 90 * @param measure Specifies whether or not to measure duration of the execution. 91 * @param deadline Optional time point. If provided, execute is expected to complete by this 92 * time point. If it is not able to be completed by the deadline, the execution may be 93 * aborted. 94 * @param loopTimeoutDuration The maximum amount of time that should be spent executing a {@link 95 * OperationType::WHILE} operation. If a loop condition model does not output `false` within 96 * this duration, the execution must be aborted. If no loop timeout duration is provided, 97 * the maximum amount of time is {@link kControlFlowTimeoutDefault}. When provided, the 98 * duration must not exceed {@link kControlFlowTimeoutMaximum}. 99 * @return A pair consisting of: 100 * - A list of shape information of model output operands. The index into "outputShapes" 101 * corresponds to the index of the output operand in the Request outputs vector. 102 * outputShapes must be empty unless the execution is successful or the ExecutionResult is 103 * {@link ErrorStatus::OUTPUT_INSUFFICIENT_SIZE}. outputShapes may be empty if the 104 * execution is successful and all model output operands are fully-specified at execution 105 * time. outputShapes must have the same number of elements as the number of model output 106 * operands if the ExecutionResult is {@link ErrorStatus::OUTPUT_INSUFFICIENT_SIZE}, or if 107 * the execution is successful and the model has at least one output operand that is not 108 * fully-specified. 109 * - Duration of execution. Unless measure is YES and the execution is successful, all times 110 * must be reported as std::nullopt. A driver may choose to report any time as 111 * std::nullopt, indicating that measurement is not available. 112 */ 113 virtual ExecutionResult<std::pair<std::vector<OutputShape>, Timing>> execute( 114 const Request& request, MeasureTiming measure, const nn::OptionalTimePoint& deadline, 115 const nn::OptionalDuration& loopTimeoutDuration) const = 0; 116 117 /** 118 * Create a reusable burst execution object. 119 * 120 * IBurst::createReusableExecution must verify the inputs to the function are correct. If there 121 * is an error, IBurst::createReusableExecution must immediately return {@link 122 * ErrorStatus::INVALID_ARGUMENT} as a GeneralError. If the inputs to the function are valid and 123 * there is no error, IBurst::createReusableExecution must construct a reusable execution. 124 * 125 * @param request The input and output information on which the prepared model is to be 126 * executed. 127 * @param measure Specifies whether or not to measure duration of the computation. 128 * @param loopTimeoutDuration The maximum amount of time that should be spent executing a {@link 129 * OperationType::WHILE} operation. If a loop condition model does not output `false` within 130 * this duration, the execution must be aborted. If no loop timeout duration is provided, 131 * the maximum amount of time is {@link kControlFlowTimeoutDefault}. When provided, the 132 * duration must not exceed {@link kControlFlowTimeoutMaximum}. 133 * @return execution An IExecution object representing a reusable burst execution that has been 134 * specialized for a fixed request, otherwise GeneralError. 135 */ 136 virtual GeneralResult<SharedExecution> createReusableExecution( 137 const Request& request, MeasureTiming measure, 138 const nn::OptionalDuration& loopTimeoutDuration) const = 0; 139 140 // Public virtual destructor to allow objects to be stored (and destroyed) as smart pointers. 141 // E.g., std::unique_ptr<IBurst>. 142 virtual ~IBurst() = default; 143 144 protected: 145 // Protect the non-destructor special member functions to prevent object slicing. 146 IBurst() = default; 147 IBurst(const IBurst&) = default; 148 IBurst(IBurst&&) noexcept = default; 149 IBurst& operator=(const IBurst&) = default; 150 IBurst& operator=(IBurst&&) noexcept = default; 151 }; 152 153 } // namespace android::nn 154 155 #endif // ANDROID_FRAMEWORKS_ML_NN_COMMON_NNAPI_IBURST_H 156