• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ANDROID_FRAMEWORKS_ML_NN_COMMON_CPU_EXECUTOR_H
18 #define ANDROID_FRAMEWORKS_ML_NN_COMMON_CPU_EXECUTOR_H
19 
20 #include <android-base/macros.h>
21 #include <nnapi/Types.h>
22 
23 #include <algorithm>
24 #include <memory>
25 #include <optional>
26 #include <vector>
27 
28 #include "ControlFlow.h"
29 #include "LegacyUtils.h"
30 #include "OperationResolver.h"
31 #include "OperationsUtils.h"
32 
33 namespace android {
34 namespace nn {
35 
36 // Information we maintain about each operand during execution that
37 // may change during execution.
38 struct RunTimeOperandInfo {
39     // TODO Storing the type here is redundant, as it won't change during execution.
40     OperandType type;
41     // The type and dimensions of the operand.  The dimensions can
42     // change at runtime.  We include the type because it's useful
43     // to pass together with the dimension to the functions implementing
44     // the operators.
45     //
46     // A dimension being zero has different meanings for different operands at different stages:
47     // - Model inputs:
48     //   * Specified in model: implies "dynamic", and must be fully-specified in request.
49     //   * Specified in request: illegal.
50     // - Constant operands: illegal.
51     // - Model outputs and internal operands:
52     //   * Before evaluation: implies unknown and to be deduced from execution.
53     //   * After evaluation:
54     //     - If isSufficient reports true: the tensor is zero-sized.
55     //     - Otherwise: implies unknown.
56     std::vector<uint32_t> dimensions;
57 
58     float scale;
59     int32_t zeroPoint;
60     // Where the operand's data is stored.  Check the corresponding
61     // location information in the model to figure out if this points
62     // to memory we have allocated for an temporary operand.
63     uint8_t* buffer;  // TODO(b/148273353): Change the type to void*.
64     // The length of the buffer.
65     uint32_t length;
66     // Whether this is a temporary variable, a model input, a constant, etc.
67     Operand::LifeTime lifetime;
68     // Keeps track of how many operations have yet to make use
69     // of this temporary variable.  When the count is decremented to 0,
70     // we free the buffer.  For non-temporary variables, this count is
71     // always 0.
72     uint32_t numberOfUsesLeft;
73 
74     Operand::ExtraParams extraParams;
75 
shapeRunTimeOperandInfo76     Shape shape() const {
77         return {
78                 .type = type,
79                 .dimensions = dimensions,
80                 .scale = scale,
81                 .offset = zeroPoint,
82                 .extraParams = extraParams,
83         };
84     }
85 
isSufficientRunTimeOperandInfo86     bool isSufficient() const {
87         if (isExtension(type)) {
88             // We don't know sizes of extension types.
89             return true;
90         }
91         return length >= nonExtensionOperandSizeOfData(type, dimensions);
92     }
93 };
94 
95 // Used to keep a pointer to each of the memory pools.
96 //
97 // RunTimePoolInfo references a region of memory. Other RunTimePoolInfo objects
98 // may reference the same region of memory by either:
99 // (1) copying an existing RunTimePoolInfo object, or
100 // (2) creating multiple RunTimePoolInfo objects from the same memory resource
101 //     (e.g., "createFromMemory" or "createFromExistingBuffer")
102 //
103 // If the underlying region of memory is mapped by "createFromMemory", the
104 // mapping will be sustained until it is no longer referenced by any
105 // RunTimePoolInfo objects.
106 class RunTimePoolInfo {
107    public:
108     static std::optional<RunTimePoolInfo> createFromMemory(const SharedMemory& memory);
109     static RunTimePoolInfo createFromExistingBuffer(uint8_t* buffer, uint32_t size = 0);
110 
111     uint8_t* getBuffer() const;
112     bool flush() const;
113     const SharedMemory& getMemory() const;
114     uint32_t getSize() const;
115 
116    private:
117     class RunTimePoolInfoImpl;
118     RunTimePoolInfo(const std::shared_ptr<const RunTimePoolInfoImpl>& impl);
119 
120     std::shared_ptr<const RunTimePoolInfoImpl> mImpl;
121 };
122 
123 bool setRunTimePoolInfosFromCanonicalMemories(std::vector<RunTimePoolInfo>* poolInfos,
124                                               const std::vector<SharedMemory>& pools);
125 
126 bool setRunTimePoolInfosFromMemoryPools(std::vector<RunTimePoolInfo>* poolInfos,
127                                         const std::vector<Request::MemoryPool>& pools);
128 
129 // This class is used to execute a model on the CPU.
130 class CpuExecutor {
131    public:
132     // This constructor allows clients of CpuExecutor to provide custom CPU
133     // operation implementations. It is used by a sample driver to test
134     // extension support.
135     //
136     // Note that it is not possible to provide custom CPU implementations for
137     // non-OperationResolver operations (b/124041202).
138     //
139     // The operation resolver must outlive the executor.
CpuExecutor(const IOperationResolver * operationResolver)140     explicit CpuExecutor(const IOperationResolver* operationResolver)
141         : mOperationResolver(operationResolver) {}
142 
CpuExecutor()143     CpuExecutor() : CpuExecutor(BuiltinOperationResolver::get()) {}
144 
145     // Executes the model. The results will be stored at the locations
146     // specified in the constructor.
147     // The model must outlive the executor.  We prevent it from being modified
148     // while this is executing.
149     int run(const Model& model, const Request& request,
150             const std::vector<RunTimePoolInfo>& modelPoolInfos,
151             const std::vector<RunTimePoolInfo>& requestPoolInfos);
152 
getOutputShapes()153     const std::vector<OutputShape>& getOutputShapes() const {
154         CHECK(mFinished) << "getOutputShapes() called by an unfinished CpuExecutor.";
155         return mOutputShapes;
156     }
157 
setDeadline(const TimePoint & deadline)158     void setDeadline(const TimePoint& deadline) { mDeadline = deadline; }
setLoopTimeout(uint64_t duration)159     void setLoopTimeout(uint64_t duration) { mLoopTimeoutDuration = duration; }
160 
161    private:
162     // Creates runtime info from what's in the model.
163     std::vector<RunTimeOperandInfo> initializeRunTimeInfo(const Model::Subgraph& subgraph);
164     // Adjusts the runtime info for the arguments passed to the model,
165     // modifying the buffer location, and possibly the dimensions.
166     void updateForArguments(const std::vector<uint32_t>& indexes,
167                             const std::vector<Request::Argument>& arguments,
168                             const std::vector<RunTimePoolInfo>& requestPoolInfos,
169                             RunTimeOperandInfo* operands);
170     // Runs one subgraph.
171     int executeSubgraph(const Model::Subgraph& subgraph, RunTimeOperandInfo* operands);
172     // Runs one operation of the graph.
173     int executeOperation(const Operation& operation, RunTimeOperandInfo* operands);
174     int executeIfOperation(const Operation& operation, RunTimeOperandInfo* operands);
175     int executeWhileOperation(const Operation& operation, RunTimeOperandInfo* operands);
176 
177     void setOutputShapes(const std::vector<uint32_t>& outputIndexes,
178                          const std::vector<RunTimeOperandInfo>& operands);
179 
180     // Compile-time operand value information used by initializeRunTimeInfo.
181     // The fields are only valid while run() is being executed.
182     const uint8_t* mModelOperandValues = nullptr;
183     const std::vector<RunTimePoolInfo>* mModelPoolInfos = nullptr;
184     const std::vector<Model::Subgraph>* mReferencedSubgraphs = nullptr;
185 
186     // The output operand shapes returning to the runtime.
187     std::vector<OutputShape> mOutputShapes;
188 
189     // Whether execution is finished and mOutputShapes is ready
190     bool mFinished = false;
191 
192     // The deadline hint for the maximum amount of time the client expects the
193     // execution will take. If this deadline is exceeded, the CpuExecutor will
194     // abort the execution if there are remaining ops to execute.
195     OptionalTimePoint mDeadline;
196 
197     // The maximum amount of time in nanoseconds that can be spent executing a
198     // WHILE loop.
199     uint64_t mLoopTimeoutDuration = operation_while::kTimeoutNsDefault;
200 
201     const IOperationResolver* mOperationResolver;
202 };
203 
204 // Class for setting reasonable OpenMP threading settings. (OpenMP is used by
205 // the Eigen matrix library.)
206 //
207 // Currently sets a low blocktime: the time OpenMP threads busy-wait for more
208 // work before going to sleep. See b/79159165, https://reviews.llvm.org/D18577.
209 // The default is 200ms, we set to 20ms here, see b/109645291. This keeps the
210 // cores enabled throughout inference computation without too much extra power
211 // consumption afterwards.
212 //
213 // The OpenMP settings are thread-local (applying only to worker threads formed
214 // from that thread), see https://software.intel.com/en-us/node/522688 and
215 // http://lists.llvm.org/pipermail/openmp-dev/2016-July/001432.html. This class
216 // ensures that within the scope in which an object is instantiated we use the
217 // right settings (scopes may be nested), as long as no other library changes
218 // them.  (Note that in current NNAPI usage only one instance is used in the
219 // CpuExecutor thread).
220 //
221 // TODO(mikie): consider also setting the number of threads used. Using as many
222 // threads as there are cores results in more variable performance: if we don't
223 // get all cores for our threads, the latency is doubled as we wait for one core
224 // to do twice the amount of work. Reality is complicated though as not all
225 // cores are the same. Decision to be based on benchmarking against a
226 // representative set of workloads and devices. I'm keeping the code here for
227 // reference.
228 // b/109953668, disable OpenMP
229 #ifdef NNAPI_OPENMP
230 class ScopedOpenmpSettings {
231    public:
232     ScopedOpenmpSettings();
233     ~ScopedOpenmpSettings();
234     DISALLOW_COPY_AND_ASSIGN(ScopedOpenmpSettings);
235 
236    private:
237     int mBlocktimeInitial;
238 #if NNAPI_LIMIT_CPU_THREADS
239     int mMaxThreadsInitial;
240 #endif
241 };
242 #endif  // NNAPI_OPENMP
243 
244 namespace {
245 
246 template <typename T>
getScalarData(const RunTimeOperandInfo & info)247 T getScalarData(const RunTimeOperandInfo& info) {
248     CHECK_GE(info.length, sizeof(T)) << "Cannot get scalar data: buffer too short";
249     T* data = reinterpret_cast<T*>(info.buffer);
250     return data[0];
251 }
252 
253 template <typename T>
getScalarDataWithDefault(const RunTimeOperandInfo & info,T defaultValue)254 T getScalarDataWithDefault(const RunTimeOperandInfo& info, T defaultValue) {
255     if (info.length < sizeof(T)) {
256         return defaultValue;
257     }
258     return getScalarData<T>(info);
259 }
260 
IsNullInput(const RunTimeOperandInfo * input)261 inline bool IsNullInput(const RunTimeOperandInfo* input) {
262     return input->lifetime == Operand::LifeTime::NO_VALUE;
263 }
264 
NumInputsWithValues(const Operation & operation,const RunTimeOperandInfo * operands)265 inline int NumInputsWithValues(const Operation& operation, const RunTimeOperandInfo* operands) {
266     const std::vector<uint32_t>& inputs = operation.inputs;
267     return std::count_if(inputs.begin(), inputs.end(),
268                          [&operands](uint32_t i) { return !IsNullInput(&operands[i]); });
269 }
270 
NumOutputs(const Operation & operation)271 inline int NumOutputs(const Operation& operation) {
272     return operation.outputs.size();
273 }
274 
NumDimensions(const RunTimeOperandInfo * operand)275 inline size_t NumDimensions(const RunTimeOperandInfo* operand) {
276     return operand->shape().dimensions.size();
277 }
278 
SizeOfDimension(const RunTimeOperandInfo * operand,int i)279 inline uint32_t SizeOfDimension(const RunTimeOperandInfo* operand, int i) {
280     return operand->shape().dimensions[i];
281 }
282 
GetInput(const Operation & operation,RunTimeOperandInfo * operands,int index)283 inline RunTimeOperandInfo* GetInput(const Operation& operation, RunTimeOperandInfo* operands,
284                                     int index) {
285     return &operands[operation.inputs[index]];
286 }
287 
GetOutput(const Operation & operation,RunTimeOperandInfo * operands,int index)288 inline RunTimeOperandInfo* GetOutput(const Operation& operation, RunTimeOperandInfo* operands,
289                                      int index) {
290     return &operands[operation.outputs[index]];
291 }
292 
293 }  // anonymous namespace
294 
295 }  // namespace nn
296 }  // namespace android
297 
298 #endif  // ANDROID_FRAMEWORKS_ML_NN_COMMON_CPU_EXECUTOR_H
299