1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Memory"
18
19 #include "Memory.h"
20
21 #include <CpuExecutor.h>
22 #include <LegacyUtils.h>
23 #include <android-base/scopeguard.h>
24 #include <android/hardware_buffer.h>
25 #include <nnapi/IBurst.h>
26 #include <nnapi/SharedMemory.h>
27 #include <nnapi/TypeUtils.h>
28 #include <nnapi/Types.h>
29
30 #include <algorithm>
31 #include <memory>
32 #include <set>
33 #include <tuple>
34 #include <utility>
35 #include <vector>
36
37 #include "CompilationBuilder.h"
38 #include "Manager.h"
39 #include "TypeManager.h"
40
41 namespace android {
42 namespace nn {
43 namespace {
44
45 // The validator for a client-managed single-dimensional memory pool with a known size.
46 // The memory may be used for request inputs, request outputs, or model constants.
47 class SizedMemoryValidator : public MemoryValidatorBase {
48 public:
SizedMemoryValidator(uint32_t size)49 explicit SizedMemoryValidator(uint32_t size) : kSize(size) {}
50
validate(const CompilationBuilder *,IOType,uint32_t,const ANeuralNetworksOperandType *,uint32_t offset,uint32_t length) const51 bool validate(const CompilationBuilder*, IOType, uint32_t, const ANeuralNetworksOperandType*,
52 uint32_t offset, uint32_t length) const override {
53 NN_RET_CHECK(offset + length <= kSize) << "request size larger than the memory size.";
54 NN_RET_CHECK(offset != 0 || length != 0) << "memory size cannot be implied.";
55 return true;
56 }
57
getMetadata() const58 Metadata getMetadata() const override { return {.logicalSize = kSize}; }
updateMetadata(const Metadata & metadata)59 bool updateMetadata(const Metadata& metadata) override {
60 return metadata.logicalSize == 0 || metadata.logicalSize == kSize;
61 }
62
63 private:
64 const uint32_t kSize;
65 };
66
67 // The validator for an AHardwareBuffer with Non-BLOB format.
68 // We require the memory only used for request inputs or request outputs,
69 // with both offset and length set to zero.
70 class AHardwareBufferNonBlobValidator : public MemoryValidatorBase {
71 public:
72 AHardwareBufferNonBlobValidator() = default;
73
validate(const CompilationBuilder * compilation,IOType,uint32_t,const ANeuralNetworksOperandType *,uint32_t offset,uint32_t length) const74 bool validate(const CompilationBuilder* compilation, IOType, uint32_t,
75 const ANeuralNetworksOperandType*, uint32_t offset,
76 uint32_t length) const override {
77 NN_RET_CHECK(compilation != nullptr)
78 << "cannot use Non-BLOB AHardwareBuffer as model constant";
79 NN_RET_CHECK(offset == 0 && length == 0)
80 << "non-zero offset (" << offset << ") and/or length (" << length
81 << ") for Non-BLOB format AHardwareBuffer.";
82 return true;
83 }
84
getMetadata() const85 Metadata getMetadata() const override { return {}; }
updateMetadata(const Metadata &)86 bool updateMetadata(const Metadata&) override { return true; }
87 };
88
89 // The validator for a memory created from ANNMemory_createFromDesc.
90 // We require the memory only used as one of the pre-specified roles,
91 // with both offset and length set to zero.
92 class DeviceMemoryValidator : public MemoryValidatorBase {
93 public:
DeviceMemoryValidator(std::set<CompilationRole> roles,Operand operand,std::vector<uint32_t> dimensions)94 DeviceMemoryValidator(std::set<CompilationRole> roles, Operand operand,
95 std::vector<uint32_t> dimensions)
96 : kCompilationRoles(std::move(roles)),
97 kOperand(std::move(operand)),
98 kInitialDimensions(std::move(dimensions)),
99 mUpdatedDimensions(kInitialDimensions) {}
100
validate(const CompilationBuilder * compilation,IOType ioType,uint32_t index,const ANeuralNetworksOperandType * type,uint32_t offset,uint32_t length) const101 bool validate(const CompilationBuilder* compilation, IOType ioType, uint32_t index,
102 const ANeuralNetworksOperandType* type, uint32_t offset,
103 uint32_t length) const override {
104 NN_RET_CHECK(kCompilationRoles.count({compilation, ioType, index}) > 0)
105 << "invalid compilation role.";
106 NN_RET_CHECK(offset == 0 && length == 0)
107 << "non-zero offset and/or length for driver-allocated memory.";
108 if (type) {
109 const bool isTensor = TypeManager::get()->isTensorType(kOperand.type);
110 NN_RET_CHECK(isTensor || type->dimensionCount == 0)
111 << "invalid dimensions for scalar memory.";
112 std::vector<uint32_t> dimensions(type->dimensions,
113 type->dimensions + type->dimensionCount);
114 // We only check against kInitialDimensions here.
115 // For input memories, mUpdatedDimensions will be checked in validateInputDimensions
116 // at the beginning of a computation.
117 const auto combined = combineDimensions(dimensions, kInitialDimensions);
118 NN_RET_CHECK(combined.has_value())
119 << "incompatible dimensions between request and memory. (request: "
120 << toString(dimensions) << ", memory: " << toString(kInitialDimensions) << ")";
121 }
122 return true;
123 }
124
validateInputDimensions(const std::vector<uint32_t> & dimensions) const125 bool validateInputDimensions(const std::vector<uint32_t>& dimensions) const override {
126 NN_RET_CHECK(mInitialized) << "using an uninitialized memory as input";
127 NN_RET_CHECK(dimensions == mUpdatedDimensions)
128 << "incompatible input dimensions between request and memory. (request: "
129 << toString(dimensions) << ", memory: " << toString(mUpdatedDimensions) << ")";
130 return true;
131 }
132
getMetadata() const133 Metadata getMetadata() const override {
134 return {.logicalSize = TypeManager::get()->getSizeOfData(kOperand.type, mUpdatedDimensions),
135 .dimensions = mUpdatedDimensions,
136 .operand = kOperand};
137 }
138
updateMetadata(const Metadata & metadata)139 bool updateMetadata(const Metadata& metadata) override {
140 NN_RET_CHECK(!metadata.operand.has_value() ||
141 (metadata.operand->type == kOperand.type &&
142 metadata.operand->scale == kOperand.scale &&
143 metadata.operand->zeroPoint == kOperand.zeroPoint &&
144 metadata.operand->extraParams == kOperand.extraParams));
145
146 NN_RET_CHECK(metadata.dimensions.empty() ||
147 TypeManager::get()->isTensorType(kOperand.type));
148 auto combined = combineDimensions(metadata.dimensions, kInitialDimensions);
149 NN_RET_CHECK(combined.has_value());
150 NN_RET_CHECK(metadata.logicalSize == 0 ||
151 metadata.logicalSize ==
152 TypeManager::get()->getSizeOfData(kOperand.type, combined.value()));
153 mUpdatedDimensions = std::move(combined.value());
154 return true;
155 }
156
createdWithUnknownShape() const157 bool createdWithUnknownShape() const override {
158 return TypeManager::get()->getSizeOfData(kOperand.type, kInitialDimensions) == 0;
159 }
160
setInitialized(bool initialized)161 void setInitialized(bool initialized) override { mInitialized = initialized; }
isInitialized() const162 bool isInitialized() const override { return mInitialized; }
163
164 private:
165 const std::set<CompilationRole> kCompilationRoles;
166
167 // Keep track of the data type, scale, zero point, and extra parameters of the target operand.
168 // Other fields will be ignored, including dimensions, lifetime, location, etc.
169 const Operand kOperand;
170
171 // The dimensions of the memory when the memory object is created.
172 // May have unknown dimensions or rank.
173 const std::vector<uint32_t> kInitialDimensions;
174
175 // The updated dimensions after a successful execution or memory copying.
176 std::vector<uint32_t> mUpdatedDimensions;
177
178 bool mInitialized = false;
179 };
180
181 } // namespace
182
RuntimeMemory(SharedMemory memory)183 RuntimeMemory::RuntimeMemory(SharedMemory memory) : kMemory(std::move(memory)) {
184 CHECK(kMemory != nullptr);
185 mValidator = std::make_unique<SizedMemoryValidator>(nn::getSize(kMemory));
186 }
187
RuntimeMemory(SharedMemory memory,std::unique_ptr<MemoryValidatorBase> validator)188 RuntimeMemory::RuntimeMemory(SharedMemory memory, std::unique_ptr<MemoryValidatorBase> validator)
189 : kMemory(std::move(memory)), mValidator(std::move(validator)) {
190 CHECK(kMemory != nullptr);
191 }
192
RuntimeMemory(SharedBuffer buffer)193 RuntimeMemory::RuntimeMemory(SharedBuffer buffer) : kBuffer(std::move(buffer)) {}
194
getMemoryPool() const195 Request::MemoryPool RuntimeMemory::getMemoryPool() const {
196 if (kBuffer != nullptr) {
197 return kBuffer->getToken();
198 }
199 return kMemory;
200 }
201
getRunTimePoolInfo() const202 std::optional<RunTimePoolInfo> RuntimeMemory::getRunTimePoolInfo() const {
203 std::lock_guard<std::mutex> guard(mMutex);
204 if (!mHasCachedRunTimePoolInfo) {
205 mCachedRunTimePoolInfo = RunTimePoolInfo::createFromMemory(kMemory);
206 mHasCachedRunTimePoolInfo = true;
207 }
208 return mCachedRunTimePoolInfo;
209 }
210
hold(const IBurst::OptionalCacheHold & cacheHold) const211 void RuntimeMemory::hold(const IBurst::OptionalCacheHold& cacheHold) const {
212 if (cacheHold != nullptr) {
213 std::lock_guard<std::mutex> guard(mMutex);
214 mHold.insert(cacheHold);
215 }
216 }
217
copyHidlMemories(const std::optional<RunTimePoolInfo> & src,const std::optional<RunTimePoolInfo> & dst)218 static int copyHidlMemories(const std::optional<RunTimePoolInfo>& src,
219 const std::optional<RunTimePoolInfo>& dst) {
220 if (!src.has_value() || !dst.has_value()) {
221 LOG(ERROR) << "ANeuralNetworksMemory_copy -- unable to map memory";
222 return ANEURALNETWORKS_UNMAPPABLE;
223 }
224 if (src->getSize() != dst->getSize()) {
225 LOG(ERROR) << "ANeuralNetworksMemory_copy -- incompatible memory size";
226 return ANEURALNETWORKS_BAD_DATA;
227 }
228 CHECK(src->getBuffer() != nullptr);
229 CHECK(dst->getBuffer() != nullptr);
230 std::copy(src->getBuffer(), src->getBuffer() + src->getSize(), dst->getBuffer());
231 dst->flush();
232 return ANEURALNETWORKS_NO_ERROR;
233 }
234
copyIBufferToMemory(const SharedBuffer & src,const SharedMemory & dst)235 int copyIBufferToMemory(const SharedBuffer& src, const SharedMemory& dst) {
236 const auto ret = src->copyTo(dst);
237 if (!ret.has_value()) {
238 LOG(ERROR) << "ANeuralNetworksMemory_copy failure: " << ret.error().message;
239 return convertErrorStatusToResultCode(ret.error().code);
240 }
241 return ANEURALNETWORKS_NO_ERROR;
242 }
243
copyMemoryToIBuffer(const SharedMemory & src,const SharedBuffer & dst,const std::vector<uint32_t> & dimensions)244 int copyMemoryToIBuffer(const SharedMemory& src, const SharedBuffer& dst,
245 const std::vector<uint32_t>& dimensions) {
246 const auto ret = dst->copyFrom(src, dimensions);
247 if (!ret.has_value()) {
248 LOG(ERROR) << "ANeuralNetworksMemory_copy failure: " << ret.error().message;
249 return convertErrorStatusToResultCode(ret.error().code);
250 }
251 return ANEURALNETWORKS_NO_ERROR;
252 }
253
copyIBuffers(const SharedBuffer & src,const SharedBuffer & dst,const MemoryValidatorBase::Metadata & srcMetadata)254 static int copyIBuffers(const SharedBuffer& src, const SharedBuffer& dst,
255 const MemoryValidatorBase::Metadata& srcMetadata) {
256 const auto [n, memoryAHWB] = MemoryRuntimeAHWB::create(srcMetadata.logicalSize);
257 NN_RETURN_IF_ERROR(n);
258 const SharedMemory& memory = memoryAHWB->getMemory();
259 if (!validate(memory).ok()) return ANEURALNETWORKS_OUT_OF_MEMORY;
260 NN_RETURN_IF_ERROR(copyIBufferToMemory(src, memory));
261 NN_RETURN_IF_ERROR(copyMemoryToIBuffer(memory, dst, srcMetadata.dimensions));
262 return ANEURALNETWORKS_NO_ERROR;
263 }
264
copyInternal(const RuntimeMemory & src,const RuntimeMemory & dst)265 static int copyInternal(const RuntimeMemory& src, const RuntimeMemory& dst) {
266 if (&src == &dst) return ANEURALNETWORKS_NO_ERROR;
267
268 if (!src.getValidator().isInitialized()) {
269 LOG(ERROR) << "ANeuralNetworksMemory_copy -- uninitialized source memory";
270 return ANEURALNETWORKS_BAD_DATA;
271 }
272
273 const auto srcMetadata = src.getValidator().getMetadata();
274 if (!dst.getValidator().updateMetadata(srcMetadata)) {
275 LOG(ERROR) << "ANeuralNetworksMemory_copy -- incompatible memories";
276 return ANEURALNETWORKS_BAD_DATA;
277 }
278
279 bool srcHasMemory = validate(src.getMemory()).ok();
280 bool dstHasMemory = validate(dst.getMemory()).ok();
281 bool srcHasIBuffer = src.getIBuffer() != nullptr;
282 bool dstHasIBuffer = dst.getIBuffer() != nullptr;
283 if (srcHasIBuffer && dstHasIBuffer) {
284 return copyIBuffers(src.getIBuffer(), dst.getIBuffer(), srcMetadata);
285 } else if (srcHasMemory && dstHasMemory) {
286 return copyHidlMemories(src.getRunTimePoolInfo(), dst.getRunTimePoolInfo());
287 } else if (srcHasMemory && dstHasIBuffer) {
288 return copyMemoryToIBuffer(src.getMemory(), dst.getIBuffer(), srcMetadata.dimensions);
289 } else if (srcHasIBuffer && dstHasMemory) {
290 return copyIBufferToMemory(src.getIBuffer(), dst.getMemory());
291 }
292 return ANEURALNETWORKS_OP_FAILED;
293 }
294
copy(const RuntimeMemory & src,const RuntimeMemory & dst)295 int RuntimeMemory::copy(const RuntimeMemory& src, const RuntimeMemory& dst) {
296 int n = copyInternal(src, dst);
297 dst.getValidator().setInitialized(n == ANEURALNETWORKS_NO_ERROR);
298 return n;
299 }
300
badState(const char * name) const301 bool MemoryBuilder::badState(const char* name) const {
302 if (mFinished) {
303 LOG(ERROR) << "ANeuralNetworksMemoryDesc_" << name << " can't modify after finished";
304 return true;
305 }
306 return false;
307 }
308
addRole(const CompilationBuilder & compilation,IOType ioType,uint32_t index,float prob)309 int MemoryBuilder::addRole(const CompilationBuilder& compilation, IOType ioType, uint32_t index,
310 float prob) {
311 const char* tag = ioType == IOType::INPUT ? "addInputRole" : "addOutputRole";
312 if (badState(tag)) {
313 return ANEURALNETWORKS_BAD_STATE;
314 }
315 if (mRoles.count({&compilation, ioType, index}) > 0) {
316 LOG(ERROR) << "ANeuralNetworksMemoryDesc_" << tag
317 << " -- the same operand is specified twice.";
318 return ANEURALNETWORKS_BAD_DATA;
319 }
320
321 std::vector<std::tuple<const RuntimePreparedModel*, IOType, uint32_t>> roles;
322 auto callback = [&roles](const auto* preparedModel, IOType type, uint32_t index) {
323 roles.emplace_back(preparedModel, type, index);
324 };
325 if (ioType == IOType::INPUT) {
326 if (compilation.forEachStepRoleOfInput(index, callback) != ANEURALNETWORKS_NO_ERROR) {
327 return ANEURALNETWORKS_BAD_DATA;
328 }
329 } else {
330 if (compilation.forEachStepRoleOfOutput(index, callback) != ANEURALNETWORKS_NO_ERROR) {
331 return ANEURALNETWORKS_BAD_DATA;
332 }
333 }
334
335 const ModelBuilder* model = compilation.getModel();
336 CHECK(model != nullptr);
337 Operand operand;
338 if (ioType == IOType::INPUT) {
339 if (index >= model->inputCount()) {
340 LOG(ERROR) << "ANeuralNetworksMemoryDesc_addInputRole -- input index out of range.";
341 return ANEURALNETWORKS_BAD_DATA;
342 }
343 operand = model->getInputOperand(index);
344 } else {
345 if (index >= model->outputCount()) {
346 LOG(ERROR) << "ANeuralNetworksMemoryDesc_addOutputRole -- output index out of range.";
347 return ANEURALNETWORKS_BAD_DATA;
348 }
349 operand = model->getOutputOperand(index);
350 }
351 if (mOperand.has_value()) {
352 if (operand.type != mOperand->type || operand.scale != mOperand->scale ||
353 operand.zeroPoint != mOperand->zeroPoint ||
354 operand.extraParams != mOperand->extraParams) {
355 LOG(ERROR) << "ANeuralNetworksMemoryDesc_" << tag
356 << " -- incompatible operand metadata.";
357 return ANEURALNETWORKS_BAD_DATA;
358 }
359 }
360 if (!TypeManager::get()->isTensorType(operand.type) && !mDesc.dimensions.empty()) {
361 LOG(ERROR) << "ANeuralNetworksMemoryDesc_" << tag << " -- incompatible dimensions.";
362 return ANEURALNETWORKS_BAD_DATA;
363 }
364 auto combined = combineDimensions(mDesc.dimensions, operand.dimensions);
365 if (!combined.has_value()) {
366 LOG(ERROR) << "ANeuralNetworksMemoryDesc_" << tag << " -- incompatible dimensions.";
367 return ANEURALNETWORKS_BAD_DATA;
368 }
369
370 if (prob > 1.0f || prob <= 0.0f) {
371 LOG(ERROR) << "ANeuralNetworksMemoryDesc_" << tag << " -- invalid frequency " << prob;
372 return ANEURALNETWORKS_BAD_DATA;
373 }
374
375 mRoles.emplace(&compilation, ioType, index);
376 for (const auto& [preparedModel, type, ind] : roles) {
377 uint32_t modelIndex = mDesc.preparedModels.add(preparedModel);
378 BufferRole role = {.modelIndex = modelIndex, .ioIndex = ind, .probability = prob};
379 if (type == IOType::INPUT) {
380 mDesc.inputRoles.push_back(role);
381 } else {
382 mDesc.outputRoles.push_back(role);
383 }
384 }
385 mOperand = std::move(operand);
386 mDesc.dimensions = std::move(combined.value());
387 return ANEURALNETWORKS_NO_ERROR;
388 }
389
setDimensions(const std::vector<uint32_t> & dimensions)390 int MemoryBuilder::setDimensions(const std::vector<uint32_t>& dimensions) {
391 if (badState("setDimensions")) return ANEURALNETWORKS_BAD_STATE;
392 if (mOperand.has_value() && !TypeManager::get()->isTensorType(mOperand->type) &&
393 !dimensions.empty()) {
394 LOG(ERROR) << "ANeuralNetworksMemoryDesc_setDimensions -- incompatible dimensions for "
395 "scalars.";
396 return ANEURALNETWORKS_BAD_DATA;
397 }
398 auto combined = combineDimensions(mDesc.dimensions, dimensions);
399 if (!combined.has_value()) {
400 LOG(ERROR) << "ANeuralNetworksMemoryDesc_setDimensions -- incompatible dimensions.";
401 return ANEURALNETWORKS_BAD_DATA;
402 }
403 mDesc.dimensions = std::move(combined.value());
404 return ANEURALNETWORKS_NO_ERROR;
405 }
406
logMemoryDescriptorToInfo(const MemoryDescriptor & desc,const Operand & operand)407 static void logMemoryDescriptorToInfo(const MemoryDescriptor& desc, const Operand& operand) {
408 LOG(INFO) << "MemoryDescriptor start";
409 LOG(INFO) << " Data type: " << operand.type;
410 LOG(INFO) << " Scale: " << operand.scale;
411 LOG(INFO) << " Zero point: " << operand.zeroPoint;
412 LOG(INFO) << " Extra params: " << operand.extraParams;
413 LOG(INFO) << " Dimensions: " << toString(desc.dimensions);
414 LOG(INFO) << " Prepared models [" << desc.preparedModels.size() << "]:";
415 for (const auto* preparedModel : desc.preparedModels) {
416 LOG(INFO) << " service = " << preparedModel->getDevice()->getName();
417 }
418 LOG(INFO) << " Input roles [" << desc.inputRoles.size() << "]:";
419 for (const auto& usage : desc.inputRoles) {
420 LOG(INFO) << " " << usage;
421 }
422 LOG(INFO) << " Output roles [" << desc.outputRoles.size() << "]:";
423 for (const auto& usage : desc.outputRoles) {
424 LOG(INFO) << " " << usage;
425 }
426 LOG(INFO) << "MemoryDescriptor end";
427 }
428
getDevices(const MemoryDescriptor & desc)429 static std::set<const Device*> getDevices(const MemoryDescriptor& desc) {
430 std::set<const Device*> devices;
431 for (const auto* preparedModel : desc.preparedModels) {
432 const auto* device = preparedModel->getDevice();
433 devices.insert(device);
434 }
435 return devices;
436 }
437
finish()438 int MemoryBuilder::finish() {
439 if (badState("finish")) return ANEURALNETWORKS_BAD_STATE;
440 if (mRoles.empty()) {
441 LOG(ERROR) << "ANeuralNetworksMemoryDesc_finish -- no role has been specified.";
442 return ANEURALNETWORKS_BAD_DATA;
443 }
444 CHECK(mOperand.has_value());
445 if (VLOG_IS_ON(MEMORY)) {
446 logMemoryDescriptorToInfo(mDesc, mOperand.value());
447 }
448 std::set<const Device*> devices = getDevices(mDesc);
449 if (devices.empty()) {
450 // This can happen with interpreted control flow.
451 mAllocator = nullptr;
452 } else if (devices.size() == 1) {
453 mAllocator = *devices.begin();
454 VLOG(MEMORY) << "Using " << mAllocator->getName() << " as allocator.";
455 } else {
456 LOG(INFO) << "MemoryBuilder::finish -- cannot handle multiple devices.";
457 mAllocator = nullptr;
458 }
459 mSupportsAhwb = std::all_of(devices.begin(), devices.end(), [](const auto* device) {
460 return device->getFeatureLevel() >= kHalVersionV1_3ToApi.featureLevel;
461 });
462 mShouldFallback = std::none_of(mRoles.begin(), mRoles.end(), [](const auto& role) {
463 const auto* cb = std::get<const CompilationBuilder*>(role);
464 return cb->createdWithExplicitDeviceList();
465 });
466 const uint32_t size = TypeManager::get()->getSizeOfData(mOperand->type, mDesc.dimensions);
467 mShouldFallback &= (size != 0);
468 mFinished = true;
469 return ANEURALNETWORKS_NO_ERROR;
470 }
471
allocate() const472 std::pair<int, std::unique_ptr<RuntimeMemory>> MemoryBuilder::allocate() const {
473 if (!mFinished) {
474 LOG(ERROR) << "ANeuralNetworksMemory_createFromDesc -- passed an unfinished descriptor";
475 return {ANEURALNETWORKS_BAD_STATE, nullptr};
476 }
477
478 int n = ANEURALNETWORKS_OP_FAILED;
479 std::unique_ptr<RuntimeMemory> memory;
480 CHECK(mOperand.has_value());
481
482 // Try allocate the memory on device.
483 if (mAllocator != nullptr) {
484 std::tie(n, memory) = mAllocator->allocate(mDesc, mOperand->type);
485 }
486
487 // If failed, fallback to ashmem or BLOB mode AHWB.
488 if (n != ANEURALNETWORKS_NO_ERROR && mShouldFallback) {
489 const uint32_t size = TypeManager::get()->getSizeOfData(mOperand->type, mDesc.dimensions);
490 if (mSupportsAhwb) {
491 VLOG(MEMORY) << "MemoryBuilder::allocate -- fallback to BLOB mode AHWB.";
492 std::tie(n, memory) = MemoryRuntimeAHWB::create(size);
493 } else {
494 VLOG(MEMORY) << "MemoryBuilder::allocate -- fallback to ashmem.";
495 std::tie(n, memory) = MemoryAshmem::create(size);
496 }
497 }
498
499 if (n == ANEURALNETWORKS_NO_ERROR) {
500 CHECK(memory != nullptr);
501 auto validator =
502 std::make_unique<DeviceMemoryValidator>(mRoles, mOperand.value(), mDesc.dimensions);
503 memory->setValidator(std::move(validator));
504 }
505 return {n, std::move(memory)};
506 }
507
create(uint32_t size)508 std::pair<int, std::unique_ptr<MemoryAshmem>> MemoryAshmem::create(uint32_t size) {
509 auto memory = createSharedMemory(size);
510 if (!memory.has_value()) {
511 LOG(ERROR) << "RuntimeMemory::create() failed: " << memory.error().message;
512 return {convertErrorStatusToResultCode(memory.error().code), nullptr};
513 }
514 auto mapping = map(memory.value());
515 if (!mapping.has_value()) {
516 LOG(ERROR) << "RuntimeMemory::create() map failed: " << mapping.error().message;
517 return {convertErrorStatusToResultCode(mapping.error().code), nullptr};
518 }
519 return {ANEURALNETWORKS_NO_ERROR,
520 std::make_unique<MemoryAshmem>(std::move(memory).value(), std::move(mapping).value())};
521 }
522
getPointer() const523 uint8_t* MemoryAshmem::getPointer() const {
524 return static_cast<uint8_t*>(std::get<void*>(kMapping.pointer));
525 }
526
MemoryAshmem(SharedMemory memory,Mapping mapping)527 MemoryAshmem::MemoryAshmem(SharedMemory memory, Mapping mapping)
528 : RuntimeMemory(std::move(memory)), kMapping(std::move(mapping)) {}
529
create(size_t size,int prot,int fd,size_t offset)530 std::pair<int, std::unique_ptr<MemoryFd>> MemoryFd::create(size_t size, int prot, int fd,
531 size_t offset) {
532 auto memory = createSharedMemoryFromFd(size, prot, fd, offset);
533 if (!memory.has_value()) {
534 LOG(ERROR) << "Failed to create memory from fd: " << memory.error().message;
535 return {convertErrorStatusToResultCode(memory.error().code), nullptr};
536 }
537 return {ANEURALNETWORKS_NO_ERROR, std::make_unique<MemoryFd>(std::move(memory).value())};
538 }
539
MemoryFd(SharedMemory memory)540 MemoryFd::MemoryFd(SharedMemory memory) : RuntimeMemory(std::move(memory)) {}
541
create(const AHardwareBuffer & ahwb)542 std::pair<int, std::unique_ptr<MemoryAHWB>> MemoryAHWB::create(const AHardwareBuffer& ahwb) {
543 auto memory = createSharedMemoryFromAHWB(const_cast<AHardwareBuffer*>(&ahwb),
544 /*takeOwnership=*/false);
545 if (!memory.has_value()) {
546 LOG(ERROR) << "Failed to create memory from AHWB: " << memory.error().message;
547 return {convertErrorStatusToResultCode(memory.error().code), nullptr};
548 }
549
550 std::unique_ptr<MemoryValidatorBase> validator;
551 if (isAhwbBlob(memory.value())) {
552 validator = std::make_unique<SizedMemoryValidator>(nn::getSize(memory.value()));
553 } else {
554 validator = std::make_unique<AHardwareBufferNonBlobValidator>();
555 }
556
557 auto memoryAHWB = std::make_unique<MemoryAHWB>(std::move(memory).value(), std::move(validator));
558 return {ANEURALNETWORKS_NO_ERROR, std::move(memoryAHWB)};
559 }
560
create(uint32_t size)561 std::pair<int, std::unique_ptr<MemoryRuntimeAHWB>> MemoryRuntimeAHWB::create(uint32_t size) {
562 AHardwareBuffer* ahwb = nullptr;
563 const auto usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN | AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN;
564 const AHardwareBuffer_Desc desc = {
565 .width = size,
566 .height = 1,
567 .layers = 1,
568 .format = AHARDWAREBUFFER_FORMAT_BLOB,
569 .usage = usage,
570 .stride = size,
571 };
572 int err = AHardwareBuffer_allocate(&desc, &ahwb);
573 if (err != 0 || ahwb == nullptr) {
574 LOG(ERROR) << "Failed to allocate BLOB mode AHWB.";
575 return {ANEURALNETWORKS_OP_FAILED, nullptr};
576 }
577
578 auto memory = createSharedMemoryFromAHWB(ahwb, /*takeOWnership=*/true);
579 if (!memory.has_value()) {
580 LOG(ERROR) << "Failed to allocate BLOB mode AHWB: " << memory.error().message;
581 return {convertErrorStatusToResultCode(memory.error().code), nullptr};
582 }
583 auto mapping = map(memory.value());
584 if (!mapping.has_value()) {
585 LOG(ERROR) << "Failed to map BLOB mode AHWB: " << mapping.error().message;
586 return {convertErrorStatusToResultCode(mapping.error().code), nullptr};
587 }
588 auto memoryAHWB = std::make_unique<MemoryRuntimeAHWB>(std::move(memory).value(),
589 std::move(mapping).value());
590 return {ANEURALNETWORKS_NO_ERROR, std::move(memoryAHWB)};
591 }
592
getPointer() const593 uint8_t* MemoryRuntimeAHWB::getPointer() const {
594 return static_cast<uint8_t*>(std::get<void*>(kMapping.pointer));
595 }
596
MemoryRuntimeAHWB(SharedMemory memory,Mapping mapping)597 MemoryRuntimeAHWB::MemoryRuntimeAHWB(SharedMemory memory, Mapping mapping)
598 : RuntimeMemory(std::move(memory)), kMapping(std::move(mapping)) {}
599
create(SharedBuffer buffer)600 std::pair<int, std::unique_ptr<MemoryFromDevice>> MemoryFromDevice::create(SharedBuffer buffer) {
601 if (buffer == nullptr) {
602 LOG(ERROR) << "nullptr IBuffer for device memory.";
603 return {ANEURALNETWORKS_OP_FAILED, nullptr};
604 }
605 return {ANEURALNETWORKS_NO_ERROR, std::make_unique<MemoryFromDevice>(std::move(buffer))};
606 }
607
MemoryFromDevice(SharedBuffer buffer)608 MemoryFromDevice::MemoryFromDevice(SharedBuffer buffer) : RuntimeMemory(std::move(buffer)) {}
609
610 } // namespace nn
611 } // namespace android
612