1 //
2 // Copyright © 2017 Arm Ltd. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5
6 #define LOG_TAG "ArmnnDriver"
7
8 #include "Utils.hpp"
9 #include "Half.hpp"
10
11 #include <armnnUtils/Permute.hpp>
12
13 #include <armnn/Utils.hpp>
14 #include <armnn/utility/Assert.hpp>
15 #include <Filesystem.hpp>
16 #include <log/log.h>
17
18 #include <cassert>
19 #include <cerrno>
20 #include <cinttypes>
21 #include <sstream>
22 #include <cstdio>
23 #include <time.h>
24
25
26
27 using namespace android;
28 using namespace android::hardware;
29 using namespace android::hidl::memory::V1_0;
30
31 namespace armnn_driver
32 {
33 const armnn::PermutationVector g_DontPermute{};
34
35 namespace
36 {
37
SwizzleAndroidNn4dTensorToArmNn(const armnn::TensorShape & inTensorShape,const void * input,void * output,size_t dataTypeSize,const armnn::PermutationVector & mappings)38 void SwizzleAndroidNn4dTensorToArmNn(const armnn::TensorShape& inTensorShape, const void* input,
39 void* output, size_t dataTypeSize, const armnn::PermutationVector& mappings)
40 {
41 assert(inTensorShape.GetNumDimensions() == 4U);
42
43 armnnUtils::Permute(armnnUtils::Permuted(inTensorShape, mappings), mappings, input, output, dataTypeSize);
44 }
45
46 } // anonymous namespace
47
SwizzleAndroidNn4dTensorToArmNn(const armnn::TensorInfo & tensor,const void * input,void * output,const armnn::PermutationVector & mappings)48 void SwizzleAndroidNn4dTensorToArmNn(const armnn::TensorInfo& tensor, const void* input, void* output,
49 const armnn::PermutationVector& mappings)
50 {
51 assert(tensor.GetNumDimensions() == 4U);
52
53 armnn::DataType dataType = tensor.GetDataType();
54 switch (dataType)
55 {
56 case armnn::DataType::Float16:
57 case armnn::DataType::Float32:
58 case armnn::DataType::QAsymmU8:
59 case armnn::DataType::QSymmS8:
60 case armnn::DataType::QAsymmS8:
61 SwizzleAndroidNn4dTensorToArmNn(tensor.GetShape(), input, output, armnn::GetDataTypeSize(dataType), mappings);
62 break;
63 default:
64 ALOGW("Unknown armnn::DataType for swizzling");
65 assert(0);
66 }
67 }
68
GetMemoryFromPool(V1_0::DataLocation location,const std::vector<android::nn::RunTimePoolInfo> & memPools)69 void* GetMemoryFromPool(V1_0::DataLocation location, const std::vector<android::nn::RunTimePoolInfo>& memPools)
70 {
71 // find the location within the pool
72 assert(location.poolIndex < memPools.size());
73
74 const android::nn::RunTimePoolInfo& memPool = memPools[location.poolIndex];
75
76 uint8_t* memPoolBuffer = memPool.getBuffer();
77
78 uint8_t* memory = memPoolBuffer + location.offset;
79
80 return memory;
81 }
82
GetTensorInfoForOperand(const V1_0::Operand & operand)83 armnn::TensorInfo GetTensorInfoForOperand(const V1_0::Operand& operand)
84 {
85 using namespace armnn;
86 DataType type;
87
88 switch (operand.type)
89 {
90 case V1_0::OperandType::TENSOR_FLOAT32:
91 type = armnn::DataType::Float32;
92 break;
93 case V1_0::OperandType::TENSOR_QUANT8_ASYMM:
94 type = armnn::DataType::QAsymmU8;
95 break;
96 case V1_0::OperandType::TENSOR_INT32:
97 type = armnn::DataType::Signed32;
98 break;
99 default:
100 throw UnsupportedOperand<V1_0::OperandType>(operand.type);
101 }
102
103 TensorInfo ret;
104 if (operand.dimensions.size() == 0)
105 {
106 TensorShape tensorShape(Dimensionality::NotSpecified);
107 ret = TensorInfo(tensorShape, type);
108 }
109 else
110 {
111 bool dimensionsSpecificity[5] = { true, true, true, true, true };
112 int count = 0;
113 std::for_each(operand.dimensions.data(),
114 operand.dimensions.data() + operand.dimensions.size(),
115 [&](const unsigned int val)
116 {
117 if (val == 0)
118 {
119 dimensionsSpecificity[count] = false;
120 }
121 count++;
122 });
123
124 TensorShape tensorShape(operand.dimensions.size(), operand.dimensions.data(), dimensionsSpecificity);
125 ret = TensorInfo(tensorShape, type);
126 }
127
128 ret.SetQuantizationScale(operand.scale);
129 ret.SetQuantizationOffset(operand.zeroPoint);
130
131 return ret;
132 }
133
134 #if defined(ARMNN_ANDROID_NN_V1_2) || defined(ARMNN_ANDROID_NN_V1_3)// Using ::android::hardware::neuralnetworks::V1_2
135
GetTensorInfoForOperand(const V1_2::Operand & operand)136 armnn::TensorInfo GetTensorInfoForOperand(const V1_2::Operand& operand)
137 {
138 using namespace armnn;
139 bool perChannel = false;
140
141 DataType type;
142 switch (operand.type)
143 {
144 case V1_2::OperandType::TENSOR_BOOL8:
145 type = armnn::DataType::Boolean;
146 break;
147 case V1_2::OperandType::TENSOR_FLOAT32:
148 type = armnn::DataType::Float32;
149 break;
150 case V1_2::OperandType::TENSOR_FLOAT16:
151 type = armnn::DataType::Float16;
152 break;
153 case V1_2::OperandType::TENSOR_QUANT8_ASYMM:
154 type = armnn::DataType::QAsymmU8;
155 break;
156 case V1_2::OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
157 perChannel=true;
158 ARMNN_FALLTHROUGH;
159 case V1_2::OperandType::TENSOR_QUANT8_SYMM:
160 type = armnn::DataType::QSymmS8;
161 break;
162 case V1_2::OperandType::TENSOR_QUANT16_SYMM:
163 type = armnn::DataType::QSymmS16;
164 break;
165 case V1_2::OperandType::TENSOR_INT32:
166 type = armnn::DataType::Signed32;
167 break;
168 default:
169 throw UnsupportedOperand<V1_2::OperandType>(operand.type);
170 }
171
172 TensorInfo ret;
173 if (operand.dimensions.size() == 0)
174 {
175 TensorShape tensorShape(Dimensionality::NotSpecified);
176 ret = TensorInfo(tensorShape, type);
177 }
178 else
179 {
180 bool dimensionsSpecificity[5] = { true, true, true, true, true };
181 int count = 0;
182 std::for_each(operand.dimensions.data(),
183 operand.dimensions.data() + operand.dimensions.size(),
184 [&](const unsigned int val)
185 {
186 if (val == 0)
187 {
188 dimensionsSpecificity[count] = false;
189 }
190 count++;
191 });
192
193 TensorShape tensorShape(operand.dimensions.size(), operand.dimensions.data(), dimensionsSpecificity);
194 ret = TensorInfo(tensorShape, type);
195 }
196
197 if (perChannel)
198 {
199 // ExtraParams is expected to be of type channelQuant
200 ARMNN_ASSERT(operand.extraParams.getDiscriminator() ==
201 V1_2::Operand::ExtraParams::hidl_discriminator::channelQuant);
202
203 auto perAxisQuantParams = operand.extraParams.channelQuant();
204
205 ret.SetQuantizationScales(perAxisQuantParams.scales);
206 ret.SetQuantizationDim(MakeOptional<unsigned int>(perAxisQuantParams.channelDim));
207 }
208 else
209 {
210 ret.SetQuantizationScale(operand.scale);
211 ret.SetQuantizationOffset(operand.zeroPoint);
212 }
213
214 return ret;
215 }
216
217 #endif
218
219 #ifdef ARMNN_ANDROID_NN_V1_3 // Using ::android::hardware::neuralnetworks::V1_3
220
GetTensorInfoForOperand(const V1_3::Operand & operand)221 armnn::TensorInfo GetTensorInfoForOperand(const V1_3::Operand& operand)
222 {
223 using namespace armnn;
224 bool perChannel = false;
225 bool isScalar = false;
226
227 DataType type;
228 switch (operand.type)
229 {
230 case V1_3::OperandType::TENSOR_BOOL8:
231 type = armnn::DataType::Boolean;
232 break;
233 case V1_3::OperandType::TENSOR_FLOAT32:
234 type = armnn::DataType::Float32;
235 break;
236 case V1_3::OperandType::TENSOR_FLOAT16:
237 type = armnn::DataType::Float16;
238 break;
239 case V1_3::OperandType::TENSOR_QUANT8_ASYMM:
240 type = armnn::DataType::QAsymmU8;
241 break;
242 case V1_3::OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
243 perChannel=true;
244 ARMNN_FALLTHROUGH;
245 case V1_3::OperandType::TENSOR_QUANT8_SYMM:
246 type = armnn::DataType::QSymmS8;
247 break;
248 case V1_3::OperandType::TENSOR_QUANT16_SYMM:
249 type = armnn::DataType::QSymmS16;
250 break;
251 case V1_3::OperandType::TENSOR_INT32:
252 type = armnn::DataType::Signed32;
253 break;
254 case V1_3::OperandType::INT32:
255 type = armnn::DataType::Signed32;
256 isScalar = true;
257 break;
258 case V1_3::OperandType::TENSOR_QUANT8_ASYMM_SIGNED:
259 type = armnn::DataType::QAsymmS8;
260 break;
261 default:
262 throw UnsupportedOperand<V1_3::OperandType>(operand.type);
263 }
264
265 TensorInfo ret;
266 if (isScalar)
267 {
268 ret = TensorInfo(TensorShape(armnn::Dimensionality::Scalar), type);
269 }
270 else
271 {
272 if (operand.dimensions.size() == 0)
273 {
274 TensorShape tensorShape(Dimensionality::NotSpecified);
275 ret = TensorInfo(tensorShape, type);
276 }
277 else
278 {
279 bool dimensionsSpecificity[5] = { true, true, true, true, true };
280 int count = 0;
281 std::for_each(operand.dimensions.data(),
282 operand.dimensions.data() + operand.dimensions.size(),
283 [&](const unsigned int val)
284 {
285 if (val == 0)
286 {
287 dimensionsSpecificity[count] = false;
288 }
289 count++;
290 });
291
292 TensorShape tensorShape(operand.dimensions.size(), operand.dimensions.data(), dimensionsSpecificity);
293 ret = TensorInfo(tensorShape, type);
294 }
295 }
296
297 if (perChannel)
298 {
299 // ExtraParams is expected to be of type channelQuant
300 ARMNN_ASSERT(operand.extraParams.getDiscriminator() ==
301 V1_2::Operand::ExtraParams::hidl_discriminator::channelQuant);
302
303 auto perAxisQuantParams = operand.extraParams.channelQuant();
304
305 ret.SetQuantizationScales(perAxisQuantParams.scales);
306 ret.SetQuantizationDim(MakeOptional<unsigned int>(perAxisQuantParams.channelDim));
307 }
308 else
309 {
310 ret.SetQuantizationScale(operand.scale);
311 ret.SetQuantizationOffset(operand.zeroPoint);
312 }
313 return ret;
314 }
315
316 #endif
317
GetOperandSummary(const V1_0::Operand & operand)318 std::string GetOperandSummary(const V1_0::Operand& operand)
319 {
320 return android::hardware::details::arrayToString(operand.dimensions, operand.dimensions.size()) + " " +
321 toString(operand.type);
322 }
323
324 #if defined(ARMNN_ANDROID_NN_V1_2) || defined(ARMNN_ANDROID_NN_V1_3) // Using ::android::hardware::neuralnetworks::V1_2
325
GetOperandSummary(const V1_2::Operand & operand)326 std::string GetOperandSummary(const V1_2::Operand& operand)
327 {
328 return android::hardware::details::arrayToString(operand.dimensions, operand.dimensions.size()) + " " +
329 toString(operand.type);
330 }
331
332 #endif
333
334 #ifdef ARMNN_ANDROID_NN_V1_3 // Using ::android::hardware::neuralnetworks::V1_3
335
GetOperandSummary(const V1_3::Operand & operand)336 std::string GetOperandSummary(const V1_3::Operand& operand)
337 {
338 return android::hardware::details::arrayToString(operand.dimensions, operand.dimensions.size()) + " " +
339 toString(operand.type);
340 }
341
342 #endif
343
344 using DumpElementFunction = void (*)(const armnn::ConstTensor& tensor,
345 unsigned int elementIndex,
346 std::ofstream& fileStream);
347
348 namespace
349 {
350 template <typename ElementType, typename PrintableType = ElementType>
DumpTensorElement(const armnn::ConstTensor & tensor,unsigned int elementIndex,std::ofstream & fileStream)351 void DumpTensorElement(const armnn::ConstTensor& tensor, unsigned int elementIndex, std::ofstream& fileStream)
352 {
353 const ElementType* elements = reinterpret_cast<const ElementType*>(tensor.GetMemoryArea());
354 fileStream << static_cast<PrintableType>(elements[elementIndex]) << ",";
355 }
356
MemoryLayoutString(const armnn::ConstTensor & tensor)357 constexpr const char* MemoryLayoutString(const armnn::ConstTensor& tensor)
358 {
359 const char* str = "";
360
361 switch (tensor.GetNumDimensions())
362 {
363 case 4: { str = "(BHWC) "; break; }
364 case 3: { str = "(HWC) "; break; }
365 case 2: { str = "(HW) "; break; }
366 default: { str = ""; break; }
367 }
368
369 return str;
370 }
371 } // namespace
372
DumpTensor(const std::string & dumpDir,const std::string & requestName,const std::string & tensorName,const armnn::ConstTensor & tensor)373 void DumpTensor(const std::string& dumpDir,
374 const std::string& requestName,
375 const std::string& tensorName,
376 const armnn::ConstTensor& tensor)
377 {
378 // The dump directory must exist in advance.
379 fs::path dumpPath = dumpDir;
380 const fs::path fileName = dumpPath / (requestName + "_" + tensorName + ".dump");
381
382 std::ofstream fileStream;
383 fileStream.open(fileName.c_str(), std::ofstream::out | std::ofstream::trunc);
384
385 if (!fileStream.good())
386 {
387 ALOGW("Could not open file %s for writing", fileName.c_str());
388 return;
389 }
390
391 DumpElementFunction dumpElementFunction = nullptr;
392
393 switch (tensor.GetDataType())
394 {
395 case armnn::DataType::Float32:
396 {
397 dumpElementFunction = &DumpTensorElement<float>;
398 break;
399 }
400 case armnn::DataType::QAsymmU8:
401 {
402 dumpElementFunction = &DumpTensorElement<uint8_t, uint32_t>;
403 break;
404 }
405 case armnn::DataType::Signed32:
406 {
407 dumpElementFunction = &DumpTensorElement<int32_t>;
408 break;
409 }
410 case armnn::DataType::Float16:
411 {
412 dumpElementFunction = &DumpTensorElement<armnn::Half>;
413 break;
414 }
415 case armnn::DataType::QAsymmS8:
416 {
417 dumpElementFunction = &DumpTensorElement<int8_t, int32_t>;
418 break;
419 }
420 case armnn::DataType::Boolean:
421 {
422 dumpElementFunction = &DumpTensorElement<bool>;
423 break;
424 }
425 default:
426 {
427 dumpElementFunction = nullptr;
428 }
429 }
430
431 if (dumpElementFunction != nullptr)
432 {
433 const unsigned int numDimensions = tensor.GetNumDimensions();
434
435 const unsigned int batch = (numDimensions == 4) ? tensor.GetShape()[numDimensions - 4] : 1;
436
437 const unsigned int height = (numDimensions >= 3)
438 ? tensor.GetShape()[numDimensions - 3]
439 : (numDimensions >= 2) ? tensor.GetShape()[numDimensions - 2] : 1;
440
441 const unsigned int width = (numDimensions >= 3)
442 ? tensor.GetShape()[numDimensions - 2]
443 : (numDimensions >= 1) ? tensor.GetShape()[numDimensions - 1] : 0;
444
445 const unsigned int channels = (numDimensions >= 3) ? tensor.GetShape()[numDimensions - 1] : 1;
446
447 fileStream << "# Number of elements " << tensor.GetNumElements() << std::endl;
448 fileStream << "# Dimensions " << MemoryLayoutString(tensor);
449 fileStream << "[" << tensor.GetShape()[0];
450 for (unsigned int d = 1; d < numDimensions; d++)
451 {
452 fileStream << "," << tensor.GetShape()[d];
453 }
454 fileStream << "]" << std::endl;
455
456 for (unsigned int e = 0, b = 0; b < batch; ++b)
457 {
458 if (numDimensions >= 4)
459 {
460 fileStream << "# Batch " << b << std::endl;
461 }
462 for (unsigned int c = 0; c < channels; c++)
463 {
464 if (numDimensions >= 3)
465 {
466 fileStream << "# Channel " << c << std::endl;
467 }
468 for (unsigned int h = 0; h < height; h++)
469 {
470 for (unsigned int w = 0; w < width; w++, e += channels)
471 {
472 (*dumpElementFunction)(tensor, e, fileStream);
473 }
474 fileStream << std::endl;
475 }
476 e -= channels - 1;
477 if (c < channels)
478 {
479 e -= ((height * width) - 1) * channels;
480 }
481 }
482 fileStream << std::endl;
483 }
484 fileStream << std::endl;
485 }
486 else
487 {
488 fileStream << "Cannot dump tensor elements: Unsupported data type "
489 << static_cast<unsigned int>(tensor.GetDataType()) << std::endl;
490 }
491
492 if (!fileStream.good())
493 {
494 ALOGW("An error occurred when writing to file %s", fileName.c_str());
495 }
496 }
497
DumpJsonProfilingIfRequired(bool gpuProfilingEnabled,const std::string & dumpDir,armnn::NetworkId networkId,const armnn::IProfiler * profiler)498 void DumpJsonProfilingIfRequired(bool gpuProfilingEnabled,
499 const std::string& dumpDir,
500 armnn::NetworkId networkId,
501 const armnn::IProfiler* profiler)
502 {
503 // Check if profiling is required.
504 if (!gpuProfilingEnabled)
505 {
506 return;
507 }
508
509 // The dump directory must exist in advance.
510 if (dumpDir.empty())
511 {
512 return;
513 }
514
515 ARMNN_ASSERT(profiler);
516
517 // Set the name of the output profiling file.
518 fs::path dumpPath = dumpDir;
519 const fs::path fileName = dumpPath / (std::to_string(networkId) + "_profiling.json");
520
521 // Open the ouput file for writing.
522 std::ofstream fileStream;
523 fileStream.open(fileName.c_str(), std::ofstream::out | std::ofstream::trunc);
524
525 if (!fileStream.good())
526 {
527 ALOGW("Could not open file %s for writing", fileName.c_str());
528 return;
529 }
530
531 // Write the profiling info to a JSON file.
532 profiler->Print(fileStream);
533 }
534
ExportNetworkGraphToDotFile(const armnn::IOptimizedNetwork & optimizedNetwork,const std::string & dumpDir)535 std::string ExportNetworkGraphToDotFile(const armnn::IOptimizedNetwork& optimizedNetwork,
536 const std::string& dumpDir)
537 {
538 std::string fileName;
539 // The dump directory must exist in advance.
540 if (dumpDir.empty())
541 {
542 return fileName;
543 }
544
545 std::string timestamp = GetFileTimestamp();
546 if (timestamp.empty())
547 {
548 return fileName;
549 }
550
551 // Set the name of the output .dot file.
552 fs::path dumpPath = dumpDir;
553 fs::path tempFilePath = dumpPath / (timestamp + "_networkgraph.dot");
554 fileName = tempFilePath.string();
555
556 ALOGV("Exporting the optimized network graph to file: %s", fileName.c_str());
557
558 // Write the network graph to a dot file.
559 std::ofstream fileStream;
560 fileStream.open(fileName, std::ofstream::out | std::ofstream::trunc);
561
562 if (!fileStream.good())
563 {
564 ALOGW("Could not open file %s for writing", fileName.c_str());
565 return fileName;
566 }
567
568 if (optimizedNetwork.SerializeToDot(fileStream) != armnn::Status::Success)
569 {
570 ALOGW("An error occurred when writing to file %s", fileName.c_str());
571 }
572 return fileName;
573 }
574
IsDynamicTensor(const armnn::TensorInfo & tensorInfo)575 bool IsDynamicTensor(const armnn::TensorInfo& tensorInfo)
576 {
577 if (tensorInfo.GetShape().GetDimensionality() == armnn::Dimensionality::NotSpecified)
578 {
579 return true;
580 }
581 // Account for the usage of the TensorShape empty constructor
582 if (tensorInfo.GetNumDimensions() == 0)
583 {
584 return true;
585 }
586 return !tensorInfo.GetShape().AreAllDimensionsSpecified();
587 }
588
AreDynamicTensorsSupported()589 bool AreDynamicTensorsSupported()
590 {
591 #if defined(ARMNN_ANDROID_NN_V1_3)
592 return true;
593 #else
594 return false;
595 #endif
596 }
597
GetFileTimestamp()598 std::string GetFileTimestamp()
599 {
600 // used to get a timestamp to name diagnostic files (the ArmNN serialized graph
601 // and getSupportedOperations.txt files)
602 timespec ts;
603 int iRet = clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
604 std::stringstream ss;
605 if (iRet == 0)
606 {
607 ss << std::to_string(ts.tv_sec) << "_" << std::to_string(ts.tv_nsec);
608 }
609 else
610 {
611 ALOGW("clock_gettime failed with errno %s : %s", std::to_string(errno).c_str(), std::strerror(errno));
612 }
613 return ss.str();
614 }
615
RenameGraphDotFile(const std::string & oldName,const std::string & dumpDir,const armnn::NetworkId networkId)616 void RenameGraphDotFile(const std::string& oldName, const std::string& dumpDir, const armnn::NetworkId networkId)
617 {
618 if (dumpDir.empty())
619 {
620 return;
621 }
622 if (oldName.empty())
623 {
624 return;
625 }
626 fs::path dumpPath = dumpDir;
627 const fs::path newFileName = dumpPath / (std::to_string(networkId) + "_networkgraph.dot");
628
629 int iRet = rename(oldName.c_str(), newFileName.c_str());
630 if (iRet != 0)
631 {
632 std::stringstream ss;
633 ss << "rename of [" << oldName << "] to [" << newFileName << "] failed with errno " << std::to_string(errno)
634 << " : " << std::strerror(errno);
635 ALOGW(ss.str().c_str());
636 }
637 }
638
CommitPools(std::vector<::android::nn::RunTimePoolInfo> & memPools)639 void CommitPools(std::vector<::android::nn::RunTimePoolInfo>& memPools)
640 {
641 if (memPools.empty())
642 {
643 return;
644 }
645 // Commit output buffers.
646 // Note that we update *all* pools, even if they aren't actually used as outputs -
647 // this is simpler and is what the CpuExecutor does.
648 for (auto& pool : memPools)
649 {
650 // Type android::nn::RunTimePoolInfo has changed between Android P & Q and Android R, where
651 // update() has been removed and flush() added.
652 #if defined(ARMNN_ANDROID_R) // Use the new Android implementation.
653 pool.flush();
654 #else
655 pool.update();
656 #endif
657 }
658 }
659 } // namespace armnn_driver
660