1 //
2 // Copyright © 2017 Arm Ltd. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5
6 #define LOG_TAG "ArmnnDriver"
7
8 #include "ModelToINetworkConverter.hpp"
9 #include "Utils.hpp"
10
11 #include <log/log.h>
12 #include <type_traits>
13
14 namespace armnn_driver
15 {
16
17 template<typename HalPolicy>
ModelToINetworkConverter(const std::vector<armnn::BackendId> & backends,const HalModel & model,const std::set<unsigned int> & forcedUnsupportedOperations)18 ModelToINetworkConverter<HalPolicy>::ModelToINetworkConverter(const std::vector<armnn::BackendId>& backends,
19 const HalModel& model,
20 const std::set<unsigned int>& forcedUnsupportedOperations)
21 : m_Data(backends)
22 , m_Model(model)
23 , m_ForcedUnsupportedOperations(forcedUnsupportedOperations)
24 , m_ConversionResult(ConversionResult::Success)
25 {
26 try
27 {
28 Convert();
29 }
30 catch (std::exception& e)
31 {
32 m_ConversionResult = ConversionResult::UnsupportedFeature;
33 ALOGE("%s: Unexpected exception: %s", __func__, e.what());
34 assert(false);
35 }
36 }
37
38 template<typename HalPolicy>
Convert()39 void ModelToINetworkConverter<HalPolicy>::Convert()
40 {
41 using HalModel = typename HalPolicy::Model;
42 using HalOperand = typename HalPolicy::Operand;
43 using HalOperandType = typename HalPolicy::OperandType;
44
45 ALOGV("ModelToINetworkConverter::Convert(): %s", GetModelSummary<HalModel>(m_Model).c_str());
46
47 // map the memory pool into shared pointers
48 m_Data.m_MemPools.clear();
49 if (!setRunTimePoolInfosFromHidlMemories(&m_Data.m_MemPools, m_Model.pools))
50 {
51 Fail("%s: Setting of run time pool infos from Hidl Memories has failed.", __func__);
52 m_ConversionResult = ConversionResult::ErrorMappingPools;
53 return;
54 }
55
56 uint32_t totalPoolSize = 0;
57 for (auto&& pool : m_Model.pools)
58 {
59 totalPoolSize += pool.size();
60 }
61
62 using NetworkOptions = std::vector<armnn::BackendOptions>;
63 NetworkOptions networkOptions;
64 armnn::BackendOptions shapeInferenceMethodOption("ShapeInferenceMethod",
65 {
66 { "InferAndValidate", true }
67 });
68
69 networkOptions.push_back(shapeInferenceMethodOption);
70
71 // Create armnn::INetwork
72 m_Data.m_Network = armnn::INetwork::Create(networkOptions);
73
74 // add operations to it
75 // track which layer outputs each operand
76 ALOGV("ModelToINetworkConverter::Convert(): m_OutputSlotForOperand");
77 m_Data.m_OutputSlotForOperand = std::vector<armnn::IOutputSlot*>(getMainModel(m_Model).operands.size(), nullptr);
78 try
79 {
80 ALOGV("ModelToINetworkConverter::Convert(): for getMainModel(m_Model).inputIndexes.size()");
81 for (uint32_t i = 0; i < getMainModel(m_Model).inputIndexes.size(); i++)
82 {
83 ALOGV("ModelToINetworkConverter::Convert(): getMainModel(m_Model).inputIndexes[i]");
84 // inputs in android nn are represented by operands
85 uint32_t inputIndex = getMainModel(m_Model).inputIndexes[i];
86 ALOGV("ModelToINetworkConverter::Convert(): getMainModel(m_Model).operands[inputIndex];");
87 const HalOperand& operand = getMainModel(m_Model).operands[inputIndex];
88 ALOGV("ModelToINetworkConverter::Convert(): GetTensorInfoForOperand(operand)");
89 const armnn::TensorInfo& tensor = GetTensorInfoForOperand(operand);
90 ALOGV("ModelToINetworkConverter::Convert(): m_Data.m_Network->AddInputLayer(i)");
91 armnn::IConnectableLayer* layer = m_Data.m_Network->AddInputLayer(i);
92
93 ALOGV("ModelToINetworkConverter::Convert(): layer->GetOutputSlot(0)");
94 armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(0);
95 ALOGV("ModelToINetworkConverter::Convert(): outputSlot.SetTensorInfo(GetTensorInfoForOperand(operand))");
96 outputSlot.SetTensorInfo(GetTensorInfoForOperand(operand));
97
98 ALOGV("ModelToINetworkConverter::Convert(): m_Data.m_OutputSlotForOperand[inputIndex] = &outputSlot");
99 // store for later layers
100 m_Data.m_OutputSlotForOperand[inputIndex] = &outputSlot;
101 }
102 }
103 catch (UnsupportedOperand<HalOperandType>& e)
104 {
105 Fail("%s: Operand type %s not supported in ArmnnDriver", __func__, toString(e.m_type).c_str());
106 m_ConversionResult = ConversionResult::UnsupportedFeature;
107 }
108 catch (const armnn::InvalidArgumentException& e)
109 {
110 Fail("%s: Failed to convert input operand to TensorShape: %s", __func__, e.what());
111 m_ConversionResult = ConversionResult::UnsupportedFeature;
112 }
113 bool UnsupportedDynamicOperation = false;
114 for (uint32_t operationIdx = 0; operationIdx < getMainModel(m_Model).operations.size(); operationIdx++)
115 {
116 const auto& operation = getMainModel(m_Model).operations[operationIdx];
117
118 bool ok = true;
119 if (m_ForcedUnsupportedOperations.find(operationIdx) != m_ForcedUnsupportedOperations.end())
120 {
121 Fail("%s: Operation at index %i has been forced to be unsupported.", __func__, operationIdx);
122 ok = false;
123 }
124
125 if (ok)
126 {
127 try
128 {
129 ok = HalPolicy::ConvertOperation(operation, m_Model, m_Data);
130 }
131 catch (UnsupportedOperand<HalOperandType>& e)
132 {
133 Fail("%s: Operand type %s not supported in ArmnnDriver", __func__, toString(e.m_type).c_str());
134 ok = false;
135 }
136 catch (const armnn::InvalidArgumentException& e)
137 {
138 Fail("%s: Failed to convert operation in %s", __func__, e.what());
139 ok = false;
140 }
141 }
142
143 // Store whether this operation was successfully converted.
144 m_OperationSupported.emplace(operationIdx, ok);
145
146 // Any single operation failing will fail the entire conversion.
147 // We still need to continue and check the other ones.
148 if (!ok)
149 {
150 if (m_Data.m_DynamicInputsEncountered)
151 {
152 Fail("%s: The unsupported operation at index %i has dynamic inputs.", __func__, operationIdx);
153 UnsupportedDynamicOperation = true;
154 }
155
156 m_ConversionResult = ConversionResult::UnsupportedFeature;
157 }
158 m_Data.m_DynamicInputsEncountered = false;
159 }
160
161 // Due to the NNAPI partitioner not supporting partition boundaries of unknown size,
162 // any operations who's outputs connect to an unsupported operation with with dynamic inputs
163 // will cause a failure.
164
165 // The simplest solution to this problem is to not support any operations in a model containing
166 // an unsupported operation with with dynamic inputs.
167 if (UnsupportedDynamicOperation)
168 {
169 Fail("%s: Unsupported operation with dynamic inputs found. Retroactively setting all operations to unsupported",
170 __func__);
171 for (auto& operation : m_OperationSupported)
172 {
173 operation.second = false;
174 }
175 }
176
177 try
178 {
179 if (m_ConversionResult == ConversionResult::Success)
180 {
181 for (uint32_t i = 0; i < getMainModel(m_Model).outputIndexes.size(); i++)
182 {
183 // outputs in android nn are represented by operands
184 uint32_t outputIndex = getMainModel(m_Model).outputIndexes[i];
185 const HalOperand& operand = getMainModel(m_Model).operands[outputIndex];
186 const armnn::TensorInfo& tensor = GetTensorInfoForOperand(operand);
187 armnn::IConnectableLayer* layer = m_Data.m_Network->AddOutputLayer(i);
188
189 assert(m_Data.m_OutputSlotForOperand[outputIndex]);
190 m_Data.m_OutputSlotForOperand[outputIndex]->Connect(layer->GetInputSlot(0));
191 }
192 }
193 }
194 catch (const armnn::InvalidArgumentException& e)
195 {
196 Fail("%s: Failed to convert output operand to TensorShape: %s", __func__, e.what());
197 m_ConversionResult = ConversionResult::UnsupportedFeature;
198 }
199 }
200
201 template<typename HalPolicy>
IsOperationSupported(uint32_t operationIndex) const202 bool ModelToINetworkConverter<HalPolicy>::IsOperationSupported(uint32_t operationIndex) const
203 {
204 std::map<uint32_t, bool>::const_iterator it = m_OperationSupported.find(operationIndex);
205 assert(it != m_OperationSupported.end());
206 return it->second;
207 }
208
209 ///
210 /// Class template specializations
211 ///
212
213 template class ModelToINetworkConverter<hal_1_0::HalPolicy>;
214
215 #ifdef ARMNN_ANDROID_NN_V1_1
216 template class ModelToINetworkConverter<hal_1_1::HalPolicy>;
217 #endif
218
219 #ifdef ARMNN_ANDROID_NN_V1_2
220 template class ModelToINetworkConverter<hal_1_1::HalPolicy>;
221 template class ModelToINetworkConverter<hal_1_2::HalPolicy>;
222 #endif
223
224 #ifdef ARMNN_ANDROID_NN_V1_3
225 template class ModelToINetworkConverter<hal_1_1::HalPolicy>;
226 template class ModelToINetworkConverter<hal_1_2::HalPolicy>;
227 template class ModelToINetworkConverter<hal_1_3::HalPolicy>;
228 #endif
229
230 } // armnn_driver
231