1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "Callbacks.h"
18 #include "TestHarness.h"
19 #include "VtsHalNeuralnetworksV1_0TargetTest.h"
20
21 #include <android-base/logging.h>
22 #include <android/hidl/memory/1.0/IMemory.h>
23 #include <hidlmemory/mapping.h>
24 #include <iostream>
25
26 namespace android {
27 namespace hardware {
28 namespace neuralnetworks {
29 namespace V1_0 {
30 namespace vts {
31 namespace functional {
32 // allocator helper
33 hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem");
34
35 namespace generated_tests {
36 using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
37 using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
38 using ::generated_tests::filter;
39 using ::generated_tests::for_all;
40 using ::generated_tests::for_each;
41 using ::generated_tests::resize_accordingly;
42 using ::generated_tests::MixedTyped;
43 using ::generated_tests::MixedTypedExampleType;
44 using ::generated_tests::Float32Operands;
45 using ::generated_tests::Int32Operands;
46 using ::generated_tests::Quant8Operands;
47 using ::generated_tests::compare;
48
49 template <typename T>
copy_back_(MixedTyped * dst,const std::vector<RequestArgument> & ra,char * src)50 void copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
51 MixedTyped& test = *dst;
52 for_each<T>(test, [&ra, src](int index, std::vector<T>& m) {
53 ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T));
54 char* begin = src + ra[index].location.offset;
55 memcpy(m.data(), begin, ra[index].location.length);
56 });
57 }
58
copy_back(MixedTyped * dst,const std::vector<RequestArgument> & ra,char * src)59 void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
60 copy_back_<float>(dst, ra, src);
61 copy_back_<int32_t>(dst, ra, src);
62 copy_back_<uint8_t>(dst, ra, src);
63 }
64
65 // Top level driver for models and examples generated by test_generator.py
66 // Test driver for those generated from ml/nn/runtime/test/spec
Execute(const sp<IDevice> & device,std::function<Model (void)> create_model,std::function<bool (int)> is_ignored,const std::vector<MixedTypedExampleType> & examples)67 void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
68 std::function<bool(int)> is_ignored,
69 const std::vector<MixedTypedExampleType>& examples) {
70 const uint32_t INPUT = 0;
71 const uint32_t OUTPUT = 1;
72 Model model = create_model();
73
74 // see if service can handle model
75 ErrorStatus supportedStatus;
76 bool fullySupportsModel = false;
77 Return<void> supportedCall = device->getSupportedOperations(
78 model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
79 supportedStatus = status;
80 ASSERT_NE(0ul, supported.size());
81 fullySupportsModel =
82 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
83 });
84 ASSERT_TRUE(supportedCall.isOk());
85 ASSERT_EQ(ErrorStatus::NONE, supportedStatus);
86
87 // launch prepare model
88 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
89 ASSERT_NE(nullptr, preparedModelCallback.get());
90 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
91 ASSERT_TRUE(prepareLaunchStatus.isOk());
92
93 // retrieve prepared model
94 preparedModelCallback->wait();
95 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
96 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
97 if (fullySupportsModel) {
98 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
99 } else {
100 EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
101 prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);
102 }
103
104 // early termination if vendor service cannot fully prepare model
105 if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) {
106 ASSERT_EQ(nullptr, preparedModel.get());
107 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
108 "prepare model that it does not support.";
109 std::cout << "[ ] Early termination of test because vendor service cannot "
110 "prepare model that it does not support."
111 << std::endl;
112 return;
113 }
114 ASSERT_NE(nullptr, preparedModel.get());
115
116 int example_no = 1;
117 for (auto& example : examples) {
118 SCOPED_TRACE(example_no++);
119
120 const MixedTyped& inputs = example.first;
121 const MixedTyped& golden = example.second;
122
123 std::vector<RequestArgument> inputs_info, outputs_info;
124 uint32_t inputSize = 0, outputSize = 0;
125
126 // This function only partially specifies the metadata (vector of RequestArguments).
127 // The contents are copied over below.
128 for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
129 if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
130 RequestArgument arg = {
131 .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
132 .dimensions = {},
133 };
134 RequestArgument arg_empty = {
135 .hasNoValue = true,
136 };
137 inputs_info[index] = s ? arg : arg_empty;
138 inputSize += s;
139 });
140 // Compute offset for inputs 1 and so on
141 {
142 size_t offset = 0;
143 for (auto& i : inputs_info) {
144 if (!i.hasNoValue) i.location.offset = offset;
145 offset += i.location.length;
146 }
147 }
148
149 MixedTyped test; // holding test results
150
151 // Go through all outputs, initialize RequestArgument descriptors
152 resize_accordingly(golden, test);
153 for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
154 if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
155 RequestArgument arg = {
156 .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
157 .dimensions = {},
158 };
159 outputs_info[index] = arg;
160 outputSize += s;
161 });
162 // Compute offset for outputs 1 and so on
163 {
164 size_t offset = 0;
165 for (auto& i : outputs_info) {
166 i.location.offset = offset;
167 offset += i.location.length;
168 }
169 }
170 std::vector<hidl_memory> pools = {allocateSharedMemory(inputSize),
171 allocateSharedMemory(outputSize)};
172 ASSERT_NE(0ull, pools[INPUT].size());
173 ASSERT_NE(0ull, pools[OUTPUT].size());
174
175 // load data
176 sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
177 sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
178 ASSERT_NE(nullptr, inputMemory.get());
179 ASSERT_NE(nullptr, outputMemory.get());
180 char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
181 char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
182 ASSERT_NE(nullptr, inputPtr);
183 ASSERT_NE(nullptr, outputPtr);
184 inputMemory->update();
185 outputMemory->update();
186
187 // Go through all inputs, copy the values
188 for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
189 char* begin = (char*)p;
190 char* end = begin + s;
191 // TODO: handle more than one input
192 std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
193 });
194
195 inputMemory->commit();
196 outputMemory->commit();
197
198 // launch execution
199 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
200 ASSERT_NE(nullptr, executionCallback.get());
201 Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(
202 {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}, executionCallback);
203 ASSERT_TRUE(executionLaunchStatus.isOk());
204 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
205
206 // retrieve execution status
207 executionCallback->wait();
208 ErrorStatus executionReturnStatus = executionCallback->getStatus();
209 EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
210
211 // validate results
212 outputMemory->read();
213 copy_back(&test, outputs_info, outputPtr);
214 outputMemory->commit();
215 // Filter out don't cares
216 MixedTyped filtered_golden = filter(golden, is_ignored);
217 MixedTyped filtered_test = filter(test, is_ignored);
218
219 // We want "close-enough" results for float
220 compare(filtered_golden, filtered_test);
221 }
222 }
223
224 } // namespace generated_tests
225
226 } // namespace functional
227 } // namespace vts
228 } // namespace V1_0
229 } // namespace neuralnetworks
230 } // namespace hardware
231 } // namespace android
232