1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "neuralnetworks_hidl_hal_test"
18
19 #include "VtsHalNeuralnetworksV1_0TargetTest.h"
20
21 #include "Callbacks.h"
22 #include "Models.h"
23 #include "TestHarness.h"
24
25 #include <android-base/logging.h>
26 #include <android/hidl/memory/1.0/IMemory.h>
27 #include <hidlmemory/mapping.h>
28
29 namespace android {
30 namespace hardware {
31 namespace neuralnetworks {
32 namespace V1_0 {
33 namespace vts {
34 namespace functional {
35
36 using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
37 using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
38 using ::generated_tests::MixedTypedExampleType;
39
40 namespace generated_tests {
41 extern void Execute(const sp<IDevice>&, std::function<Model(void)>, std::function<bool(int)>,
42 const std::vector<MixedTypedExampleType>&);
43 }
44
45 // A class for test environment setup
NeuralnetworksHidlEnvironment()46 NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {}
47
~NeuralnetworksHidlEnvironment()48 NeuralnetworksHidlEnvironment::~NeuralnetworksHidlEnvironment() {}
49
getInstance()50 NeuralnetworksHidlEnvironment* NeuralnetworksHidlEnvironment::getInstance() {
51 // This has to return a "new" object because it is freed inside
52 // ::testing::AddGlobalTestEnvironment when the gtest is being torn down
53 static NeuralnetworksHidlEnvironment* instance = new NeuralnetworksHidlEnvironment();
54 return instance;
55 }
56
registerTestServices()57 void NeuralnetworksHidlEnvironment::registerTestServices() {
58 registerTestService<IDevice>();
59 }
60
61 // The main test class for NEURALNETWORK HIDL HAL.
~NeuralnetworksHidlTest()62 NeuralnetworksHidlTest::~NeuralnetworksHidlTest() {}
63
SetUp()64 void NeuralnetworksHidlTest::SetUp() {
65 device = ::testing::VtsHalHidlTargetTestBase::getService<IDevice>(
66 NeuralnetworksHidlEnvironment::getInstance());
67 ASSERT_NE(nullptr, device.get());
68 }
69
TearDown()70 void NeuralnetworksHidlTest::TearDown() {}
71
doPrepareModelShortcut()72 sp<IPreparedModel> NeuralnetworksHidlTest::doPrepareModelShortcut() {
73 Model model = createValidTestModel();
74
75 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
76 if (preparedModelCallback == nullptr) {
77 return nullptr;
78 }
79 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
80 if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) {
81 return nullptr;
82 }
83
84 preparedModelCallback->wait();
85 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
86 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
87 if (prepareReturnStatus != ErrorStatus::NONE || preparedModel == nullptr) {
88 return nullptr;
89 }
90
91 return preparedModel;
92 }
93
94 // create device test
TEST_F(NeuralnetworksHidlTest,CreateDevice)95 TEST_F(NeuralnetworksHidlTest, CreateDevice) {}
96
97 // status test
TEST_F(NeuralnetworksHidlTest,StatusTest)98 TEST_F(NeuralnetworksHidlTest, StatusTest) {
99 Return<DeviceStatus> status = device->getStatus();
100 ASSERT_TRUE(status.isOk());
101 EXPECT_EQ(DeviceStatus::AVAILABLE, static_cast<DeviceStatus>(status));
102 }
103
104 // initialization
TEST_F(NeuralnetworksHidlTest,GetCapabilitiesTest)105 TEST_F(NeuralnetworksHidlTest, GetCapabilitiesTest) {
106 Return<void> ret =
107 device->getCapabilities([](ErrorStatus status, const Capabilities& capabilities) {
108 EXPECT_EQ(ErrorStatus::NONE, status);
109 EXPECT_LT(0.0f, capabilities.float32Performance.execTime);
110 EXPECT_LT(0.0f, capabilities.float32Performance.powerUsage);
111 EXPECT_LT(0.0f, capabilities.quantized8Performance.execTime);
112 EXPECT_LT(0.0f, capabilities.quantized8Performance.powerUsage);
113 });
114 EXPECT_TRUE(ret.isOk());
115 }
116
117 // supported operations positive test
TEST_F(NeuralnetworksHidlTest,SupportedOperationsPositiveTest)118 TEST_F(NeuralnetworksHidlTest, SupportedOperationsPositiveTest) {
119 Model model = createValidTestModel();
120 Return<void> ret = device->getSupportedOperations(
121 model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
122 EXPECT_EQ(ErrorStatus::NONE, status);
123 EXPECT_EQ(model.operations.size(), supported.size());
124 });
125 EXPECT_TRUE(ret.isOk());
126 }
127
128 // supported operations negative test 1
TEST_F(NeuralnetworksHidlTest,SupportedOperationsNegativeTest1)129 TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest1) {
130 Model model = createInvalidTestModel1();
131 Return<void> ret = device->getSupportedOperations(
132 model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
133 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status);
134 (void)supported;
135 });
136 EXPECT_TRUE(ret.isOk());
137 }
138
139 // supported operations negative test 2
TEST_F(NeuralnetworksHidlTest,SupportedOperationsNegativeTest2)140 TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest2) {
141 Model model = createInvalidTestModel2();
142 Return<void> ret = device->getSupportedOperations(
143 model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
144 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status);
145 (void)supported;
146 });
147 EXPECT_TRUE(ret.isOk());
148 }
149
150 // prepare simple model positive test
TEST_F(NeuralnetworksHidlTest,SimplePrepareModelPositiveTest)151 TEST_F(NeuralnetworksHidlTest, SimplePrepareModelPositiveTest) {
152 Model model = createValidTestModel();
153 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
154 ASSERT_NE(nullptr, preparedModelCallback.get());
155 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
156 ASSERT_TRUE(prepareLaunchStatus.isOk());
157 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
158
159 preparedModelCallback->wait();
160 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
161 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
162 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
163 EXPECT_NE(nullptr, preparedModel.get());
164 }
165
166 // prepare simple model negative test 1
TEST_F(NeuralnetworksHidlTest,SimplePrepareModelNegativeTest1)167 TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest1) {
168 Model model = createInvalidTestModel1();
169 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
170 ASSERT_NE(nullptr, preparedModelCallback.get());
171 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
172 ASSERT_TRUE(prepareLaunchStatus.isOk());
173 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(prepareLaunchStatus));
174
175 preparedModelCallback->wait();
176 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
177 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus);
178 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
179 EXPECT_EQ(nullptr, preparedModel.get());
180 }
181
182 // prepare simple model negative test 2
TEST_F(NeuralnetworksHidlTest,SimplePrepareModelNegativeTest2)183 TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest2) {
184 Model model = createInvalidTestModel2();
185 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
186 ASSERT_NE(nullptr, preparedModelCallback.get());
187 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
188 ASSERT_TRUE(prepareLaunchStatus.isOk());
189 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(prepareLaunchStatus));
190
191 preparedModelCallback->wait();
192 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
193 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus);
194 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
195 EXPECT_EQ(nullptr, preparedModel.get());
196 }
197
198 // execute simple graph positive test
TEST_F(NeuralnetworksHidlTest,SimpleExecuteGraphPositiveTest)199 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) {
200 std::vector<float> outputData = {-1.0f, -1.0f, -1.0f, -1.0f};
201 std::vector<float> expectedData = {6.0f, 8.0f, 10.0f, 12.0f};
202 const uint32_t OUTPUT = 1;
203
204 sp<IPreparedModel> preparedModel = doPrepareModelShortcut();
205 ASSERT_NE(nullptr, preparedModel.get());
206 Request request = createValidTestRequest();
207
208 auto postWork = [&] {
209 sp<IMemory> outputMemory = mapMemory(request.pools[OUTPUT]);
210 if (outputMemory == nullptr) {
211 return false;
212 }
213 float* outputPtr = reinterpret_cast<float*>(static_cast<void*>(outputMemory->getPointer()));
214 if (outputPtr == nullptr) {
215 return false;
216 }
217 outputMemory->read();
218 std::copy(outputPtr, outputPtr + outputData.size(), outputData.begin());
219 outputMemory->commit();
220 return true;
221 };
222
223 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
224 ASSERT_NE(nullptr, executionCallback.get());
225 executionCallback->on_finish(postWork);
226 Return<ErrorStatus> executeLaunchStatus = preparedModel->execute(request, executionCallback);
227 ASSERT_TRUE(executeLaunchStatus.isOk());
228 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executeLaunchStatus));
229
230 executionCallback->wait();
231 ErrorStatus executionReturnStatus = executionCallback->getStatus();
232 EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
233 EXPECT_EQ(expectedData, outputData);
234 }
235
236 // execute simple graph negative test 1
TEST_F(NeuralnetworksHidlTest,SimpleExecuteGraphNegativeTest1)237 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) {
238 sp<IPreparedModel> preparedModel = doPrepareModelShortcut();
239 ASSERT_NE(nullptr, preparedModel.get());
240 Request request = createInvalidTestRequest1();
241
242 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
243 ASSERT_NE(nullptr, executionCallback.get());
244 Return<ErrorStatus> executeLaunchStatus = preparedModel->execute(request, executionCallback);
245 ASSERT_TRUE(executeLaunchStatus.isOk());
246 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(executeLaunchStatus));
247
248 executionCallback->wait();
249 ErrorStatus executionReturnStatus = executionCallback->getStatus();
250 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus);
251 }
252
253 // execute simple graph negative test 2
TEST_F(NeuralnetworksHidlTest,SimpleExecuteGraphNegativeTest2)254 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest2) {
255 sp<IPreparedModel> preparedModel = doPrepareModelShortcut();
256 ASSERT_NE(nullptr, preparedModel.get());
257 Request request = createInvalidTestRequest2();
258
259 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
260 ASSERT_NE(nullptr, executionCallback.get());
261 Return<ErrorStatus> executeLaunchStatus = preparedModel->execute(request, executionCallback);
262 ASSERT_TRUE(executeLaunchStatus.isOk());
263 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(executeLaunchStatus));
264
265 executionCallback->wait();
266 ErrorStatus executionReturnStatus = executionCallback->getStatus();
267 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus);
268 }
269
270 // Mixed-typed examples
271 typedef MixedTypedExampleType MixedTypedExample;
272
273 // in frameworks/ml/nn/runtime/tests/generated/
274 #include "all_generated_vts_tests.cpp"
275
276 // TODO: Add tests for execution failure, or wait_for/wait_until timeout.
277 // Discussion:
278 // https://googleplex-android-review.git.corp.google.com/#/c/platform/hardware/interfaces/+/2654636/5/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp@222
279
280 } // namespace functional
281 } // namespace vts
282 } // namespace V1_0
283 } // namespace neuralnetworks
284 } // namespace hardware
285 } // namespace android
286
287 using android::hardware::neuralnetworks::V1_0::vts::functional::NeuralnetworksHidlEnvironment;
288
main(int argc,char ** argv)289 int main(int argc, char** argv) {
290 ::testing::AddGlobalTestEnvironment(NeuralnetworksHidlEnvironment::getInstance());
291 ::testing::InitGoogleTest(&argc, argv);
292 NeuralnetworksHidlEnvironment::getInstance()->init(&argc, argv);
293
294 int status = RUN_ALL_TESTS();
295 return status;
296 }
297