1#!/usr/bin/env python 2# 3# Copyright (C) 2019 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import logging 19import re 20 21from time import sleep 22from vts.runners.host import asserts 23from vts.runners.host import const 24from vts.runners.host import test_runner 25from vts.testcases.template.hal_hidl_host_test import hal_hidl_host_test 26 27 28class VtsHalNeuralnetworksV1_2Benchmark(hal_hidl_host_test.HalHidlHostTest): 29 """A test case that runs the accuracy benchmark for every NN device driver.""" 30 31 TEST_HAL_SERVICES = {"android.hardware.neuralnetworks@1.2::IDevice"} 32 33 def testAccuracy(self): 34 """Test that the driver meets accuracy requirements.""" 35 hal_service_instance = self.cur_param[0] 36 command = ( 37 "am instrument -e halServiceInstance '{}' " + 38 "-w com.android.nn.benchmark.vts.v1_2/androidx.test.runner.AndroidJUnitRunner" 39 ).format(hal_service_instance) 40 logging.info("Executing command: %s", command) 41 logging.info("Check logcat for more information") 42 results = self.shell.Execute([command]) 43 stdout = results[const.STDOUT][0] 44 stderr = results[const.STDERR][0] 45 exit_code = results[const.EXIT_CODE][0] 46 if stdout: 47 logging.info("stdout: %s", stdout) 48 if stderr: 49 logging.error("stderr: %s", stderr) 50 if exit_code: 51 logging.error("exit_code: %s", exit_code) 52 # The "am instrument" command exits with 0 even when there are failed tests. 53 asserts.assertTrue(re.search(r"^OK \([0-9]+ tests?\)$", stdout, re.M), 54 "Benchmark failed") 55 56 57if __name__ == "__main__": 58 test_runner.main() 59