1#!/bin/bash 2# 3# Build benchmark app and run it, mimicking a user-initiated run 4# 5# Output is logged to a temporary folder and summarized in txt and JSON formats. 6# parallel-inference-stress tests produce no output except for the success or failure notification, 7# which is not logged. 8 9if [[ "$OSTYPE" == "darwin"* ]]; then 10 OPTS="$(getopt f:rbsm:x -- "$*")" 11else 12 OPTS="$(getopt -o f:rbsm:x -l filter-driver:,include-nnapi-reference,nnapi-reference-only,skip-build,use-nnapi-sl,filter-model,extract-nnapi-sl -- "$@")" 13fi 14 15if [ $? -ne 0 ]; then 16 echo "Invalid arguments, accepted options are" 17 if [[ "$OSTYPE" == "darwin"* ]]; then 18 echo " -f <regex> : to run crash tests only on the drivers (ignoring nnapi-reference) matching the specified regular expression" 19 echo " -r : to include nnapi-reference in target drivers" 20 echo " -b : skip build and installation of tests" 21 echo " -s : use NNAPI Support Library drivers (embedded in the benchmark APK unless -x is specified)" 22 echo " -x : extract NNAPI Support Library drivers from the APK" 23 echo " -m <regex> : to filter the models used in the tests" 24 else 25 echo " -f <regex> | --filter-driver <regex> : to run crash tests only on the drivers (ignoring nnapi-reference) matching the specified regular expression" 26 echo " -r | --include-nnapi-reference : to include nnapi-reference in target drivers" 27 echo " --nnapi-reference-only : to run tests only vs nnapi-reference" 28 echo " -b | --skip-build : skip build and installation of tests" 29 echo " -s | --use-nnapi-sl : use NNAPI Support Library drivers (embedded in the benchmark APK unless -x is specified)" 30 echo " -x | --extract-nnapi-sl : extract NNAPI Support Library drivers from the APK" 31 echo " -m <regex> : to filter the models used in the tests" 32 fi 33 exit 34fi 35 36eval set -- "$OPTS" 37 38DRIVER_FILTER_OPT="" 39INCLUDE_NNAPI_REF_OPT="" 40BUILD_AND_INSTALL=true 41NNAPI_SL_FILTER_OPT="" 42MODEL_FILTER_OPT="" 43while [ $# -gt 0 ] ; do 44 case "$1" in 45 -f|--filter-driver) 46 DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter $2" 47 shift 2 48 ;; 49 -r|--include-nnapi-reference) 50 INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true" 51 shift 52 ;; 53 --nnapi-reference-only) 54 DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter no-device" 55 INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true" 56 shift 57 ;; 58 -m|--filter-model) 59 MODEL_FILTER_OPT="-e nnBenchmarkModelFilter $2" 60 shift 2 61 ;; 62 -b|--skip-build) 63 BUILD_AND_INSTALL=false 64 shift 65 ;; 66 -s|--use-nnapi-sl) 67 NNAPI_SL_FILTER_OPT+=" -e useNnApiSupportLibrary true" 68 shift 69 ;; 70 -x|--extract-nnapi-sl) 71 NNAPI_SL_FILTER_OPT+=" -e extractNnApiSupportLibrary true" 72 shift 73 74 echo "Creating configuration file with list of libraries" 75 mkdir sl_prebuilt/assets 76 ls sl_prebuilt/ 2>/dev/null | grep '.so' >sl_prebuilt/assets/sl_prebuilt_filelist.txt 77 ;; 78 --) 79 shift 80 break 81 ;; 82 *) 83 echo "Unsupported arg $1" 84 exit 1 85 esac 86done 87 88MODE="${1:-scoring}" 89INSTALL_NATIVE_TESTS=false 90CRASH_TEST_APP="NeuralNetworksApiCrashTest" 91APP="NeuralNetworksApiBenchmark" 92case "$MODE" in 93 scoring) 94 CLASS=com.android.nn.benchmark.app.NNScoringTest 95 ;; 96 inference-stress) 97 CLASS=com.android.nn.benchmark.app.NNInferenceStressTest 98 ;; 99 model-loading-stress) 100 CLASS=com.android.nn.benchmark.app.NNModelLoadingStressTest 101 ;; 102 parallel-inference-stress) 103 CLASS=com.android.nn.crashtest.app.NNParallelCrashResistantInferenceTest 104 APP="$CRASH_TEST_APP" 105 ;; 106 parallel-inference-stress-in-process) 107 CLASS=com.android.nn.crashtest.app.NNParallelInProcessInferenceTest 108 APP="$CRASH_TEST_APP" 109 ;; 110 client-early-termination-stress) 111 CLASS=com.android.nn.crashtest.app.NNClientEarlyTerminationTest 112 APP="$CRASH_TEST_APP" 113 ;; 114 multi-process-inference-stress) 115 CLASS=com.android.nn.crashtest.app.NNMultipleProcessInferenceTest 116 APP="$CRASH_TEST_APP" 117 INSTALL_NATIVE_TESTS=true 118 ;; 119 multi-process-model-load-stress) 120 CLASS=com.android.nn.crashtest.app.NNMultipleProcessModelLoadTest 121 APP="$CRASH_TEST_APP" 122 INSTALL_NATIVE_TESTS=true 123 ;; 124 memory-mapped-model-load-stress) 125 CLASS=com.android.nn.crashtest.app.NNMemoryMappedModelCompilationTest 126 APP="$CRASH_TEST_APP" 127 ;; 128 model-load-random-stress) 129 APP="$CRASH_TEST_APP" 130 CLASS=com.android.nn.crashtest.app.NNRandomGraphLoadTest 131 ;; 132 inference-random-stress) 133 APP="$CRASH_TEST_APP" 134 CLASS=com.android.nn.crashtest.app.NNRandomGraphExecutionTest 135 ;; 136 performance-degradation-stress) 137 APP="$CRASH_TEST_APP" 138 CLASS=com.android.nn.crashtest.app.NNPerformanceDegradationTest 139 ;; 140 *) 141 echo "Unknown execution mode: $1" 142 echo "Known modes: scoring (default), inference-stress, model-loading-stress, " \ 143 "parallel-inference-stress, parallel-inference-stress-in-process, " \ 144 "client-early-termination-stress, multi-process-inference-stress, " \ 145 "multi-process-model-load-stress memory-mapped-model-load-stress, " \ 146 "model-load-random-stress, inference-random-stress, performance-degradation-stress" 147 exit 1 148 ;; 149esac 150 151if [[ -z "$ANDROID_BUILD_TOP" ]]; then 152 echo ANDROID_BUILD_TOP not set, bailing out 153 echo you must run lunch before running this script 154 exit 1 155fi 156 157set -e 158cd $ANDROID_BUILD_TOP 159 160if [ "$BUILD_AND_INSTALL" = true ]; then 161 if [ ! -z "$NNAPI_SL_FILTER_OPT" ]; then 162 SL_PREBUILT=test/mlts/benchmark/sl_prebuilt 163 if [ ! -n "$(ls -A $SL_PREBUILT/*.so 2>/dev/null)" ]; then 164 echo "There is no NNAPI SL binary file under $ANDROID_BUILD_TOP/$SL_PREBUILT, cannot test using NNAPI SL" 165 exit 166 fi 167 if [ ! -f "$SL_PREBUILT/Android.bp" ]; then 168 echo "================================================================" 169 echo "Enabling build of NNAPI SL libraries using template definition." 170 echo "If the definitions in $SL_PREBUILT/Android.bp don't match the libraries you copied" 171 echo " please define your own version of $SL_PREBUILT/Android.bp" 172 echo "================================================================" 173 mv $SL_PREBUILT/Android.bp.template $SL_PREBUILT/Android.bp 174 fi 175 fi 176 177 # Build and install benchmark app 178 TMPFILE=$(mktemp) 179 build/soong/soong_ui.bash --make-mode ${APP} 2>&1 | tee ${TMPFILE} 180 TARGET_ARCH=$(cat ${TMPFILE} | grep TARGET_ARCH= | sed -e 's/TARGET_ARCH=//') 181 if [ "${TARGET_ARCH}" = "aarch64" ]; then 182 APK_DIR=arm64 183 else 184 APK_DIR=${TARGET_ARCH} 185 fi 186 187 if [ ! -z "$NNAPI_SL_FILTER_OPT" ]; then 188 if [ "$(unzip -l $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk | grep libnnapi_sl_driver | wc -l)" -ne 1 ]; then 189 echo "NNAPI SL Libraries are not included in the APK" \ 190 "please check the library list is included in the LOCAL_JNI_SHARED_LIBRARIES list " \ 191 "for ${APP}. Please check the value of SL_LIBS in Android.mk" 192 exit 193 fi 194 fi 195 196 if ! adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk; then 197 adb uninstall com.android.nn.benchmark.app 198 adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk 199 fi 200 201 if [ "$INSTALL_NATIVE_TESTS" = true ]; then 202 build/soong/soong_ui.bash --make-mode nn_stress_test 203 adb push $OUT/system/bin/nn_stress_test /bin/ 204 fi 205fi 206 207# Should we figure out if we run on release device 208if [ -z "$MLTS_RELEASE_DEVICE" ]; then 209 BUILD_DESCRIPTION=`adb shell getprop ro.build.description` 210 if [[ $BUILD_DESCRIPTION =~ .*release.* ]] 211 then 212 MLTS_RELEASE_DEVICE=True 213 else 214 MLTS_RELEASE_DEVICE=False 215 fi 216fi 217 218# Pass --no-isolated-storage to am instrument? 219BUILD_VERSION_RELEASE=`adb shell getprop ro.build.version.release` 220AM_INSTRUMENT_FLAGS="$DRIVER_FILTER_OPT $INCLUDE_NNAPI_REF_OPT $NNAPI_SL_FILTER_OPT $MODEL_FILTER_OPT" 221if [[ $BUILD_VERSION_RELEASE == "Q" ]]; then 222 AM_INSTRUMENT_FLAGS+=" --no-isolated-storage" 223fi 224 225if [[ "$MODE" == "scoring" ]]; then 226 if [[ "$MLTS_RELEASE_DEVICE" == "True" ]]; then 227 TEST_EXTENRAL_STORAGE="com.android.nn.benchmark.app/com.android.nn.benchmark.util.TestExternalStorageActivity" 228 while ! adb shell "am start -W $TEST_EXTENRAL_STORAGE && rm /sdcard/mlts_write_external_storage" > /dev/null 2>&1; do 229 echo "************************************************************" 230 echo "Grant External storage write permissions to MLTS to proceed!" 231 echo "************************************************************" 232 read -n 1 -r -p "Continue? (press any key)" 233 echo 234 done 235 else 236 adb root 237 adb shell "pm grant com.android.nn.benchmark.app android.permission.WRITE_EXTERNAL_STORAGE" 238 # Skip setup wizard and remount (read-write) 239 if ! adb shell test -f /data/local.prop; then 240 adb shell 'echo ro.setupwizard.mode=DISABLED > /data/local.prop' 241 adb shell 'chmod 644 /data/local.prop' 242 adb shell 'settings put global device_provisioned 1*' 243 adb shell 'settings put secure user_setup_complete 1' 244 adb disable-verity 245 adb reboot 246 sleep 5 247 adb wait-for-usb-device root 248 adb wait-for-usb-device remount 249 sleep 5 250 fi 251 set +e 252 # Enable menu key press through adb 253 adb shell 'echo testing > /data/local/enable_menu_key' 254 # Leave screen on (affects scheduling) 255 adb shell settings put system screen_off_timeout 86400000 256 # Stop background apps, seem to take ~10% CPU otherwise 257 adb shell 'pm disable com.google.android.googlequicksearchbox' 258 adb shell 'pm list packages -f' | sed -e 's/.*=//' | sed 's/\r//g' | grep "com.breel.wallpapers" | while read pkg; do adb shell "pm disable $pkg"; done; 259 set -e 260 fi 261fi 262 263adb shell setprop debug.nn.cpuonly 0 264adb shell setprop debug.nn.vlog "''" 265 266# Menukey - make sure screen is on 267adb shell "input keyevent 82" 268# Show homescreen 269adb shell wm dismiss-keyguard 270 271if [[ "$MODE" == "scoring" ]]; then 272 LOGDIR=$(mktemp -d)/mlts-logs 273 HOST_CSV=$LOGDIR/benchmark.csv 274 RESULT_HTML=$LOGDIR/result.html 275 DEVICE_CSV=/sdcard/mlts_benchmark.csv 276 277 mkdir -p $LOGDIR 278 echo Creating logs in $LOGDIR 279 280 # Remove old benchmark csv data 281 adb shell rm -f ${DEVICE_CSV} 282fi 283 284# Set the shell pid as a top-app and run tests 285time adb shell "echo $$ > /dev/stune/top-app/tasks; am instrument ${AM_INSTRUMENT_FLAGS} -w -e class $CLASS com.android.nn.benchmark.app/androidx.test.runner.AndroidJUnitRunner" 286 287if [[ "$MODE" == "scoring" ]]; then 288 adb pull $DEVICE_CSV $HOST_CSV 289 echo Benchmark data saved in $HOST_CSV 290 291 $ANDROID_BUILD_TOP/test/mlts/benchmark/results/generate_result.py $HOST_CSV $RESULT_HTML 292 echo Results stored in $RESULT_HTML 293 xdg-open $RESULT_HTML 294fi 295