1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.nn.crashtest.app; 18 19 import android.app.Activity; 20 import android.content.Intent; 21 import android.os.Bundle; 22 import android.test.ActivityInstrumentationTestCase2; 23 import android.test.UiThreadTest; 24 import android.test.suitebuilder.annotation.LargeTest; 25 import android.util.Log; 26 27 import androidx.test.InstrumentationRegistry; 28 29 import com.android.nn.benchmark.app.BenchmarkTestBase; 30 import com.android.nn.benchmark.core.NnApiDelegationFailure; 31 import com.android.nn.benchmark.core.TestModels; 32 33 import org.junit.After; 34 import org.junit.Before; 35 import org.junit.Rule; 36 import org.junit.Test; 37 import org.junit.rules.TestName; 38 import org.junit.runners.Parameterized.Parameters; 39 40 import java.time.Duration; 41 import java.util.Arrays; 42 import java.util.stream.IntStream; 43 44 abstract class NNParallelInferenceTest 45 extends ActivityInstrumentationTestCase2<NNParallelTestActivity> 46 implements AcceleratorSpecificTestSupport { 47 static final String TAG = "NNParallelInferenceTest"; 48 49 @Rule 50 public TestName mTestName = new TestName(); 51 52 private final int mThreadCount; 53 private final Duration mTestDuration; 54 private final String mAcceleratorName; 55 runTestsInSeparateProcess()56 protected abstract boolean runTestsInSeparateProcess(); 57 NNParallelInferenceTest(int threadCount, Duration testDuration, String acceleratorName)58 protected NNParallelInferenceTest(int threadCount, Duration testDuration, 59 String acceleratorName) { 60 super(NNParallelTestActivity.class); 61 mThreadCount = threadCount; 62 mTestDuration = testDuration; 63 mAcceleratorName = acceleratorName; 64 } 65 66 @Before 67 @Override setUp()68 public void setUp() { 69 injectInstrumentation(InstrumentationRegistry.getInstrumentation()); 70 BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 90); 71 try { 72 setActivityIntent( 73 runAllModelsOnNThreadsForOnAccelerator(mThreadCount, mTestDuration, 74 mAcceleratorName)); 75 } catch (NnApiDelegationFailure nnApiDelegationFailure) { 76 throw new RuntimeException( 77 "Cannot initialize test, failure looking for supported models, please check " 78 + "the driver status", 79 nnApiDelegationFailure); 80 } 81 } 82 83 @Test 84 @LargeTest 85 @UiThreadTest shouldNotFailWithParallelThreads()86 public void shouldNotFailWithParallelThreads() { 87 Bundle testData = new Bundle(); 88 testData.putString("Test name", mTestName.getMethodName()); 89 testData.putString("Test status", "Started"); 90 getInstrumentation().sendStatus(Activity.RESULT_FIRST_USER, testData); 91 92 CrashTestStatus.TestResult testResult = getActivity().testResult(); 93 assertEquals("Test didn't complete successfully", CrashTestStatus.TestResult.SUCCESS, 94 testResult); 95 96 testData.putString("Test status", "Completed"); 97 getInstrumentation().sendStatus(Activity.RESULT_OK, testData); 98 } 99 100 @After 101 @Override tearDown()102 public void tearDown() throws Exception { 103 Log.i(TAG, "Tearing down test"); 104 super.tearDown(); 105 } 106 107 @Parameters(name = "{0} threads for {1} on accelerator {2}") threadCountValues()108 public static Iterable<Object[]> threadCountValues() { 109 return AcceleratorSpecificTestSupport.perAcceleratorTestConfig( 110 Arrays.asList( 111 new Object[]{8, Duration.ofMinutes(30)}, 112 new Object[]{12, Duration.ofMinutes(20)})); 113 } 114 runAllModelsOnNThreadsForOnAccelerator(int threadCount, Duration testDuration, String acceleratorName)115 private Intent runAllModelsOnNThreadsForOnAccelerator(int threadCount, Duration testDuration, 116 String acceleratorName) throws NnApiDelegationFailure { 117 Intent intent = new Intent(); 118 119 int modelsCount = TestModels.modelsList().size(); 120 intent.putExtra( 121 NNParallelTestActivity.EXTRA_TEST_LIST, IntStream.range(0, modelsCount).toArray()); 122 intent.putExtra(NNParallelTestActivity.EXTRA_THREAD_COUNT, threadCount); 123 intent.putExtra(NNParallelTestActivity.EXTRA_TEST_DURATION_MILLIS, testDuration.toMillis()); 124 intent.putExtra(NNParallelTestActivity.EXTRA_RUN_IN_SEPARATE_PROCESS, 125 runTestsInSeparateProcess()); 126 intent.putExtra(NNParallelTestActivity.EXTRA_TEST_NAME, mTestName.getMethodName()); 127 if (acceleratorName != null) { 128 intent.putExtra(NNParallelTestActivity.EXTRA_ACCELERATOR_NAME, acceleratorName); 129 intent.putExtra(NNParallelTestActivity.EXTRA_IGNORE_UNSUPPORTED_MODELS, true); 130 } 131 return intent; 132 } 133 } 134