1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.nn.crashtest.app; 18 19 import static com.android.nn.crashtest.app.CrashTestStatus.TestResult.SUCCESS; 20 21 import android.content.Intent; 22 import android.test.ActivityInstrumentationTestCase2; 23 import android.test.UiThreadTest; 24 import android.test.suitebuilder.annotation.LargeTest; 25 26 import androidx.test.InstrumentationRegistry; 27 28 import com.android.nn.benchmark.app.BenchmarkTestBase; 29 import com.android.nn.benchmark.core.NnApiDelegationFailure; 30 import com.android.nn.benchmark.core.TestModels; 31 import com.android.nn.crashtest.core.test.RandomGraphTest; 32 33 import org.junit.Before; 34 import org.junit.Rule; 35 import org.junit.Test; 36 import org.junit.rules.TestName; 37 import org.junit.runner.RunWith; 38 import org.junit.runners.Parameterized; 39 import org.junit.runners.Parameterized.Parameters; 40 41 import java.time.Duration; 42 import java.util.Arrays; 43 import java.util.Optional; 44 import java.util.concurrent.ExecutionException; 45 46 @RunWith(Parameterized.class) 47 public abstract class NNRandomGraphTest 48 extends ActivityInstrumentationTestCase2<NNRandomGraphTestActivity> 49 implements AcceleratorSpecificTestSupport { 50 private static final String TAG = "NN_RAND_MODEL"; 51 52 protected final String mAcceleratorName; 53 private final int mModelCount; 54 private final int mGraphSize; 55 private final Duration mDuration; 56 private final int mDimensionRange; 57 58 private final static int SMALL_MODEL_SIZE = 10; 59 private final static int LARGE_MODEL_SIZE = 600; 60 private final static int NARROW_DIMENSIONS_RANGE = 10; 61 private final static int WIDE_DIMENSIONS_RANGE = 1000; 62 private final static Duration MAX_TEST_DURATION = Duration.ofMinutes(15); 63 private final static int NUMBER_OF_MODELS = 600; 64 private final boolean mRunModelCompilationOnly; 65 66 @Parameters(name = "{0} models of size {1} and dimensions range {2} for max duration of {3} " 67 + "on accelerator {4}") testConfiguration()68 public static Iterable<Object[]> testConfiguration() { 69 return AcceleratorSpecificTestSupport.perAcceleratorTestConfig( 70 Arrays.asList( 71 new Object[]{NUMBER_OF_MODELS, SMALL_MODEL_SIZE, WIDE_DIMENSIONS_RANGE, 72 MAX_TEST_DURATION}, 73 new Object[]{NUMBER_OF_MODELS, LARGE_MODEL_SIZE, NARROW_DIMENSIONS_RANGE, 74 MAX_TEST_DURATION})); 75 } 76 77 @Rule 78 public TestName mTestName = new TestName(); 79 NNRandomGraphTest(int modelCount, int graphSize, int dimensionRange, Duration duration, String acceleratorName, boolean runModelCompilationOnly)80 public NNRandomGraphTest(int modelCount, int graphSize, int dimensionRange, 81 Duration duration, String acceleratorName, boolean runModelCompilationOnly) { 82 super(NNRandomGraphTestActivity.class); 83 mModelCount = modelCount; 84 mGraphSize = graphSize; 85 mDuration = duration; 86 mAcceleratorName = acceleratorName; 87 mDimensionRange = dimensionRange; 88 mRunModelCompilationOnly = runModelCompilationOnly; 89 } 90 91 @Before 92 @Override setUp()93 public void setUp() { 94 injectInstrumentation(InstrumentationRegistry.getInstrumentation()); 95 BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 60); 96 setActivityIntent(getTestModelsOfSizeAndRangeForMaxTimeIntent(mGraphSize, mDimensionRange, 97 mModelCount, mAcceleratorName, mDuration, mTestName.getMethodName())); 98 } 99 findModelForLivenessTest()100 protected Optional<TestModels.TestModelEntry> findModelForLivenessTest() 101 throws NnApiDelegationFailure { 102 return AcceleratorSpecificTestSupport.findTestModelRunningOnAccelerator( 103 getInstrumentation().getTargetContext(), mAcceleratorName); 104 } 105 106 @Test 107 @LargeTest 108 @UiThreadTest testDriverDoesNotFailWithParallelWorkload()109 public void testDriverDoesNotFailWithParallelWorkload() 110 throws ExecutionException, InterruptedException, NnApiDelegationFailure { 111 final NNRandomGraphTestActivity activity = getActivity(); 112 113 assertEquals(SUCCESS, activity.testResult()); 114 } 115 116 /** 117 * @return the intent to use to initialise the RandomGraphTest test class 118 */ getTestModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange, int modelsCount, String deviceName, Duration maxTestDuration, String testName)119 protected Intent getTestModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange, 120 int modelsCount, String deviceName, Duration maxTestDuration, String testName) { 121 Intent result = new Intent(); 122 RandomGraphTest 123 .intentInitializer(graphSize, dimensionsRange, modelsCount, 124 RandomGraphTest.DEFAULT_PAUSE_BETWEEN_MODELS_MILLIS, 125 mRunModelCompilationOnly, deviceName, maxTestDuration.toMillis(), testName) 126 .addIntentParams(result); 127 return result; 128 } 129 }