1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.collectors; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertTrue; 21 import static org.mockito.Mockito.doReturn; 22 import static org.mockito.Mockito.mock; 23 24 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 25 import com.android.tradefed.invoker.IInvocationContext; 26 import com.android.tradefed.result.CollectingTestListener; 27 import com.android.tradefed.result.TestDescription; 28 import com.android.tradefed.result.TestResult; 29 import com.android.tradefed.result.TestRunResult; 30 import com.android.tradefed.testtype.AndroidJUnitTest; 31 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; 32 import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test; 33 import com.android.tradefed.testtype.junit4.DeviceTestRunOptions; 34 35 import org.junit.Before; 36 import org.junit.Test; 37 import org.junit.runner.RunWith; 38 39 import java.util.Arrays; 40 import java.util.Collection; 41 import java.util.Map.Entry; 42 43 /** 44 * Host side tests for the core device collectors, this ensure that we are able to use the 45 * collectors in a similar way as the infra. 46 * 47 * Command: 48 * mm CollectorHostsideLibTest CollectorDeviceLibTest -j16 49 * tradefed.sh run commandAndExit template/local_min --template:map test=CollectorHostsideLibTest 50 */ 51 @RunWith(DeviceJUnit4ClassRunner.class) 52 public class DeviceCollectorsTest extends BaseHostJUnit4Test { 53 private static final String TEST_APK = "CollectorDeviceLibTest.apk"; 54 private static final String PACKAGE_NAME = "android.device.collectors"; 55 private static final String AJUR_RUNNER = "androidx.test.runner.AndroidJUnitRunner"; 56 57 private static final String STUB_BASE_COLLECTOR = 58 "android.device.collectors.StubTestMetricListener"; 59 private static final String SCHEDULED_COLLECTOR = 60 "android.device.collectors.StubScheduledRunMetricListener"; 61 62 private RemoteAndroidTestRunner mTestRunner; 63 private IInvocationContext mContext; 64 65 @Before setUp()66 public void setUp() throws Exception { 67 installPackage(TEST_APK); 68 assertTrue(isPackageInstalled(PACKAGE_NAME)); 69 mTestRunner = 70 new RemoteAndroidTestRunner(PACKAGE_NAME, AJUR_RUNNER, getDevice().getIDevice()); 71 // Set the new runListener order to ensure test cases can show their metrics. 72 mTestRunner.addInstrumentationArg(AndroidJUnitTest.NEW_RUN_LISTENER_ORDER_KEY, "true"); 73 mTestRunner.addInstrumentationArg("notClass", "android.device.tests.TestEvents"); 74 mContext = mock(IInvocationContext.class); 75 doReturn(Arrays.asList(getDevice())).when(mContext).getDevices(); 76 doReturn(Arrays.asList(getBuild())).when(mContext).getBuildInfos(); 77 } 78 79 /** 80 * Test that our base metric listener can output metrics. 81 */ 82 @Test testBaseListenerRuns()83 public void testBaseListenerRuns() throws Exception { 84 mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR); 85 mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 86 CollectingTestListener listener = new CollectingTestListener(); 87 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 88 Collection<TestRunResult> results = listener.getRunResults(); 89 assertEquals(1, results.size()); 90 TestRunResult result = results.iterator().next(); 91 assertFalse(result.isRunFailure()); 92 assertFalse(result.hasFailedTests()); 93 // Ensure the listener added a metric at test run start and end. 94 assertTrue(result.getRunMetrics().containsKey("run_start")); 95 assertTrue(result.getRunMetrics().containsKey("run_end")); 96 // Check that each test case has results with the metrics associated. 97 for (TestResult res : result.getTestResults().values()) { 98 assertTrue(res.getMetrics().containsKey("test_start")); 99 assertTrue(res.getMetrics().containsKey("test_end")); 100 } 101 } 102 103 /** 104 * Test that our base metric listener can filter metrics to run only against some groups tagged 105 * with an annotation. All annotation of BaseMetricListenerInstrumentedTest are annotated with 106 * the group 'testGroup'. 107 */ 108 @Test testBaseListenerRuns_withExcludeFilters()109 public void testBaseListenerRuns_withExcludeFilters() throws Exception { 110 mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR); 111 mTestRunner.addInstrumentationArg("exclude-filter-group", "testGroup"); 112 mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 113 CollectingTestListener listener = new CollectingTestListener(); 114 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 115 Collection<TestRunResult> results = listener.getRunResults(); 116 assertEquals(1, results.size()); 117 TestRunResult result = results.iterator().next(); 118 assertFalse(result.isRunFailure()); 119 assertFalse(result.hasFailedTests()); 120 // Ensure the listener added a metric at test run start and end. 121 assertTrue(result.getRunMetrics().containsKey("run_start")); 122 assertTrue(result.getRunMetrics().containsKey("run_end")); 123 // We did run some tests 124 assertTrue(!result.getTestResults().isEmpty()); 125 // After filtering none of the test case should contain any of the metrics since it was 126 // filtered. 127 for (TestResult testCaseResult : result.getTestResults().values()) { 128 assertFalse(testCaseResult.getMetrics().containsKey("test_start")); 129 assertFalse(testCaseResult.getMetrics().containsKey("test_fail")); 130 assertFalse(testCaseResult.getMetrics().containsKey("test_end")); 131 } 132 } 133 134 /** 135 * Test that if an include and exclude filters are provided, the exclude filters has priority. 136 */ 137 @Test testBaseListenerRuns_withIncludeAndExcludeFilters()138 public void testBaseListenerRuns_withIncludeAndExcludeFilters() throws Exception { 139 mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR); 140 mTestRunner.addInstrumentationArg("include-filter-group", "testGroup"); 141 mTestRunner.addInstrumentationArg("exclude-filter-group", "testGroup"); 142 mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 143 CollectingTestListener listener = new CollectingTestListener(); 144 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 145 Collection<TestRunResult> results = listener.getRunResults(); 146 assertEquals(1, results.size()); 147 TestRunResult result = results.iterator().next(); 148 assertFalse(result.isRunFailure()); 149 assertFalse(result.hasFailedTests()); 150 // Ensure the listener added a metric at test run start and end. 151 assertTrue(result.getRunMetrics().containsKey("run_start")); 152 assertTrue(result.getRunMetrics().containsKey("run_end")); 153 // We did run some tests 154 assertTrue(!result.getTestResults().isEmpty()); 155 // After filtering none of the test case should contain any of the metrics since it was 156 // filtered. Exclusion has priority over inclusion. 157 for (TestResult testCaseResult : result.getTestResults().values()) { 158 assertFalse(testCaseResult.getMetrics().containsKey("test_start")); 159 assertFalse(testCaseResult.getMetrics().containsKey("test_fail")); 160 assertFalse(testCaseResult.getMetrics().containsKey("test_end")); 161 } 162 } 163 164 /** 165 * Test that if an include filter is provided, only method part of the included group will 166 * run the collection. 167 */ 168 @Test testBaseListenerRuns_withIncludeFilters()169 public void testBaseListenerRuns_withIncludeFilters() throws Exception { 170 mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR); 171 mTestRunner.addInstrumentationArg("include-filter-group", "testGroup1"); 172 mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 173 CollectingTestListener listener = new CollectingTestListener(); 174 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 175 Collection<TestRunResult> results = listener.getRunResults(); 176 assertEquals(1, results.size()); 177 TestRunResult result = results.iterator().next(); 178 assertFalse(result.isRunFailure()); 179 assertFalse(result.hasFailedTests()); 180 // Ensure the listener added a metric at test run start and end. 181 assertTrue(result.getRunMetrics().containsKey("run_start")); 182 assertTrue(result.getRunMetrics().containsKey("run_end")); 183 // We did run some tests 184 assertTrue(!result.getTestResults().isEmpty()); 185 // After filtering none of the test case should contain any of the metrics since it was 186 // filtered. Exclusion has priority over inclusion. 187 for (Entry<TestDescription, TestResult> testResult : result.getTestResults().entrySet()) { 188 // testReportMetrics method is the only one annotated with 'testGroup1' it should be the 189 // only one collecting metrics. 190 if ("testReportMetrics".equals(testResult.getKey().getTestName())) { 191 assertTrue(testResult.getValue().getMetrics().containsKey("test_start")); 192 assertTrue(testResult.getValue().getMetrics().containsKey("test_end")); 193 } else { 194 assertFalse(testResult.getValue().getMetrics().containsKey("test_start")); 195 assertFalse(testResult.getValue().getMetrics().containsKey("test_fail")); 196 assertFalse(testResult.getValue().getMetrics().containsKey("test_end")); 197 } 198 } 199 } 200 201 /** 202 * Test that our base scheduled listener can output metrics periodically. 203 */ 204 @Test testScheduledListenerRuns()205 public void testScheduledListenerRuns() throws Exception { 206 mTestRunner.addInstrumentationArg("listener", SCHEDULED_COLLECTOR); 207 mTestRunner.addInstrumentationArg("interval", "15"); 208 mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 209 CollectingTestListener listener = new CollectingTestListener(); 210 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 211 Collection<TestRunResult> results = listener.getRunResults(); 212 assertEquals(1, results.size()); 213 TestRunResult result = results.iterator().next(); 214 assertFalse(result.isRunFailure()); 215 assertFalse(result.hasFailedTests()); 216 // There is time during the test to output at least a handful of periodic metrics. 217 assertTrue(result.getRunMetrics().containsKey("collect0")); 218 assertTrue(result.getRunMetrics().containsKey("collect1")); 219 assertTrue(result.getRunMetrics().containsKey("collect2")); 220 } 221 222 /** 223 * Test that our base scheduled listener can use its default period to run when the interval 224 * given is not valid. 225 */ 226 @Test testScheduledListenerRuns_defaultValue()227 public void testScheduledListenerRuns_defaultValue() throws Exception { 228 mTestRunner.addInstrumentationArg("listener", SCHEDULED_COLLECTOR); 229 // Invalid interval will results in the default period to be used. 230 mTestRunner.addInstrumentationArg("interval", "-100"); 231 mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 232 CollectingTestListener listener = new CollectingTestListener(); 233 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 234 Collection<TestRunResult> results = listener.getRunResults(); 235 assertEquals(1, results.size()); 236 TestRunResult result = results.iterator().next(); 237 assertFalse(result.isRunFailure()); 238 assertFalse(result.hasFailedTests()); 239 // The default interval value is one minute so it will only have time to run once. 240 assertEquals(1, result.getRunMetrics().size()); 241 assertTrue(result.getRunMetrics().containsKey("collect0")); 242 } 243 244 /** 245 * Test that when using -e log true. Nothing gets collected. 246 */ 247 @Test testLogOnly()248 public void testLogOnly() throws Exception { 249 DeviceTestRunOptions options = new DeviceTestRunOptions(PACKAGE_NAME); 250 options.addInstrumentationArg("listener", STUB_BASE_COLLECTOR); 251 options.addInstrumentationArg("log", "true"); 252 options.setTestClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 253 boolean res = runDeviceTests(options); 254 assertTrue(res); 255 TestRunResult result = getLastDeviceRunResults(); 256 assertTrue(result.getRunMetrics().isEmpty()); 257 for (Entry<TestDescription, TestResult> testResult : result.getTestResults().entrySet()) { 258 assertTrue(testResult.getValue().getMetrics().isEmpty()); 259 } 260 } 261 } 262