1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.platform.test.rule; 17 18 import static com.google.common.truth.Truth.assertThat; 19 20 import android.app.Instrumentation; 21 import android.device.collectors.BaseMetricListener; 22 import android.device.collectors.DataRecord; 23 import android.os.Bundle; 24 25 import org.junit.Before; 26 import org.junit.Rule; 27 import org.junit.Test; 28 import org.junit.rules.ExpectedException; 29 import org.junit.runner.Description; 30 import org.junit.runner.notification.Failure; 31 import org.junit.runners.model.Statement; 32 33 import java.util.ArrayList; 34 import java.util.List; 35 36 /** Tests for {@link TestMetricRule}. */ 37 public class TestMetricRuleTest { 38 // A static variable is used so that the metric listeners in this test can modify its state 39 // while being static and able to be created via an empty constructor. This route is taken as 40 // the metric collectors themselves are not directly observable due to being created via 41 // reflection fron TestMetricRule. 42 private static List<String> sLogs = new ArrayList<String>(); 43 44 private static final Description DESCRIPTION = 45 Description.createTestDescription("class", "method"); 46 47 private static final String TEST_FAILURE_MESSAGE = "Failed!"; 48 49 private static final Statement PASSING_STATEMENT = 50 new Statement() { 51 @Override 52 public void evaluate() { 53 sLogs.add("Test execution"); 54 } 55 }; 56 57 private static final Statement FAILING_STATEMENT = 58 new Statement() { 59 @Override 60 public void evaluate() { 61 sLogs.add("Test execution"); 62 throw new RuntimeException(TEST_FAILURE_MESSAGE); 63 } 64 }; 65 66 @Rule public ExpectedException expectedException = ExpectedException.none(); 67 68 @Before setUp()69 public void setUp() { 70 sLogs.clear(); 71 } 72 73 @Test testValidListener_testPasses()74 public void testValidListener_testPasses() throws Throwable { 75 TestMetricRule rule = 76 createWithMetricCollectorNames( 77 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1"); 78 rule.apply(PASSING_STATEMENT, DESCRIPTION).evaluate(); 79 assertThat(sLogs) 80 .containsExactly( 81 "TestableCollector1#setInstrumentation", 82 "TestableCollector1#setupAdditionalArgs", 83 "TestableCollector1#onSetUp", 84 String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION), 85 "Test execution", 86 String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION), 87 "TestableCollector1#onCleanUp") 88 .inOrder(); 89 } 90 91 @Test testValidListener_testFails()92 public void testValidListener_testFails() throws Throwable { 93 TestMetricRule rule = 94 createWithMetricCollectorNames( 95 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1"); 96 expectedException.expectMessage(TEST_FAILURE_MESSAGE); 97 rule.apply(FAILING_STATEMENT, DESCRIPTION).evaluate(); 98 Failure failure = new Failure(DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE)); 99 assertThat(sLogs) 100 .containsExactly( 101 "TestableCollector1#setInstrumentation", 102 "TestableCollector1#setupAdditionalArgs", 103 "TestableCollector1#onSetUp", 104 String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION), 105 "Test execution", 106 String.format( 107 "Test %s: TestableCollector1#onTestFail with failure %s", 108 DESCRIPTION, 109 new Failure( 110 DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE))), 111 String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION), 112 "TestableCollector1#onCleanUp") 113 .inOrder(); 114 } 115 116 @Test testMultipleListeners_allValid()117 public void testMultipleListeners_allValid() throws Throwable { 118 TestMetricRule rule = 119 createWithMetricCollectorNames( 120 "android.platform.test.rule.TestMetricRuleTest$TestableCollector2", 121 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1"); 122 expectedException.expectMessage(TEST_FAILURE_MESSAGE); 123 rule.apply(FAILING_STATEMENT, DESCRIPTION).evaluate(); 124 Failure failure = new Failure(DESCRIPTION, new RuntimeException(TEST_FAILURE_MESSAGE)); 125 assertThat(sLogs) 126 .containsExactly( 127 "TestableCollector1#setInstrumentation", 128 "TestableCollector2#setInstrumentation", 129 "TestableCollector1#setupAdditionalArgs", 130 "TestableCollector1#onSetUp", 131 "TestableCollector2#setupAdditionalArgs", 132 "TestableCollector2#onSetUp", 133 String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION), 134 String.format("Test %s: TestableCollector2#onTestStart", DESCRIPTION), 135 "Test execution", 136 String.format( 137 "Test %s: TestableCollector1#onTestFail with failure %s", 138 DESCRIPTION, failure), 139 String.format( 140 "Test %s: TestableCollector2#onTestFail with failure %s", 141 DESCRIPTION, failure), 142 String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION), 143 String.format("Test %s: TestableCollector2#onTestEnd", DESCRIPTION), 144 "TestableCollector1#onCleanUp", 145 "TestableCollector2#onCleanUp") 146 .inOrder(); 147 } 148 149 @Test testInvalidListenerNameThrows()150 public void testInvalidListenerNameThrows() { 151 String invalidName = "not.a.Collector"; 152 expectedException.expectMessage( 153 String.format( 154 "Failed to dynamically load metric collector with fully qualified name %s.", 155 invalidName)); 156 // The creation should fail. 157 TestMetricRule rule = 158 createWithMetricCollectorNames( 159 "android.platform.test.rule.TestMetricRuleTest$TestableCollector1", 160 invalidName); 161 } 162 163 @Test testSimpleClassNameAttemptedWithKnownPackage()164 public void testSimpleClassNameAttemptedWithKnownPackage() { 165 String simpleName = "NonExistentCollector"; 166 // We can't validate real collectors here due to test logistics, so we proxy it by using 167 // an invalid name, and checking that the full name we give in the exception. 168 expectedException.expectMessage( 169 String.format("%s.%s", TestMetricRule.METRIC_COLLECTORS_PACKAGE, simpleName)); 170 // The creation should fail. 171 TestMetricRule rule = createWithMetricCollectorNames(simpleName); 172 } 173 174 @Test testInitWithDifferentOptionName()175 public void testInitWithDifferentOptionName() throws Throwable { 176 String optionName = "another-" + TestMetricRule.METRIC_COLLECTORS_OPTION; 177 178 Bundle args = new Bundle(); 179 args.putString( 180 optionName, "android.platform.test.rule.TestMetricRuleTest$TestableCollector1"); 181 TestMetricRule rule = 182 new TestMetricRule(args, new Instrumentation(), optionName, "log tag"); 183 184 rule.apply(PASSING_STATEMENT, DESCRIPTION).evaluate(); 185 assertThat(sLogs) 186 .containsExactly( 187 "TestableCollector1#setInstrumentation", 188 "TestableCollector1#setupAdditionalArgs", 189 "TestableCollector1#onSetUp", 190 String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION), 191 "Test execution", 192 String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION), 193 "TestableCollector1#onCleanUp") 194 .inOrder(); 195 } 196 createWithMetricCollectorNames(String... names)197 private TestMetricRule createWithMetricCollectorNames(String... names) { 198 Bundle args = new Bundle(); 199 args.putString(TestMetricRule.METRIC_COLLECTORS_OPTION, String.join(",", names)); 200 return new TestMetricRule(args); 201 } 202 203 public static class BaseTestableCollector extends BaseMetricListener { 204 private final String mName; 205 BaseTestableCollector(String name)206 public BaseTestableCollector(String name) { 207 mName = name; 208 } 209 210 @Override setInstrumentation(Instrumentation instr)211 public void setInstrumentation(Instrumentation instr) { 212 sLogs.add(String.format("%s#%s", mName, "setInstrumentation")); 213 } 214 215 @Override setupAdditionalArgs()216 public void setupAdditionalArgs() { 217 sLogs.add(String.format("%s#%s", mName, "setupAdditionalArgs")); 218 } 219 220 @Override onSetUp()221 public void onSetUp() { 222 sLogs.add(String.format("%s#%s", mName, "onSetUp")); 223 } 224 225 @Override onCleanUp()226 public void onCleanUp() { 227 sLogs.add(String.format("%s#%s", mName, "onCleanUp")); 228 } 229 230 @Override onTestStart(DataRecord testData, Description description)231 public void onTestStart(DataRecord testData, Description description) { 232 sLogs.add(String.format("Test %s: %s#%s", description, mName, "onTestStart")); 233 } 234 235 @Override onTestEnd(DataRecord testData, Description description)236 public void onTestEnd(DataRecord testData, Description description) { 237 sLogs.add(String.format("Test %s: %s#%s", description, mName, "onTestEnd")); 238 } 239 240 @Override onTestFail(DataRecord testData, Description description, Failure failure)241 public void onTestFail(DataRecord testData, Description description, Failure failure) { 242 sLogs.add( 243 String.format( 244 "Test %s: %s#%s with failure %s", 245 description, mName, "onTestFail", failure)); 246 } 247 } 248 249 public static class TestableCollector1 extends BaseTestableCollector { TestableCollector1()250 public TestableCollector1() { 251 super(TestableCollector1.class.getSimpleName()); 252 } 253 } 254 255 public static class TestableCollector2 extends BaseTestableCollector { TestableCollector2()256 public TestableCollector2() { 257 super(TestableCollector2.class.getSimpleName()); 258 } 259 } 260 } 261