1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package vogar.target; 18 19 import com.google.common.base.Function; 20 import com.google.common.base.Functions; 21 import java.util.List; 22 import java.util.Properties; 23 import java.util.concurrent.atomic.AtomicInteger; 24 import org.junit.After; 25 import org.junit.Before; 26 import org.junit.Rule; 27 import vogar.Result; 28 import vogar.target.junit.JUnitUtils; 29 import vogar.testing.InterceptOutputStreams; 30 import vogar.testing.InterceptOutputStreams.Stream; 31 32 import static org.junit.Assert.assertEquals; 33 import static org.junit.Assert.assertTrue; 34 35 /** 36 * Provides support for testing {@link TestRunner} class. 37 * 38 * <p>Subclasses provide the individual test methods, each test method has the following structure. 39 * 40 * <p>It is annotated with {@link TestRunnerProperties @TestRunnerProperties} that specifies the 41 * properties that the main Vogar process supplies to the client process that actually runs the 42 * tests. It must specify either a {@link TestRunnerProperties#testClass()} or 43 * {@link TestRunnerProperties#testClassOrPackage()}, all the remaining properties are optional. 44 * 45 * <p>It calls {@code testRunnerRule.createTestRunner(...)} to create a {@link TestRunner}; passing 46 * in any additional command line arguments that {@link TestRunner#TestRunner(Properties, List)} 47 * accepts. 48 * 49 * <p>It calls {@link TestRunner#run()} to actually run the tests. 50 * 51 * <p>It calls {@link #expectedResults()} to obtain a {@link ExpectedResults} instance that it uses 52 * to specify the expected results for the test. Once it has specified the expected results then it 53 * must call either {@link ExpectedResults#completedNormally()} or 54 * {@link ExpectedResults#aborted()}. They indicate whether the test process completed normally or 55 * would abort (due to a test timing out) and cause the actual results to be checked against the 56 * expected results. 57 */ 58 public abstract class AbstractTestRunnerTest { 59 60 @Rule 61 public InterceptOutputStreams ios = new InterceptOutputStreams(Stream.OUT); 62 63 @Rule 64 public TestRunnerRule testRunnerRule = new TestRunnerRule(); 65 66 /** 67 * Keeps track of number of times {@link #expectedResults()} has been called without 68 * {@link ExpectedResults#checkFilteredOutput(String)} 69 * also being called. If it is {@code > 0} then the test is in error. 70 */ 71 private AtomicInteger checkCount; 72 73 @Before beforeTest()74 public void beforeTest() { 75 checkCount = new AtomicInteger(); 76 } 77 78 @After afterTest()79 public void afterTest() { 80 if (checkCount.get() != 0) { 81 throw new IllegalStateException("Test called expectedResults() but failed to call" 82 + "either aborted() or completedNormally()"); 83 } 84 } 85 expectedResults()86 protected ExpectedResults expectedResults() { 87 checkCount.incrementAndGet(); 88 return new ExpectedResults(testRunnerRule.testClass(), ios, 89 checkCount); 90 } 91 92 protected static class ExpectedResults { 93 94 private final StringBuilder builder = new StringBuilder(); 95 private final InterceptOutputStreams ios; 96 private final AtomicInteger checkCount; 97 private String testClassName; 98 private Function<String, String> filter; 99 ExpectedResults( Class<?> testClass, InterceptOutputStreams ios, AtomicInteger checkCount)100 private ExpectedResults( 101 Class<?> testClass, InterceptOutputStreams ios, AtomicInteger checkCount) { 102 this.testClassName = testClass.getName(); 103 this.checkCount = checkCount; 104 // Automatically strip out methods from a stack trace to avoid making tests dependent 105 // on either the call hierarchy or on source line numbers which would make the tests 106 // incredibly fragile. If a test fails then the unfiltered output containing the full 107 // stack trace will be output so this will not lose information needed to debug errors. 108 filter = new Function<String, String>() { 109 @Override 110 public String apply(String input) { 111 // Remove stack trace from output. 112 return input.replaceAll("\\t(at[^\\n]+|\\.\\.\\. [0-9]+ more)\\n", ""); 113 } 114 }; 115 this.ios = ios; 116 } 117 text(String message)118 public ExpectedResults text(String message) { 119 builder.append(message); 120 return this; 121 } 122 addFilter(Function<String, String> function)123 private ExpectedResults addFilter(Function<String, String> function) { 124 filter = Functions.compose(filter, function); 125 return this; 126 } 127 forTestClass(Class<?> testClass)128 public ExpectedResults forTestClass(Class<?> testClass) { 129 this.testClassName = testClass.getName(); 130 return this; 131 } 132 forTestClass(String testClassName)133 public ExpectedResults forTestClass(String testClassName) { 134 this.testClassName = testClassName; 135 return this; 136 } 137 failure(String methodName, String message)138 public ExpectedResults failure(String methodName, String message) { 139 String output = outcome(testClassName, methodName, message, Result.EXEC_FAILED); 140 return text(output); 141 } 142 success(String methodName)143 public ExpectedResults success(String methodName) { 144 String output = outcome(testClassName, methodName, null, Result.SUCCESS); 145 return text(output); 146 } 147 success(String methodName, String message)148 public ExpectedResults success(String methodName, String message) { 149 String output = outcome(testClassName, methodName, message, Result.SUCCESS); 150 return text(output); 151 } 152 153 unsupported()154 public ExpectedResults unsupported() { 155 String output = outcome( 156 testClassName, null, 157 "Skipping " + testClassName + ": no associated runner class\n", 158 Result.UNSUPPORTED); 159 return text(output); 160 } 161 noRunner()162 public ExpectedResults noRunner() { 163 String message = 164 String.format("Skipping %s: no associated runner class\n", testClassName); 165 String output = outcome(testClassName, null, message, Result.UNSUPPORTED); 166 return text(output); 167 } 168 completedNormally()169 public void completedNormally() { 170 text("//00xx{\"completedNormally\":true}\n"); 171 checkFilteredOutput(builder.toString()); 172 } 173 aborted()174 public void aborted() { 175 checkFilteredOutput(builder.toString()); 176 } 177 178 outcome( String testClassName, String methodName, String message, Result result)179 private static String outcome( 180 String testClassName, String methodName, String message, Result result) { 181 String testName = JUnitUtils.getTestName(testClassName, methodName); 182 183 return String.format("//00xx{\"outcome\":\"%s\"}\n" 184 + "%s" 185 + "//00xx{\"result\":\"%s\"}\n", 186 testName, message == null ? "" : message, result); 187 } 188 checkFilteredOutput(String expected)189 private void checkFilteredOutput(String expected) { 190 checkCount.decrementAndGet(); 191 String output = ios.contents(Stream.OUT); 192 String filtered = filter.apply(output); 193 if (!expected.equals(filtered)) { 194 assertEquals(expected, output); 195 } 196 } 197 } 198 } 199