1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package vogar.target; 18 19 import com.google.caliper.Benchmark; 20 import com.google.caliper.runner.UserCodeException; 21 import junit.framework.TestCase; 22 import org.junit.Rule; 23 import org.junit.Test; 24 import org.junit.runner.RunWith; 25 import org.junit.runners.JUnit4; 26 import vogar.target.junit.JUnitRunner; 27 import vogar.testing.InterceptOutputStreams; 28 import vogar.testing.InterceptOutputStreams.Stream; 29 30 import static org.junit.Assert.assertEquals; 31 import static org.junit.Assert.assertTrue; 32 33 /** 34 * Tests for {@link TestRunner} 35 */ 36 @RunWith(JUnit4.class) 37 public class TestRunnerTest { 38 39 @Rule public InterceptOutputStreams ios = new InterceptOutputStreams(Stream.OUT); 40 41 @Rule public TestRunnerRule testRunnerRule = new TestRunnerRule(); 42 43 @TestRunnerProperties(testClass = JUnit3Test.class) 44 @Test testConstructor_JUnit3Test()45 public void testConstructor_JUnit3Test() throws Exception { 46 TestRunner runner = testRunnerRule.createTestRunner(); 47 runner.run(); 48 49 assertEquals("" 50 + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testMethodName\"," 51 + "\"runner\":\"" + JUnitRunner.class.getName() + "\"}\n" 52 + "//00xx{\"result\":\"SUCCESS\"}\n" 53 + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testOtherName\"," 54 + "\"runner\":\"" + JUnitRunner.class.getName() + "\"}\n" 55 + "//00xx{\"result\":\"SUCCESS\"}\n" 56 + "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT)); 57 } 58 59 /** 60 * Make sure that the {@code --monitorPort <port>} command line option overrides the default 61 * specified in the properties. 62 */ 63 @TestRunnerProperties(testClassOrPackage = "vogar.DummyTest", monitorPort = 2345) 64 @Test testConstructor_MonitorPortOverridden()65 public void testConstructor_MonitorPortOverridden() throws Exception { 66 TestRunner runner = testRunnerRule.createTestRunner(); 67 assertEquals(2345, (int) runner.monitorPort); 68 69 runner = testRunnerRule.createTestRunner("--monitorPort", "10"); 70 assertEquals(10, (int) runner.monitorPort); 71 } 72 73 @TestRunnerProperties(testClass = JUnit3Test.class) 74 @Test testConstructor_SkipPastJUnitRunner()75 public void testConstructor_SkipPastJUnitRunner() throws Exception { 76 String failingTestName = JUnit3Test.class.getName() + "#testMethodName"; 77 TestRunner runner = testRunnerRule.createTestRunner("--skipPast", failingTestName); 78 String skipPast = runner.skipPastReference.get(); 79 assertEquals(failingTestName, skipPast); 80 81 runner.run(); 82 assertEquals("" 83 + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#testOtherName\"," 84 + "\"runner\":\"" + JUnitRunner.class.getName() + "\"}\n" 85 + "//00xx{\"result\":\"SUCCESS\"}\n" 86 + "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT)); 87 } 88 89 public static class JUnit3Test extends TestCase { testMethodName()90 public void testMethodName() { 91 } 92 testOtherName()93 public void testOtherName() { 94 } 95 } 96 97 @TestRunnerProperties(testClass = CaliperBenchmark.class) 98 @Test testConstructor_CaliperBenchmark()99 public void testConstructor_CaliperBenchmark() throws Exception { 100 TestRunner runner = testRunnerRule.createTestRunner("-i", "runtime"); 101 runner.run(); 102 103 String out = ios.contents(Stream.OUT); 104 // Remove stack trace from output. 105 out = out.replaceAll("\t[^\n]+\\n", ""); 106 assertEquals("" 107 + "//00xx{\"outcome\":\"" + CaliperBenchmark.class.getName() + "\"," 108 + "\"runner\":\"" + CaliperRunner.class.getName() + "\"}\n" 109 + "Experiment selection: \n" 110 + " Benchmark Methods: [timeMethod]\n" 111 + " Instruments: [runtime]\n" 112 + " User parameters: {}\n" 113 + " Virtual machines: [default]\n" 114 + " Selection type: Full cartesian product\n" 115 + "\n" 116 + "This selection yields 1 experiments.\n" 117 + UserCodeException.class.getName() 118 + ": An exception was thrown from the benchmark code\n" 119 + "Caused by: " + IllegalStateException.class.getName() 120 + ": " + CaliperBenchmark.CALIPER_BENCHMARK_MESSAGE + "\n" 121 + "//00xx{\"result\":\"SUCCESS\"}\n" 122 + "//00xx{\"completedNormally\":true}\n", out); 123 } 124 125 /** 126 * Ensure that requesting profiling doesn't send an invalid option to Caliper. 127 * 128 * <p>Cannot check that profiling works because it will only work on Android and these tests 129 * do not run on android yet. 130 */ 131 @TestRunnerProperties(testClass = CaliperBenchmark.class, profile = true) 132 @Test testConstructor_CaliperBenchmark_Profile()133 public void testConstructor_CaliperBenchmark_Profile() throws Exception { 134 TestRunner runner = testRunnerRule.createTestRunner("-i", "runtime"); 135 runner.run(); 136 137 String out = ios.contents(Stream.OUT); 138 139 // Make sure that profiling is requested (even though it's not supported). 140 assertTrue(out.startsWith("Profiling is disabled: ")); 141 142 // Remove warning about profiling being disabled. 143 out = out.replaceAll("^Profiling is disabled:[^\n]+\\n", ""); 144 145 // Remove stack trace from output. 146 out = out.replaceAll("\t[^\n]+\\n", ""); 147 148 assertEquals("" 149 + "//00xx{\"outcome\":\"" + CaliperBenchmark.class.getName() + "\"," 150 + "\"runner\":\"" + CaliperRunner.class.getName() + "\"}\n" 151 + "Experiment selection: \n" 152 + " Benchmark Methods: [timeMethod]\n" 153 + " Instruments: [runtime]\n" 154 + " User parameters: {}\n" 155 + " Virtual machines: [default]\n" 156 + " Selection type: Full cartesian product\n" 157 + "\n" 158 + "This selection yields 1 experiments.\n" 159 + UserCodeException.class.getName() 160 + ": An exception was thrown from the benchmark code\n" 161 + "Caused by: " + IllegalStateException.class.getName() 162 + ": " + CaliperBenchmark.CALIPER_BENCHMARK_MESSAGE + "\n" 163 + "//00xx{\"result\":\"SUCCESS\"}\n" 164 + "//00xx{\"completedNormally\":true}\n", out); 165 } 166 167 public static class CaliperBenchmark { 168 169 static final String CALIPER_BENCHMARK_MESSAGE = "Aborting test to save time"; 170 171 @Benchmark timeMethod(long reps)172 public long timeMethod(long reps) { 173 throw new IllegalStateException(CALIPER_BENCHMARK_MESSAGE); 174 } 175 } 176 177 @TestRunnerProperties(testClass = Main.class) 178 @Test testConstructor_Main()179 public void testConstructor_Main() throws Exception { 180 TestRunner runner = testRunnerRule.createTestRunner(); 181 runner.run(); 182 183 assertEquals("" 184 + "//00xx{\"outcome\":\"" + Main.class.getName() + "\"," 185 + "\"runner\":\"" + MainRunner.class.getName() + "\"}\n" 186 + "//00xx{\"result\":\"SUCCESS\"}\n" 187 + "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT)); 188 } 189 190 public static class Main { main(String[] args)191 public static void main(String[] args) { 192 } 193 } 194 195 @TestRunnerProperties(testClass = JUnit3Test.class) 196 @Test testConstructor_WithMethodName()197 public void testConstructor_WithMethodName() throws Exception { 198 String methodName = "testMethodName"; 199 TestRunner runner = testRunnerRule.createTestRunner(methodName); 200 runner.run(); 201 202 assertEquals("" 203 + "Warning: Arguments are invalid for Caliper: " 204 + "Extra stuff, did not expect non-option arguments: [" + methodName + "]\n" 205 + "//00xx{\"outcome\":\"" + JUnit3Test.class.getName() + "#" + methodName + "\"," 206 + "\"runner\":\"" + JUnitRunner.class.getName() + "\"}\n" 207 + "//00xx{\"result\":\"SUCCESS\"}\n" 208 + "//00xx{\"completedNormally\":true}\n", ios.contents(Stream.OUT)); 209 } 210 } 211