1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertNotEquals; 21 import static org.junit.Assert.assertTrue; 22 import static org.junit.Assert.fail; 23 import static org.mockito.Mockito.any; 24 import static org.mockito.Mockito.when; 25 26 import com.android.tradefed.build.DeviceBuildInfo; 27 import com.android.tradefed.build.BuildInfoKey; 28 import com.android.tradefed.config.ConfigurationException; 29 import com.android.tradefed.config.OptionSetter; 30 import com.android.tradefed.device.DeviceNotAvailableException; 31 import com.android.tradefed.result.ITestInvocationListener; 32 import com.android.tradefed.util.CommandResult; 33 import com.android.tradefed.util.CommandStatus; 34 import com.android.tradefed.util.FileUtil; 35 import com.android.tradefed.util.FakeShellOutputReceiver; 36 37 import org.junit.After; 38 import org.junit.Before; 39 import org.junit.Test; 40 import org.junit.runner.RunWith; 41 import org.junit.runners.JUnit4; 42 import org.mockito.Mockito; 43 44 import java.io.File; 45 import java.io.IOException; 46 import java.nio.charset.StandardCharsets; 47 import java.nio.file.Path; 48 import java.nio.file.Paths; 49 import java.nio.charset.StandardCharsets; 50 51 /** Unit tests for {@link HostGTest}. */ 52 @RunWith(JUnit4.class) 53 public class HostGTestTest { 54 private File mTestsDir; 55 private HostGTest mHostGTest; 56 private ITestInvocationListener mMockInvocationListener; 57 private FakeShellOutputReceiver mFakeReceiver; 58 private OptionSetter mSetter; 59 60 /** Helper to initialize the object or folder for unittest need. */ 61 @Before setUp()62 public void setUp() throws Exception { 63 mTestsDir = FileUtil.createTempDir("test_folder_for_unittest"); 64 mMockInvocationListener = Mockito.mock(ITestInvocationListener.class); 65 mFakeReceiver = new FakeShellOutputReceiver(); 66 67 mHostGTest = Mockito.spy(new HostGTest()); 68 GTestXmlResultParser mockXmlParser = Mockito.mock(GTestXmlResultParser.class); 69 when(mHostGTest.createXmlParser(any(), any())).thenReturn(mockXmlParser); 70 when(mHostGTest.createResultParser(any(), any())).thenReturn(mFakeReceiver); 71 72 mSetter = new OptionSetter(mHostGTest); 73 } 74 75 @After afterMethod()76 public void afterMethod() { 77 FileUtil.recursiveDelete(mTestsDir); 78 } 79 80 /** 81 * Helper to create a executable script for use in these unit tests. 82 * 83 * <p>This method will create a executable file for unittest. This executable file is shell 84 * script file. It will output all arguments to a file like "file.called" when it has been 85 * called. It will also output to both stdout and stderr. Unit tests can check the .called file 86 * or the test output (or both) to determine test success or not. 87 * 88 * @param folderName The path where to create. 89 * @param fileName The file name you want to create. 90 * @return The file path of "file.called", it is used to check if the file has been called 91 * correctly or not. 92 */ createTestScript(String folderName, String fileName)93 private File createTestScript(String folderName, String fileName) throws IOException { 94 final Path outputPath = Paths.get(folderName, fileName + ".called"); 95 final String script = 96 String.format( 97 "echo \"$@\" > %s; echo \"stdout: %s\"; echo \"stderr: %s\" >&2", 98 outputPath, fileName, fileName); 99 100 final Path scriptPath = Paths.get(folderName, fileName); 101 createExecutableFile(scriptPath, script); 102 103 return outputPath.toFile(); 104 } 105 106 /** 107 * Helper to create an executable file with the given contents. 108 * 109 * @param folderName The path where to create. 110 * @param fileName The file name you want to create. 111 * @param contents Contents to write to the file 112 */ createExecutableFile(Path outPath, String contents)113 private void createExecutableFile(Path outPath, String contents) throws IOException { 114 final File outFile = outPath.toFile(); 115 FileUtil.writeToFile(contents, outFile); 116 outFile.setExecutable(true); 117 } 118 119 /** 120 * Helper to create a sub folder in mTestsDir. 121 * 122 * @param folderName The path where to create. 123 * @return Sub folder File. 124 */ createSubFolder(String folderName)125 private File createSubFolder(String folderName) throws IOException { 126 return FileUtil.createTempDir(folderName, mTestsDir); 127 } 128 129 /** Test the executeHostCommand method. */ 130 @Test testExecuteHostCommand_success()131 public void testExecuteHostCommand_success() { 132 CommandResult lsResult = mHostGTest.executeHostCommand("ls"); 133 assertNotEquals("", lsResult.getStdout()); 134 assertEquals(CommandStatus.SUCCESS, lsResult.getStatus()); 135 } 136 137 /** Test the executeHostCommand method. */ 138 @Test testExecuteHostCommand_fail()139 public void testExecuteHostCommand_fail() { 140 CommandResult cmdResult = mHostGTest.executeHostCommand(""); 141 assertNotEquals(CommandStatus.SUCCESS, cmdResult.getStatus()); 142 } 143 144 /** Test the loadFilter method. */ 145 @Test testLoadFilter()146 public void testLoadFilter() throws ConfigurationException, IOException { 147 String moduleName = "hello_world_test"; 148 String testFilterKey = "presubmit"; 149 OptionSetter setter = new OptionSetter(mHostGTest); 150 setter.setOptionValue("test-filter-key", testFilterKey); 151 152 String filter = "LayerTransactionTest.*:LayerUpdateTest.*"; 153 String json_content = 154 "{\n" 155 + " \"" 156 + testFilterKey 157 + "\": {\n" 158 + " \"filter\": \"" 159 + filter 160 + "\"\n" 161 + " }\n" 162 + "}\n"; 163 Path path = Paths.get(mTestsDir.getAbsolutePath(), moduleName + GTestBase.FILTER_EXTENSION); 164 File filterFile = path.toFile(); 165 filterFile.createNewFile(); 166 filterFile.setReadable(true); 167 FileUtil.writeToFile(json_content, filterFile); 168 assertEquals( 169 mHostGTest.loadFilter(filterFile.getParent() + File.separator + moduleName), 170 filter); 171 } 172 173 /** Test runTest method. */ 174 @Test testRunTest()175 public void testRunTest() 176 throws ConfigurationException, IOException, DeviceNotAvailableException { 177 String moduleName = "hello_world_test"; 178 String dirPath = mTestsDir.getAbsolutePath(); 179 File cmd1 = createTestScript(dirPath, "cmd1"); 180 File cmd2 = createTestScript(dirPath, "cmd2"); 181 File cmd3 = createTestScript(dirPath, "cmd3"); 182 File cmd4 = createTestScript(dirPath, "cmd4"); 183 184 mSetter.setOptionValue("before-test-cmd", dirPath + File.separator + "cmd1"); 185 mSetter.setOptionValue("before-test-cmd", dirPath + File.separator + "cmd2"); 186 mSetter.setOptionValue("after-test-cmd", dirPath + File.separator + "cmd3"); 187 mSetter.setOptionValue("after-test-cmd", dirPath + File.separator + "cmd4"); 188 mSetter.setOptionValue("module-name", moduleName); 189 190 File hostLinkedFolder = createSubFolder("hosttestcases"); 191 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 192 193 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 194 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 195 mHostGTest.setBuild(buildInfo); 196 197 mHostGTest.run(mMockInvocationListener); 198 199 assertTrue(cmd1.exists()); 200 assertTrue(cmd2.exists()); 201 assertTrue(cmd3.exists()); 202 assertTrue(cmd4.exists()); 203 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 204 } 205 206 /** Test the run method for host linked folder is set. */ 207 @Test testRun_priority_get_testcase_from_hostlinked_folder()208 public void testRun_priority_get_testcase_from_hostlinked_folder() 209 throws IOException, ConfigurationException, DeviceNotAvailableException { 210 String moduleName = "hello_world_test"; 211 String hostLinkedFolderName = "hosttestcases"; 212 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 213 File hostTestcaseExecutedCheckFile = 214 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 215 216 String testFolderName = "testcases"; 217 File testcasesFolder = createSubFolder(testFolderName); 218 File testfolderTestcaseCheckExecuted = 219 createTestScript(testcasesFolder.getAbsolutePath(), moduleName); 220 221 mSetter.setOptionValue("module-name", moduleName); 222 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 223 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 224 buildInfo.setTestsDir(testcasesFolder, "0.0"); 225 mHostGTest.setBuild(buildInfo); 226 227 mHostGTest.run(mMockInvocationListener); 228 assertTrue(hostTestcaseExecutedCheckFile.exists()); 229 assertFalse(testfolderTestcaseCheckExecuted.exists()); 230 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 231 } 232 233 /** Test the run method for host linked folder is not set. */ 234 @Test testRun_get_testcase_from_testcases_folder_if_no_hostlinked_dir_set()235 public void testRun_get_testcase_from_testcases_folder_if_no_hostlinked_dir_set() 236 throws IOException, ConfigurationException, DeviceNotAvailableException { 237 String moduleName = "hello_world_test"; 238 String hostLinkedFolderName = "hosttestcases"; 239 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 240 File hostTestcaseExecutedCheckFile = 241 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 242 243 String testFolderName = "testcases"; 244 File testcasesFolder = createSubFolder(testFolderName); 245 File testfolderTestcaseCheckExecuted = 246 createTestScript(testcasesFolder.getAbsolutePath(), moduleName); 247 248 mSetter.setOptionValue("module-name", moduleName); 249 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 250 buildInfo.setTestsDir(testcasesFolder, "0.0"); 251 mHostGTest.setBuild(buildInfo); 252 253 mHostGTest.run(mMockInvocationListener); 254 assertFalse(hostTestcaseExecutedCheckFile.exists()); 255 assertTrue(testfolderTestcaseCheckExecuted.exists()); 256 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 257 } 258 259 /** Test can't find testcase. */ 260 @Test(expected = RuntimeException.class) testRun_can_not_find_testcase()261 public void testRun_can_not_find_testcase() 262 throws ConfigurationException, DeviceNotAvailableException { 263 String moduleName = "hello_world_test"; 264 mSetter.setOptionValue("module-name", moduleName); 265 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 266 mHostGTest.setBuild(buildInfo); 267 268 mHostGTest.run(mMockInvocationListener); 269 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 270 } 271 272 /** Test the run method for a binary with a suffix. */ 273 @Test testRun_withSuffix()274 public void testRun_withSuffix() throws Exception { 275 String moduleName = "hello_world_test"; 276 String hostLinkedFolderName = "hosttestcases"; 277 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 278 // The actual execution file has a suffix 279 File hostTestcaseExecutedCheckFile = 280 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName + "32"); 281 282 String testFolderName = "testcases"; 283 File testcasesFolder = createSubFolder(testFolderName); 284 File testfolderTestcaseCheckExecuted = 285 createTestScript(testcasesFolder.getAbsolutePath(), moduleName + "32"); 286 287 mSetter.setOptionValue("module-name", moduleName); 288 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 289 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 290 buildInfo.setTestsDir(testcasesFolder, "0.0"); 291 mHostGTest.setBuild(buildInfo); 292 293 mHostGTest.run(mMockInvocationListener); 294 assertTrue(hostTestcaseExecutedCheckFile.exists()); 295 assertFalse(testfolderTestcaseCheckExecuted.exists()); 296 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 297 } 298 299 /* Test that some command in the test run fails, an exception is thrown and the run stops. */ 300 @Test testBeforeCmdError()301 public void testBeforeCmdError() throws Exception { 302 String moduleName = "hello_world_test"; 303 String hostLinkedFolderName = "hosttestcases"; 304 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 305 File hostTestcaseExecutedCheckFile = 306 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 307 308 String testDir = mTestsDir.getAbsolutePath(); 309 Path errorScriptPath = Paths.get(testDir, "bad_cmd"); 310 createExecutableFile(errorScriptPath, "exit 1"); 311 312 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 313 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 314 mHostGTest.setBuild(buildInfo); 315 316 mSetter.setOptionValue("module-name", moduleName); 317 mSetter.setOptionValue("before-test-cmd", errorScriptPath.toString()); 318 319 try { 320 mHostGTest.run(mMockInvocationListener); 321 fail("Didn't throw RuntimeException for before cmd with non-zero exit code"); 322 } catch (RuntimeException e) { 323 // Expected exception 324 } 325 assertFalse(hostTestcaseExecutedCheckFile.exists()); 326 } 327 328 /* Test that if the test module exits with code 1, no exception is thrown and the run completes 329 * normally. */ 330 @Test testTestFailureHandledCorrectly()331 public void testTestFailureHandledCorrectly() throws Exception { 332 String moduleName = "hello_world_test"; 333 String hostLinkedFolderName = "hosttestcases"; 334 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 335 Path errorScriptPath = Paths.get(hostLinkedFolder.getAbsolutePath(), moduleName); 336 createExecutableFile(errorScriptPath, "echo 'TEST FAILED'; exit 1"); 337 338 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 339 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 340 mHostGTest.setBuild(buildInfo); 341 342 mSetter.setOptionValue("module-name", moduleName); 343 344 mHostGTest.run(mMockInvocationListener); 345 String testOutput = new String(mFakeReceiver.getReceivedOutput(), StandardCharsets.UTF_8); 346 assertEquals("TEST FAILED\n", testOutput); 347 } 348 349 /* Test that if the test module exits with a non-zero code other than 1, an exception is thrown 350 * and the run stops. */ 351 @Test testAbnormalTestCmdExitHandled()352 public void testAbnormalTestCmdExitHandled() throws Exception { 353 String moduleName = "hello_world_test"; 354 String hostLinkedFolderName = "hosttestcases"; 355 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 356 Path errorScriptPath = Paths.get(hostLinkedFolder.getAbsolutePath(), moduleName); 357 createExecutableFile(errorScriptPath, "echo 'TEST BLOWING UP'; exit 2"); 358 359 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 360 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 361 mHostGTest.setBuild(buildInfo); 362 363 mSetter.setOptionValue("module-name", moduleName); 364 365 try { 366 mHostGTest.run(mMockInvocationListener); 367 fail("Didn't throw RuntimeException for test cmd with bad exit code"); 368 } catch (RuntimeException e) { 369 // Expected exception 370 } 371 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 372 } 373 374 @Test testBothStdoutAndStderrCollected()375 public void testBothStdoutAndStderrCollected() throws Exception { 376 String moduleName = "hello_world_test"; 377 String hostLinkedFolderName = "hosttestcases"; 378 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 379 File hostTestcaseExecutedCheckFile = 380 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 381 382 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 383 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 384 mHostGTest.setBuild(buildInfo); 385 386 mSetter.setOptionValue("module-name", moduleName); 387 mHostGTest.run(mMockInvocationListener); 388 assertTrue(hostTestcaseExecutedCheckFile.exists()); 389 390 String expected = String.format("stdout: %s\nstderr: %s\n", moduleName, moduleName); 391 String testOutput = new String(mFakeReceiver.getReceivedOutput(), StandardCharsets.UTF_8); 392 assertEquals(expected, testOutput); 393 } 394 } 395