1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import com.android.tradefed.result.ITestLifeCycleReceiver; 19 import com.android.tradefed.result.TestDescription; 20 import com.android.tradefed.util.StreamUtil; 21 22 import org.easymock.EasyMock; 23 import org.json.JSONException; 24 import org.json.JSONObject; 25 import org.junit.Assert; 26 import org.junit.Test; 27 import org.junit.runner.RunWith; 28 import org.junit.runners.JUnit4; 29 30 import java.io.IOException; 31 import java.io.InputStream; 32 import java.text.ParseException; 33 import java.text.SimpleDateFormat; 34 import java.util.Collections; 35 import java.util.Date; 36 37 /** 38 * Unit tests for {@link VtsMultiDeviceTestResultParser}. 39 */ 40 @RunWith(JUnit4.class) 41 public class VtsMultiDeviceTestResultParserTest { 42 // Test file paths in the resources 43 private static final String OUTPUT_FILE_1 = "/testtype/vts_multi_device_test_parser_output.txt"; 44 private static final String OUTPUT_FILE_2 = 45 "/testtype/vts_multi_device_test_parser_output_timeout.txt"; 46 private static final String OUTPUT_FILE_3 = 47 "/testtype/vts_multi_device_test_parser_output_error.txt"; 48 private static final String SUMMARY_FILE_NORMAL = "/testtype/test_run_summary_normal.json"; 49 private static final String SUMMARY_FILE_CLASS_ERRORS = 50 "/testtype/test_run_summary_class_errors.json"; 51 52 // test name 53 private static final String RUN_NAME = "SampleLightFuzzTest"; 54 private static final String TEST_NAME_1 = "testTurnOnLightBlackBoxFuzzing"; 55 private static final String TEST_NAME_2 = "testTurnOnLightWhiteBoxFuzzing"; 56 57 // error messages in the summary files. 58 private static final String FAILURE_MESSAGE = "unit test"; 59 private static final String CLASS_ERROR_MESSAGE = "class error"; 60 61 /** 62 * Test the run method with a normal input. 63 */ 64 @Test testRunTimeoutInput()65 public void testRunTimeoutInput() throws IOException { 66 String[] contents = getOutput(OUTPUT_FILE_2); 67 long totalTime = getTotalTime(contents); 68 69 // prepare the mock object 70 ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class); 71 mockRunListener.testRunStarted(TEST_NAME_2, 2); 72 TestDescription test1 = new TestDescription(RUN_NAME, TEST_NAME_2); 73 mockRunListener.testStarted(test1); 74 mockRunListener.testFailed(test1, "TIMEOUT"); 75 mockRunListener.testEnded(test1, Collections.emptyMap()); 76 77 TestDescription test2 = new TestDescription(RUN_NAME, TEST_NAME_1); 78 mockRunListener.testStarted(test2); 79 mockRunListener.testEnded(test2, Collections.emptyMap()); 80 mockRunListener.testRunEnded(totalTime, Collections.emptyMap()); 81 82 EasyMock.replay(mockRunListener); 83 VtsMultiDeviceTestResultParser resultParser = 84 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME); 85 resultParser.processNewLines(contents); 86 resultParser.done(); 87 EasyMock.verify(mockRunListener); 88 } 89 90 /** 91 * Test the run method with a normal input. 92 */ 93 @Test testRunNormalInput()94 public void testRunNormalInput() throws IOException { 95 String[] contents = getOutput(OUTPUT_FILE_1); 96 long totalTime = getTotalTime(contents); 97 98 // prepare the mock object 99 ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class); 100 mockRunListener.testRunStarted(TEST_NAME_2, 2); 101 TestDescription test1 = new TestDescription(RUN_NAME, TEST_NAME_2); 102 mockRunListener.testStarted(test1); 103 mockRunListener.testEnded(test1, Collections.emptyMap()); 104 105 TestDescription test2 = new TestDescription(RUN_NAME, TEST_NAME_1); 106 mockRunListener.testStarted(test2); 107 mockRunListener.testEnded(test2, Collections.emptyMap()); 108 mockRunListener.testRunEnded(totalTime, Collections.emptyMap()); 109 110 EasyMock.replay(mockRunListener); 111 VtsMultiDeviceTestResultParser resultParser = 112 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME); 113 resultParser.processNewLines(contents); 114 resultParser.done(); 115 EasyMock.verify(mockRunListener); 116 } 117 118 /** 119 * Test the run method with a erroneous input. 120 */ 121 @Test testRunErrorInput()122 public void testRunErrorInput() throws IOException { 123 String[] contents = getOutput(OUTPUT_FILE_3); 124 long totalTime = getTotalTime(contents); 125 126 // prepare the mock object 127 ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class); 128 mockRunListener.testRunStarted(null, 0); 129 mockRunListener.testRunEnded(totalTime, Collections.<String, String>emptyMap()); 130 131 EasyMock.replay(mockRunListener); 132 VtsMultiDeviceTestResultParser resultParser = 133 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME); 134 resultParser.processNewLines(contents); 135 resultParser.done(); 136 EasyMock.verify(mockRunListener); 137 } 138 /** 139 * @param contents The logs that are used for a test case. 140 * @return {long} total running time of the test. 141 */ getTotalTime(String[] contents)142 private long getTotalTime(String[] contents) { 143 Date startDate = getDate(contents, true); 144 Date endDate = getDate(contents, false); 145 146 if (startDate == null || endDate == null) { 147 return 0; 148 } 149 return endDate.getTime() - startDate.getTime(); 150 } 151 152 /** 153 * Read a resource file as a string. 154 * 155 * @param resource the name of the resource. 156 * @return {String} the contents of the resource. 157 * @throws IOException if fails to read. 158 */ getResourceAsString(String resource)159 private String getResourceAsString(String resource) throws IOException { 160 InputStream input = getClass().getResourceAsStream(resource); 161 Assert.assertNotNull("Cannot load " + resource, input); 162 try { 163 return StreamUtil.getStringFromStream(input); 164 } finally { 165 input.close(); 166 } 167 } 168 169 /** 170 * Returns the sample shell output for a test command. 171 * 172 * @return {String} shell output 173 * @throws IOException 174 */ getOutput(String filePath)175 private String[] getOutput(String filePath) throws IOException { 176 return getResourceAsString(filePath).split("\n"); 177 } 178 179 /** 180 * Return the time in milliseconds to calculate the time elapsed in a particular test. 181 * 182 * @param lines The logs that need to be parsed. 183 * @param calculateStartDate flag which is true if we need to calculate start date. 184 * @return {Date} the start and end time corresponding to a test. 185 */ getDate(String[] lines, boolean calculateStartDate)186 private Date getDate(String[] lines, boolean calculateStartDate) { 187 Date date = null; 188 int begin = calculateStartDate ? 0 : lines.length - 1; 189 int diff = calculateStartDate ? 1 : -1; 190 191 for (int index = begin; index >= 0 && index < lines.length; index += diff) { 192 lines[index].trim(); 193 String[] toks = lines[index].split(" "); 194 195 // set the start time from the first line 196 // the loop should continue if exception occurs, else it can break 197 if (toks.length < 3) { 198 continue; 199 } 200 String time = toks[2]; 201 SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss.SSS"); 202 try { 203 date = sdf.parse(time); 204 } catch (ParseException e) { 205 continue; 206 } 207 break; 208 } 209 return date; 210 } 211 212 /* 213 * Test parsing a summary containing passing and failing records. 214 */ 215 @Test testNormalSummary()216 public void testNormalSummary() throws IOException, JSONException { 217 JSONObject object = new JSONObject(getResourceAsString(SUMMARY_FILE_NORMAL)); 218 219 ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class); 220 mockRunListener.testRunStarted(RUN_NAME, 2); 221 TestDescription test1 = new TestDescription(RUN_NAME, TEST_NAME_1); 222 mockRunListener.testStarted(test1, 1525425222367l); 223 mockRunListener.testEnded(test1, 1525425223793l, Collections.emptyMap()); 224 225 TestDescription test2 = new TestDescription(RUN_NAME, TEST_NAME_2); 226 mockRunListener.testStarted(test2, 1525425749536l); 227 mockRunListener.testFailed(test2, FAILURE_MESSAGE); 228 mockRunListener.testEnded(test2, 1525425749537l, Collections.emptyMap()); 229 mockRunListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(Collections.emptyMap())); 230 231 EasyMock.replay(mockRunListener); 232 VtsMultiDeviceTestResultParser resultParser = 233 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME); 234 resultParser.processJsonFile(object); 235 } 236 237 /* 238 * Test parsing a summary containing class error message. 239 */ 240 @Test testClassErrorSummary()241 public void testClassErrorSummary() throws IOException, JSONException { 242 JSONObject object = new JSONObject(getResourceAsString(SUMMARY_FILE_CLASS_ERRORS)); 243 244 ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class); 245 mockRunListener.testRunStarted(RUN_NAME, 1); 246 TestDescription test1 = new TestDescription(RUN_NAME, TEST_NAME_1); 247 mockRunListener.testStarted(test1, 1525424790227l); 248 mockRunListener.testFailed(test1, FAILURE_MESSAGE); 249 mockRunListener.testEnded(test1, 1525424790227l, Collections.emptyMap()); 250 mockRunListener.testRunFailed(CLASS_ERROR_MESSAGE); 251 mockRunListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(Collections.emptyMap())); 252 253 EasyMock.replay(mockRunListener); 254 VtsMultiDeviceTestResultParser resultParser = 255 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME); 256 resultParser.processJsonFile(object); 257 } 258 } 259