1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.command; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider; 20 import com.android.compatibility.common.tradefed.result.SubPlanHelper; 21 import com.android.compatibility.common.tradefed.result.suite.CertificationResultXml; 22 import com.android.compatibility.common.tradefed.testtype.ModuleRepo; 23 import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite; 24 import com.android.compatibility.common.util.ResultHandler; 25 import com.android.tradefed.build.BuildRetrievalError; 26 import com.android.tradefed.build.IBuildInfo; 27 import com.android.tradefed.command.Console; 28 import com.android.tradefed.config.ArgsOptionParser; 29 import com.android.tradefed.config.ConfigurationException; 30 import com.android.tradefed.config.ConfigurationFactory; 31 import com.android.tradefed.config.IConfiguration; 32 import com.android.tradefed.config.IConfigurationFactory; 33 import com.android.tradefed.device.DeviceNotAvailableException; 34 import com.android.tradefed.device.ITestDevice; 35 import com.android.tradefed.log.LogUtil.CLog; 36 import com.android.tradefed.result.suite.SuiteResultHolder; 37 import com.android.tradefed.testtype.Abi; 38 import com.android.tradefed.testtype.IAbi; 39 import com.android.tradefed.testtype.IRemoteTest; 40 import com.android.tradefed.testtype.IRuntimeHintProvider; 41 import com.android.tradefed.testtype.suite.TestSuiteInfo; 42 import com.android.tradefed.util.AbiUtils; 43 import com.android.tradefed.util.FileUtil; 44 import com.android.tradefed.util.MultiMap; 45 import com.android.tradefed.util.Pair; 46 import com.android.tradefed.util.RegexTrie; 47 import com.android.tradefed.util.TableFormatter; 48 import com.android.tradefed.util.TimeUtil; 49 import com.android.tradefed.util.VersionParser; 50 51 import com.google.common.base.Joiner; 52 53 import java.io.File; 54 import java.io.FileNotFoundException; 55 import java.io.IOException; 56 import java.io.PrintWriter; 57 import java.util.ArrayList; 58 import java.util.Arrays; 59 import java.util.Collections; 60 import java.util.Comparator; 61 import java.util.HashSet; 62 import java.util.Iterator; 63 import java.util.LinkedHashMap; 64 import java.util.LinkedHashSet; 65 import java.util.List; 66 import java.util.Map; 67 import java.util.Set; 68 69 /** 70 * An extension of Tradefed's console which adds features specific to compatibility testing. 71 */ 72 public class CompatibilityConsole extends Console { 73 74 /** 75 * Hard coded list of modules to be excluded from manual module sharding 76 * @see #splitModules(int) 77 */ 78 private final static Set<String> MODULE_SPLIT_EXCLUSIONS = new HashSet<>(); 79 static { 80 MODULE_SPLIT_EXCLUSIONS.add("CtsDeqpTestCases"); 81 } 82 private final static String ADD_PATTERN = "a(?:dd)?"; 83 private static final String LATEST_RESULT_DIR = "latest"; 84 private CompatibilityBuildHelper mBuildHelper; 85 private IBuildInfo mBuildInfo; 86 87 /** 88 * {@inheritDoc} 89 */ 90 @Override run()91 public void run() { 92 String buildNumber = TestSuiteInfo.getInstance().getBuildNumber(); 93 String versionFile = VersionParser.fetchVersion(); 94 if (versionFile != null) { 95 buildNumber = versionFile; 96 } 97 printLine( 98 String.format( 99 "Android %s %s (%s)", 100 TestSuiteInfo.getInstance().getFullName(), 101 TestSuiteInfo.getInstance().getVersion(), 102 buildNumber)); 103 printLine("Use \"help\" or \"help all\" to get more information on running commands."); 104 super.run(); 105 } 106 107 /** 108 * Adds the 'list plans', 'list modules' and 'list results' commands 109 */ 110 @Override setCustomCommands(RegexTrie<Runnable> trie, List<String> genericHelp, Map<String, String> commandHelp)111 protected void setCustomCommands(RegexTrie<Runnable> trie, List<String> genericHelp, 112 Map<String, String> commandHelp) { 113 114 genericHelp.add("Enter 'help add' for help with 'add subplan' commands"); 115 genericHelp.add("\t----- " + TestSuiteInfo.getInstance().getFullName() + " usage ----- "); 116 genericHelp.add("Usage: run <plan> [--module <module name>] [ options ]"); 117 genericHelp.add("Example: run cts --module CtsGestureTestCases --bugreport-on-failure"); 118 genericHelp.add(""); 119 120 trie.put(new Runnable() { 121 @Override 122 public void run() { 123 listPlans(); 124 } 125 }, LIST_PATTERN, "p(?:lans)?"); 126 trie.put(new Runnable() { 127 @Override 128 public void run() { 129 listModules(); 130 } 131 }, LIST_PATTERN, "m(?:odules)?"); 132 trie.put(new Runnable() { 133 @Override 134 public void run() { 135 listResults(); 136 } 137 }, LIST_PATTERN, "r(?:esults)?"); 138 trie.put(new Runnable() { 139 @Override 140 public void run() { 141 listSubPlans(); 142 } 143 }, LIST_PATTERN, "s(?:ubplans)?"); 144 trie.put(new ArgRunnable<CaptureList>() { 145 @Override 146 public void run(CaptureList args) { 147 // Skip 2 tokens to get past split and modules pattern 148 String arg = args.get(2).get(0); 149 int shards = Integer.parseInt(arg); 150 if (shards <= 1) { 151 printLine("number of shards should be more than 1"); 152 return; 153 } 154 splitModules(shards); 155 } 156 }, "split", "m(?:odules)?", "(\\d+)"); 157 trie.put(new ArgRunnable<CaptureList>() { 158 @Override 159 public void run(CaptureList args) { 160 // Skip 2 tokens to get past "add" and "subplan" 161 String[] flatArgs = new String[args.size() - 2]; 162 for (int i = 2; i < args.size(); i++) { 163 flatArgs[i - 2] = args.get(i).get(0); 164 } 165 addSubPlan(flatArgs); 166 } 167 }, ADD_PATTERN, "s(?:ubplan)?", null); 168 trie.put(new ArgRunnable<CaptureList>() { 169 @Override 170 public void run(CaptureList args) { 171 printLine("'add subplan' requires parameters, use 'help add' to get more details"); 172 } 173 }, ADD_PATTERN, "s(?:ubplan)?"); 174 trie.put(new Runnable() { 175 @Override 176 public void run() { 177 printLine(String.format("Android %s %s (%s)", 178 TestSuiteInfo.getInstance().getFullName(), 179 TestSuiteInfo.getInstance().getVersion(), 180 TestSuiteInfo.getInstance().getBuildNumber())); 181 } 182 }, "version"); // override tradefed 'version' command to print test suite name and version 183 184 // find existing help for 'LIST_PATTERN' commands, and append these commands help 185 String listHelp = commandHelp.get(LIST_PATTERN); 186 if (listHelp == null) { 187 // no help? Unexpected, but soldier on 188 listHelp = new String(); 189 } 190 String combinedHelp = listHelp + 191 LINE_SEPARATOR + 192 "\t----- " + TestSuiteInfo.getInstance().getFullName() 193 + " specific options ----- " + LINE_SEPARATOR + 194 "\tp[lans] List all plans available" + LINE_SEPARATOR + 195 "\tm[odules] List all modules available" + LINE_SEPARATOR + 196 "\tr[esults] List all results" + LINE_SEPARATOR; 197 commandHelp.put(LIST_PATTERN, combinedHelp); 198 199 // Update existing RUN_PATTERN with CTS specific extra run possibilities. 200 String runHelp = commandHelp.get(RUN_PATTERN); 201 if (runHelp == null) { 202 runHelp = new String(); 203 } 204 String combinedRunHelp = runHelp + 205 LINE_SEPARATOR + 206 "\t----- " + TestSuiteInfo.getInstance().getFullName() 207 + " specific options ----- " + LINE_SEPARATOR + 208 "\t<plan> --module/-m <module> Run a test module" + LINE_SEPARATOR + 209 "\t<plan> --module/-m <module> --test/-t <test_name> Run a specific test from" + 210 " the module. Test name can be <package>.<class>, <package>.<class>#<method> or " 211 + "<native_binary_name>" + LINE_SEPARATOR + 212 "\t\tAvailable Options:" + LINE_SEPARATOR + 213 "\t\t\t--serial/-s <device_id>: The device to run the test on" + LINE_SEPARATOR + 214 "\t\t\t--abi/-a <abi> : The ABI to run the test against" + LINE_SEPARATOR + 215 "\t\t\t--logcat-on-failure : Capture logcat when a test fails" 216 + LINE_SEPARATOR + 217 "\t\t\t--bugreport-on-failure : Capture a bugreport when a test fails" 218 + LINE_SEPARATOR + 219 "\t\t\t--screenshot-on-failure: Capture a screenshot when a test fails" 220 + LINE_SEPARATOR + 221 "\t\t\t--shard-count <shards>: Shards a run into the given number of independent " + 222 "chunks, to run on multiple devices in parallel." + LINE_SEPARATOR + 223 "\t ----- In order to retry a previous run -----" + LINE_SEPARATOR + 224 "\tretry --retry <session id to retry> [--retry-type <FAILED | NOT_EXECUTED>]" 225 + LINE_SEPARATOR + 226 "\t\tWithout --retry-type, retry will run both FAIL and NOT_EXECUTED tests" 227 + LINE_SEPARATOR; 228 commandHelp.put(RUN_PATTERN, combinedRunHelp); 229 230 commandHelp.put(ADD_PATTERN, String.format( 231 "%s help:" + LINE_SEPARATOR + 232 "\tadd s[ubplan]: create a subplan from a previous session" + LINE_SEPARATOR + 233 "\t\tAvailable Options:" + LINE_SEPARATOR + 234 "\t\t\t--session <session_id>: The session used to create a subplan" 235 + LINE_SEPARATOR + 236 "\t\t\t--name/-n <subplan_name>: The name of the new subplan" + LINE_SEPARATOR + 237 "\t\t\t--result-type <status>: Which results to include in the subplan. " 238 + "One of passed, failed, not_executed. Repeatable" + LINE_SEPARATOR, 239 ADD_PATTERN)); 240 } 241 242 /** 243 * {@inheritDoc} 244 */ 245 @Override getConsolePrompt()246 protected String getConsolePrompt() { 247 return String.format("%s-tf > ", TestSuiteInfo.getInstance().getName().toLowerCase()); 248 } 249 listModules()250 private void listModules() { 251 CompatibilityTestSuite test = new CompatibilityTestSuite() { 252 @Override 253 public Set<IAbi> getAbis(ITestDevice device) throws DeviceNotAvailableException { 254 Set<String> abiStrings = getAbisForBuildTargetArch(); 255 Set<IAbi> abis = new LinkedHashSet<>(); 256 for (String abi : abiStrings) { 257 if (AbiUtils.isAbiSupportedByCompatibility(abi)) { 258 abis.add(new Abi(abi, AbiUtils.getBitness(abi))); 259 } 260 } 261 return abis; 262 } 263 }; 264 if (getBuild() != null) { 265 test.setBuild(getBuild()); 266 LinkedHashMap<String, IConfiguration> configs = test.loadTests(); 267 printLine(String.format("%s", Joiner.on("\n").join(configs.keySet()))); 268 } else { 269 printLine("Error fetching information about modules."); 270 } 271 } 272 listPlans()273 private void listPlans() { 274 printLine("Available plans include:"); 275 ConfigurationFactory.getInstance().printHelp(System.out); 276 } 277 splitModules(int shards)278 private void splitModules(int shards) { 279 File[] files = null; 280 try { 281 files = getBuildHelper().getTestsDir().listFiles(new ModuleRepo.ConfigFilter()); 282 } catch (FileNotFoundException e) { 283 printLine(e.getMessage()); 284 e.printStackTrace(); 285 } 286 // parse through all config files to get runtime hints 287 if (files != null && files.length > 0) { 288 IConfigurationFactory configFactory = ConfigurationFactory.getInstance(); 289 List<Pair<String, Long>> moduleRuntime = new ArrayList<>(); 290 // parse through all config files to calculate module execution time 291 for (File file : files) { 292 IConfiguration config = null; 293 String moduleName = file.getName().split("\\.")[0]; 294 if (MODULE_SPLIT_EXCLUSIONS.contains(moduleName)) { 295 continue; 296 } 297 try { 298 config = configFactory.createConfigurationFromArgs(new String[]{ 299 file.getAbsolutePath(), 300 }); 301 } catch (ConfigurationException ce) { 302 printLine("Error loading config file: " + file.getAbsolutePath()); 303 CLog.e(ce); 304 continue; 305 } 306 long runtime = 0; 307 for (IRemoteTest test : config.getTests()) { 308 if (test instanceof IRuntimeHintProvider) { 309 runtime += ((IRuntimeHintProvider) test).getRuntimeHint(); 310 } else { 311 CLog.w("Using default 1m runtime estimation for test type %s", 312 test.getClass().getSimpleName()); 313 runtime += 60 * 1000; 314 } 315 } 316 moduleRuntime.add(new Pair<String, Long>(moduleName, runtime)); 317 } 318 // sort list modules in descending order of runtime hint 319 Collections.sort(moduleRuntime, new Comparator<Pair<String, Long>>() { 320 @Override 321 public int compare(Pair<String, Long> o1, Pair<String, Long> o2) { 322 return o2.second.compareTo(o1.second); 323 } 324 }); 325 // partition list of modules based on the runtime hint 326 List<List<Pair<String, Long>>> splittedModules = new ArrayList<>(); 327 for (int i = 0; i < shards; i++) { 328 splittedModules.add(new ArrayList<>()); 329 } 330 int shardIndex = 0; 331 int increment = 1; 332 long[] shardTimes = new long[shards]; 333 // go through the sorted list, distribute modules into shards in zig-zag pattern to get 334 // an even execution time among shards 335 for (Pair<String, Long> module : moduleRuntime) { 336 splittedModules.get(shardIndex).add(module); 337 // also collect total runtime per shard 338 shardTimes[shardIndex] += module.second; 339 shardIndex += increment; 340 // zig-zagging: first distribute modules from shard 0 to N, then N down to 0, repeat 341 if (shardIndex == shards) { 342 increment = -1; 343 shardIndex = shards - 1; 344 } 345 if (shardIndex == -1) { 346 increment = 1; 347 shardIndex = 0; 348 } 349 } 350 shardIndex = 0; 351 // print the final shared lists 352 for (List<Pair<String, Long>> shardedModules : splittedModules) { 353 StringBuilder lineBuffer = new StringBuilder(); 354 lineBuffer.append(String.format("shard #%d (%s):", 355 shardIndex, TimeUtil.formatElapsedTime(shardTimes[shardIndex]))); 356 Iterator<Pair<String, Long>> itr = shardedModules.iterator(); 357 lineBuffer.append(itr.next().first); 358 while (itr.hasNext()) { 359 lineBuffer.append(','); 360 lineBuffer.append(itr.next().first); 361 } 362 shardIndex++; 363 printLine(lineBuffer.toString()); 364 } 365 } else { 366 printLine("No modules found"); 367 } 368 } 369 listResults()370 private void listResults() { 371 TableFormatter tableFormatter = new TableFormatter(); 372 List<List<String>> table = new ArrayList<>(); 373 374 List<File> resultDirs = null; 375 Map<SuiteResultHolder, File> holders = new LinkedHashMap<>(); 376 try { 377 resultDirs = getResults(getBuildHelper().getResultsDir()); 378 } catch (FileNotFoundException e) { 379 throw new RuntimeException("Error while parsing results directory", e); 380 } 381 CertificationResultXml xmlParser = new CertificationResultXml(); 382 for (File resultDir : resultDirs) { 383 if (LATEST_RESULT_DIR.equals(resultDir.getName())) { 384 continue; 385 } 386 try { 387 holders.put(xmlParser.parseResults(resultDir, true), resultDir); 388 } catch (IOException e) { 389 e.printStackTrace(); 390 } 391 } 392 393 if (holders.isEmpty()) { 394 printLine(String.format("No results found")); 395 return; 396 } 397 int i = 0; 398 for (SuiteResultHolder holder : holders.keySet()) { 399 String moduleProgress = String.format("%d of %d", 400 holder.completeModules, holder.totalModules); 401 402 table.add( 403 Arrays.asList( 404 Integer.toString(i), 405 Long.toString(holder.passedTests), 406 Long.toString(holder.failedTests), 407 moduleProgress, 408 holders.get(holder).getName(), 409 holder.context 410 .getAttributes() 411 .get(CertificationResultXml.SUITE_PLAN_ATTR) 412 .get(0), 413 Joiner.on(", ").join(holder.context.getShardsSerials().values()), 414 printAttributes(holder.context.getAttributes(), "build_id"), 415 printAttributes(holder.context.getAttributes(), "build_product"))); 416 i++; 417 } 418 419 // add the table header to the beginning of the list 420 table.add(0, Arrays.asList("Session", "Pass", "Fail", "Modules Complete", 421 "Result Directory", "Test Plan", "Device serial(s)", "Build ID", "Product")); 422 tableFormatter.displayTable(table, new PrintWriter(System.out, true)); 423 } 424 printAttributes(MultiMap<String, String> map, String key)425 private String printAttributes(MultiMap<String, String> map, String key) { 426 if (map.get(key) == null) { 427 return "unknown"; 428 } 429 return map.get(key).get(0); 430 } 431 432 /** 433 * Returns the list of all results directories. 434 */ getResults(File resultsDir)435 private List<File> getResults(File resultsDir) { 436 return ResultHandler.getResultDirectories(resultsDir); 437 } 438 listSubPlans()439 private void listSubPlans() { 440 File[] files = null; 441 try { 442 files = getBuildHelper().getSubPlansDir().listFiles(); 443 } catch (FileNotFoundException e) { 444 printLine(e.getMessage()); 445 e.printStackTrace(); 446 } 447 if (files != null && files.length > 0) { 448 List<String> subPlans = new ArrayList<>(); 449 for (File subPlanFile : files) { 450 subPlans.add(FileUtil.getBaseName(subPlanFile.getName())); 451 } 452 Collections.sort(subPlans); 453 for (String subPlan : subPlans) { 454 printLine(subPlan); 455 } 456 } else { 457 printLine("No subplans found"); 458 } 459 } 460 addSubPlan(String[] flatArgs)461 private void addSubPlan(String[] flatArgs) { 462 SubPlanHelper creator = new SubPlanHelper(); 463 try { 464 ArgsOptionParser optionParser = new ArgsOptionParser(creator); 465 optionParser.parse(Arrays.asList(flatArgs)); 466 creator.createAndSerializeSubPlan(getBuildHelper()); 467 } catch (ConfigurationException e) { 468 printLine("Error: " + e.getMessage()); 469 printLine(ArgsOptionParser.getOptionHelp(false, creator)); 470 } 471 472 } 473 getBuildHelper()474 private CompatibilityBuildHelper getBuildHelper() { 475 if (mBuildHelper == null) { 476 IBuildInfo build = getBuild(); 477 if (build == null) { 478 return null; 479 } 480 mBuildHelper = new CompatibilityBuildHelper(build); 481 } 482 return mBuildHelper; 483 } 484 getBuild()485 private IBuildInfo getBuild() { 486 if (mBuildInfo == null) { 487 try { 488 CompatibilityBuildProvider buildProvider = new CompatibilityBuildProvider(); 489 mBuildInfo = buildProvider.getBuild(); 490 } catch (BuildRetrievalError e) { 491 e.printStackTrace(); 492 } 493 } 494 return mBuildInfo; 495 } 496 main(String[] args)497 public static void main(String[] args) throws InterruptedException, ConfigurationException { 498 Console console = new CompatibilityConsole(); 499 Console.startConsole(console, args); 500 } 501 } 502