1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.invoker; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.build.BuildRetrievalError; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.build.StubBuildProvider; 22 import com.android.tradefed.clearcut.ClearcutClient; 23 import com.android.tradefed.command.CommandOptions; 24 import com.android.tradefed.command.CommandRunner; 25 import com.android.tradefed.config.GlobalConfiguration; 26 import com.android.tradefed.config.IConfiguration; 27 import com.android.tradefed.config.IDeviceConfiguration; 28 import com.android.tradefed.config.OptionCopier; 29 import com.android.tradefed.device.DeviceNotAvailableException; 30 import com.android.tradefed.device.DeviceSelectionOptions; 31 import com.android.tradefed.device.TestDeviceOptions; 32 import com.android.tradefed.device.cloud.GceAvdInfo; 33 import com.android.tradefed.device.cloud.GceManager; 34 import com.android.tradefed.device.cloud.LaunchCvdHelper; 35 import com.android.tradefed.device.cloud.ManagedRemoteDevice; 36 import com.android.tradefed.device.cloud.MultiUserSetupUtil; 37 import com.android.tradefed.device.cloud.RemoteFileUtil; 38 import com.android.tradefed.log.ITestLogger; 39 import com.android.tradefed.log.LogUtil.CLog; 40 import com.android.tradefed.result.FileInputStreamSource; 41 import com.android.tradefed.result.ITestInvocationListener; 42 import com.android.tradefed.result.InputStreamSource; 43 import com.android.tradefed.result.LogDataType; 44 import com.android.tradefed.result.proto.FileProtoResultReporter; 45 import com.android.tradefed.result.proto.ProtoResultParser; 46 import com.android.tradefed.targetprep.BuildError; 47 import com.android.tradefed.targetprep.TargetSetupError; 48 import com.android.tradefed.util.CommandResult; 49 import com.android.tradefed.util.CommandStatus; 50 import com.android.tradefed.util.FileUtil; 51 import com.android.tradefed.util.IRunUtil; 52 import com.android.tradefed.util.RunUtil; 53 import com.android.tradefed.util.TimeUtil; 54 import com.android.tradefed.util.proto.TestRecordProtoUtil; 55 56 import com.google.common.base.Joiner; 57 import com.google.common.base.Strings; 58 import com.google.protobuf.InvalidProtocolBufferException; 59 60 import java.io.File; 61 import java.io.IOException; 62 import java.io.PrintWriter; 63 import java.util.ArrayList; 64 import java.util.Arrays; 65 import java.util.HashSet; 66 import java.util.List; 67 import java.util.concurrent.Semaphore; 68 69 /** Implementation of {@link InvocationExecution} that drives a remote execution. */ 70 public class RemoteInvocationExecution extends InvocationExecution { 71 72 public static final long PUSH_TF_TIMEOUT = 150000L; 73 public static final long PULL_RESULT_TIMEOUT = 180000L; 74 public static final long REMOTE_PROCESS_RUNNING_WAIT = 15000L; 75 public static final long LAUNCH_EXTRA_DEVICE = 10 * 60 * 1000L; 76 public static final long NEW_USER_TIMEOUT = 5 * 60 * 1000L; 77 public static final String REMOTE_VM_VARIABLE = "REMOTE_VM_ENV"; 78 79 public static final String REMOTE_USER_DIR = "/home/{$USER}/"; 80 public static final String PROTO_RESULT_NAME = "output.pb"; 81 public static final String STDOUT_FILE = "screen-VM_tradefed-stdout.txt"; 82 public static final String STDERR_FILE = "screen-VM_tradefed-stderr.txt"; 83 public static final String REMOTE_CONFIG = "configuration"; 84 public static final String GLOBAL_REMOTE_CONFIG = "global-remote-configuration"; 85 public static final String SHARDING_DEVICE_SETUP_TIME = "sharding-device-setup-ms"; 86 87 private static final int MAX_CONNECTION_REFUSED_COUNT = 3; 88 private static final int MAX_PUSH_TF_ATTEMPTS = 3; 89 private static final int MAX_WORKER_THREAD = 3; 90 91 private String mRemoteTradefedDir = null; 92 private String mRemoteAdbPath = null; 93 94 @Override fetchBuild( IInvocationContext context, IConfiguration config, IRescheduler rescheduler, ITestInvocationListener listener)95 public boolean fetchBuild( 96 IInvocationContext context, 97 IConfiguration config, 98 IRescheduler rescheduler, 99 ITestInvocationListener listener) 100 throws DeviceNotAvailableException, BuildRetrievalError { 101 // TODO: handle multiple devices/build config 102 updateInvocationContext(context, config); 103 StubBuildProvider stubProvider = new StubBuildProvider(); 104 105 String deviceName = config.getDeviceConfig().get(0).getDeviceName(); 106 OptionCopier.copyOptionsNoThrow( 107 config.getDeviceConfig().get(0).getBuildProvider(), stubProvider); 108 109 IBuildInfo info = stubProvider.getBuild(); 110 if (info == null) { 111 return false; 112 } 113 context.addDeviceBuildInfo(deviceName, info); 114 updateBuild(info, config); 115 return true; 116 } 117 118 @Override runTests( IInvocationContext context, IConfiguration config, ITestInvocationListener listener)119 public void runTests( 120 IInvocationContext context, IConfiguration config, ITestInvocationListener listener) 121 throws Throwable { 122 ManagedRemoteDevice device = (ManagedRemoteDevice) context.getDevices().get(0); 123 GceAvdInfo info = device.getRemoteAvdInfo(); 124 125 // Run remote TF (new tests?) 126 IRunUtil runUtil = new RunUtil(); 127 128 TestDeviceOptions options = device.getOptions(); 129 String mainRemoteDir = getRemoteMainDir(options); 130 // Handle sharding 131 if (config.getCommandOptions().getShardCount() != null 132 && config.getCommandOptions().getShardIndex() == null) { 133 if (config.getCommandOptions().getShardCount() > 1) { 134 boolean parallel = config.getCommandOptions().shouldUseParallelRemoteSetup(); 135 long startTime = System.currentTimeMillis(); 136 // For each device after the first one we need to start a new device. 137 if (!parallel) { 138 for (int i = 2; i < config.getCommandOptions().getShardCount() + 1; i++) { 139 boolean res = startDevice(listener, i, info, options, runUtil, null); 140 if (!res) { 141 return; 142 } 143 } 144 } else { 145 // Parallel setup of devices 146 Semaphore token = new Semaphore(MAX_WORKER_THREAD); 147 List<StartDeviceThread> threads = new ArrayList<>(); 148 for (int i = 2; i < config.getCommandOptions().getShardCount() + 1; i++) { 149 StartDeviceThread sdt = 150 new StartDeviceThread(listener, i, info, options, runUtil, token); 151 threads.add(sdt); 152 sdt.start(); 153 } 154 155 boolean res = true; 156 for (StartDeviceThread t : threads) { 157 t.join(); 158 res = res & t.getFinalStatus(); 159 } 160 if (!res) { 161 return; 162 } 163 } 164 165 // Log the overhead to start the device 166 long elapsedTime = System.currentTimeMillis() - startTime; 167 context.getBuildInfos() 168 .get(0) 169 .addBuildAttribute(SHARDING_DEVICE_SETUP_TIME, Long.toString(elapsedTime)); 170 } 171 } 172 173 mRemoteAdbPath = String.format("/home/%s/bin/adb", options.getInstanceUser()); 174 175 String tfPath = System.getProperty("TF_JAR_DIR"); 176 if (tfPath == null) { 177 listener.invocationFailed(new RuntimeException("Failed to find $TF_JAR_DIR.")); 178 return; 179 } 180 File currentTf = new File(tfPath).getAbsoluteFile(); 181 if (tfPath.equals(".")) { 182 currentTf = new File("").getAbsoluteFile(); 183 } 184 mRemoteTradefedDir = mainRemoteDir + "tradefed/"; 185 CommandResult createRemoteDir = 186 GceManager.remoteSshCommandExecution( 187 info, options, runUtil, 120000L, "mkdir", "-p", mRemoteTradefedDir); 188 if (!CommandStatus.SUCCESS.equals(createRemoteDir.getStatus())) { 189 listener.invocationFailed(new RuntimeException("Failed to create remote dir.")); 190 return; 191 } 192 193 // Push Tradefed to the remote 194 int attempt = 0; 195 boolean result = false; 196 while (!result && attempt < MAX_PUSH_TF_ATTEMPTS) { 197 result = 198 RemoteFileUtil.pushFileToRemote( 199 info, 200 options, 201 Arrays.asList("-r"), 202 runUtil, 203 PUSH_TF_TIMEOUT, 204 mRemoteTradefedDir, 205 currentTf); 206 attempt++; 207 } 208 if (!result) { 209 CLog.e("Failed to push Tradefed."); 210 listener.invocationFailed(new RuntimeException("Failed to push Tradefed.")); 211 return; 212 } 213 214 mRemoteTradefedDir = mRemoteTradefedDir + currentTf.getName() + "/"; 215 CommandResult listRemoteDir = 216 GceManager.remoteSshCommandExecution( 217 info, options, runUtil, 120000L, "ls", "-l", mRemoteTradefedDir); 218 CLog.d("stdout: %s", listRemoteDir.getStdout()); 219 CLog.d("stderr: %s", listRemoteDir.getStderr()); 220 221 File configFile = createRemoteConfig(config, listener, mRemoteTradefedDir); 222 File globalConfig = null; 223 try { 224 CLog.d("Pushing Tradefed XML configuration to remote."); 225 boolean resultPush = 226 RemoteFileUtil.pushFileToRemote( 227 info, 228 options, 229 null, 230 runUtil, 231 PUSH_TF_TIMEOUT, 232 mRemoteTradefedDir, 233 configFile); 234 if (!resultPush) { 235 CLog.e("Failed to push Tradefed Configuration."); 236 listener.invocationFailed( 237 new RuntimeException("Failed to push Tradefed Configuration.")); 238 return; 239 } 240 241 String[] whitelistConfigs = 242 new String[] { 243 GlobalConfiguration.SCHEDULER_TYPE_NAME, 244 GlobalConfiguration.HOST_OPTIONS_TYPE_NAME, 245 "android-build" 246 }; 247 try { 248 globalConfig = 249 GlobalConfiguration.getInstance() 250 .cloneConfigWithFilter(new HashSet<>(), whitelistConfigs); 251 } catch (IOException e) { 252 listener.invocationFailed(e); 253 return; 254 } 255 try (InputStreamSource source = new FileInputStreamSource(globalConfig)) { 256 listener.testLog(GLOBAL_REMOTE_CONFIG, LogDataType.XML, source); 257 } 258 // Push the global configuration 259 boolean resultPushGlobal = 260 RemoteFileUtil.pushFileToRemote( 261 info, 262 options, 263 null, 264 runUtil, 265 PUSH_TF_TIMEOUT, 266 mRemoteTradefedDir, 267 globalConfig); 268 if (!resultPushGlobal) { 269 CLog.e("Failed to push Tradefed Global Configuration."); 270 listener.invocationFailed( 271 new RuntimeException("Failed to push Tradefed Global Configuration.")); 272 return; 273 } 274 275 resetAdb(info, options, runUtil); 276 runRemote(listener, context, configFile, info, options, runUtil, config, globalConfig); 277 collectAdbLogs(info, options, runUtil, listener); 278 } finally { 279 FileUtil.recursiveDelete(configFile); 280 FileUtil.recursiveDelete(globalConfig); 281 } 282 } 283 284 @Override doSetup( IInvocationContext context, IConfiguration config, ITestInvocationListener listener)285 public void doSetup( 286 IInvocationContext context, IConfiguration config, ITestInvocationListener listener) 287 throws TargetSetupError, BuildError, DeviceNotAvailableException { 288 // Skip 289 } 290 291 @Override doTeardown( IInvocationContext context, IConfiguration config, ITestLogger logger, Throwable exception)292 public void doTeardown( 293 IInvocationContext context, 294 IConfiguration config, 295 ITestLogger logger, 296 Throwable exception) 297 throws Throwable { 298 // Only run device post invocation teardown 299 super.runDevicePostInvocationTearDown(context, config); 300 } 301 302 @Override doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception)303 public void doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception) { 304 // Skip 305 } 306 307 @Override getAdbVersion()308 protected String getAdbVersion() { 309 // Do not report the adb version from the parent, the remote child will remote its own. 310 return null; 311 } 312 runRemote( ITestInvocationListener currentInvocationListener, IInvocationContext context, File configFile, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, IConfiguration config, File globalConfig)313 private void runRemote( 314 ITestInvocationListener currentInvocationListener, 315 IInvocationContext context, 316 File configFile, 317 GceAvdInfo info, 318 TestDeviceOptions options, 319 IRunUtil runUtil, 320 IConfiguration config, 321 File globalConfig) 322 throws InvalidProtocolBufferException, IOException { 323 List<String> remoteTfCommand = new ArrayList<>(); 324 remoteTfCommand.add("pushd"); 325 remoteTfCommand.add(mRemoteTradefedDir + ";"); 326 remoteTfCommand.add(String.format("PATH=%s:$PATH", new File(mRemoteAdbPath).getParent())); 327 remoteTfCommand.add("screen -dmSU tradefed sh -c"); 328 329 StringBuilder tfCmdBuilder = 330 new StringBuilder("TF_GLOBAL_CONFIG=" + globalConfig.getName()); 331 // Set an env variable to notify that this a remote environment. 332 tfCmdBuilder.append(" " + REMOTE_VM_VARIABLE + "=1"); 333 // Disable clearcut in the remote 334 tfCmdBuilder.append(" " + ClearcutClient.DISABLE_CLEARCUT_KEY + "=1"); 335 tfCmdBuilder.append(" ENTRY_CLASS=" + CommandRunner.class.getCanonicalName()); 336 tfCmdBuilder.append(" ./tradefed.sh " + mRemoteTradefedDir + configFile.getName()); 337 if (config.getCommandOptions().shouldUseRemoteSandboxMode()) { 338 tfCmdBuilder.append(" --" + CommandOptions.USE_SANDBOX); 339 } 340 tfCmdBuilder.append(" > " + STDOUT_FILE + " 2> " + STDERR_FILE); 341 remoteTfCommand.add("\"" + tfCmdBuilder.toString() + "\""); 342 // Kick off the actual remote run 343 CommandResult resultRemoteExecution = 344 GceManager.remoteSshCommandExecution( 345 info, options, runUtil, 0L, remoteTfCommand.toArray(new String[0])); 346 if (!CommandStatus.SUCCESS.equals(resultRemoteExecution.getStatus())) { 347 CLog.e("Error running the remote command: %s", resultRemoteExecution.getStdout()); 348 currentInvocationListener.invocationFailed( 349 new RuntimeException(resultRemoteExecution.getStderr())); 350 return; 351 } 352 // Sleep a bit to let the process start 353 RunUtil.getDefault().sleep(10000L); 354 355 // Monitor the remote invocation to ensure it's completing. Block until timeout or stops 356 // running. 357 boolean stillRunning = 358 isStillRunning( 359 currentInvocationListener, configFile, info, options, runUtil, config); 360 361 // Fetch the logs 362 File stdoutFile = 363 RemoteFileUtil.fetchRemoteFile( 364 info, 365 options, 366 runUtil, 367 PULL_RESULT_TIMEOUT, 368 mRemoteTradefedDir + STDOUT_FILE); 369 if (stdoutFile != null) { 370 try (InputStreamSource source = new FileInputStreamSource(stdoutFile, true)) { 371 currentInvocationListener.testLog(STDOUT_FILE, LogDataType.TEXT, source); 372 } 373 } 374 375 File stderrFile = 376 RemoteFileUtil.fetchRemoteFile( 377 info, 378 options, 379 runUtil, 380 PULL_RESULT_TIMEOUT, 381 mRemoteTradefedDir + STDERR_FILE); 382 if (stderrFile != null) { 383 try (InputStreamSource source = new FileInputStreamSource(stderrFile, true)) { 384 currentInvocationListener.testLog(STDERR_FILE, LogDataType.TEXT, source); 385 } 386 } 387 388 fetchAndProcessResults( 389 stillRunning, 390 currentInvocationListener, 391 context, 392 info, 393 options, 394 runUtil, 395 mRemoteTradefedDir); 396 } 397 isStillRunning( ITestInvocationListener currentInvocationListener, File configFile, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, IConfiguration config)398 private boolean isStillRunning( 399 ITestInvocationListener currentInvocationListener, 400 File configFile, 401 GceAvdInfo info, 402 TestDeviceOptions options, 403 IRunUtil runUtil, 404 IConfiguration config) { 405 long maxTimeout = config.getCommandOptions().getInvocationTimeout(); 406 Long endTime = null; 407 if (maxTimeout > 0L) { 408 endTime = System.currentTimeMillis() + maxTimeout; 409 } 410 boolean stillRunning = true; 411 int errorConnectCount = 0; 412 while (stillRunning) { 413 CommandResult psRes = 414 GceManager.remoteSshCommandExecution( 415 info, 416 options, 417 runUtil, 418 120000L, 419 "ps", 420 "-ef", 421 "| grep", 422 CommandRunner.class.getCanonicalName()); 423 if (!CommandStatus.SUCCESS.equals(psRes.getStatus())) { 424 errorConnectCount++; 425 // If we get several connection errors in a row, give up. 426 if (errorConnectCount > MAX_CONNECTION_REFUSED_COUNT) { 427 CLog.e("Failed to connect to the remote to check running status."); 428 return false; 429 } 430 } else { 431 // Reset the error count 432 errorConnectCount = 0; 433 CLog.d("ps -ef: stdout: %s\nstderr: %s\n", psRes.getStdout(), psRes.getStderr()); 434 stillRunning = psRes.getStdout().contains(configFile.getName()); 435 CLog.d("still running: %s", stillRunning); 436 if (endTime != null && System.currentTimeMillis() > endTime) { 437 currentInvocationListener.invocationFailed( 438 new RuntimeException( 439 String.format( 440 "Remote invocation timeout after %s", 441 TimeUtil.formatElapsedTime(maxTimeout)))); 442 break; 443 } 444 } 445 if (stillRunning) { 446 RunUtil.getDefault().sleep(REMOTE_PROCESS_RUNNING_WAIT); 447 } 448 } 449 return stillRunning; 450 } 451 452 /** Returns the main remote working directory. */ getRemoteMainDir(TestDeviceOptions options)453 private String getRemoteMainDir(TestDeviceOptions options) { 454 return REMOTE_USER_DIR.replace("{$USER}", options.getInstanceUser()); 455 } 456 457 /** 458 * Sometimes remote adb version is a bit weird and is not running properly the first time. Try 459 * it out once to ensure it starts. 460 */ resetAdb(GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil)461 private void resetAdb(GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil) { 462 CommandResult probAdb = 463 GceManager.remoteSshCommandExecution( 464 info, options, runUtil, 120000L, mRemoteAdbPath, "devices"); 465 CLog.d("remote adb prob: %s", probAdb.getStdout()); 466 CLog.d("%s", probAdb.getStderr()); 467 468 CommandResult versionAdb = 469 GceManager.remoteSshCommandExecution( 470 info, options, runUtil, 120000L, mRemoteAdbPath, "version"); 471 CLog.d("version adb: %s", versionAdb.getStdout()); 472 CLog.d("%s", versionAdb.getStderr()); 473 } 474 475 /** 476 * Remote invocation relies on the adb of the remote, so always collect its logs to make sure we 477 * can debug it appropriately. 478 */ collectAdbLogs( GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, ITestLogger logger)479 private void collectAdbLogs( 480 GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, ITestLogger logger) { 481 CommandResult tmpDirFolder = 482 GceManager.remoteSshCommandExecution( 483 info, options, runUtil, 120000L, "bash -c \"echo \\$TMPDIR\""); 484 String folder = tmpDirFolder.getStdout().trim(); 485 CLog.d("Remote TMPDIR folder is: %s", folder); 486 if (Strings.isNullOrEmpty(folder)) { 487 // If TMPDIR is not set, default to /tmp/ location. 488 folder = "/tmp"; 489 } 490 CommandResult uid = 491 GceManager.remoteSshCommandExecution( 492 info, options, new RunUtil(), 120000L, "bash -c \"echo \\$UID\""); 493 String uidString = uid.getStdout().trim(); 494 CLog.d("Remote $UID for adb is: %s", uidString); 495 496 if (Strings.isNullOrEmpty(uidString)) { 497 CLog.w("Could not determine adb log path."); 498 return; 499 } 500 501 GceManager.logNestedRemoteFile( 502 logger, 503 info, 504 options, 505 runUtil, 506 folder + "/adb." + uidString + ".log", 507 LogDataType.TEXT, 508 "full_adb.log"); 509 } 510 511 /** 512 * Create the configuration that will run in the remote VM. 513 * 514 * @param config The main {@link IConfiguration}. 515 * @param logger A logger where to save the XML configuration for debugging. 516 * @param resultDirPath the remote result dir where results should be saved. 517 * @return A file containing the dumped remote XML configuration. 518 * @throws IOException 519 */ 520 @VisibleForTesting createRemoteConfig(IConfiguration config, ITestLogger logger, String resultDirPath)521 File createRemoteConfig(IConfiguration config, ITestLogger logger, String resultDirPath) 522 throws IOException { 523 // Setup the remote reporting to a proto file 524 List<ITestInvocationListener> reporters = new ArrayList<>(); 525 FileProtoResultReporter protoReporter = new FileProtoResultReporter(); 526 protoReporter.setFileOutput(new File(resultDirPath + PROTO_RESULT_NAME)); 527 reporters.add(protoReporter); 528 529 config.setTestInvocationListeners(reporters); 530 531 for (IDeviceConfiguration deviceConfig : config.getDeviceConfig()) { 532 deviceConfig.getDeviceRequirements().setSerial(); 533 if (deviceConfig.getDeviceRequirements() instanceof DeviceSelectionOptions) { 534 ((DeviceSelectionOptions) deviceConfig.getDeviceRequirements()) 535 .setDeviceTypeRequested(null); 536 } 537 } 538 539 // Dump and log the configuration 540 File configFile = FileUtil.createTempFile(config.getName(), ".xml"); 541 config.dumpXml(new PrintWriter(configFile)); 542 try (InputStreamSource source = new FileInputStreamSource(configFile)) { 543 logger.testLog(REMOTE_CONFIG, LogDataType.XML, source); 544 } 545 return configFile; 546 } 547 fetchAndProcessResults( boolean wasStillRunning, ITestInvocationListener invocationListener, IInvocationContext context, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, String resultDirPath)548 private void fetchAndProcessResults( 549 boolean wasStillRunning, 550 ITestInvocationListener invocationListener, 551 IInvocationContext context, 552 GceAvdInfo info, 553 TestDeviceOptions options, 554 IRunUtil runUtil, 555 String resultDirPath) 556 throws InvalidProtocolBufferException, IOException { 557 File resultFile = null; 558 if (wasStillRunning) { 559 CLog.d("Remote invocation was still running. No result can be pulled."); 560 return; 561 } 562 resultFile = 563 RemoteFileUtil.fetchRemoteFile( 564 info, 565 options, 566 runUtil, 567 PULL_RESULT_TIMEOUT, 568 resultDirPath + PROTO_RESULT_NAME); 569 if (resultFile == null) { 570 invocationListener.invocationFailed( 571 new RuntimeException( 572 String.format( 573 "Could not find remote result file at %s", 574 resultDirPath + PROTO_RESULT_NAME))); 575 return; 576 } 577 CLog.d("Fetched remote result file!"); 578 // Report result to listener. 579 try { 580 ProtoResultParser parser = 581 new ProtoResultParser(invocationListener, context, false, "remote-"); 582 parser.processFinalizedProto(TestRecordProtoUtil.readFromFile(resultFile)); 583 } finally { 584 FileUtil.deleteFile(resultFile); 585 } 586 } 587 588 /** 589 * Method that handles starting an extra Android Virtual Device inside a given remote VM. 590 * 591 * @param listener The invocation {@link ITestInvocationListener}. 592 * @param userId The username id to associate the device with. 593 * @param info The {@link GceAvdInfo} describing the remote VM. 594 * @param options The {@link TestDeviceOptions} of the virtual device. 595 * @param runUtil A {@link IRunUtil} to run host commands 596 * @return True if the device is started successfully, false otherwise. 597 */ startDevice( ITestInvocationListener listener, int userId, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, Semaphore token)598 private boolean startDevice( 599 ITestInvocationListener listener, 600 int userId, 601 GceAvdInfo info, 602 TestDeviceOptions options, 603 IRunUtil runUtil, 604 Semaphore token) 605 throws InterruptedException { 606 String useridString = MultiUserSetupUtil.getUserNumber(userId); 607 String username = String.format("vsoc-%s", useridString); 608 CommandResult userSetup = 609 MultiUserSetupUtil.prepareRemoteUser( 610 username, info, options, runUtil, NEW_USER_TIMEOUT); 611 if (userSetup != null) { 612 String errorMsg = String.format("Failed to setup user: %s", userSetup.getStderr()); 613 CLog.e(errorMsg); 614 listener.invocationFailed(new RuntimeException(errorMsg)); 615 return false; 616 } 617 618 CommandResult homeDirSetup = 619 MultiUserSetupUtil.prepareRemoteHomeDir( 620 options.getInstanceUser(), 621 username, 622 info, 623 options, 624 runUtil, 625 NEW_USER_TIMEOUT); 626 if (homeDirSetup != null) { 627 String errorMsg = 628 String.format("Failed to setup home dir: %s", homeDirSetup.getStderr()); 629 CLog.e(errorMsg); 630 listener.invocationFailed(new RuntimeException(errorMsg)); 631 return false; 632 } 633 634 // Create the cvd user if missing 635 CommandResult cvdSetup = 636 MultiUserSetupUtil.addExtraCvdUser( 637 userId, info, options, runUtil, NEW_USER_TIMEOUT); 638 if (cvdSetup != null) { 639 String errorMsg = String.format("Failed to setup user: %s", cvdSetup.getStderr()); 640 CLog.e(errorMsg); 641 listener.invocationFailed(new RuntimeException(errorMsg)); 642 return false; 643 } 644 645 // Setup the tuntap interface if needed 646 CommandResult tapSetup = 647 MultiUserSetupUtil.setupNetworkInterface( 648 userId, info, options, runUtil, NEW_USER_TIMEOUT); 649 if (tapSetup != null) { 650 String errorMsg = 651 String.format("Failed to setup network interface: %s", tapSetup.getStderr()); 652 CLog.e(errorMsg); 653 listener.invocationFailed(new RuntimeException(errorMsg)); 654 return false; 655 } 656 657 List<String> startCommand = LaunchCvdHelper.createSimpleDeviceCommand(username, true); 658 if (token != null) { 659 token.acquire(); 660 } 661 CommandResult startDeviceRes = null; 662 try { 663 startDeviceRes = 664 GceManager.remoteSshCommandExecution( 665 info, 666 options, 667 runUtil, 668 LAUNCH_EXTRA_DEVICE, 669 Joiner.on(" ").join(startCommand)); 670 } finally { 671 if (token != null) { 672 token.release(); 673 } 674 } 675 if (!CommandStatus.SUCCESS.equals(startDeviceRes.getStatus())) { 676 String errorMsg = 677 String.format("Failed to start %s: %s", username, startDeviceRes.getStderr()); 678 CLog.e(errorMsg); 679 listener.invocationFailed(new RuntimeException(errorMsg)); 680 return false; 681 } 682 return true; 683 } 684 685 /** Thread class that allows to start a device asynchronously. */ 686 private class StartDeviceThread extends Thread { 687 688 private ITestInvocationListener mListener; 689 private int mUserId; 690 private GceAvdInfo mInfo; 691 private TestDeviceOptions mOptions; 692 private IRunUtil mRunUtil; 693 private Semaphore mToken; 694 695 private boolean mFinalResult = false; 696 StartDeviceThread( ITestInvocationListener listener, int userId, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, Semaphore token)697 public StartDeviceThread( 698 ITestInvocationListener listener, 699 int userId, 700 GceAvdInfo info, 701 TestDeviceOptions options, 702 IRunUtil runUtil, 703 Semaphore token) { 704 super(); 705 setDaemon(true); 706 setName(String.format("start-device-thread-vsoc-%s", userId)); 707 mListener = listener; 708 mUserId = userId; 709 mInfo = info; 710 mOptions = options; 711 mRunUtil = runUtil; 712 mToken = token; 713 } 714 715 @Override run()716 public void run() { 717 try { 718 mFinalResult = startDevice(mListener, mUserId, mInfo, mOptions, mRunUtil, mToken); 719 } catch (InterruptedException e) { 720 CLog.e(e); 721 } 722 } 723 724 /** 725 * Returns the final status of the startDevice. Returns true if it succeeded, false 726 * otherwise. 727 */ getFinalStatus()728 boolean getFinalStatus() { 729 return mFinalResult; 730 } 731 } 732 } 733