1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.invoker; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.build.BuildRetrievalError; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.build.StubBuildProvider; 22 import com.android.tradefed.clearcut.ClearcutClient; 23 import com.android.tradefed.command.CommandOptions; 24 import com.android.tradefed.command.CommandRunner; 25 import com.android.tradefed.config.ConfigurationException; 26 import com.android.tradefed.config.GlobalConfiguration; 27 import com.android.tradefed.config.IConfigOptionValueTransformer; 28 import com.android.tradefed.config.IConfiguration; 29 import com.android.tradefed.config.IDeviceConfiguration; 30 import com.android.tradefed.config.Option; 31 import com.android.tradefed.config.OptionCopier; 32 import com.android.tradefed.config.OptionSetter; 33 import com.android.tradefed.device.DeviceNotAvailableException; 34 import com.android.tradefed.device.DeviceSelectionOptions; 35 import com.android.tradefed.device.DeviceSelectionOptions.DeviceRequestedType; 36 import com.android.tradefed.device.ITestDevice; 37 import com.android.tradefed.device.TestDeviceOptions; 38 import com.android.tradefed.device.TestDeviceOptions.InstanceType; 39 import com.android.tradefed.device.cloud.GceAvdInfo; 40 import com.android.tradefed.device.cloud.GceManager; 41 import com.android.tradefed.device.cloud.ManagedRemoteDevice; 42 import com.android.tradefed.device.cloud.RemoteFileUtil; 43 import com.android.tradefed.device.connection.AdbSshConnection; 44 import com.android.tradefed.error.IHarnessException; 45 import com.android.tradefed.invoker.logger.CurrentInvocation; 46 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 47 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 48 import com.android.tradefed.log.ITestLogger; 49 import com.android.tradefed.log.LogUtil.CLog; 50 import com.android.tradefed.result.FailureDescription; 51 import com.android.tradefed.result.FileInputStreamSource; 52 import com.android.tradefed.result.ITestInvocationListener; 53 import com.android.tradefed.result.InputStreamSource; 54 import com.android.tradefed.result.LogDataType; 55 import com.android.tradefed.result.error.ErrorIdentifier; 56 import com.android.tradefed.result.proto.FileProtoResultReporter; 57 import com.android.tradefed.result.proto.ProtoResultParser; 58 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus; 59 import com.android.tradefed.service.TradefedFeatureServer; 60 import com.android.tradefed.targetprep.BuildError; 61 import com.android.tradefed.targetprep.TargetSetupError; 62 import com.android.tradefed.testtype.SubprocessTfLauncher; 63 import com.android.tradefed.util.CommandResult; 64 import com.android.tradefed.util.CommandStatus; 65 import com.android.tradefed.util.FileUtil; 66 import com.android.tradefed.util.IRunUtil; 67 import com.android.tradefed.util.RunUtil; 68 import com.android.tradefed.util.SerializationUtil; 69 import com.android.tradefed.util.SubprocessExceptionParser; 70 import com.android.tradefed.util.SystemUtil; 71 import com.android.tradefed.util.TimeUtil; 72 import com.android.tradefed.util.proto.TestRecordProtoUtil; 73 74 import com.google.common.base.Strings; 75 import com.google.protobuf.InvalidProtocolBufferException; 76 77 import java.io.File; 78 import java.io.IOException; 79 import java.io.PrintWriter; 80 import java.util.ArrayList; 81 import java.util.Arrays; 82 import java.util.HashMap; 83 import java.util.HashSet; 84 import java.util.List; 85 import java.util.Map; 86 import java.util.UUID; 87 88 /** Implementation of {@link InvocationExecution} that drives a remote execution. */ 89 public class RemoteInvocationExecution extends InvocationExecution { 90 91 public static final String START_FEATURE_SERVER = "START_FEATURE_SERVER"; 92 public static final long PUSH_TF_TIMEOUT = 150000L; 93 public static final long PULL_RESULT_TIMEOUT = 180000L; 94 public static final long REMOTE_PROCESS_RUNNING_WAIT = 15000L; 95 public static final long LAUNCH_EXTRA_DEVICE = 15 * 60 * 1000L; 96 public static final long SETUP_REMOTE_DIR_TIMEOUT = 10 * 60 * 1000L; 97 public static final long NEW_USER_TIMEOUT = 5 * 60 * 1000L; 98 public static final long JOIN_CLEAN_TIMEOUT_MS = 2 * 60 * 1000L; 99 100 public static final String REMOTE_USER_DIR = "/home/{$USER}/"; 101 public static final String PROTO_RESULT_NAME = "output.pb"; 102 public static final String STDOUT_FILE = "screen-VM_tradefed-stdout.txt"; 103 public static final String STDERR_FILE = "screen-VM_tradefed-stderr.txt"; 104 public static final String REMOTE_CONFIG = "configuration"; 105 public static final String GLOBAL_REMOTE_CONFIG = "global-remote-configuration"; 106 107 private static final int MAX_CONNECTION_REFUSED_COUNT = 3; 108 private static final int MAX_PUSH_TF_ATTEMPTS = 3; 109 private static final String TRADEFED_EARLY_TERMINATION = 110 "Remote Tradefed might have terminated early.\nRemote Stderr:\n%s"; 111 /** 112 * Pass these invocation context attributes to remote if they are not part of invocation data 113 */ 114 private static final String[] INVOCATION_CONTEXT_ATTR_TO_DATA = { 115 "invocation_id", "work_unit_id" 116 }; 117 118 private String mRemoteTradefedDir = null; 119 private String mRemoteAdbPath = null; 120 private ProtoResultParser mProtoParser = null; 121 private String mRemoteConsoleStdErr = null; 122 123 @Override fetchBuild( TestInformation testInfo, IConfiguration config, IRescheduler rescheduler, ITestInvocationListener listener)124 public boolean fetchBuild( 125 TestInformation testInfo, 126 IConfiguration config, 127 IRescheduler rescheduler, 128 ITestInvocationListener listener) 129 throws DeviceNotAvailableException, BuildRetrievalError { 130 // TODO: handle multiple devices/build config 131 StubBuildProvider stubProvider = new StubBuildProvider(); 132 133 String deviceName = config.getDeviceConfig().get(0).getDeviceName(); 134 OptionCopier.copyOptionsNoThrow( 135 config.getDeviceConfig().get(0).getBuildProvider(), stubProvider); 136 137 IBuildInfo info = stubProvider.getBuild(); 138 if (info == null) { 139 return false; 140 } 141 testInfo.getContext().addDeviceBuildInfo(deviceName, info); 142 updateBuild(info, config); 143 return true; 144 } 145 146 @Override customizeDevicePreInvocation(IConfiguration config, IInvocationContext context)147 protected void customizeDevicePreInvocation(IConfiguration config, IInvocationContext context) { 148 super.customizeDevicePreInvocation(config, context); 149 150 if (config.getCommandOptions().getShardCount() != null 151 && config.getCommandOptions().getShardIndex() == null 152 && !config.getCommandOptions().isRemoteInvocationDeviceless()) { 153 ITestDevice device = context.getDevices().get(0); 154 TestDeviceOptions options = device.getOptions(); 155 // Trigger the multi-tenant start in the VM 156 options.addGceDriverParams("--num-avds-per-instance"); 157 String count = config.getCommandOptions().getShardCount().toString(); 158 options.addGceDriverParams(count); 159 InvocationMetricLogger.addInvocationMetrics( 160 InvocationMetricKey.CF_INSTANCE_COUNT, count); 161 } 162 } 163 164 @Override runTests( TestInformation info, IConfiguration config, ITestInvocationListener listener)165 public void runTests( 166 TestInformation info, IConfiguration config, ITestInvocationListener listener) 167 throws Throwable { 168 ManagedRemoteDevice device = (ManagedRemoteDevice) info.getDevice(); 169 GceAvdInfo gceInfo = ((AdbSshConnection) device.getConnection()).getAvdInfo(); 170 171 // Run remote TF (new tests?) 172 IRunUtil runUtil = new RunUtil(); 173 174 TestDeviceOptions options = device.getOptions(); 175 String mainRemoteDir = getRemoteMainDir(options); 176 mRemoteAdbPath = String.format("/home/%s/bin/adb", options.getInstanceUser()); 177 // Select the TF version that should be pushed to the remote VM 178 File tfToPush = getLocalTradefedPath(listener, options.getRemoteTf()); 179 if (tfToPush == null) { 180 return; 181 } 182 183 String invocationWorkDir = 184 String.format("%stf-invocation-%s/", mainRemoteDir, UUID.randomUUID().toString()); 185 CLog.d("Remote invocation work directory is at %s", invocationWorkDir); 186 187 CommandResult cr = 188 GceManager.remoteSshCommandExecution( 189 gceInfo, options, runUtil, 120000L, "mkdir", "-p", invocationWorkDir); 190 if (!CommandStatus.SUCCESS.equals(cr.getStatus())) { 191 CLog.e("Creation of %s failed.", invocationWorkDir); 192 CLog.e("Command stdout: %s, stderr: %s", cr.getStdout(), cr.getStderr()); 193 listener.invocationFailed( 194 createInvocationFailure( 195 "Failed to create remote tradefed dir.", FailureStatus.INFRA_FAILURE)); 196 return; 197 } 198 199 // Push Tradefed to the remote 200 int attempt = 0; 201 boolean result = false; 202 while (!result && attempt < MAX_PUSH_TF_ATTEMPTS) { 203 result = 204 RemoteFileUtil.pushFileToRemote( 205 gceInfo, 206 options, 207 Arrays.asList("-r"), 208 runUtil, 209 PUSH_TF_TIMEOUT, 210 invocationWorkDir, 211 tfToPush); 212 attempt++; 213 } 214 if (!result) { 215 CLog.e("Failed to push Tradefed."); 216 listener.invocationFailed( 217 createInvocationFailure( 218 "Failed to push Tradefed.", FailureStatus.INFRA_FAILURE)); 219 return; 220 } 221 222 mRemoteTradefedDir = invocationWorkDir + tfToPush.getName() + "/"; 223 CommandResult listRemoteDir = 224 GceManager.remoteSshCommandExecution( 225 gceInfo, options, runUtil, 120000L, "ls", "-l", mRemoteTradefedDir); 226 CLog.d("stdout: %s", listRemoteDir.getStdout()); 227 CLog.d("stderr: %s", listRemoteDir.getStderr()); 228 229 File configFile = 230 createRemoteConfig(info.getContext(), config, listener, mRemoteTradefedDir); 231 File globalConfig = null; 232 try { 233 CLog.d("Pushing Tradefed XML configuration to remote."); 234 boolean resultPush = 235 RemoteFileUtil.pushFileToRemote( 236 gceInfo, 237 options, 238 null, 239 runUtil, 240 PUSH_TF_TIMEOUT, 241 mRemoteTradefedDir, 242 configFile); 243 if (!resultPush) { 244 CLog.e("Failed to push Tradefed Configuration."); 245 listener.invocationFailed( 246 createInvocationFailure( 247 "Failed to push Tradefed Configuration.", 248 FailureStatus.INFRA_FAILURE)); 249 return; 250 } 251 252 String[] allowListConfigs = 253 new String[] { 254 GlobalConfiguration.SANDBOX_FACTORY_TYPE_NAME, 255 GlobalConfiguration.HOST_OPTIONS_TYPE_NAME, 256 "android-build" 257 }; 258 // use an IConfigOptionValueTransformer to collect+rewrite options that are File type 259 FileOptionValueTransformer fileTransformer = 260 new FileOptionValueTransformer(mRemoteTradefedDir); 261 try { 262 globalConfig = 263 GlobalConfiguration.getInstance() 264 .cloneConfigWithFilter( 265 new HashSet<>(), fileTransformer, true, allowListConfigs); 266 } catch (IOException e) { 267 listener.invocationFailed(createInvocationFailure(e, FailureStatus.INFRA_FAILURE)); 268 return; 269 } 270 try (InputStreamSource source = new FileInputStreamSource(globalConfig)) { 271 listener.testLog(GLOBAL_REMOTE_CONFIG, LogDataType.HARNESS_CONFIG, source); 272 } 273 // Push the global configuration 274 boolean resultPushGlobal = 275 RemoteFileUtil.pushFileToRemote( 276 gceInfo, 277 options, 278 null, 279 runUtil, 280 PUSH_TF_TIMEOUT, 281 mRemoteTradefedDir, 282 globalConfig); 283 if (!resultPushGlobal) { 284 CLog.e("Failed to push Tradefed Global Configuration."); 285 listener.invocationFailed( 286 createInvocationFailure( 287 "Failed to push Tradefed Global Configuration.", 288 FailureStatus.INFRA_FAILURE)); 289 return; 290 } 291 // Push files referenced in global config over to remote 292 if (!fileTransformer.getRenamedFiles().isEmpty()) { 293 List<String> pushErrors = new ArrayList<>(); 294 CLog.d("Pushing files referenced in global config to remote."); 295 for (Map.Entry<String, String> entry : 296 fileTransformer.getRenamedFiles().entrySet()) { 297 boolean pushResult = 298 RemoteFileUtil.pushFileToRemote( 299 gceInfo, 300 options, 301 null, 302 runUtil, 303 PUSH_TF_TIMEOUT, 304 entry.getValue(), 305 new File(entry.getKey())); 306 if (!pushResult) { 307 pushErrors.add(entry.getKey()); 308 } 309 } 310 CLog.d("Done pushing files."); 311 if (!pushErrors.isEmpty()) { 312 listener.invocationFailed( 313 createInvocationFailure( 314 "Failed to push some files to remote: " + pushErrors, 315 FailureStatus.INFRA_FAILURE)); 316 } 317 } 318 319 resetAdb(gceInfo, options, runUtil); 320 runRemote( 321 listener, 322 info.getContext(), 323 configFile, 324 gceInfo, 325 options, 326 runUtil, 327 config, 328 globalConfig); 329 collectAdbLogs(gceInfo, options, runUtil, listener); 330 } finally { 331 FileUtil.recursiveDelete(configFile); 332 FileUtil.recursiveDelete(globalConfig); 333 cr = 334 GceManager.remoteSshCommandExecution( 335 gceInfo, options, runUtil, 120000L, "rm", "-rf", invocationWorkDir); 336 if (!CommandStatus.SUCCESS.equals(cr.getStatus())) { 337 CLog.w("Clean up of %s failed.", invocationWorkDir); 338 CLog.w("Command stdout: %s, stderr: %s", cr.getStdout(), cr.getStderr()); 339 } 340 } 341 } 342 343 @Override doSetup(TestInformation testInfo, IConfiguration config, ITestLogger logger)344 public void doSetup(TestInformation testInfo, IConfiguration config, ITestLogger logger) 345 throws TargetSetupError, BuildError, DeviceNotAvailableException { 346 // Skip 347 } 348 349 @Override doTeardown( TestInformation testInfo, IConfiguration config, ITestLogger logger, Throwable exception)350 public void doTeardown( 351 TestInformation testInfo, 352 IConfiguration config, 353 ITestLogger logger, 354 Throwable exception) 355 throws Throwable { 356 super.runDevicePostInvocationTearDown(testInfo.getContext(), config, exception); 357 } 358 359 @Override doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception)360 public void doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception) { 361 // Skip 362 } 363 364 @Override getAdbVersion()365 protected String getAdbVersion() { 366 // Do not report the adb version from the parent, the remote child will remote its own. 367 return null; 368 } 369 runRemote( ITestInvocationListener currentInvocationListener, IInvocationContext context, File configFile, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, IConfiguration config, File globalConfig)370 private void runRemote( 371 ITestInvocationListener currentInvocationListener, 372 IInvocationContext context, 373 File configFile, 374 GceAvdInfo info, 375 TestDeviceOptions options, 376 IRunUtil runUtil, 377 IConfiguration config, 378 File globalConfig) 379 throws InvalidProtocolBufferException, IOException { 380 List<String> remoteTfCommand = new ArrayList<>(); 381 remoteTfCommand.add("pushd"); 382 remoteTfCommand.add(mRemoteTradefedDir + ";"); 383 remoteTfCommand.add(String.format("PATH=%s:$PATH", new File(mRemoteAdbPath).getParent())); 384 remoteTfCommand.add("screen -dmSU tradefed sh -c"); 385 386 StringBuilder tfCmdBuilder = 387 new StringBuilder("TF_GLOBAL_CONFIG=" + globalConfig.getName()); 388 // Set an env variable to notify that this a remote environment. 389 tfCmdBuilder.append(" " + SystemUtil.REMOTE_VM_VARIABLE + "=1"); 390 // Disable clearcut in the remote 391 tfCmdBuilder.append(" " + ClearcutClient.DISABLE_CLEARCUT_KEY + "=1"); 392 tfCmdBuilder.append(" " + START_FEATURE_SERVER + "=1"); 393 tfCmdBuilder.append(" ENTRY_CLASS=" + CommandRunner.class.getCanonicalName()); 394 tfCmdBuilder.append(" ./tradefed.sh " + mRemoteTradefedDir + configFile.getName()); 395 if (config.getCommandOptions().shouldUseRemoteSandboxMode()) { 396 tfCmdBuilder.append(" --" + CommandOptions.USE_SANDBOX); 397 } 398 tfCmdBuilder.append(" > " + STDOUT_FILE + " 2> " + STDERR_FILE); 399 remoteTfCommand.add("\"" + tfCmdBuilder.toString() + "\""); 400 // Kick off the actual remote run 401 CommandResult resultRemoteExecution = 402 GceManager.remoteSshCommandExecution( 403 info, options, runUtil, 0L, remoteTfCommand.toArray(new String[0])); 404 if (!CommandStatus.SUCCESS.equals(resultRemoteExecution.getStatus())) { 405 CLog.e("Error running the remote command: %s", resultRemoteExecution.getStdout()); 406 currentInvocationListener.invocationFailed( 407 createInvocationFailure( 408 resultRemoteExecution.getStderr(), FailureStatus.INFRA_FAILURE)); 409 return; 410 } 411 // Sleep a bit to let the process start 412 RunUtil.getDefault().sleep(10000L); 413 414 mProtoParser = new ProtoResultParser(currentInvocationListener, context, false, "remote-"); 415 // Print when parsing 416 mProtoParser.setQuiet(false); 417 // Do not report accounting again, it should have been done in the remote 418 mProtoParser.setSkipParsingAccounting(true); 419 // Monitor the remote invocation to ensure it's completing. Block until timeout or stops 420 // running. 421 boolean stillRunning = true; 422 try { 423 stillRunning = 424 isStillRunning( 425 currentInvocationListener, configFile, info, options, runUtil, config); 426 } finally { 427 // Fetch the logs for debugging 428 File stdout = 429 fetchRemoteAndLogFile( 430 currentInvocationListener, 431 STDOUT_FILE, 432 STDOUT_FILE, 433 info, 434 options, 435 runUtil); 436 FileUtil.recursiveDelete(stdout); 437 File stderr = 438 fetchRemoteAndLogFile( 439 currentInvocationListener, 440 STDERR_FILE, 441 STDERR_FILE, 442 info, 443 options, 444 runUtil); 445 if (stderr != null && stderr.exists()) { 446 mRemoteConsoleStdErr = FileUtil.readStringFromFile(stderr); 447 FileUtil.recursiveDelete(stderr); 448 } else { 449 mRemoteConsoleStdErr = "Failed to fetch stderr from remote."; 450 } 451 } 452 453 // If no result in progress are reported, parse the full results at the end. 454 if (!config.getCommandOptions().shouldReportModuleProgression()) { 455 fetchAndProcessResults( 456 stillRunning, 457 currentInvocationListener, 458 info, 459 options, 460 runUtil, 461 mRemoteTradefedDir); 462 } else if (!mProtoParser.invocationEndedReached()) { 463 String message = 464 String.format( 465 "Parsing of results protos might be incomplete: invocation ended " 466 + "of remote execution was not found. " 467 + TRADEFED_EARLY_TERMINATION, 468 mRemoteConsoleStdErr); 469 String exceptionRemoteFilePath = 470 SubprocessExceptionParser.getPathFromStderr(mRemoteConsoleStdErr); 471 FailureDescription failureDescription = 472 createInvocationFailure(message, FailureStatus.INFRA_FAILURE); 473 if (exceptionRemoteFilePath != null) { 474 File remoteSerialized = 475 RemoteFileUtil.fetchRemoteFile( 476 info, 477 options, 478 runUtil, 479 PULL_RESULT_TIMEOUT, 480 exceptionRemoteFilePath); 481 if (remoteSerialized != null) { 482 try { 483 Throwable obj = 484 (Throwable) SerializationUtil.deserialize(remoteSerialized, true); 485 failureDescription = 486 createInvocationFailure(obj, FailureStatus.INFRA_FAILURE); 487 } catch (IOException e) { 488 // Ignored 489 CLog.w("Could not parse the stderr as a particular exception."); 490 } 491 } 492 } 493 currentInvocationListener.invocationFailed(failureDescription); 494 } 495 } 496 isStillRunning( ITestInvocationListener currentInvocationListener, File configFile, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, IConfiguration config)497 private boolean isStillRunning( 498 ITestInvocationListener currentInvocationListener, 499 File configFile, 500 GceAvdInfo info, 501 TestDeviceOptions options, 502 IRunUtil runUtil, 503 IConfiguration config) 504 throws IOException { 505 long maxTimeout = config.getCommandOptions().getInvocationTimeout(); 506 Long endTime = null; 507 if (maxTimeout > 0L) { 508 endTime = System.currentTimeMillis() + maxTimeout; 509 } 510 boolean stillRunning = true; 511 int errorConnectCount = 0; 512 int currentIndex = 0; 513 long currentTimeOnProto = 0L; 514 while (stillRunning) { 515 if (config.getCommandOptions().shouldReportModuleProgression()) { 516 String remoteFilePath = mRemoteTradefedDir + PROTO_RESULT_NAME + currentIndex; 517 if (RemoteFileUtil.doesRemoteFileExist( 518 info, options, runUtil, PULL_RESULT_TIMEOUT, remoteFilePath)) { 519 File resultFile = 520 RemoteFileUtil.fetchRemoteFile( 521 info, options, runUtil, PULL_RESULT_TIMEOUT, remoteFilePath); 522 if (resultFile != null) { 523 currentIndex++; 524 currentTimeOnProto = System.currentTimeMillis(); 525 try { 526 mProtoParser.processFileProto(resultFile); 527 } finally { 528 FileUtil.deleteFile(resultFile); 529 } 530 // Don't sleep in that case since we might have more file to process, this 531 // will sleep next time we don't find a file to process on the remote. 532 continue; 533 } 534 } 535 } 536 if (System.currentTimeMillis() - currentTimeOnProto > 7200000) { // 2 hours 537 // If we are stuck on waiting the same proto for over 2 hours, collect some logs 538 File stdout = 539 fetchRemoteAndLogFile( 540 currentInvocationListener, 541 STDOUT_FILE, 542 STDOUT_FILE + "-early", 543 info, 544 options, 545 runUtil); 546 FileUtil.recursiveDelete(stdout); 547 currentTimeOnProto = System.currentTimeMillis(); 548 } 549 550 CommandResult psRes = 551 GceManager.remoteSshCommandExecution( 552 info, 553 options, 554 runUtil, 555 120000L, 556 "ps", 557 "-ef", 558 "| grep", 559 CommandRunner.class.getCanonicalName()); 560 if (!CommandStatus.SUCCESS.equals(psRes.getStatus())) { 561 errorConnectCount++; 562 // If we get several connection errors in a row, give up. 563 if (errorConnectCount > MAX_CONNECTION_REFUSED_COUNT) { 564 CLog.e("Failed to connect to the remote to check running status."); 565 return false; 566 } 567 } else { 568 // Reset the error count 569 errorConnectCount = 0; 570 CLog.d("ps -ef: stdout: %s\nstderr: %s\n", psRes.getStdout(), psRes.getStderr()); 571 stillRunning = psRes.getStdout().contains(configFile.getName()); 572 CLog.d("still running: %s", stillRunning); 573 if (endTime != null && System.currentTimeMillis() > endTime) { 574 currentInvocationListener.invocationFailed( 575 createInvocationFailure( 576 String.format( 577 "Remote invocation timeout after %s", 578 TimeUtil.formatElapsedTime(maxTimeout)), 579 FailureStatus.TIMED_OUT)); 580 break; 581 } 582 } 583 if (stillRunning) { 584 RunUtil.getDefault().sleep(REMOTE_PROCESS_RUNNING_WAIT); 585 } 586 } 587 588 File resultFile = null; 589 if (config.getCommandOptions().shouldReportModuleProgression()) { 590 // Process all remaining proto files available 591 do { 592 String remoteFilePath = mRemoteTradefedDir + PROTO_RESULT_NAME + currentIndex; 593 if (RemoteFileUtil.doesRemoteFileExist( 594 info, options, runUtil, PULL_RESULT_TIMEOUT, remoteFilePath)) { 595 resultFile = 596 RemoteFileUtil.fetchRemoteFile( 597 info, options, runUtil, PULL_RESULT_TIMEOUT, remoteFilePath); 598 if (resultFile != null) { 599 currentIndex++; 600 try { 601 mProtoParser.processFileProto(resultFile); 602 } finally { 603 FileUtil.deleteFile(resultFile); 604 } 605 } 606 } else { 607 break; 608 } 609 } while (resultFile != null); 610 } 611 return stillRunning; 612 } 613 614 /** Returns the main remote working directory. */ getRemoteMainDir(TestDeviceOptions options)615 private String getRemoteMainDir(TestDeviceOptions options) { 616 return REMOTE_USER_DIR.replace("{$USER}", options.getInstanceUser()); 617 } 618 619 /** 620 * Sometimes remote adb version is a bit weird and is not running properly the first time. Try 621 * it out once to ensure it starts. 622 */ resetAdb(GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil)623 private void resetAdb(GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil) { 624 CommandResult probAdb = 625 GceManager.remoteSshCommandExecution( 626 info, options, runUtil, 120000L, mRemoteAdbPath, "devices"); 627 CLog.d("remote adb prob: %s", probAdb.getStdout()); 628 CLog.d("%s", probAdb.getStderr()); 629 630 CommandResult versionAdb = 631 GceManager.remoteSshCommandExecution( 632 info, options, runUtil, 120000L, mRemoteAdbPath, "version"); 633 CLog.d("version adb: %s", versionAdb.getStdout()); 634 CLog.d("%s", versionAdb.getStderr()); 635 } 636 637 /** 638 * Remote invocation relies on the adb of the remote, so always collect its logs to make sure we 639 * can debug it appropriately. 640 */ collectAdbLogs( GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, ITestLogger logger)641 private void collectAdbLogs( 642 GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, ITestLogger logger) { 643 CommandResult tmpDirFolder = 644 GceManager.remoteSshCommandExecution( 645 info, options, runUtil, 120000L, "bash -c \"echo \\$TMPDIR\""); 646 String folder = tmpDirFolder.getStdout().trim(); 647 CLog.d("Remote TMPDIR folder is: %s", folder); 648 if (Strings.isNullOrEmpty(folder)) { 649 // If TMPDIR is not set, default to /tmp/ location. 650 folder = "/tmp"; 651 } 652 CommandResult uid = 653 GceManager.remoteSshCommandExecution( 654 info, options, new RunUtil(), 120000L, "bash -c \"echo \\$UID\""); 655 String uidString = uid.getStdout().trim(); 656 CLog.d("Remote $UID for adb is: %s", uidString); 657 658 if (Strings.isNullOrEmpty(uidString)) { 659 CLog.w("Could not determine adb log path."); 660 return; 661 } 662 663 GceManager.logNestedRemoteFile( 664 logger, 665 info, 666 options, 667 runUtil, 668 folder + "/adb." + uidString + ".log", 669 LogDataType.TEXT, 670 "full_adb.log"); 671 } 672 673 /** 674 * Create the configuration that will run in the remote VM. 675 * 676 * @param context the {@link IInvocationContext} for the current invocation 677 * @param config The main {@link IConfiguration}. 678 * @param logger A logger where to save the XML configuration for debugging. 679 * @param resultDirPath the remote result dir where results should be saved. 680 * @return A file containing the dumped remote XML configuration. 681 * @throws IOException 682 */ 683 @VisibleForTesting createRemoteConfig( IInvocationContext context, IConfiguration config, ITestLogger logger, String resultDirPath)684 File createRemoteConfig( 685 IInvocationContext context, 686 IConfiguration config, 687 ITestLogger logger, 688 String resultDirPath) 689 throws IOException, ConfigurationException { 690 // Setup the remote reporting to a proto file 691 List<ITestInvocationListener> reporters = new ArrayList<>(); 692 FileProtoResultReporter protoReporter = new FileProtoResultReporter(); 693 OptionSetter protoResSetter = new OptionSetter(protoReporter); 694 if (config.getCommandOptions().shouldReportModuleProgression()) { 695 protoResSetter.setOptionValue( 696 FileProtoResultReporter.PERIODIC_PROTO_WRITING_OPTION, "true"); 697 } 698 protoResSetter.setOptionValue( 699 FileProtoResultReporter.PROTO_OUTPUT_FILE, 700 new File(resultDirPath + PROTO_RESULT_NAME).getPath()); 701 reporters.add(protoReporter); 702 703 config.setTestInvocationListeners(reporters); 704 705 for (IDeviceConfiguration deviceConfig : config.getDeviceConfig()) { 706 deviceConfig.getDeviceRequirements().setSerial(); 707 if (deviceConfig.getDeviceRequirements() instanceof DeviceSelectionOptions) { 708 ((DeviceSelectionOptions) deviceConfig.getDeviceRequirements()) 709 .setDeviceTypeRequested(null); 710 if (config.getCommandOptions().isRemoteInvocationDeviceless()) { 711 ((DeviceSelectionOptions) deviceConfig.getDeviceRequirements()) 712 .setDeviceTypeRequested(DeviceRequestedType.NULL_DEVICE); 713 } 714 } 715 // For deviceless reset instance type so remote has right type 716 if (config.getCommandOptions().isRemoteInvocationDeviceless()) { 717 deviceConfig.getDeviceOptions().setInstanceType(InstanceType.GCE); 718 } 719 } 720 721 if (config.getCommandOptions().getShardCount() != null 722 && config.getCommandOptions().getShardIndex() == null) { 723 config.getCommandOptions().setReplicateSetup(true); 724 } 725 726 // Lower the remote invocation timeout to trigger an interrupt 727 long invocationTimeout = 728 Math.max(config.getCommandOptions().getInvocationTimeout() - 120000L, 0L); 729 config.getCommandOptions().setInvocationTimeout(invocationTimeout); 730 731 // Mark the remote invocation as subprocess 732 config.getCommandOptions() 733 .getInvocationData() 734 .put(SubprocessTfLauncher.SUBPROCESS_TAG_NAME, "true"); 735 736 // Pass invocation and work unit ids for local invocation since they are not provided as 737 // command line invocation-data options 738 for (String key : INVOCATION_CONTEXT_ATTR_TO_DATA) { 739 if (!config.getCommandOptions().getInvocationData().containsKey(key)) { 740 String value = context.getAttribute(key); 741 if (!Strings.isNullOrEmpty(value)) { 742 config.getCommandOptions().getInvocationData().put(key, value); 743 } 744 } 745 } 746 747 // Clear the server reference as remote will run its own. 748 if (GlobalConfiguration.getInstance().getFeatureServer() != null) { 749 GlobalConfiguration.getInstance().getFeatureServer().unregisterInvocation(config); 750 } 751 config.getConfigurationDescription().removeMetadata(TradefedFeatureServer.SERVER_REFERENCE); 752 753 // Unset remote-tf-version to avoid re-downloading from remote VM. 754 OptionSetter deviceOptions = 755 new OptionSetter(config.getDeviceConfig().get(0).getDeviceOptions()); 756 deviceOptions.setOptionValue(TestDeviceOptions.REMOTE_TF_VERSION_OPTION, ""); 757 758 // Dump and log the configuration 759 File configFile = FileUtil.createTempFile(config.getName(), ".xml"); 760 config.dumpXml( 761 new PrintWriter(configFile), 762 new ArrayList<String>(), 763 /* print deprecated */ true, 764 /* print unchanged*/ false); 765 try (InputStreamSource source = new FileInputStreamSource(configFile)) { 766 logger.testLog(REMOTE_CONFIG, LogDataType.HARNESS_CONFIG, source); 767 } 768 return configFile; 769 } 770 771 /** Returns the Tradefed version that should be pushed to the remote to drive the invocation. */ getLocalTradefedPath(ITestInvocationListener listener, File remoteTf)772 private File getLocalTradefedPath(ITestInvocationListener listener, File remoteTf) { 773 if (remoteTf != null && remoteTf.exists()) { 774 return remoteTf; 775 } 776 777 String tfPath = System.getProperty("TF_JAR_DIR"); 778 if (tfPath == null) { 779 listener.invocationFailed( 780 createInvocationFailure( 781 "Failed to find $TF_JAR_DIR.", FailureStatus.INFRA_FAILURE)); 782 return null; 783 } 784 File currentTf = new File(tfPath).getAbsoluteFile(); 785 if (tfPath.equals(".")) { 786 currentTf = new File("").getAbsoluteFile(); 787 } 788 return currentTf; 789 } 790 fetchAndProcessResults( boolean wasStillRunning, ITestInvocationListener invocationListener, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil, String resultDirPath)791 private void fetchAndProcessResults( 792 boolean wasStillRunning, 793 ITestInvocationListener invocationListener, 794 GceAvdInfo info, 795 TestDeviceOptions options, 796 IRunUtil runUtil, 797 String resultDirPath) 798 throws InvalidProtocolBufferException, IOException { 799 File resultFile = null; 800 if (wasStillRunning) { 801 CLog.d("Remote invocation was still running. No result can be pulled."); 802 return; 803 } 804 resultFile = 805 RemoteFileUtil.fetchRemoteFile( 806 info, 807 options, 808 runUtil, 809 PULL_RESULT_TIMEOUT, 810 resultDirPath + PROTO_RESULT_NAME); 811 if (resultFile == null) { 812 invocationListener.invocationFailed( 813 createInvocationFailure( 814 String.format( 815 "Could not find remote result file at %s. " 816 + TRADEFED_EARLY_TERMINATION, 817 resultDirPath + PROTO_RESULT_NAME, 818 mRemoteConsoleStdErr), 819 FailureStatus.INFRA_FAILURE)); 820 return; 821 } 822 CLog.d("Fetched remote result file!"); 823 // Report result to listener. 824 try { 825 mProtoParser.processFinalizedProto(TestRecordProtoUtil.readFromFile(resultFile)); 826 } finally { 827 FileUtil.deleteFile(resultFile); 828 } 829 } 830 fetchRemoteAndLogFile( ITestLogger logger, String fileName, String logName, GceAvdInfo info, TestDeviceOptions options, IRunUtil runUtil)831 private File fetchRemoteAndLogFile( 832 ITestLogger logger, 833 String fileName, 834 String logName, 835 GceAvdInfo info, 836 TestDeviceOptions options, 837 IRunUtil runUtil) { 838 File file = 839 RemoteFileUtil.fetchRemoteFile( 840 info, options, runUtil, PULL_RESULT_TIMEOUT, mRemoteTradefedDir + fileName); 841 if (file != null) { 842 try (InputStreamSource source = new FileInputStreamSource(file, false)) { 843 logger.testLog(logName, LogDataType.HARNESS_STD_LOG, source); 844 } 845 } 846 return file; 847 } 848 createInvocationFailure(String errorMessage, FailureStatus status)849 private FailureDescription createInvocationFailure(String errorMessage, FailureStatus status) { 850 FailureDescription failure = FailureDescription.create(errorMessage); 851 failure.setFailureStatus(status); 852 failure.setCause(new RuntimeException(errorMessage)); 853 return failure; 854 } 855 createInvocationFailure( Throwable exception, FailureStatus defaultStatus)856 private FailureDescription createInvocationFailure( 857 Throwable exception, FailureStatus defaultStatus) { 858 ErrorIdentifier id = null; 859 if (exception instanceof IHarnessException) { 860 id = ((IHarnessException) exception).getErrorId(); 861 } 862 String message = exception.getMessage(); 863 if (message == null) { 864 message = "No error message"; 865 } 866 FailureDescription failure = 867 CurrentInvocation.createFailure(message, id).setCause(exception); 868 if (id == null) { 869 // Use default status if none available 870 failure.setFailureStatus(defaultStatus); 871 } 872 return failure; 873 } 874 875 protected static class FileOptionValueTransformer implements IConfigOptionValueTransformer { 876 877 private Map<String, String> mRenamedFiles = new HashMap<>(); 878 private String mBaseRemoteDir; 879 FileOptionValueTransformer(String baseRemoteDir)880 public FileOptionValueTransformer(String baseRemoteDir) { 881 mBaseRemoteDir = baseRemoteDir; 882 } 883 getRenamedFiles()884 public Map<String, String> getRenamedFiles() { 885 return mRenamedFiles; 886 } 887 shouldTransformValueForOption(Option option)888 private boolean shouldTransformValueForOption(Option option) { 889 // seems sufficient for now, may need more configurable mechanism eventually 890 return option.name().contains("key-file"); 891 } 892 handleFileTypeValue(File file)893 private File handleFileTypeValue(File file) { 894 // use String here because it might contain unresolved value e.g. "gs://" 895 String filePath = file.toString(); 896 if (filePath.startsWith("/")) { 897 // strip the leading '/' of the file path and replace the rest with '_' 898 // i.e. /path/to/file becomes path_to_file 899 String remoteName = mBaseRemoteDir + filePath.substring(1).replace('/', '_'); 900 mRenamedFiles.put(filePath, remoteName); 901 CLog.v("File to be transferred: local=%s -> remote=%s", filePath, remoteName); 902 return new File(remoteName); 903 } 904 return file; 905 } 906 907 @Override transform(Object configObj, Option option, Object fieldValue)908 public Object transform(Object configObj, Option option, Object fieldValue) { 909 if (fieldValue instanceof File) { 910 if (shouldTransformValueForOption(option)) { 911 return handleFileTypeValue((File) fieldValue); 912 } 913 } 914 return fieldValue; 915 } 916 } 917 } 918