1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.sandbox; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.command.CommandOptions; 20 import com.android.tradefed.config.Configuration; 21 import com.android.tradefed.config.ConfigurationException; 22 import com.android.tradefed.config.ConfigurationFactory; 23 import com.android.tradefed.config.GlobalConfiguration; 24 import com.android.tradefed.config.IConfiguration; 25 import com.android.tradefed.config.IConfigurationFactory; 26 import com.android.tradefed.config.IGlobalConfiguration; 27 import com.android.tradefed.invoker.IInvocationContext; 28 import com.android.tradefed.invoker.InvocationContext; 29 import com.android.tradefed.invoker.proto.InvocationContext.Context; 30 import com.android.tradefed.log.ITestLogger; 31 import com.android.tradefed.log.LogUtil.CLog; 32 import com.android.tradefed.result.FileInputStreamSource; 33 import com.android.tradefed.result.ITestInvocationListener; 34 import com.android.tradefed.result.InputStreamSource; 35 import com.android.tradefed.result.LogDataType; 36 import com.android.tradefed.result.proto.StreamProtoReceiver; 37 import com.android.tradefed.result.proto.StreamProtoResultReporter; 38 import com.android.tradefed.sandbox.SandboxConfigDump.DumpCmd; 39 import com.android.tradefed.util.CommandResult; 40 import com.android.tradefed.util.CommandStatus; 41 import com.android.tradefed.util.FileUtil; 42 import com.android.tradefed.util.IRunUtil; 43 import com.android.tradefed.util.PrettyPrintDelimiter; 44 import com.android.tradefed.util.QuotationAwareTokenizer; 45 import com.android.tradefed.util.RunUtil; 46 import com.android.tradefed.util.StreamUtil; 47 import com.android.tradefed.util.SubprocessTestResultsParser; 48 import com.android.tradefed.util.keystore.IKeyStoreClient; 49 50 import java.io.File; 51 import java.io.FileOutputStream; 52 import java.io.IOException; 53 import java.io.OutputStream; 54 import java.io.PrintWriter; 55 import java.lang.reflect.InvocationTargetException; 56 import java.lang.reflect.Method; 57 import java.util.ArrayList; 58 import java.util.HashSet; 59 import java.util.List; 60 import java.util.Set; 61 62 /** 63 * Sandbox container that can run a Trade Federation invocation. TODO: Allow Options to be passed to 64 * the sandbox. 65 */ 66 public class TradefedSandbox implements ISandbox { 67 68 private static final String SANDBOX_PREFIX = "sandbox-"; 69 70 private File mStdoutFile = null; 71 private File mStderrFile = null; 72 private OutputStream mStdout = null; 73 private FileOutputStream mStderr = null; 74 75 private File mSandboxTmpFolder = null; 76 private File mRootFolder = null; 77 private File mGlobalConfig = null; 78 private File mSerializedContext = null; 79 private File mSerializedConfiguration = null; 80 81 private SubprocessTestResultsParser mEventParser = null; 82 private StreamProtoReceiver mProtoReceiver = null; 83 84 private IRunUtil mRunUtil; 85 private boolean mCollectStdout = true; 86 87 @Override run(IConfiguration config, ITestLogger logger)88 public CommandResult run(IConfiguration config, ITestLogger logger) throws Throwable { 89 List<String> mCmdArgs = new ArrayList<>(); 90 mCmdArgs.add("java"); 91 mCmdArgs.add(String.format("-Djava.io.tmpdir=%s", mSandboxTmpFolder.getAbsolutePath())); 92 mCmdArgs.add(String.format("-DTF_JAR_DIR=%s", mRootFolder.getAbsolutePath())); 93 mCmdArgs.add("-cp"); 94 mCmdArgs.add(createClasspath(mRootFolder)); 95 mCmdArgs.add(TradefedSandboxRunner.class.getCanonicalName()); 96 mCmdArgs.add(mSerializedContext.getAbsolutePath()); 97 mCmdArgs.add(mSerializedConfiguration.getAbsolutePath()); 98 if (mProtoReceiver != null) { 99 mCmdArgs.add("--" + StreamProtoResultReporter.PROTO_REPORT_PORT_OPTION); 100 mCmdArgs.add(Integer.toString(mProtoReceiver.getSocketServerPort())); 101 } else { 102 mCmdArgs.add("--subprocess-report-port"); 103 mCmdArgs.add(Integer.toString(mEventParser.getSocketServerPort())); 104 } 105 if (config.getCommandOptions().shouldUseSandboxTestMode()) { 106 // In test mode, re-add the --use-sandbox to trigger a sandbox run again in the process 107 mCmdArgs.add("--" + CommandOptions.USE_SANDBOX); 108 } 109 110 long timeout = config.getCommandOptions().getInvocationTimeout(); 111 mRunUtil.allowInterrupt(false); 112 CommandResult result = 113 mRunUtil.runTimedCmd(timeout, mStdout, mStderr, mCmdArgs.toArray(new String[0])); 114 // Log stdout and stderr 115 if (mStdoutFile != null) { 116 try (InputStreamSource sourceStdOut = new FileInputStreamSource(mStdoutFile)) { 117 logger.testLog("sandbox-stdout", LogDataType.TEXT, sourceStdOut); 118 } 119 } 120 try (InputStreamSource sourceStdErr = new FileInputStreamSource(mStderrFile)) { 121 logger.testLog("sandbox-stderr", LogDataType.TEXT, sourceStdErr); 122 } 123 124 boolean failedStatus = false; 125 String stderrText; 126 try { 127 stderrText = FileUtil.readStringFromFile(mStderrFile); 128 } catch (IOException e) { 129 stderrText = "Could not read the stderr output from process."; 130 } 131 if (!CommandStatus.SUCCESS.equals(result.getStatus())) { 132 failedStatus = true; 133 result.setStderr(stderrText); 134 } 135 // Log the configuration used to run 136 try (InputStreamSource configFile = new FileInputStreamSource(mSerializedConfiguration)) { 137 logger.testLog("sandbox-config", LogDataType.XML, configFile); 138 } 139 140 boolean joinResult = false; 141 long waitTime = getSandboxOptions(config).getWaitForEventsTimeout(); 142 if (mProtoReceiver != null) { 143 joinResult = mProtoReceiver.joinReceiver(waitTime); 144 } else { 145 joinResult = mEventParser.joinReceiver(waitTime); 146 } 147 148 if (!joinResult) { 149 if (!failedStatus) { 150 result.setStatus(CommandStatus.EXCEPTION); 151 } 152 result.setStderr( 153 String.format("Event receiver thread did not complete.:\n%s", stderrText)); 154 } 155 PrettyPrintDelimiter.printStageDelimiter( 156 String.format( 157 "Execution of the tests occurred in the sandbox, you can find its logs " 158 + "under the name pattern '%s*'", 159 SANDBOX_PREFIX)); 160 161 return result; 162 } 163 164 @Override prepareEnvironment( IInvocationContext context, IConfiguration config, ITestInvocationListener listener)165 public Exception prepareEnvironment( 166 IInvocationContext context, IConfiguration config, ITestInvocationListener listener) { 167 // Check for local sharding, avoid redirecting several stdout (from each shards) to the 168 // sandbox stdout as it creates a lot of I/O to the same output. 169 if (config.getCommandOptions().getShardCount() != null 170 && config.getCommandOptions().getShardIndex() == null) { 171 mCollectStdout = false; 172 } 173 // Create our temp directories. 174 try { 175 if (mCollectStdout) { 176 mStdoutFile = FileUtil.createTempFile("stdout_subprocess_", ".log"); 177 mStdout = new FileOutputStream(mStdoutFile); 178 } else { 179 mStdout = 180 new OutputStream() { 181 @Override 182 public void write(int b) throws IOException { 183 // Ignore stdout 184 } 185 }; 186 } 187 188 mStderrFile = FileUtil.createTempFile("stderr_subprocess_", ".log"); 189 mStderr = new FileOutputStream(mStderrFile); 190 191 mSandboxTmpFolder = FileUtil.createTempDir("tradefed-container"); 192 } catch (IOException e) { 193 return e; 194 } 195 // Unset the current global environment 196 mRunUtil = createRunUtil(); 197 mRunUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_VARIABLE); 198 mRunUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_SERVER_CONFIG_VARIABLE); 199 // TODO: add handling of setting and creating the subprocess global configuration 200 201 try { 202 mRootFolder = 203 getTradefedSandboxEnvironment( 204 context, 205 config, 206 QuotationAwareTokenizer.tokenizeLine( 207 config.getCommandLine(), 208 /** no logging */ 209 false)); 210 } catch (ConfigurationException e) { 211 return e; 212 } 213 214 // Prepare the configuration 215 Exception res = prepareConfiguration(context, config, listener); 216 if (res != null) { 217 return res; 218 } 219 // Prepare the context 220 try { 221 mSerializedContext = prepareContext(context, config); 222 } catch (IOException e) { 223 return e; 224 } 225 226 return null; 227 } 228 229 @Override tearDown()230 public void tearDown() { 231 StreamUtil.close(mEventParser); 232 StreamUtil.close(mProtoReceiver); 233 StreamUtil.close(mStdout); 234 StreamUtil.close(mStderr); 235 FileUtil.deleteFile(mStdoutFile); 236 FileUtil.deleteFile(mStderrFile); 237 FileUtil.recursiveDelete(mSandboxTmpFolder); 238 FileUtil.deleteFile(mSerializedContext); 239 FileUtil.deleteFile(mSerializedConfiguration); 240 FileUtil.deleteFile(mGlobalConfig); 241 } 242 243 @Override getTradefedSandboxEnvironment( IInvocationContext context, IConfiguration nonVersionedConfig, String[] args)244 public File getTradefedSandboxEnvironment( 245 IInvocationContext context, IConfiguration nonVersionedConfig, String[] args) 246 throws ConfigurationException { 247 SandboxOptions options = getSandboxOptions(nonVersionedConfig); 248 // Check that we have no args conflicts. 249 if (options.getSandboxTfDirectory() != null && options.getSandboxBuildId() != null) { 250 throw new ConfigurationException( 251 String.format( 252 "Sandbox options %s and %s cannot be set at the same time", 253 SandboxOptions.TF_LOCATION, SandboxOptions.SANDBOX_BUILD_ID)); 254 } 255 256 if (options.getSandboxTfDirectory() != null) { 257 return options.getSandboxTfDirectory(); 258 } 259 String tfDir = System.getProperty("TF_JAR_DIR"); 260 if (tfDir == null || tfDir.isEmpty()) { 261 throw new ConfigurationException( 262 "Could not read TF_JAR_DIR to get current Tradefed instance."); 263 } 264 return new File(tfDir); 265 } 266 267 /** 268 * Create a classpath based on the environment and the working directory returned by {@link 269 * #getTradefedSandboxEnvironment(IInvocationContext, IConfiguration, String[])}. 270 * 271 * @param workingDir the current working directory for the sandbox. 272 * @return The classpath to be use. 273 */ 274 @Override createClasspath(File workingDir)275 public String createClasspath(File workingDir) throws ConfigurationException { 276 // Get the classpath property. 277 String classpathStr = System.getProperty("java.class.path"); 278 if (classpathStr == null) { 279 throw new ConfigurationException( 280 "Could not find the classpath property: java.class.path"); 281 } 282 return classpathStr; 283 } 284 285 /** 286 * Prepare the {@link IConfiguration} that will be passed to the subprocess and will drive the 287 * container execution. 288 * 289 * @param context The current {@link IInvocationContext}. 290 * @param config the {@link IConfiguration} to be prepared. 291 * @param listener The current invocation {@link ITestInvocationListener}. 292 * @return an Exception if anything went wrong, null otherwise. 293 */ prepareConfiguration( IInvocationContext context, IConfiguration config, ITestInvocationListener listener)294 protected Exception prepareConfiguration( 295 IInvocationContext context, IConfiguration config, ITestInvocationListener listener) { 296 try { 297 // TODO: switch reporting of parent and subprocess to proto 298 String commandLine = config.getCommandLine(); 299 if (getSandboxOptions(config).shouldUseProtoReporter()) { 300 mProtoReceiver = 301 new StreamProtoReceiver(listener, context, false, false, SANDBOX_PREFIX); 302 // Force the child to the same mode as the parent. 303 commandLine = commandLine + " --" + SandboxOptions.USE_PROTO_REPORTER; 304 } else { 305 mEventParser = new SubprocessTestResultsParser(listener, true, context); 306 commandLine = commandLine + " --no-" + SandboxOptions.USE_PROTO_REPORTER; 307 } 308 String[] args = 309 QuotationAwareTokenizer.tokenizeLine(commandLine, /* No Logging */ false); 310 mGlobalConfig = dumpGlobalConfig(config, new HashSet<>()); 311 try (InputStreamSource source = new FileInputStreamSource(mGlobalConfig)) { 312 listener.testLog("sandbox-global-config", LogDataType.XML, source); 313 } 314 DumpCmd mode = DumpCmd.RUN_CONFIG; 315 if (config.getCommandOptions().shouldUseSandboxTestMode()) { 316 mode = DumpCmd.TEST_MODE; 317 } 318 319 try { 320 mSerializedConfiguration = 321 SandboxConfigUtil.dumpConfigForVersion( 322 createClasspath(mRootFolder), mRunUtil, args, mode, mGlobalConfig); 323 } catch (SandboxConfigurationException e) { 324 // TODO: Improve our detection of that scenario 325 if (e.getMessage().contains(String.format("Can not find local config %s", args[0])) 326 || e.getMessage() 327 .contains( 328 String.format( 329 "Could not find configuration '%s'", args[0]))) { 330 File parentConfig = handleChildMissingConfig(args); 331 if (parentConfig != null) { 332 try { 333 mSerializedConfiguration = 334 SandboxConfigUtil.dumpConfigForVersion( 335 createClasspath(mRootFolder), 336 mRunUtil, 337 new String[] {parentConfig.getAbsolutePath()}, 338 mode, 339 mGlobalConfig); 340 } finally { 341 FileUtil.deleteFile(parentConfig); 342 } 343 return null; 344 } 345 } 346 throw e; 347 } 348 } catch (IOException | ConfigurationException e) { 349 StreamUtil.close(mEventParser); 350 StreamUtil.close(mProtoReceiver); 351 return e; 352 } 353 return null; 354 } 355 356 @VisibleForTesting createRunUtil()357 IRunUtil createRunUtil() { 358 return new RunUtil(); 359 } 360 361 /** 362 * Prepare and serialize the {@link IInvocationContext}. 363 * 364 * @param context the {@link IInvocationContext} to be prepared. 365 * @param config The {@link IConfiguration} of the sandbox. 366 * @return the serialized {@link IInvocationContext}. 367 * @throws IOException 368 */ prepareContext(IInvocationContext context, IConfiguration config)369 protected File prepareContext(IInvocationContext context, IConfiguration config) 370 throws IOException { 371 // In test mode we need to keep the context unlocked for the next layer. 372 if (config.getCommandOptions().shouldUseSandboxTestMode()) { 373 try { 374 Method unlock = InvocationContext.class.getDeclaredMethod("unlock"); 375 unlock.setAccessible(true); 376 unlock.invoke(context); 377 unlock.setAccessible(false); 378 } catch (NoSuchMethodException 379 | SecurityException 380 | IllegalAccessException 381 | IllegalArgumentException 382 | InvocationTargetException e) { 383 throw new IOException("Couldn't unlock the context.", e); 384 } 385 } 386 File protoFile = 387 FileUtil.createTempFile( 388 "context-proto", "." + LogDataType.PB.getFileExt(), mSandboxTmpFolder); 389 Context contextProto = context.toProto(); 390 contextProto.writeDelimitedTo(new FileOutputStream(protoFile)); 391 return protoFile; 392 } 393 394 /** Dump the global configuration filtered from some objects. */ dumpGlobalConfig(IConfiguration config, Set<String> exclusionPatterns)395 protected File dumpGlobalConfig(IConfiguration config, Set<String> exclusionPatterns) 396 throws IOException, ConfigurationException { 397 SandboxOptions options = getSandboxOptions(config); 398 if (options.getChildGlobalConfig() != null) { 399 IConfigurationFactory factory = ConfigurationFactory.getInstance(); 400 IGlobalConfiguration globalConfig = 401 factory.createGlobalConfigurationFromArgs( 402 new String[] {options.getChildGlobalConfig()}, new ArrayList<>()); 403 CLog.d( 404 "Using %s directly as global config without filtering", 405 options.getChildGlobalConfig()); 406 return globalConfig.cloneConfigWithFilter(); 407 } 408 return SandboxConfigUtil.dumpFilteredGlobalConfig(exclusionPatterns); 409 } 410 411 /** {@inheritDoc} */ 412 @Override createThinLauncherConfig( String[] args, IKeyStoreClient keyStoreClient, IRunUtil runUtil, File globalConfig)413 public IConfiguration createThinLauncherConfig( 414 String[] args, IKeyStoreClient keyStoreClient, IRunUtil runUtil, File globalConfig) { 415 // Default thin launcher cannot do anything, since this sandbox uses the same version as 416 // the parent version. 417 return null; 418 } 419 getSandboxOptions(IConfiguration config)420 private SandboxOptions getSandboxOptions(IConfiguration config) { 421 return (SandboxOptions) 422 config.getConfigurationObject(Configuration.SANBOX_OPTIONS_TYPE_NAME); 423 } 424 handleChildMissingConfig(String[] args)425 private File handleChildMissingConfig(String[] args) { 426 IConfiguration parentConfig = null; 427 try { 428 parentConfig = ConfigurationFactory.getInstance().createConfigurationFromArgs(args); 429 File tmpParentConfig = 430 FileUtil.createTempFile("parent-config", ".xml", mSandboxTmpFolder); 431 PrintWriter pw = new PrintWriter(tmpParentConfig); 432 // Do not print deprecated options to avoid compatibility issues 433 parentConfig.dumpXml(pw, new ArrayList<>(), false); 434 return tmpParentConfig; 435 } catch (ConfigurationException | IOException e) { 436 CLog.e("Parent doesn't understand the command either:"); 437 CLog.e(e); 438 return null; 439 } 440 } 441 } 442