1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.invoker; 17 18 import com.android.tradefed.command.CommandRunner; 19 import com.android.tradefed.config.ConfigurationException; 20 import com.android.tradefed.config.GlobalConfiguration; 21 import com.android.tradefed.config.IConfiguration; 22 import com.android.tradefed.config.proxy.AutomatedReporters; 23 import com.android.tradefed.config.proxy.TradefedDelegator; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.error.HarnessRuntimeException; 27 import com.android.tradefed.invoker.TestInvocation.Stage; 28 import com.android.tradefed.invoker.logger.CurrentInvocation; 29 import com.android.tradefed.log.ITestLogger; 30 import com.android.tradefed.log.LogUtil.CLog; 31 import com.android.tradefed.result.FileInputStreamSource; 32 import com.android.tradefed.result.ITestInvocationListener; 33 import com.android.tradefed.result.LogDataType; 34 import com.android.tradefed.result.error.InfraErrorIdentifier; 35 import com.android.tradefed.result.proto.StreamProtoReceiver; 36 import com.android.tradefed.service.TradefedFeatureServer; 37 import com.android.tradefed.targetprep.BuildError; 38 import com.android.tradefed.targetprep.TargetSetupError; 39 import com.android.tradefed.util.CommandResult; 40 import com.android.tradefed.util.CommandStatus; 41 import com.android.tradefed.util.FileUtil; 42 import com.android.tradefed.util.IRunUtil; 43 import com.android.tradefed.util.IRunUtil.EnvPriority; 44 import com.android.tradefed.util.RunUtil; 45 import com.android.tradefed.util.StreamUtil; 46 import com.android.tradefed.util.SubprocessExceptionParser; 47 import com.android.tradefed.util.SystemUtil; 48 49 import java.io.File; 50 import java.io.FileOutputStream; 51 import java.io.IOException; 52 import java.io.OutputStream; 53 import java.io.PrintWriter; 54 import java.net.ServerSocket; 55 import java.util.ArrayList; 56 import java.util.Arrays; 57 import java.util.List; 58 59 /** {@link InvocationExecution} which delegate the execution to another Tradefed binary. */ 60 public class DelegatedInvocationExecution extends InvocationExecution { 61 62 /** If present the invocation is executing within a delegated mode */ 63 public static final String DELEGATED_MODE_VAR = "DELEGATED_MODE"; 64 65 /** Timeout to wait for the events received from subprocess to finish being processed. */ 66 private static final long EVENT_THREAD_JOIN_TIMEOUT_MS = 30 * 1000; 67 68 private File mTmpDelegatedDir = null; 69 private File mGlobalConfig = null; 70 // Output reporting 71 private File mStdoutFile = null; 72 private File mStderrFile = null; 73 private OutputStream mStderr = null; 74 private OutputStream mStdout = null; 75 76 @Override reportLogs(ITestDevice device, ITestLogger logger, Stage stage)77 public void reportLogs(ITestDevice device, ITestLogger logger, Stage stage) { 78 // Do nothing 79 } 80 81 @Override shardConfig( IConfiguration config, TestInformation testInfo, IRescheduler rescheduler, ITestLogger logger)82 public boolean shardConfig( 83 IConfiguration config, 84 TestInformation testInfo, 85 IRescheduler rescheduler, 86 ITestLogger logger) { 87 return false; 88 } 89 90 @Override doSetup(TestInformation testInfo, IConfiguration config, ITestLogger listener)91 public void doSetup(TestInformation testInfo, IConfiguration config, ITestLogger listener) 92 throws TargetSetupError, BuildError, DeviceNotAvailableException { 93 // Do nothing 94 } 95 96 @Override runDevicePreInvocationSetup( IInvocationContext context, IConfiguration config, ITestLogger logger)97 public void runDevicePreInvocationSetup( 98 IInvocationContext context, IConfiguration config, ITestLogger logger) 99 throws DeviceNotAvailableException, TargetSetupError { 100 // Do nothing 101 } 102 103 @Override runDevicePostInvocationTearDown( IInvocationContext context, IConfiguration config, Throwable exception)104 public void runDevicePostInvocationTearDown( 105 IInvocationContext context, IConfiguration config, Throwable exception) { 106 // Do nothing 107 } 108 109 @Override doTeardown( TestInformation testInfo, IConfiguration config, ITestLogger logger, Throwable exception)110 public void doTeardown( 111 TestInformation testInfo, 112 IConfiguration config, 113 ITestLogger logger, 114 Throwable exception) 115 throws Throwable { 116 // Do nothing 117 } 118 119 @Override runTests( TestInformation info, IConfiguration config, ITestInvocationListener listener)120 public void runTests( 121 TestInformation info, IConfiguration config, ITestInvocationListener listener) 122 throws Throwable { 123 // Dump the delegated config for debugging 124 File dumpConfig = FileUtil.createTempFile("delegated-config", ".xml"); 125 try (PrintWriter pw = new PrintWriter(dumpConfig)) { 126 config.dumpXml(pw); 127 } 128 logAndCleanFile(dumpConfig, LogDataType.HARNESS_CONFIG, listener); 129 130 if (config.getConfigurationObject(TradefedDelegator.DELEGATE_OBJECT) == null) { 131 throw new ConfigurationException( 132 "Delegate object should not be null in DelegatedInvocation"); 133 } 134 TradefedDelegator delegator = 135 (TradefedDelegator) 136 config.getConfigurationObject(TradefedDelegator.DELEGATE_OBJECT); 137 if (!delegator.getTfRootDir().exists() || !delegator.getTfRootDir().isDirectory()) { 138 throw new ConfigurationException( 139 String.format( 140 "delegated-tf was misconfigured and doesn't point to a valid" 141 + " location: %s", 142 delegator.getTfRootDir()), 143 InfraErrorIdentifier.OPTION_CONFIGURATION_ERROR); 144 } 145 List<String> commandLine = new ArrayList<>(); 146 commandLine.add(SystemUtil.getRunningJavaBinaryPath().getAbsolutePath()); 147 mTmpDelegatedDir = 148 FileUtil.createTempDir("delegated-invocation", CurrentInvocation.getWorkFolder()); 149 commandLine.add( 150 String.format("-Doriginal.tf.tmpdir=%s", System.getProperty("java.io.tmpdir"))); 151 commandLine.add(String.format("-Djava.io.tmpdir=%s", mTmpDelegatedDir.getAbsolutePath())); 152 commandLine.add("-cp"); 153 // Add classpath 154 commandLine.add(delegator.createClasspath()); 155 // Carry the updated TF_JAR_DIR to delegate, this will simulate tradefed.sh environment. 156 commandLine.add( 157 String.format("-DTF_JAR_DIR=%s", delegator.getTfRootDir().getAbsolutePath())); 158 commandLine.add("com.android.tradefed.command.CommandRunner"); 159 // Add command line 160 commandLine.addAll(Arrays.asList(delegator.getCommandLine())); 161 162 try (StreamProtoReceiver receiver = createReceiver(listener, info.getContext())) { 163 mStdoutFile = FileUtil.createTempFile("stdout_delegate_", ".log", mTmpDelegatedDir); 164 mStderrFile = FileUtil.createTempFile("stderr_delegate_", ".log", mTmpDelegatedDir); 165 mStderr = new FileOutputStream(mStderrFile); 166 mStdout = new FileOutputStream(mStdoutFile); 167 IRunUtil runUtil = createRunUtil(receiver.getSocketServerPort(), config); 168 CommandResult result = null; 169 RuntimeException runtimeException = null; 170 CLog.d("Command line: %s", commandLine); 171 try { 172 result = 173 runUtil.runTimedCmd( 174 config.getCommandOptions().getInvocationTimeout(), 175 mStdout, 176 mStderr, 177 commandLine.toArray(new String[0])); 178 } catch (RuntimeException e) { 179 CLog.e("Delegated runtimedCmd threw an exception"); 180 CLog.e(e); 181 runtimeException = e; 182 result = new CommandResult(CommandStatus.EXCEPTION); 183 result.setStdout(StreamUtil.getStackTrace(e)); 184 } 185 boolean failedStatus = false; 186 String stderrText; 187 try { 188 stderrText = FileUtil.readStringFromFile(mStderrFile); 189 } catch (IOException e) { 190 stderrText = "Could not read the stderr output from process."; 191 } 192 if (!CommandStatus.SUCCESS.equals(result.getStatus())) { 193 failedStatus = true; 194 result.setStderr(stderrText); 195 } 196 boolean joinResult = receiver.joinReceiver(EVENT_THREAD_JOIN_TIMEOUT_MS); 197 if (runtimeException != null) { 198 throw runtimeException; 199 } 200 if (!joinResult) { 201 if (!failedStatus) { 202 result.setStatus(CommandStatus.EXCEPTION); 203 } 204 result.setStderr( 205 String.format("Event receiver thread did not complete.:\n%s", stderrText)); 206 } 207 receiver.completeModuleEvents(); 208 if (result.getStatus().equals(CommandStatus.TIMED_OUT)) { 209 throw new HarnessRuntimeException( 210 "Delegated invocation timed out.", InfraErrorIdentifier.INVOCATION_TIMEOUT); 211 } 212 if (!CommandStatus.SUCCESS.equals(result.getStatus())) { 213 CLog.e( 214 "Sandbox finished with status: %s and exit code: %s", 215 result.getStatus(), result.getExitCode()); 216 SubprocessExceptionParser.handleStderrException(result); 217 } 218 } finally { 219 StreamUtil.close(mStderr); 220 StreamUtil.close(mStdout); 221 logAndCleanFile(mStdoutFile, LogDataType.HARNESS_STD_LOG, listener); 222 logAndCleanFile(mStderrFile, LogDataType.HARNESS_STD_LOG, listener); 223 logAndCleanFile(mGlobalConfig, LogDataType.HARNESS_CONFIG, listener); 224 } 225 } 226 227 @Override doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception)228 public void doCleanUp(IInvocationContext context, IConfiguration config, Throwable exception) { 229 super.doCleanUp(context, config, exception); 230 FileUtil.recursiveDelete(mTmpDelegatedDir); 231 FileUtil.deleteFile(mGlobalConfig); 232 } 233 createRunUtil(int port, IConfiguration config)234 private IRunUtil createRunUtil(int port, IConfiguration config) throws IOException { 235 IRunUtil runUtil = new RunUtil(); 236 // Handle the global configs for the subprocess 237 runUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_VARIABLE); 238 runUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_SERVER_CONFIG_VARIABLE); 239 runUtil.setEnvVariablePriority(EnvPriority.SET); 240 mGlobalConfig = createGlobalConfig(); 241 runUtil.setEnvVariable( 242 GlobalConfiguration.GLOBAL_CONFIG_VARIABLE, mGlobalConfig.getAbsolutePath()); 243 runUtil.setEnvVariable(AutomatedReporters.PROTO_REPORTING_PORT, Integer.toString(port)); 244 // Set a variable to detect delegated mode 245 runUtil.setEnvVariable(DELEGATED_MODE_VAR, "1"); 246 // Trigger the feature server to be restarted in the delegate 247 // this ensures all the code is being delegated. 248 runUtil.setEnvVariable(CommandRunner.START_FEATURE_SERVER, "1"); 249 ServerSocket s = new ServerSocket(0); 250 s.setReuseAddress(true); 251 int servicePort = s.getLocalPort(); 252 s.close(); 253 runUtil.setEnvVariable( 254 TradefedFeatureServer.TF_SERVICE_PORT, Integer.toString(servicePort)); 255 return runUtil; 256 } 257 createReceiver( ITestInvocationListener listener, IInvocationContext mainContext)258 private StreamProtoReceiver createReceiver( 259 ITestInvocationListener listener, IInvocationContext mainContext) throws IOException { 260 StreamProtoReceiver receiver = 261 new StreamProtoReceiver( 262 listener, mainContext, false, false, /* report logs */ false, ""); 263 return receiver; 264 } 265 createGlobalConfig()266 private File createGlobalConfig() throws IOException { 267 String[] configList = 268 new String[] { 269 GlobalConfiguration.DEVICE_MANAGER_TYPE_NAME, 270 GlobalConfiguration.KEY_STORE_TYPE_NAME, 271 GlobalConfiguration.HOST_OPTIONS_TYPE_NAME, 272 GlobalConfiguration.SANDBOX_FACTORY_TYPE_NAME, 273 "android-build" 274 }; 275 File filteredGlobalConfig = 276 GlobalConfiguration.getInstance().cloneConfigWithFilter(configList); 277 return filteredGlobalConfig; 278 } 279 280 /** 281 * Log the content of given file to listener, then remove the file. 282 * 283 * @param fileToExport the {@link File} pointing to the file to log. 284 * @param type the {@link LogDataType} of the data 285 * @param listener the {@link ITestInvocationListener} where to report the test. 286 */ logAndCleanFile( File fileToExport, LogDataType type, ITestInvocationListener listener)287 private void logAndCleanFile( 288 File fileToExport, LogDataType type, ITestInvocationListener listener) { 289 if (fileToExport == null) { return; } 290 291 try (FileInputStreamSource inputStream = new FileInputStreamSource(fileToExport, true)) { 292 listener.testLog(fileToExport.getName(), type, inputStream); 293 } 294 } 295 } 296