1 /* 2 * Copyright (C) 2025 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.invoker; 17 18 import com.android.tradefed.build.BuildInfoKey.BuildInfoFileKey; 19 import com.android.tradefed.cache.ExecutableAction; 20 import com.android.tradefed.cache.ExecutableActionResult; 21 import com.android.tradefed.cache.ICacheClient; 22 import com.android.tradefed.config.ConfigurationException; 23 import com.android.tradefed.config.IConfiguration; 24 import com.android.tradefed.config.proxy.TradefedDelegator; 25 import com.android.tradefed.invoker.logger.CurrentInvocation; 26 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 27 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 28 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 29 import com.android.tradefed.log.LogUtil.CLog; 30 import com.android.tradefed.result.proto.ModuleProtoResultReporter; 31 import com.android.tradefed.util.CacheClientFactory; 32 import com.android.tradefed.util.FileUtil; 33 import com.android.tradefed.util.QuotationAwareTokenizer; 34 35 import build.bazel.remote.execution.v2.Digest; 36 37 import java.io.File; 38 import java.io.IOException; 39 import java.util.Arrays; 40 import java.util.HashMap; 41 import java.util.List; 42 import java.util.Map; 43 import java.util.Map.Entry; 44 45 /** Utility to handle uploading and looking up invocation cache results. */ 46 public class InvocationCacheHelper { 47 48 /** Describes the cache results. */ 49 public static class CacheInvocationResultDescriptor { 50 private final boolean cacheHit; 51 private final String cacheExplanation; 52 CacheInvocationResultDescriptor(boolean cacheHit, String explanation)53 public CacheInvocationResultDescriptor(boolean cacheHit, String explanation) { 54 this.cacheHit = cacheHit; 55 this.cacheExplanation = explanation; 56 } 57 isCacheHit()58 public boolean isCacheHit() { 59 return cacheHit; 60 } 61 getDetails()62 public String getDetails() { 63 return cacheExplanation; 64 } 65 } 66 67 /** 68 * Upload invocation results 69 * 70 * @param mainConfig 71 * @param protoResults 72 * @param testInfo 73 */ uploadInvocationResults( IConfiguration mainConfig, File protoResults, TestInformation testInfo)74 public static void uploadInvocationResults( 75 IConfiguration mainConfig, File protoResults, TestInformation testInfo) { 76 if (testInfo.getDevices().size() > 1) { 77 return; 78 } 79 boolean emptyTestsDir = false; 80 File invocationTestsDir = null; 81 if (testInfo.getBuildInfo().getFile(BuildInfoFileKey.ROOT_DIRECTORY) == null 82 || testInfo.getBuildInfo().getFile(BuildInfoFileKey.TESTDIR_IMAGE) == null) { 83 emptyTestsDir = true; 84 } 85 if (!emptyTestsDir 86 && (mainConfig.getSkipManager().getTestArtifactsToDigest().isEmpty() 87 || mainConfig 88 .getSkipManager() 89 .getTestArtifactsToDigest() 90 .containsValue(null))) { 91 CLog.d("Cannot handle testsdir."); 92 return; 93 } 94 try (CloseableTraceScope ignored = new CloseableTraceScope("upload_invocation_results")) { 95 String cacheInstance = mainConfig.getCommandOptions().getRemoteCacheInstanceName(); 96 ICacheClient cacheClient = 97 CacheClientFactory.createCacheClient( 98 CurrentInvocation.getWorkFolder(), cacheInstance); 99 invocationTestsDir = 100 FileUtil.createNamedTempDir(CurrentInvocation.getWorkFolder(), "invoc-cache"); 101 ExecutableAction action = 102 ExecutableAction.create( 103 invocationTestsDir, 104 getCommonCommandLine(mainConfig.getCommandLine()), 105 computeEnvironment(mainConfig), 106 60000L); 107 ExecutableActionResult result = ExecutableActionResult.create(0, protoResults, null); 108 CLog.d("Uploading cache for %s and %s", action, protoResults); 109 cacheClient.uploadCache(action, result); 110 } catch (IOException | RuntimeException | InterruptedException e) { 111 CLog.e(e); 112 } finally { 113 FileUtil.recursiveDelete(invocationTestsDir); 114 } 115 } 116 lookupInvocationResults( IConfiguration mainConfig, TestInformation testInfo)117 public static CacheInvocationResultDescriptor lookupInvocationResults( 118 IConfiguration mainConfig, TestInformation testInfo) { 119 if (testInfo.getDevices().size() > 1) { 120 return null; 121 } 122 if (mainConfig.getSkipManager().getImageToDigest().containsValue(null)) { 123 CLog.d("No digest for device."); 124 return new CacheInvocationResultDescriptor(false, null); 125 } 126 boolean emptyTestsDir = false; 127 File invocationTestsDir = null; 128 if (testInfo.getBuildInfo().getFile(BuildInfoFileKey.ROOT_DIRECTORY) == null 129 && testInfo.getBuildInfo().getFile(BuildInfoFileKey.TESTDIR_IMAGE) == null) { 130 emptyTestsDir = true; 131 } 132 if (!emptyTestsDir 133 && (mainConfig.getSkipManager().getTestArtifactsToDigest().isEmpty() 134 || mainConfig 135 .getSkipManager() 136 .getTestArtifactsToDigest() 137 .containsValue(null))) { 138 CLog.d("Cannot handle testsdir."); 139 return new CacheInvocationResultDescriptor(false, null); 140 } 141 try (CloseableTraceScope ignored = new CloseableTraceScope("lookup_invocation_results")) { 142 String cacheInstance = mainConfig.getCommandOptions().getRemoteCacheInstanceName(); 143 ICacheClient cacheClient = 144 CacheClientFactory.createCacheClient( 145 CurrentInvocation.getWorkFolder(), cacheInstance); 146 invocationTestsDir = 147 FileUtil.createNamedTempDir(CurrentInvocation.getWorkFolder(), "invoc-cache"); 148 ExecutableAction action = 149 ExecutableAction.create( 150 invocationTestsDir, 151 getCommonCommandLine(mainConfig.getCommandLine()), 152 computeEnvironment(mainConfig), 153 60000L); 154 CLog.d("Looking up cache for %s", action); 155 InvocationMetricLogger.addInvocationMetrics( 156 InvocationMetricKey.INVOCATION_RESULTS_CHECKING_CACHE, 1); 157 ExecutableActionResult cachedResults = cacheClient.lookupCache(action); 158 if (cachedResults == null) { 159 CLog.d("No cached results for the invocation."); 160 return null; 161 } else { 162 InvocationMetricLogger.addInvocationMetrics( 163 InvocationMetricKey.INVOCATION_CACHE_HIT, 1); 164 String details = "Cached results."; 165 Map<String, String> metadata = 166 ModuleProtoResultReporter.parseResultsMetadata(cachedResults.stdOut()); 167 if (metadata.containsKey(ModuleProtoResultReporter.INVOCATION_ID_KEY)) { 168 details += 169 String.format( 170 " origin of results: http://ab/%s", 171 metadata.get(ModuleProtoResultReporter.INVOCATION_ID_KEY)); 172 CLog.d(details); 173 } 174 FileUtil.deleteFile(cachedResults.stdOut()); 175 FileUtil.deleteFile(cachedResults.stdErr()); 176 return new CacheInvocationResultDescriptor(true, details); 177 } 178 } catch (IOException | RuntimeException | InterruptedException e) { 179 CLog.e(e); 180 } finally { 181 FileUtil.recursiveDelete(invocationTestsDir); 182 } 183 return null; 184 } 185 computeEnvironment(IConfiguration mainConfig)186 private static Map<String, String> computeEnvironment(IConfiguration mainConfig) { 187 Map<String, String> environment = new HashMap<>(); 188 for (Entry<String, Digest> entry : 189 mainConfig.getSkipManager().getImageToDigest().entrySet()) { 190 environment.put(entry.getKey(), entry.getValue().getHash()); 191 } 192 for (Entry<String, Digest> entry : 193 mainConfig.getSkipManager().getTestArtifactsToDigest().entrySet()) { 194 environment.put(entry.getKey(), entry.getValue().getHash()); 195 } 196 String atpTestName = 197 mainConfig 198 .getCommandOptions() 199 .getInvocationData() 200 .getUniqueMap() 201 .get("atp_test_name"); 202 if (atpTestName != null) { 203 environment.put("atp_test_name", atpTestName); 204 } 205 // add tradefed.jar version 206 return environment; 207 } 208 getCommonCommandLine(String commandLine)209 private static List<String> getCommonCommandLine(String commandLine) { 210 String[] commandArray = QuotationAwareTokenizer.tokenizeLine(commandLine, false); 211 try { 212 commandArray = 213 TradefedDelegator.clearCommandlineFromOneArg(commandArray, "invocation-data"); 214 commandArray = TradefedDelegator.clearCommandlineFromOneArg(commandArray, "build-id"); 215 commandArray = TradefedDelegator.clearCommandlineFromOneArg(commandArray, "serial"); 216 } catch (ConfigurationException e) { 217 throw new RuntimeException(e); 218 } 219 return Arrays.asList(commandArray); 220 } 221 } 222