1 /* 2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.tradefed.cache.DigestCalculator; 19 import com.android.tradefed.cache.ExecutableAction; 20 import com.android.tradefed.cache.ExecutableActionResult; 21 import com.android.tradefed.cache.ICacheClient; 22 import com.android.tradefed.config.IConfiguration; 23 import com.android.tradefed.device.NullDevice; 24 import com.android.tradefed.invoker.TestInformation; 25 import com.android.tradefed.invoker.logger.CurrentInvocation; 26 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 27 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 28 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 29 import com.android.tradefed.log.LogUtil.CLog; 30 import com.android.tradefed.result.proto.ModuleProtoResultReporter; 31 import com.android.tradefed.result.skipped.SkipContext; 32 import com.android.tradefed.testtype.IRemoteTest; 33 import com.android.tradefed.testtype.ITestFileFilterReceiver; 34 import com.android.tradefed.util.CacheClientFactory; 35 import com.android.tradefed.util.FileUtil; 36 37 import build.bazel.remote.execution.v2.Digest; 38 39 import com.google.common.collect.ImmutableSet; 40 import com.google.common.hash.HashCode; 41 42 import java.io.File; 43 import java.io.IOException; 44 import java.io.InputStream; 45 import java.security.MessageDigest; 46 import java.security.NoSuchAlgorithmException; 47 import java.util.Arrays; 48 import java.util.Enumeration; 49 import java.util.HashMap; 50 import java.util.LinkedHashSet; 51 import java.util.Map; 52 import java.util.Map.Entry; 53 import java.util.Set; 54 import java.util.concurrent.ConcurrentHashMap; 55 import java.util.jar.JarEntry; 56 import java.util.jar.JarFile; 57 58 /** Utility to upload and download cache results for a test module. */ 59 public class SuiteResultCacheUtil { 60 61 public static final String DEVICE_IMAGE_KEY = "device_image"; 62 public static final String MODULE_CONFIG_KEY = "module_config"; 63 public static final String TRADEFED_JAR_VERSION_KEY = "tradefed.jar_version"; 64 65 private static final Set<String> REMOVE_APKS = 66 ImmutableSet.of("TradefedContentProvider.apk", "TelephonyUtility.apk", "WifiUtil.apk"); 67 private static final Map<String, Digest> COMPUTE_CACHE = new ConcurrentHashMap<String, Digest>(); 68 69 /** Describes the cache results. */ 70 public static class CacheResultDescriptor { 71 private final boolean cacheHit; 72 private final String cacheExplanation; 73 CacheResultDescriptor(boolean cacheHit, String explanation)74 public CacheResultDescriptor(boolean cacheHit, String explanation) { 75 this.cacheHit = cacheHit; 76 this.cacheExplanation = explanation; 77 } 78 isCacheHit()79 public boolean isCacheHit() { 80 return cacheHit; 81 } 82 getDetails()83 public String getDetails() { 84 return cacheExplanation; 85 } 86 } 87 88 /** 89 * Upload results to RBE 90 * 91 * @param mainConfig 92 * @param testInfo 93 * @param module 94 * @param moduleConfig 95 * @param protoResults 96 * @param moduleDir 97 * @param skipContext 98 */ uploadModuleResults( IConfiguration mainConfig, TestInformation testInfo, ModuleDefinition module, File moduleConfig, File protoResults, File moduleDir, SkipContext skipContext)99 public static void uploadModuleResults( 100 IConfiguration mainConfig, 101 TestInformation testInfo, 102 ModuleDefinition module, 103 File moduleConfig, 104 File protoResults, 105 File moduleDir, 106 SkipContext skipContext) { 107 // TODO: We don't support multi-devices 108 if (testInfo.getDevices().size() > 1) { 109 return; 110 } 111 if (!(testInfo.getDevice().getIDevice() instanceof NullDevice) 112 && !skipContext.getImageToDigest().containsKey(DEVICE_IMAGE_KEY)) { 113 CLog.d("We have device but no device digest."); 114 InvocationMetricLogger.addInvocationMetrics( 115 InvocationMetricKey.MODULE_RESULTS_CACHE_DEVICE_MISMATCH, 1); 116 return; 117 } 118 if (skipContext.getImageToDigest().containsValue(null)) { 119 CLog.d("No digest for device."); 120 InvocationMetricLogger.addInvocationMetrics( 121 InvocationMetricKey.MODULE_RESULTS_CACHE_DEVICE_MISMATCH, 1); 122 return; 123 } 124 String moduleId = module.getId(); 125 long startTime = System.currentTimeMillis(); 126 try (CloseableTraceScope ignored = new CloseableTraceScope("upload_module_results")) { 127 String cacheInstance = mainConfig.getCommandOptions().getRemoteCacheInstanceName(); 128 ICacheClient cacheClient = 129 CacheClientFactory.createCacheClient( 130 CurrentInvocation.getWorkFolder(), cacheInstance); 131 Map<String, String> environment = new HashMap<>(); 132 for (Entry<String, Digest> entry : skipContext.getImageToDigest().entrySet()) { 133 environment.put(entry.getKey(), entry.getValue().getHash()); 134 } 135 Digest configDigest = DigestCalculator.compute(moduleConfig); 136 environment.put(MODULE_CONFIG_KEY, configDigest.getHash()); 137 Digest tradefedDigest = computeTradefedVersion(); 138 if (tradefedDigest != null) { 139 environment.put(TRADEFED_JAR_VERSION_KEY, tradefedDigest.getHash()); 140 } 141 int i = 0; 142 for (Digest d : filterFileDigest(module)) { 143 environment.put("filter_" + i, d.getHash()); 144 i++; 145 } 146 if (module.getIntraModuleShardCount() != null 147 && module.getIntraModuleShardIndex() != null) { 148 environment.put( 149 "intra_module_shard_index", 150 Integer.toString(module.getIntraModuleShardIndex())); 151 environment.put( 152 "intra_module_shard_count", 153 Integer.toString(module.getIntraModuleShardCount())); 154 } 155 ExecutableAction action = 156 ExecutableAction.create( 157 moduleDir, Arrays.asList(moduleId), environment, 60000L); 158 ExecutableActionResult result = ExecutableActionResult.create(0, protoResults, null); 159 CLog.d("Uploading cache for %s and %s", action, protoResults); 160 cacheClient.uploadCache(action, result); 161 } catch (IOException | RuntimeException | InterruptedException e) { 162 CLog.e(e); 163 InvocationMetricLogger.addInvocationMetrics( 164 InvocationMetricKey.MODULE_CACHE_UPLOAD_ERROR, 1); 165 } finally { 166 InvocationMetricLogger.addInvocationPairMetrics( 167 InvocationMetricKey.MODULE_CACHE_UPLOAD_TIME, 168 startTime, 169 System.currentTimeMillis()); 170 } 171 } 172 173 /** 174 * Look up results in RBE for the test module. 175 * 176 * @param mainConfig 177 * @param module 178 * @param moduleConfig 179 * @param moduleDir 180 * @param skipContext 181 * @return a {@link CacheResultDescriptor} describing the cache result. 182 */ lookUpModuleResults( IConfiguration mainConfig, ModuleDefinition module, File moduleConfig, File moduleDir, SkipContext skipContext)183 public static CacheResultDescriptor lookUpModuleResults( 184 IConfiguration mainConfig, 185 ModuleDefinition module, 186 File moduleConfig, 187 File moduleDir, 188 SkipContext skipContext) { 189 InvocationMetricLogger.addInvocationMetrics( 190 InvocationMetricKey.MODULE_RESULTS_CHECKING_CACHE, 1); 191 if (skipContext.getImageToDigest().containsValue(null)) { 192 CLog.d("No digest for device."); 193 return new CacheResultDescriptor(false, null); 194 } 195 String moduleId = module.getId(); 196 long startTime = System.currentTimeMillis(); 197 try (CloseableTraceScope ignored = new CloseableTraceScope("lookup_module_results")) { 198 String cacheInstance = mainConfig.getCommandOptions().getRemoteCacheInstanceName(); 199 ICacheClient cacheClient = 200 CacheClientFactory.createCacheClient( 201 CurrentInvocation.getWorkFolder(), cacheInstance); 202 Map<String, String> environment = new HashMap<>(); 203 for (Entry<String, Digest> entry : skipContext.getImageToDigest().entrySet()) { 204 environment.put(entry.getKey(), entry.getValue().getHash()); 205 } 206 try (CloseableTraceScope computeDigest = new CloseableTraceScope("compute_digest")) { 207 Digest configDigest = DigestCalculator.compute(moduleConfig); 208 environment.put(MODULE_CONFIG_KEY, configDigest.getHash()); 209 Digest tradefedDigest = computeTradefedVersion(); 210 if (tradefedDigest != null) { 211 environment.put(TRADEFED_JAR_VERSION_KEY, tradefedDigest.getHash()); 212 } 213 } 214 int i = 0; 215 for (Digest d : filterFileDigest(module)) { 216 environment.put("filter_" + i, d.getHash()); 217 i++; 218 } 219 if (module.getIntraModuleShardCount() != null 220 && module.getIntraModuleShardIndex() != null) { 221 environment.put( 222 "intra_module_shard_index", 223 Integer.toString(module.getIntraModuleShardIndex())); 224 environment.put( 225 "intra_module_shard_count", 226 Integer.toString(module.getIntraModuleShardCount())); 227 } 228 ExecutableAction action = 229 ExecutableAction.create( 230 moduleDir, Arrays.asList(moduleId), environment, 60000L); 231 CLog.d("Looking up cache for %s", action); 232 ExecutableActionResult cachedResults = cacheClient.lookupCache(action); 233 if (cachedResults == null) { 234 CLog.d("No cached results for %s", moduleId); 235 InvocationMetricLogger.addInvocationMetrics( 236 InvocationMetricKey.MODULE_CACHE_MISS_ID, moduleId); 237 } else { 238 InvocationMetricLogger.addInvocationMetrics( 239 InvocationMetricKey.MODULE_RESULTS_CACHE_HIT, 1); 240 InvocationMetricLogger.addInvocationMetrics( 241 InvocationMetricKey.MODULE_CACHE_HIT_ID, moduleId); 242 String details = "Cached results."; 243 Map<String, String> metadata = 244 ModuleProtoResultReporter.parseResultsMetadata(cachedResults.stdOut()); 245 if (metadata.containsKey(ModuleProtoResultReporter.INVOCATION_ID_KEY)) { 246 details += 247 String.format( 248 " origin of results: http://ab/%s", 249 metadata.get(ModuleProtoResultReporter.INVOCATION_ID_KEY)); 250 CLog.d(details); 251 } 252 FileUtil.deleteFile(cachedResults.stdOut()); 253 FileUtil.deleteFile(cachedResults.stdErr()); 254 return new CacheResultDescriptor(true, details); 255 } 256 } catch (IOException | RuntimeException | InterruptedException e) { 257 CLog.e(e); 258 InvocationMetricLogger.addInvocationMetrics( 259 InvocationMetricKey.MODULE_CACHE_DOWNLOAD_ERROR, 1); 260 } finally { 261 InvocationMetricLogger.addInvocationPairMetrics( 262 InvocationMetricKey.MODULE_CACHE_DOWNLOAD_TIME, 263 startTime, 264 System.currentTimeMillis()); 265 } 266 return new CacheResultDescriptor(false, null); 267 } 268 269 /** 270 * Hash Tradefed.jar as a denominator to keep results. This helps consider changes to Tradefed. 271 */ computeTradefedVersion()272 private static Digest computeTradefedVersion() throws IOException { 273 String classpathStr = System.getProperty("java.class.path"); 274 if (classpathStr == null) { 275 return null; 276 } 277 for (String file : classpathStr.split(":")) { 278 File currentJar = new File(file); 279 if (currentJar.exists() && "tradefed.jar".equals(currentJar.getName())) { 280 return processJarFile(currentJar.getAbsolutePath()); 281 } 282 } 283 return null; 284 } 285 processJarFile(String jarFilePath)286 private static Digest processJarFile(String jarFilePath) throws IOException { 287 if (COMPUTE_CACHE.containsKey(jarFilePath)) { 288 return COMPUTE_CACHE.get(jarFilePath); 289 } 290 try (JarFile jarFile = new JarFile(jarFilePath)) { 291 Enumeration<JarEntry> entries = jarFile.entries(); 292 MessageDigest digest = MessageDigest.getInstance("SHA-256"); 293 294 while (entries.hasMoreElements()) { 295 JarEntry entry = entries.nextElement(); 296 if (REMOVE_APKS.contains(entry.getName())) { 297 continue; 298 } 299 if (!entry.isDirectory()) { 300 try (InputStream is = jarFile.getInputStream(entry)) { 301 byte[] buffer = new byte[8192]; 302 int bytesRead; 303 304 while ((bytesRead = is.read(buffer)) != -1) { 305 digest.update(buffer, 0, bytesRead); 306 } 307 } catch (IOException e) { 308 CLog.e(e); 309 } 310 } 311 } 312 Digest tfDigest = 313 Digest.newBuilder() 314 .setHash(HashCode.fromBytes(digest.digest()).toString()) 315 .setSizeBytes(digest.getDigestLength()) 316 .build(); 317 COMPUTE_CACHE.put(jarFilePath, tfDigest); 318 return tfDigest; 319 } catch (NoSuchAlgorithmException e) { 320 throw new IOException(e); 321 } 322 } 323 filterFileDigest(ModuleDefinition m)324 private static Set<Digest> filterFileDigest(ModuleDefinition m) throws IOException { 325 Set<Digest> filterDigest = new LinkedHashSet<Digest>(); 326 for (IRemoteTest t : m.getTests()) { 327 if (t instanceof ITestFileFilterReceiver) { 328 ITestFileFilterReceiver fileFilterTest = ((ITestFileFilterReceiver) t); 329 330 File includeFilter = fileFilterTest.getIncludeTestFile(); 331 if (includeFilter != null && includeFilter.exists()) { 332 filterDigest.add(DigestCalculator.compute(includeFilter)); 333 } 334 File excludeFilter = fileFilterTest.getExcludeTestFile(); 335 if (excludeFilter != null && excludeFilter.exists()) { 336 filterDigest.add(DigestCalculator.compute(excludeFilter)); 337 } 338 } 339 } 340 return filterDigest; 341 } 342 } 343