1 /* 2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.util; 18 19 import com.android.tradefed.build.BuildInfoKey.BuildInfoFileKey; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.build.IDeviceBuildInfo; 22 import com.android.tradefed.config.ConfigurationDescriptor; 23 import com.android.tradefed.invoker.ExecutionFiles; 24 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 25 import com.android.tradefed.invoker.IInvocationContext; 26 import com.android.tradefed.invoker.TestInformation; 27 import com.android.tradefed.invoker.logger.CurrentInvocation; 28 import com.android.tradefed.invoker.logger.CurrentInvocation.InvocationInfo; 29 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 30 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 31 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 32 import com.android.tradefed.log.LogUtil.CLog; 33 import com.android.tradefed.targetprep.AltDirBehavior; 34 import com.android.tradefed.testtype.Abi; 35 import com.android.tradefed.testtype.IAbi; 36 37 import com.google.common.annotations.VisibleForTesting; 38 import com.google.common.base.Strings; 39 40 import java.io.File; 41 import java.io.IOException; 42 import java.util.Collections; 43 import java.util.LinkedList; 44 import java.util.List; 45 import java.util.Set; 46 47 /** A utility class that can be used to search for test artifacts. */ 48 public class SearchArtifactUtil { 49 // The singleton is used for mocking the non-static methods during testing.. 50 @VisibleForTesting public static SearchArtifactUtil singleton = new SearchArtifactUtil(); 51 private static final String MODULE_NAME = "module-name"; 52 private static final String MODULE_ABI = "module-abi"; 53 54 /** 55 * Searches for a test artifact/dependency file from the test directory. 56 * 57 * @param fileName The name of the file to look for. 58 * @param targetFirst Whether we are favoring target-side files vs. host-side files for the 59 * search. 60 * @return The found artifact file or null if none. 61 */ searchFile(String fileName, boolean targetFirst)62 public static File searchFile(String fileName, boolean targetFirst) { 63 return searchFile(fileName, targetFirst, null, null, null, null); 64 } 65 66 /** 67 * Searches for a test artifact/dependency file from the test directory. 68 * 69 * @param fileName The name of the file to look for. 70 * @param targetFirst Whether we are favoring target-side files vs. host-side files for the 71 * search. 72 * @param testInfo The {@link TestInformation} of the current test when available. 73 * @return The found artifact file or null if none. 74 */ searchFile(String fileName, boolean targetFirst, TestInformation testInfo)75 public static File searchFile(String fileName, boolean targetFirst, TestInformation testInfo) { 76 return searchFile(fileName, targetFirst, null, null, null, testInfo); 77 } 78 79 /** 80 * Searches for a test artifact/dependency file from the test directory. 81 * 82 * @param fileName The name of the file to look for. 83 * @param targetFirst Whether we are favoring target-side files vs. host-side files for the 84 * search. 85 * @param abi The {@link IAbi} to match the file. 86 * @return The found artifact file or null if none. 87 */ searchFile(String fileName, boolean targetFirst, IAbi abi)88 public static File searchFile(String fileName, boolean targetFirst, IAbi abi) { 89 return searchFile(fileName, targetFirst, abi, null, null, null); 90 } 91 92 /** 93 * Searches for a test artifact/dependency file from the test directory. 94 * 95 * @param fileName The name of the file to look for. 96 * @param targetFirst Whether we are favoring target-side files vs. host-side files for the 97 * search. 98 * @param altDirs Alternative search paths, in addition to the default search paths. 99 * @param altDirBehavior how alternative search paths should be used against default paths: as 100 * fallback, or as override; if unspecified, fallback will be used 101 * @return The found artifact file or null if none. 102 */ searchFile( String fileName, boolean targetFirst, List<File> altDirs, AltDirBehavior altDirBehavior)103 public static File searchFile( 104 String fileName, 105 boolean targetFirst, 106 List<File> altDirs, 107 AltDirBehavior altDirBehavior) { 108 return searchFile(fileName, targetFirst, null, altDirs, altDirBehavior, null); 109 } 110 111 /** 112 * Searches for a test artifact/dependency file from the test directory. 113 * 114 * @param fileName The name of the file to look for. 115 * @param targetFirst Whether we are favoring target-side files vs. host-side files for the 116 * search. 117 * @param abi The {@link IAbi} to match the file. 118 * @param altDirs Alternative search paths, in addition to the default search paths. 119 * @param altDirBehavior how alternative search paths should be used against default paths: as 120 * fallback, or as override; if unspecified, fallback will be used 121 * @param testInfo The {@link TestInformation} of the current test when available. 122 * @return The found artifact file or null if none. 123 */ searchFile( String fileName, boolean targetFirst, IAbi abi, List<File> altDirs, AltDirBehavior altDirBehavior, TestInformation testInfo)124 public static File searchFile( 125 String fileName, 126 boolean targetFirst, 127 IAbi abi, 128 List<File> altDirs, 129 AltDirBehavior altDirBehavior, 130 TestInformation testInfo) { 131 return searchFile(fileName, targetFirst, abi, altDirs, altDirBehavior, testInfo, false); 132 } 133 134 /** 135 * Searches for a test artifact/dependency file from the test directory. 136 * 137 * @param fileName The name of the file to look for. 138 * @param targetFirst Whether we are favoring target-side files vs. host-side files for the 139 * search. 140 * @param abi The {@link IAbi} to match the file. 141 * @param altDirs Alternative search paths, in addition to the default search paths. 142 * @param altDirBehavior how alternative search paths should be used against default paths: as 143 * fallback, or as override; if unspecified, fallback will be used 144 * @param testInfo The {@link TestInformation} of the current test when available. 145 * @param includeDirectory whether to include directories in the search result. 146 * @return The found artifact file or null if none. 147 */ searchFile( String fileName, boolean targetFirst, IAbi abi, List<File> altDirs, AltDirBehavior altDirBehavior, TestInformation testInfo, boolean includeDirectory)148 public static File searchFile( 149 String fileName, 150 boolean targetFirst, 151 IAbi abi, 152 List<File> altDirs, 153 AltDirBehavior altDirBehavior, 154 TestInformation testInfo, 155 boolean includeDirectory) { 156 List<File> searchDirectories = 157 singleton.getSearchDirectories(targetFirst, altDirs, altDirBehavior, testInfo); 158 CLog.d("Searching for file %s. Search directories: %s", fileName, searchDirectories); 159 // Search in the test directories 160 for (File dir : searchDirectories) { 161 File file = findFile(fileName, abi, dir, includeDirectory); 162 if (fileExists(file)) { 163 CLog.d( 164 "Found file %s in search directory %s.", 165 file.getAbsolutePath(), dir.getAbsolutePath()); 166 return file; 167 } 168 } 169 // Search in the execution files directly 170 ExecutionFiles executionFiles = singleton.getExecutionFiles(testInfo); 171 if (executionFiles != null) { 172 File file = executionFiles.get(fileName); 173 if (fileExists(file)) { 174 CLog.d("Found file %s in execution files object.", file.getAbsolutePath()); 175 return file; 176 } 177 } 178 179 // Search in the build info or stage remote file as fallback 180 IBuildInfo buildInfo = singleton.getBuildInfo(); 181 if (buildInfo != null) { 182 File file = buildInfo.getFile(fileName); 183 if (fileExists(file)) { 184 CLog.d("Found file %s in build info.", file.getAbsolutePath()); 185 return file; 186 } else { 187 // fallback to staging from remote zip files. 188 File stagingDir = getModuleDirFromConfig(); 189 if (stagingDir == null) { 190 stagingDir = getWorkFolder(testInfo); 191 } 192 if (fileExists(stagingDir)) { 193 buildInfo.stageRemoteFile(fileName, stagingDir); 194 // multiple matching files can be staged. So do a search with module name and 195 // abi in consideration. 196 file = findFile(fileName, abi, stagingDir, includeDirectory); 197 if (fileExists(file)) { 198 InvocationMetricLogger.addInvocationMetrics( 199 InvocationMetricKey.STAGE_UNDEFINED_DEPENDENCY, fileName); 200 CLog.d("Found file %s after staging remote file.", file.getAbsolutePath()); 201 return file; 202 } 203 } 204 } 205 } 206 CLog.e("Could not find an artifact file associated with %s.", fileName); 207 return null; 208 } 209 210 /** Returns the list of search locations in correct order. */ 211 @VisibleForTesting getSearchDirectories( boolean targetFirst, List<File> altDirs, AltDirBehavior altDirBehavior, TestInformation testInfo)212 List<File> getSearchDirectories( 213 boolean targetFirst, 214 List<File> altDirs, 215 AltDirBehavior altDirBehavior, 216 TestInformation testInfo) { 217 List<File> dirs = new LinkedList<>(); 218 // Prioritize the module directory retrieved from the config obj, as this is the ideal place 219 // for all test artifacts. 220 File moduleDir = getModuleDirFromConfig(); 221 if (moduleDir != null) { 222 dirs.add(moduleDir); 223 } 224 225 ExecutionFiles executionFiles = singleton.getExecutionFiles(testInfo); 226 if (executionFiles != null) { 227 // Add host/testcases or target/testcases directory first 228 FilesKey hostOrTarget = FilesKey.HOST_TESTS_DIRECTORY; 229 if (targetFirst) { 230 hostOrTarget = FilesKey.TARGET_TESTS_DIRECTORY; 231 } 232 File testcasesDir = executionFiles.get(hostOrTarget); 233 if (fileExists(testcasesDir)) { 234 dirs.add(testcasesDir); 235 } 236 237 // Add root test directory 238 File rootTestDir = executionFiles.get(FilesKey.TESTS_DIRECTORY); 239 if (fileExists(rootTestDir)) { 240 dirs.add(rootTestDir); 241 } 242 } else { 243 // try getting the search directories from the build info. 244 IBuildInfo buildInfo = singleton.getBuildInfo(); 245 if (buildInfo != null) { 246 247 // Add host/testcases or target/testcases directory first 248 BuildInfoFileKey hostOrTarget = BuildInfoFileKey.HOST_LINKED_DIR; 249 if (targetFirst) { 250 hostOrTarget = BuildInfoFileKey.TARGET_LINKED_DIR; 251 } 252 File testcasesDir = buildInfo.getFile(hostOrTarget); 253 if (fileExists(testcasesDir)) { 254 dirs.add(testcasesDir); 255 } 256 257 // Add root test directory 258 File rootTestDir = null; 259 if (buildInfo instanceof IDeviceBuildInfo) { 260 rootTestDir = ((IDeviceBuildInfo) buildInfo).getTestsDir(); 261 } 262 if (!fileExists(rootTestDir)) { 263 rootTestDir = buildInfo.getFile(BuildInfoFileKey.TESTDIR_IMAGE); 264 } 265 if (!fileExists(rootTestDir)) { 266 rootTestDir = buildInfo.getFile(BuildInfoFileKey.ROOT_DIRECTORY); 267 } 268 if (fileExists(rootTestDir)) { 269 dirs.add(rootTestDir); 270 } 271 } 272 } 273 274 // Add alternative directories based on the alt dir behavior 275 if (altDirs != null) { 276 // reverse the order so ones provided via command line last can be searched first 277 Collections.reverse(altDirs); 278 if (altDirBehavior == null || AltDirBehavior.FALLBACK.equals(altDirBehavior)) { 279 dirs.addAll(altDirs); 280 } else { 281 altDirs.addAll(dirs); 282 dirs = altDirs; 283 } 284 } 285 286 // Add working directory at the end as a last resort 287 File workDir = getWorkFolder(testInfo); 288 if (fileExists(workDir)) { 289 dirs.add(workDir); 290 } 291 return dirs; 292 } 293 294 /** Searches for the file in the given search directory and possibly matching the abi. */ findFile( String filename, IAbi abi, File searchDirectory, boolean includeDirectory)295 private static File findFile( 296 String filename, IAbi abi, File searchDirectory, boolean includeDirectory) { 297 if (filename == null || searchDirectory == null || !searchDirectory.exists()) { 298 return null; 299 } 300 // Try looking for abi if not provided. 301 if (abi == null) { 302 abi = findModuleAbi(); 303 } 304 File retFile; 305 String moduleName = singleton.findModuleName(); 306 // Check under module subdirectory first if it is present. 307 if (!Strings.isNullOrEmpty(moduleName)) { 308 try { 309 File moduleDir = FileUtil.findDirectory(moduleName, searchDirectory); 310 if (moduleDir != null) { 311 // return the entire module directory if it matches the search file name 312 if (includeDirectory && moduleName.equals(filename)) { 313 return moduleDir; 314 } 315 CLog.d("Searching the module dir: %s", moduleDir); 316 Set<File> allMatch = 317 FileUtil.findFiles(filename, abi, includeDirectory, moduleDir); 318 if (!allMatch.isEmpty()) { 319 if (allMatch.size() != 1) { 320 // when directories are included in the search, return any top 321 // level directory if present, otherwise return any file. 322 if (includeDirectory) { 323 List<File> directoriesMatched = new LinkedList<>(); 324 for (File f : allMatch) { 325 if (f.isDirectory()) { 326 directoriesMatched.add(f); 327 } 328 } 329 if (!directoriesMatched.isEmpty()) { 330 for (File directory : directoriesMatched) { 331 if (isTopLevelDirectory(directory, allMatch)) { 332 return directory; 333 } 334 } 335 } 336 } 337 } 338 // when only one file is found, OR 339 // when only files were searched (no dir) and multiple files matched, OR 340 // when directory and files both were searched, but no directory is present 341 // or no directory is top level 342 // return any file/directory since we do not know which to return. 343 return allMatch.iterator().next(); 344 } 345 } else { 346 CLog.w( 347 "we have a module name: %s but no directory found in %s.", 348 moduleName, searchDirectory); 349 } 350 } catch (IOException e) { 351 CLog.w( 352 "Something went wrong while searching for the module '%s' directory.", 353 moduleName); 354 CLog.e(e); 355 } 356 } 357 358 // if module subdirectory not present or file not found, search under the entire directory 359 try { 360 Set<File> allMatch = 361 FileUtil.findFilesObject(searchDirectory, filename, includeDirectory); 362 if (allMatch.size() == 1) { 363 // if only one file found, return this one since we can not filter anymore. 364 return allMatch.iterator().next(); 365 } else if (allMatch.size() > 1) { 366 // prioritize the top level file to avoid selecting from a wrong module directory. 367 for (File f : allMatch) { 368 if (searchDirectory.getAbsolutePath().equals(f.getParent())) { 369 return f; 370 } 371 } 372 } 373 // Fall-back to searching everything 374 if (!includeDirectory) { 375 allMatch = FileUtil.findFiles(filename, abi, false, searchDirectory); 376 if (!allMatch.isEmpty()) { 377 return allMatch.iterator().next(); 378 } 379 } else { 380 retFile = FileUtil.findFile(filename, null, searchDirectory); 381 if (retFile != null) { 382 // Search again with filtering on ABI 383 File fileWithAbi = FileUtil.findFile(filename, abi, searchDirectory); 384 if (fileWithAbi != null 385 && !fileWithAbi 386 .getAbsolutePath() 387 .startsWith(retFile.getAbsolutePath())) { 388 // When multiple matches are found, return the one with matching 389 // ABI unless src is its parent directory. 390 return fileWithAbi; 391 } 392 return retFile; 393 } 394 } 395 } catch (IOException e) { 396 CLog.w( 397 "Something went wrong while searching for file %s under the directory '%s'.", 398 filename, moduleName); 399 CLog.e(e); 400 } 401 CLog.w("Failed to find test file %s from directory %s.", filename, searchDirectory); 402 return null; 403 } 404 getModuleDirFromConfig(IInvocationContext moduleContext)405 public static File getModuleDirFromConfig(IInvocationContext moduleContext) { 406 if (moduleContext != null) { 407 return getModuleDirFromConfig(moduleContext.getConfigurationDescriptor()); 408 } 409 return null; 410 } 411 getModuleDirFromConfig(ConfigurationDescriptor descriptor)412 public static File getModuleDirFromConfig(ConfigurationDescriptor descriptor) { 413 if (descriptor != null) { 414 List<String> moduleDirPath = 415 descriptor.getMetaData(ConfigurationDescriptor.MODULE_DIR_PATH_KEY); 416 if (moduleDirPath != null && !moduleDirPath.isEmpty()) { 417 File moduleDir = new File(moduleDirPath.get(0)); 418 if (moduleDir.exists()) { 419 return moduleDir; 420 } 421 } 422 } 423 return null; 424 } 425 426 /** Returns the module directory if present, when called inside a module scope. */ getModuleDirFromConfig()427 public static File getModuleDirFromConfig() { 428 IInvocationContext moduleContext = CurrentInvocation.getModuleContext(); 429 return getModuleDirFromConfig(moduleContext); 430 } 431 432 /** 433 * Finds the module directory that matches the given module name 434 * 435 * @param moduleName The name of the module. 436 * @param targetFirst Whether we are favoring target-side vs. host-side for the search. 437 * @return the module directory. Can be null. 438 */ findModuleDir(String moduleName, boolean targetFirst)439 public static File findModuleDir(String moduleName, boolean targetFirst) { 440 try (CloseableTraceScope ignored = new CloseableTraceScope("findModuleDir")) { 441 List<File> searchDirectories = 442 singleton.getSearchDirectories(targetFirst, null, null, null); 443 for (File searchDirectory : searchDirectories) { 444 try { 445 File moduleDir = FileUtil.findDirectory(moduleName, searchDirectory); 446 if (moduleDir != null && moduleDir.exists()) { 447 return moduleDir; 448 } 449 } catch (IOException e) { 450 CLog.w( 451 "Something went wrong while searching for the module '%s' directory in" 452 + " %s.", 453 moduleName, searchDirectory); 454 CLog.e(e); 455 } 456 } 457 return null; 458 } 459 } 460 461 /** returns the module name for the current test invocation if present. */ 462 @VisibleForTesting findModuleName()463 String findModuleName() { 464 IInvocationContext moduleContext = CurrentInvocation.getModuleContext(); 465 if (moduleContext != null && moduleContext.getAttributes().get(MODULE_NAME) != null) { 466 return moduleContext.getAttributes().get(MODULE_NAME).get(0); 467 } else if (moduleContext != null 468 && moduleContext.getConfigurationDescriptor().getModuleName() != null) { 469 return moduleContext.getConfigurationDescriptor().getModuleName(); 470 } 471 return null; 472 } 473 474 /** returns the abi for the current module if present. */ findModuleAbi()475 private static IAbi findModuleAbi() { 476 IInvocationContext moduleContext = CurrentInvocation.getModuleContext(); 477 if (moduleContext != null && moduleContext.getAttributes().get(MODULE_ABI) != null) { 478 String abiName = moduleContext.getAttributes().get(MODULE_ABI).get(0); 479 return new Abi(abiName, AbiUtils.getBitness(abiName)); 480 } 481 return null; 482 } 483 484 /** returns the primary build info for the current invocation. */ 485 @VisibleForTesting getBuildInfo()486 IBuildInfo getBuildInfo() { 487 IInvocationContext context = CurrentInvocation.getInvocationContext(); 488 if (context != null 489 && context.getBuildInfos() != null 490 && !context.getBuildInfos().isEmpty()) { 491 return context.getBuildInfos().get(0); 492 } 493 return null; 494 } 495 496 @VisibleForTesting getExecutionFiles(TestInformation testInfo)497 ExecutionFiles getExecutionFiles(TestInformation testInfo) { 498 if (testInfo != null && testInfo.executionFiles() != null) { 499 return testInfo.executionFiles(); 500 } 501 return CurrentInvocation.getInvocationFiles(); 502 } 503 getWorkFolder(TestInformation testInfo)504 private static File getWorkFolder(TestInformation testInfo) { 505 if (testInfo != null && testInfo.dependenciesFolder() != null) { 506 return testInfo.dependenciesFolder(); 507 } 508 return CurrentInvocation.getInfo(InvocationInfo.WORK_FOLDER); 509 } 510 fileExists(File file)511 private static boolean fileExists(File file) { 512 return file != null && file.exists(); 513 } 514 515 /** 516 * Checks whether a directory can be considered a top level directory. A top level directory 517 * will contain all the files that are given in the list. 518 */ isTopLevelDirectory(File directoryToCheck, Set<File> files)519 private static boolean isTopLevelDirectory(File directoryToCheck, Set<File> files) { 520 for (File f : files) { 521 if (!f.getAbsolutePath().startsWith(directoryToCheck.getAbsolutePath())) { 522 return false; 523 } 524 } 525 return true; 526 } 527 } 528