1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.compatibility.common.tradefed.targetprep; 18 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 20 import com.android.compatibility.common.util.DeviceInfo; 21 import com.android.compatibility.common.util.HostInfoStore; 22 import com.android.tradefed.build.IBuildInfo; 23 import com.android.tradefed.config.Option; 24 import com.android.tradefed.config.OptionClass; 25 import com.android.tradefed.device.DeviceNotAvailableException; 26 import com.android.tradefed.device.ITestDevice; 27 import com.android.tradefed.invoker.IInvocationContext; 28 import com.android.tradefed.invoker.TestInformation; 29 import com.android.tradefed.log.LogUtil.CLog; 30 import com.android.tradefed.result.error.TestErrorIdentifier; 31 import com.android.tradefed.targetprep.BaseTargetPreparer; 32 import com.android.tradefed.targetprep.BuildError; 33 import com.android.tradefed.targetprep.TargetSetupError; 34 import com.android.tradefed.util.FileUtil; 35 import com.android.tradefed.util.StreamUtil; 36 37 import java.io.BufferedReader; 38 import java.io.File; 39 import java.io.FileNotFoundException; 40 import java.io.FileReader; 41 import java.io.IOException; 42 import java.util.Arrays; 43 import java.util.HashMap; 44 import java.util.HashSet; 45 import java.util.List; 46 import java.util.Map; 47 import java.util.Set; 48 import java.util.regex.Pattern; 49 50 /** Collects the dEQP dependencies and compares the builds. */ 51 @OptionClass(alias = "incremental-deqp-preparer") 52 public class IncrementalDeqpPreparer extends BaseTargetPreparer { 53 @Option(name = "run-mode", description = "The run mode for incremental dEQP.") 54 private RunMode mRunMode = RunMode.LIGHTWEIGHT_RUN; 55 56 @Option( 57 name = "fallback-strategy", 58 description = 59 "The fallback strategy to apply if the incremental dEQP qualification testing " 60 + "for the builds fails.") 61 private FallbackStrategy mFallbackStrategy = FallbackStrategy.ABORT_IF_ANY_EXCEPTION; 62 63 public enum RunMode { 64 // Collects the dependencies information for the build via the full dEQP tests. 65 FULL_RUN, 66 // Collects the dependencies information for the build via the representative dEQP tests. 67 LIGHTWEIGHT_RUN 68 } 69 70 private enum FallbackStrategy { 71 // Continues to run full dEQP tests no matter an exception is thrown or not. 72 RUN_FULL_DEQP, 73 // Aborts if an exception is thrown in the preparer. Otherwise, runs full dEQP tests due to 74 // dependency modifications. 75 ABORT_IF_ANY_EXCEPTION 76 } 77 78 private static final String MODULE_NAME = "CtsDeqpTestCases"; 79 private static final String DEVICE_DEQP_DIR = "/data/local/tmp"; 80 private static final List<String> BASELINE_DEQP_TEST_LIST = 81 Arrays.asList( 82 "gles2-incremental-deqp-baseline", 83 "gles3-incremental-deqp-baseline", 84 "gles31-incremental-deqp-baseline", 85 "vk-incremental-deqp-baseline"); 86 private static final List<String> REPRESENTATIVE_DEQP_TEST_LIST = 87 Arrays.asList("vk-incremental-deqp", "gles3-incremental-deqp"); 88 private static final List<String> DEQP_BINARY_LIST = 89 Arrays.asList("deqp-binary32", "deqp-binary64"); 90 private static final String DEQP_CASE_LIST_FILE_EXTENSION = ".txt"; 91 private static final String PERF_FILE_EXTENSION = ".data"; 92 private static final String LOG_FILE_EXTENSION = ".qpa"; 93 private static final String RUN_MODE_ATTRIBUTE = "run_mode"; 94 private static final String MODULE_ATTRIBUTE = "module"; 95 private static final String MODULE_NAME_ATTRIBUTE = "module_name"; 96 private static final String DEPENDENCY_DETAILS_ATTRIBUTE = "deps_details"; 97 private static final String DEPENDENCY_NAME_ATTRIBUTE = "dep_name"; 98 private static final String DEPENDENCY_FILE_HASH_ATTRIBUTE = "file_hash"; 99 100 private static final Pattern EXCLUDE_DEQP_PATTERN = 101 Pattern.compile("(^/data/|^/apex/|^\\[vdso" + "\\]|^/dmabuf|^/kgsl-3d0|^/mali csf)"); 102 public static final String INCREMENTAL_DEQP_ATTRIBUTE_NAME = "incremental-deqp"; 103 public static final String INCREMENTAL_DEQP_REPORT_NAME = 104 "IncrementalCtsDeviceInfo.deviceinfo.json"; 105 106 @Override setUp(TestInformation testInfo)107 public void setUp(TestInformation testInfo) 108 throws TargetSetupError, BuildError, DeviceNotAvailableException { 109 try { 110 ITestDevice device = testInfo.getDevice(); 111 CompatibilityBuildHelper buildHelper = 112 new CompatibilityBuildHelper(testInfo.getBuildInfo()); 113 IInvocationContext context = testInfo.getContext(); 114 runIncrementalDeqp(context, device, buildHelper, mRunMode); 115 } catch (Exception e) { 116 if (mFallbackStrategy == FallbackStrategy.ABORT_IF_ANY_EXCEPTION) { 117 // Rethrows the exception to abort the task. 118 throw e; 119 } 120 // Ignores the exception and continues to run full dEQP tests. 121 } 122 } 123 124 /** 125 * Collects dEQP dependencies and generate an incremental cts report with more details. 126 * 127 * <p>Synchronize this method so that multiple shards won't run it multiple times. 128 */ runIncrementalDeqp( IInvocationContext context, ITestDevice device, CompatibilityBuildHelper buildHelper, RunMode runMode)129 protected void runIncrementalDeqp( 130 IInvocationContext context, 131 ITestDevice device, 132 CompatibilityBuildHelper buildHelper, 133 RunMode runMode) 134 throws TargetSetupError, DeviceNotAvailableException { 135 // Make sure synchronization is on the class not the object. 136 synchronized (IncrementalDeqpPreparer.class) { 137 File jsonFile; 138 try { 139 File deviceInfoDir = 140 new File(buildHelper.getResultDir(), DeviceInfo.RESULT_DIR_NAME); 141 jsonFile = new File(deviceInfoDir, INCREMENTAL_DEQP_REPORT_NAME); 142 if (jsonFile.exists()) { 143 CLog.i("Another shard has already checked dEQP dependencies."); 144 // Add an attribute to the shard's build info. 145 addBuildAttribute(context, INCREMENTAL_DEQP_ATTRIBUTE_NAME); 146 return; 147 } 148 } catch (FileNotFoundException e) { 149 throw new TargetSetupError( 150 "Fail to read invocation result directory.", 151 device.getDeviceDescriptor(), 152 TestErrorIdentifier.TEST_ABORTED); 153 } 154 155 List<String> deqpTestList = 156 RunMode.FULL_RUN.equals(mRunMode) 157 ? BASELINE_DEQP_TEST_LIST 158 : REPRESENTATIVE_DEQP_TEST_LIST; 159 Set<String> dependencies = getDeqpDependencies(device, deqpTestList); 160 161 // Identify and write dependencies to device info report. 162 try (HostInfoStore store = new HostInfoStore(jsonFile)) { 163 store.open(); 164 store.addResult(RUN_MODE_ATTRIBUTE, runMode.name()); 165 store.startArray(MODULE_ATTRIBUTE); 166 store.startGroup(); // Module 167 store.addResult(MODULE_NAME_ATTRIBUTE, MODULE_NAME); 168 store.startArray(DEPENDENCY_DETAILS_ATTRIBUTE); 169 Map<String, String> currentBuildHashMap = getFileHash(dependencies, device); 170 for (String dependency : dependencies) { 171 store.startGroup(); 172 store.addResult(DEPENDENCY_NAME_ATTRIBUTE, dependency); 173 store.addResult( 174 DEPENDENCY_FILE_HASH_ATTRIBUTE, currentBuildHashMap.get(dependency)); 175 store.endGroup(); 176 } 177 store.endArray(); // dEQP details 178 store.endGroup(); // Module 179 store.endArray(); 180 addBuildAttribute(context, INCREMENTAL_DEQP_ATTRIBUTE_NAME); 181 } catch (IOException e) { 182 throw new TargetSetupError( 183 "Failed to collect dependencies", 184 e, 185 device.getDeviceDescriptor(), 186 TestErrorIdentifier.TEST_ABORTED); 187 } catch (Exception e) { 188 throw new TargetSetupError( 189 "Failed to write incremental dEQP report", 190 e, 191 device.getDeviceDescriptor(), 192 TestErrorIdentifier.TEST_ABORTED); 193 } finally { 194 if (jsonFile.exists() && jsonFile.length() == 0) { 195 FileUtil.deleteFile(jsonFile); 196 } 197 } 198 } 199 } 200 201 /** Gets the filename of dEQP dependencies in build. */ getDeqpDependencies(ITestDevice device, List<String> testList)202 private Set<String> getDeqpDependencies(ITestDevice device, List<String> testList) 203 throws TargetSetupError, DeviceNotAvailableException { 204 Set<String> result = new HashSet<>(); 205 206 for (String test : testList) { 207 for (String binaryName : DEQP_BINARY_LIST) { 208 String fileNamePrefix = test + "-" + binaryName; 209 String perfFile = DEVICE_DEQP_DIR + "/" + fileNamePrefix + PERF_FILE_EXTENSION; 210 String binaryFile = DEVICE_DEQP_DIR + "/" + binaryName; 211 String testFile = DEVICE_DEQP_DIR + "/" + test + DEQP_CASE_LIST_FILE_EXTENSION; 212 String logFile = DEVICE_DEQP_DIR + "/" + fileNamePrefix + LOG_FILE_EXTENSION; 213 214 String command = 215 String.format( 216 "cd %s && simpleperf record -o %s %s --deqp-caselist-file=%s" 217 + " --deqp-log-images=disable --deqp-log-shader-sources=disable" 218 + " --deqp-log-filename=%s --deqp-surface-type=fbo" 219 + " --deqp-surface-width=2048 --deqp-surface-height=2048", 220 DEVICE_DEQP_DIR, perfFile, binaryFile, testFile, logFile); 221 device.executeShellCommand(command); 222 223 String dumpFile = DEVICE_DEQP_DIR + "/" + fileNamePrefix + "-perf-dump.txt"; 224 String dumpCommand = String.format("simpleperf dump %s > %s", perfFile, dumpFile); 225 device.executeShellCommand(dumpCommand); 226 227 File localDumpFile = device.pullFile(dumpFile); 228 try { 229 result.addAll(parseDump(localDumpFile)); 230 } finally { 231 if (localDumpFile != null) { 232 localDumpFile.delete(); 233 } 234 } 235 } 236 } 237 238 return result; 239 } 240 241 /** Gets the hash value of the specified file's content from the device. */ getFileHash(Set<String> fileNames, ITestDevice device)242 protected Map<String, String> getFileHash(Set<String> fileNames, ITestDevice device) 243 throws DeviceNotAvailableException, TargetSetupError { 244 Map<String, String> fileHashes = new HashMap<>(); 245 for (String file : fileNames) { 246 File localFile = device.pullFile(file); 247 if (localFile == null) { 248 throw new TargetSetupError( 249 String.format("Fail to load file: %s from the device.", file), 250 TestErrorIdentifier.TEST_ABORTED); 251 } 252 String md5 = FileUtil.calculateMd5(localFile); 253 fileHashes.put(file, md5); 254 } 255 return fileHashes; 256 } 257 258 /** Parses the dump file and gets list of dependencies. */ parseDump(File localDumpFile)259 protected Set<String> parseDump(File localDumpFile) throws TargetSetupError { 260 boolean binaryExecuted = false; 261 boolean correctMmap = false; 262 Set<String> result = new HashSet<>(); 263 if (localDumpFile == null) { 264 return result; 265 } 266 BufferedReader br = null; 267 try { 268 br = new BufferedReader(new FileReader(localDumpFile)); 269 String line; 270 while ((line = br.readLine()) != null) { 271 if (!binaryExecuted) { 272 // dEQP binary has first been executed. 273 Pattern pattern = Pattern.compile(" comm .*deqp-binary"); 274 if (pattern.matcher(line).find()) { 275 binaryExecuted = true; 276 } 277 } else { 278 // New perf event 279 if (!line.startsWith(" ")) { 280 // Ignore mmap with misc 1, they are not related to deqp binary 281 correctMmap = line.startsWith("record mmap") && !line.contains("misc 1"); 282 } 283 284 // We have reached the filename for a valid perf event, add to the dependency 285 // map if it isn't in the exclusion pattern 286 if (line.contains("filename") && correctMmap) { 287 String dependency = line.substring(line.indexOf("filename") + 9).trim(); 288 if (!EXCLUDE_DEQP_PATTERN.matcher(dependency).find()) { 289 result.add(dependency); 290 } 291 } 292 } 293 } 294 } catch (IOException e) { 295 throw new TargetSetupError( 296 String.format("Could not parse file: %s", localDumpFile.getAbsoluteFile()), 297 e, 298 TestErrorIdentifier.TEST_ABORTED); 299 } finally { 300 StreamUtil.close(br); 301 } 302 return result; 303 } 304 305 /** Adds a build attribute to all the {@link IBuildInfo} tracked for the invocation. */ addBuildAttribute(IInvocationContext context, String buildAttributeName)306 private static void addBuildAttribute(IInvocationContext context, String buildAttributeName) { 307 for (IBuildInfo bi : context.getBuildInfos()) { 308 bi.addBuildAttribute(buildAttributeName, ""); 309 } 310 } 311 } 312