1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.result.skipped; 17 18 import com.android.tradefed.build.content.ContentAnalysisContext; 19 import com.android.tradefed.build.content.ContentAnalysisContext.AnalysisMethod; 20 import com.android.tradefed.build.content.ContentModuleLister; 21 import com.android.tradefed.config.IConfiguration; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.config.OptionClass; 24 import com.android.tradefed.device.ITestDevice; 25 import com.android.tradefed.invoker.IInvocationContext; 26 import com.android.tradefed.invoker.InvocationContext; 27 import com.android.tradefed.invoker.TestInformation; 28 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 29 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 30 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 31 import com.android.tradefed.log.LogUtil.CLog; 32 import com.android.tradefed.result.skipped.SkipReason.DemotionTrigger; 33 import com.android.tradefed.service.TradefedFeatureClient; 34 import com.android.tradefed.util.IDisableable; 35 import com.android.tradefed.util.MultiMap; 36 37 import build.bazel.remote.execution.v2.Digest; 38 39 import com.proto.tradefed.feature.FeatureResponse; 40 import com.proto.tradefed.feature.PartResponse; 41 42 import java.util.ArrayList; 43 import java.util.HashMap; 44 import java.util.HashSet; 45 import java.util.LinkedHashMap; 46 import java.util.List; 47 import java.util.Map; 48 import java.util.Map.Entry; 49 import java.util.Set; 50 51 /** 52 * Based on a variety of criteria the skip manager helps to decide what should be skipped at 53 * different levels: invocation, modules and tests. 54 */ 55 @OptionClass(alias = "skip-manager") 56 public class SkipManager implements IDisableable { 57 58 @Option(name = "disable-skip-manager", description = "Disable the skip manager feature.") 59 private boolean mIsDisabled = false; 60 61 @Option( 62 name = "demotion-filters", 63 description = 64 "An option to manually inject demotion filters. Intended for testing and" 65 + " validation, not for production demotion.") 66 private Map<String, String> mDemotionFilterOption = new LinkedHashMap<>(); 67 68 @Option( 69 name = "skip-on-no-change", 70 description = "Enable the layer of skipping when there is no changes to artifacts.") 71 private boolean mSkipOnNoChange = false; 72 73 @Option( 74 name = "skip-on-no-tests-discovered", 75 description = "Enable the layer of skipping when there is no discovered tests to run.") 76 private boolean mSkipOnNoTestsDiscovered = true; 77 78 @Option( 79 name = "skip-on-no-change-presubmit-only", 80 description = "Allow enabling the skip logic only in presubmit.") 81 private boolean mSkipOnNoChangePresubmitOnly = true; 82 83 @Option( 84 name = "considered-for-content-analysis", 85 description = "Some tests do not directly rely on content for being relevant.") 86 private boolean mConsideredForContent = true; 87 88 @Option(name = "analysis-level", description = "Alter assumptions level of the analysis.") 89 private AnalysisHeuristic mAnalysisLevel = AnalysisHeuristic.REMOVE_EXEMPTION; 90 91 @Option( 92 name = "report-invocation-skipped-module", 93 description = 94 "Report a placeholder skip when module are skipped as part of invocation" 95 + " skipped.") 96 private boolean mReportInvocationModuleSkipped = true; 97 98 // Contains the filter and reason for demotion 99 private final Map<String, SkipReason> mDemotionFilters = new LinkedHashMap<>(); 100 101 private boolean mNoTestsDiscovered = false; 102 private MultiMap<ITestDevice, ContentAnalysisContext> mImageAnalysis = new MultiMap<>(); 103 private List<ContentAnalysisContext> mTestArtifactsAnalysisContent = new ArrayList<>(); 104 private List<String> mModulesDiscovered = new ArrayList<String>(); 105 private List<String> mDependencyFiles = new ArrayList<String>(); 106 107 private String mReasonForSkippingInvocation = "SkipManager decided to skip."; 108 private Set<String> mUnchangedModules = new HashSet<>(); 109 private Map<String, Digest> mImageFileToDigest = new LinkedHashMap<>(); 110 private Map<String, Digest> mTestArtifactsToDigest = new LinkedHashMap<>(); 111 112 /** Setup and initialize the skip manager. */ setup(IConfiguration config, IInvocationContext context)113 public void setup(IConfiguration config, IInvocationContext context) { 114 if (config.getCommandOptions().getInvocationData().containsKey("subprocess")) { 115 // Information is going to flow through GlobalFilters mechanism 116 return; 117 } 118 for (Entry<String, String> filterReason : mDemotionFilterOption.entrySet()) { 119 mDemotionFilters.put( 120 filterReason.getKey(), 121 new SkipReason(filterReason.getValue(), DemotionTrigger.UNKNOWN_TRIGGER)); 122 } 123 fetchDemotionInformation(context); 124 } 125 126 /** Returns the demoted tests and the reason for demotion */ getDemotedTests()127 public Map<String, SkipReason> getDemotedTests() { 128 return mDemotionFilters; 129 } 130 131 /** 132 * Returns the list of unchanged modules. Modules are only unchanged if device image is also 133 * unchanged. 134 */ getUnchangedModules()135 public Set<String> getUnchangedModules() { 136 return mUnchangedModules; 137 } 138 getImageToDigest()139 public Map<String, Digest> getImageToDigest() { 140 return mImageFileToDigest; 141 } 142 getTestArtifactsToDigest()143 public Map<String, Digest> getTestArtifactsToDigest() { 144 return mTestArtifactsToDigest; 145 } 146 setImageAnalysis(ITestDevice device, ContentAnalysisContext analysisContext)147 public void setImageAnalysis(ITestDevice device, ContentAnalysisContext analysisContext) { 148 CLog.d( 149 "Received image artifact analysis '%s' for %s", 150 analysisContext.contentEntry(), device.getSerialNumber()); 151 mImageAnalysis.put(device, analysisContext); 152 } 153 setTestArtifactsAnalysis(ContentAnalysisContext analysisContext)154 public void setTestArtifactsAnalysis(ContentAnalysisContext analysisContext) { 155 CLog.d("Received test artifact analysis '%s'", analysisContext.contentEntry()); 156 mTestArtifactsAnalysisContent.add(analysisContext); 157 } 158 159 /** 160 * In the early download and discovery process, report to the skip manager that no tests are 161 * expected to be run. This should lead to skipping the invocation. 162 */ reportDiscoveryWithNoTests()163 public void reportDiscoveryWithNoTests() { 164 CLog.d("Test discovery reported that no tests were found."); 165 mNoTestsDiscovered = true; 166 } 167 reportDiscoveryDependencies(List<String> modules, List<String> depFiles)168 public void reportDiscoveryDependencies(List<String> modules, List<String> depFiles) { 169 mModulesDiscovered.addAll(modules); 170 mDependencyFiles.addAll(depFiles); 171 } 172 173 /** Reports whether we should skip the current invocation. */ shouldSkipInvocation(TestInformation information, IConfiguration configuration)174 public boolean shouldSkipInvocation(TestInformation information, IConfiguration configuration) { 175 if (InvocationContext.isOnDemand(information.getContext())) { 176 // Avoid skipping invocation for on-demand testing 177 return false; 178 } 179 try (CloseableTraceScope ignored = 180 new CloseableTraceScope("SkipManager#shouldSkipInvocation")) { 181 // Build heuristic for skipping invocation 182 if (!mNoTestsDiscovered && !mModulesDiscovered.isEmpty()) { 183 Set<String> possibleModules = new HashSet<>(); 184 for (ContentAnalysisContext context : mTestArtifactsAnalysisContent) { 185 if (context.analysisMethod().equals(AnalysisMethod.SANDBOX_WORKDIR)) { 186 Set<String> modules = ContentModuleLister.buildModuleList(context); 187 if (modules == null) { 188 // If some sort of error occurs, never skip invocation 189 InvocationMetricLogger.addInvocationMetrics( 190 InvocationMetricKey.ERROR_INVOCATION_SKIP, 1); 191 return false; 192 } 193 possibleModules.addAll(modules); 194 } 195 } 196 if (!possibleModules.isEmpty()) { 197 CLog.d("Module existing in the zips: %s", possibleModules); 198 Set<String> runnableModules = new HashSet<String>(mModulesDiscovered); 199 runnableModules.retainAll(possibleModules); 200 if (runnableModules.isEmpty()) { 201 mNoTestsDiscovered = true; 202 CLog.d( 203 "discovered modules '%s' do not exists in zips.", 204 mModulesDiscovered); 205 } 206 } 207 } 208 209 if (mNoTestsDiscovered) { 210 InvocationMetricLogger.addInvocationMetrics( 211 InvocationMetricKey.SKIP_NO_TESTS_DISCOVERED, 1); 212 if (mSkipOnNoTestsDiscovered) { 213 mReasonForSkippingInvocation = 214 "No tests to be executed where found in the configuration."; 215 return true; 216 } else { 217 InvocationMetricLogger.addInvocationMetrics( 218 InvocationMetricKey.SILENT_INVOCATION_SKIP_COUNT, 1); 219 return false; 220 } 221 } 222 223 ArtifactsAnalyzer analyzer = 224 new ArtifactsAnalyzer( 225 information, 226 configuration, 227 mImageAnalysis, 228 mTestArtifactsAnalysisContent, 229 mModulesDiscovered, 230 mDependencyFiles, 231 mAnalysisLevel); 232 return buildAnalysisDecision(information, analyzer.analyzeArtifacts()); 233 } 234 } 235 236 /** 237 * Request to fetch the demotion information for the invocation. This should only be done once 238 * in the parent process. 239 */ fetchDemotionInformation(IInvocationContext context)240 private void fetchDemotionInformation(IInvocationContext context) { 241 if (isDisabled()) { 242 return; 243 } 244 if (InvocationContext.isPresubmit(context)) { 245 try (TradefedFeatureClient client = new TradefedFeatureClient()) { 246 Map<String, String> args = new HashMap<>(); 247 FeatureResponse response = client.triggerFeature("FetchDemotionInformation", args); 248 if (response.hasErrorInfo()) { 249 InvocationMetricLogger.addInvocationMetrics( 250 InvocationMetricKey.DEMOTION_ERROR_RESPONSE, 1); 251 } else { 252 for (PartResponse part : 253 response.getMultiPartResponse().getResponsePartList()) { 254 String filter = part.getKey(); 255 mDemotionFilters.put(filter, SkipReason.fromString(part.getValue())); 256 } 257 } 258 } 259 } 260 if (!mDemotionFilters.isEmpty()) { 261 CLog.d("Demotion filters size '%s': %s", mDemotionFilters.size(), mDemotionFilters); 262 InvocationMetricLogger.addInvocationMetrics( 263 InvocationMetricKey.DEMOTION_FILTERS_RECEIVED_COUNT, mDemotionFilters.size()); 264 } 265 } 266 267 /** Based on environment of the run and the build analysis, decide to skip or not. */ buildAnalysisDecision(TestInformation information, BuildAnalysis results)268 private boolean buildAnalysisDecision(TestInformation information, BuildAnalysis results) { 269 if (results == null) { 270 return false; 271 } 272 mImageFileToDigest.putAll(results.getImageToDigest()); 273 mTestArtifactsToDigest.putAll(results.getArtifactsToDigest()); 274 boolean presubmit = InvocationContext.isPresubmit(information.getContext()); 275 if (results.deviceImageChanged()) { 276 return false; 277 } 278 InvocationMetricLogger.addInvocationMetrics( 279 InvocationMetricKey.DEVICE_IMAGE_NOT_CHANGED, 1); 280 if (results.hasTestsArtifacts()) { 281 // Keep track of the set or sub-set of modules that didn't change. 282 mUnchangedModules.addAll(results.getUnchangedModules()); 283 if (results.hasChangesInTestsArtifacts()) { 284 InvocationMetricLogger.addInvocationMetrics( 285 InvocationMetricKey.TEST_ARTIFACT_CHANGE_ONLY, 1); 286 return false; 287 } else { 288 InvocationMetricLogger.addInvocationMetrics( 289 InvocationMetricKey.TEST_ARTIFACT_NOT_CHANGED, 1); 290 } 291 } else { 292 InvocationMetricLogger.addInvocationMetrics( 293 InvocationMetricKey.PURE_DEVICE_IMAGE_UNCHANGED, 1); 294 } 295 // If we get here, it means both device image and test artifacts are unaffected. 296 if (!mConsideredForContent) { 297 return false; 298 } 299 if (!presubmit) { 300 // Eventually support postsubmit analysis. 301 InvocationMetricLogger.addInvocationMetrics( 302 InvocationMetricKey.NO_CHANGES_POSTSUBMIT, 1); 303 return false; 304 } 305 // Currently only consider skipping in presubmit 306 InvocationMetricLogger.addInvocationMetrics(InvocationMetricKey.SKIP_NO_CHANGES, 1); 307 if (mSkipOnNoChange) { 308 mReasonForSkippingInvocation = 309 "No relevant changes to device image or test artifacts detected."; 310 return true; 311 } 312 if (presubmit && mSkipOnNoChangePresubmitOnly) { 313 mReasonForSkippingInvocation = 314 "No relevant changes to device image or test artifacts detected."; 315 return true; 316 } 317 InvocationMetricLogger.addInvocationMetrics( 318 InvocationMetricKey.SILENT_INVOCATION_SKIP_COUNT, 1); 319 return false; 320 } 321 clearManager()322 public void clearManager() { 323 mDemotionFilters.clear(); 324 mDemotionFilterOption.clear(); 325 mModulesDiscovered.clear(); 326 mDependencyFiles.clear(); 327 for (ContentAnalysisContext request : mTestArtifactsAnalysisContent) { 328 if (request.contentInformation() != null) { 329 request.contentInformation().clean(); 330 } 331 } 332 for (ContentAnalysisContext request : mImageAnalysis.values()) { 333 if (request.contentInformation() != null) { 334 request.contentInformation().clean(); 335 } 336 } 337 mTestArtifactsAnalysisContent.clear(); 338 mImageAnalysis.clear(); 339 } 340 341 @Override isDisabled()342 public boolean isDisabled() { 343 return mIsDisabled; 344 } 345 346 @Override setDisable(boolean isDisabled)347 public void setDisable(boolean isDisabled) { 348 mIsDisabled = isDisabled; 349 } 350 setSkipDecision(boolean shouldSkip)351 public void setSkipDecision(boolean shouldSkip) { 352 mSkipOnNoChange = shouldSkip; 353 mSkipOnNoTestsDiscovered = shouldSkip; 354 } 355 getInvocationSkipReason()356 public String getInvocationSkipReason() { 357 return mReasonForSkippingInvocation; 358 } 359 reportInvocationSkippedModule()360 public boolean reportInvocationSkippedModule() { 361 return mReportInvocationModuleSkipped; 362 } 363 } 364