1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.tradefed.config.ConfigurationException; 19 import com.android.tradefed.config.IConfiguration; 20 import com.android.tradefed.config.IDeviceConfiguration; 21 import com.android.tradefed.config.OptionCopier; 22 import com.android.tradefed.targetprep.ITargetPreparer; 23 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer; 24 import com.android.tradefed.testtype.IAbiReceiver; 25 import com.android.tradefed.testtype.IRemoteTest; 26 import com.android.tradefed.testtype.IShardableTest; 27 28 import java.util.ArrayList; 29 import java.util.Collection; 30 import java.util.LinkedHashMap; 31 import java.util.List; 32 import java.util.Map; 33 import java.util.Map.Entry; 34 35 /** 36 * Helper to split a list of modules represented by {@link IConfiguration} into a list of execution 37 * units represented by {@link ModuleDefinition}. 38 * 39 * <p>Each configuration may generate 1 or more {@link ModuleDefinition} depending on its options 40 * and test types: 41 * 42 * <ul> 43 * <li>A non-shardable {@link IConfiguration} will generate a single {@link ModuleDefinition}. 44 * <li>A shardable {@link IConfiguration} will generate a number of ModuleDefinition linked to the 45 * {@link IRemoteTest} properties: 46 * <ul> 47 * <li>A non - {@link IShardableTest} will generate a single ModuleDefinition. 48 * <li>A {@link IShardableTest} generates one ModuleDefinition per tests returned by {@link 49 * IShardableTest#split()}. 50 * </ul> 51 * 52 * </ul> 53 */ 54 public class ModuleSplitter { 55 56 /** 57 * Create a List of executable unit {@link ModuleDefinition}s based on the map of configuration 58 * that was loaded. 59 * 60 * @param runConfig {@link LinkedHashMap} loaded from {@link ITestSuite#loadTests()}. 61 * @param shardCount a shard count hint to help with sharding. 62 * @param dynamicModule Whether or not module can be shared in pool or must be independent 63 * (strict sharding). 64 * @param intraModuleSharding Whether or not to shard within the modules. 65 * @return List of {@link ModuleDefinition} 66 */ splitConfiguration( LinkedHashMap<String, IConfiguration> runConfig, int shardCount, boolean dynamicModule, boolean intraModuleSharding)67 public static List<ModuleDefinition> splitConfiguration( 68 LinkedHashMap<String, IConfiguration> runConfig, 69 int shardCount, 70 boolean dynamicModule, 71 boolean intraModuleSharding) { 72 if (dynamicModule) { 73 // We maximize the sharding for dynamic to reduce time difference between first and 74 // last shard as much as possible. Overhead is low due to our test pooling. 75 shardCount *= 2; 76 } 77 List<ModuleDefinition> runModules = new ArrayList<>(); 78 for (Entry<String, IConfiguration> configMap : runConfig.entrySet()) { 79 // Check that it's a valid configuration for suites, throw otherwise. 80 ValidateSuiteConfigHelper.validateConfig(configMap.getValue()); 81 82 createAndAddModule( 83 runModules, 84 configMap.getKey(), 85 configMap.getValue(), 86 shardCount, 87 dynamicModule, 88 intraModuleSharding); 89 } 90 return runModules; 91 } 92 createAndAddModule( List<ModuleDefinition> currentList, String moduleName, IConfiguration config, int shardCount, boolean dynamicModule, boolean intraModuleSharding)93 private static void createAndAddModule( 94 List<ModuleDefinition> currentList, 95 String moduleName, 96 IConfiguration config, 97 int shardCount, 98 boolean dynamicModule, 99 boolean intraModuleSharding) { 100 // If this particular configuration module is declared as 'not shardable' we take it whole 101 // but still split the individual IRemoteTest in a pool. 102 if (!intraModuleSharding 103 || config.getConfigurationDescription().isNotShardable() 104 || (!dynamicModule 105 && config.getConfigurationDescription().isNotStrictShardable())) { 106 for (int i = 0; i < config.getTests().size(); i++) { 107 if (dynamicModule) { 108 ModuleDefinition module = 109 new ModuleDefinition( 110 moduleName, 111 config.getTests(), 112 clonePreparersMap(config), 113 clonePreparers(config.getMultiTargetPreparers()), 114 config); 115 currentList.add(module); 116 } else { 117 addModuleToListFromSingleTest( 118 currentList, config.getTests().get(i), moduleName, config); 119 } 120 } 121 return; 122 } 123 124 // If configuration is possibly shardable we attempt to shard it. 125 for (IRemoteTest test : config.getTests()) { 126 if (test instanceof IShardableTest) { 127 Collection<IRemoteTest> shardedTests = ((IShardableTest) test).split(shardCount); 128 if (shardedTests != null) { 129 // Test did shard we put the shard pool in ModuleDefinition which has a polling 130 // behavior on the pool. 131 if (dynamicModule) { 132 for (int i = 0; i < shardCount; i++) { 133 ModuleDefinition module = 134 new ModuleDefinition( 135 moduleName, 136 shardedTests, 137 clonePreparersMap(config), 138 clonePreparers(config.getMultiTargetPreparers()), 139 config); 140 currentList.add(module); 141 } 142 } else { 143 // We create independent modules with each sharded test. 144 for (IRemoteTest moduleTest : shardedTests) { 145 addModuleToListFromSingleTest( 146 currentList, moduleTest, moduleName, config); 147 } 148 } 149 continue; 150 } 151 } 152 // test is not shardable or did not shard 153 addModuleToListFromSingleTest(currentList, test, moduleName, config); 154 } 155 } 156 157 /** 158 * Helper to add a new {@link ModuleDefinition} to our list of Modules from a single {@link 159 * IRemoteTest}. 160 */ addModuleToListFromSingleTest( List<ModuleDefinition> currentList, IRemoteTest test, String moduleName, IConfiguration config)161 private static void addModuleToListFromSingleTest( 162 List<ModuleDefinition> currentList, 163 IRemoteTest test, 164 String moduleName, 165 IConfiguration config) { 166 List<IRemoteTest> testList = new ArrayList<>(); 167 testList.add(test); 168 ModuleDefinition module = 169 new ModuleDefinition( 170 moduleName, 171 testList, 172 clonePreparersMap(config), 173 clonePreparers(config.getMultiTargetPreparers()), 174 config); 175 currentList.add(module); 176 } 177 178 /** 179 * Deep clone a list of {@link ITargetPreparer} or {@link IMultiTargetPreparer}. We are ensured 180 * to find a default constructor with no arguments since that's the expectation from Tradefed 181 * when loading configuration. Cloning preparers is required since they may be stateful and we 182 * cannot share instance across devices. 183 */ clonePreparers(List<T> preparerList)184 private static <T> List<T> clonePreparers(List<T> preparerList) { 185 List<T> clones = new ArrayList<>(); 186 for (T prep : preparerList) { 187 try { 188 @SuppressWarnings("unchecked") 189 T clone = (T) prep.getClass().newInstance(); 190 OptionCopier.copyOptions(prep, clone); 191 // Ensure we copy the Abi too. 192 if (clone instanceof IAbiReceiver) { 193 ((IAbiReceiver) clone).setAbi(((IAbiReceiver) prep).getAbi()); 194 } 195 clones.add(clone); 196 } catch (InstantiationException | IllegalAccessException | ConfigurationException e) { 197 throw new RuntimeException(e); 198 } 199 } 200 return clones; 201 } 202 203 /** Deep cloning of potentially multi-device preparers. */ clonePreparersMap(IConfiguration config)204 private static Map<String, List<ITargetPreparer>> clonePreparersMap(IConfiguration config) { 205 Map<String, List<ITargetPreparer>> res = new LinkedHashMap<>(); 206 for (IDeviceConfiguration holder : config.getDeviceConfig()) { 207 List<ITargetPreparer> preparers = new ArrayList<>(); 208 res.put(holder.getDeviceName(), preparers); 209 preparers.addAll(clonePreparers(holder.getTargetPreparers())); 210 } 211 return res; 212 } 213 } 214