1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.game.qualification.testtype; 18 19 import com.android.game.qualification.ApkInfo; 20 import com.android.game.qualification.CertificationRequirements; 21 import com.android.game.qualification.GameCoreConfiguration; 22 import com.android.game.qualification.GameCoreConfigurationXmlParser; 23 import com.android.game.qualification.metric.BaseGameQualificationMetricCollector; 24 import com.android.game.qualification.reporter.GameQualificationResultReporter; 25 import com.android.game.qualification.test.PerformanceTest; 26 import com.android.tradefed.config.IConfiguration; 27 import com.android.tradefed.config.IConfigurationReceiver; 28 import com.android.tradefed.config.Option; 29 import com.android.tradefed.device.DeviceNotAvailableException; 30 import com.android.tradefed.device.ITestDevice; 31 import com.android.tradefed.device.metric.IMetricCollector; 32 import com.android.tradefed.device.metric.IMetricCollectorReceiver; 33 import com.android.tradefed.invoker.IInvocationContext; 34 import com.android.tradefed.log.LogUtil.CLog; 35 import com.android.tradefed.metrics.proto.MetricMeasurement; 36 import com.android.tradefed.result.ITestInvocationListener; 37 import com.android.tradefed.result.TestDescription; 38 import com.android.tradefed.testtype.IDeviceTest; 39 import com.android.tradefed.testtype.IInvocationContextReceiver; 40 import com.android.tradefed.testtype.IRemoteTest; 41 import com.android.tradefed.testtype.IShardableTest; 42 import com.android.tradefed.testtype.ITestFilterReceiver; 43 44 import com.google.common.io.ByteStreams; 45 46 import org.junit.AssumptionViolatedException; 47 import org.xml.sax.SAXException; 48 49 import java.io.File; 50 import java.io.FileNotFoundException; 51 import java.io.FileOutputStream; 52 import java.io.IOException; 53 import java.io.InputStream; 54 import java.io.OutputStream; 55 import java.util.ArrayList; 56 import java.util.Collection; 57 import java.util.HashMap; 58 import java.util.HashSet; 59 import java.util.List; 60 import java.util.Set; 61 62 import javax.xml.parsers.ParserConfigurationException; 63 64 public class GameQualificationHostsideController implements 65 IShardableTest, 66 IDeviceTest, 67 IMetricCollectorReceiver, 68 IInvocationContextReceiver, 69 IConfigurationReceiver, 70 ITestFilterReceiver { 71 // Package and class of the device side test. 72 public static final String PACKAGE = "com.android.game.qualification.device"; 73 public static final String CLASS = PACKAGE + ".GameQualificationTest"; 74 75 private ITestDevice mDevice; 76 private IConfiguration mConfiguration = null; 77 private GameCoreConfiguration mGameCoreConfiguration = null; 78 private List<ApkInfo> mApks = null; 79 private File mApkInfoFile; 80 private Collection<IMetricCollector> mCollectors; 81 private GameQualificationResultReporter mResultReporter; 82 private IInvocationContext mContext; 83 private ArrayList<BaseGameQualificationMetricCollector> mAGQMetricCollectors; 84 private Set<String> mIncludeFilters = new HashSet<>(); 85 private Set<String> mExcludeFilters = new HashSet<>(); 86 87 @Override setDevice(ITestDevice device)88 public void setDevice(ITestDevice device) { 89 mDevice = device; 90 } 91 92 @Override getDevice()93 public ITestDevice getDevice() { 94 return mDevice; 95 } 96 97 @Option(name = "apk-info", 98 description = "An XML file describing the list of APKs for qualifications.", 99 importance = Option.Importance.ALWAYS) 100 private String mApkInfoFileName; 101 102 @Option(name = "apk-dir", 103 description = 104 "Directory contains the APKs for qualifications. If --apk-info is not " 105 + "specified and a file named 'apk-info.xml' exists in --apk-dir, that " 106 + "file will be used as the apk-info.", 107 importance = Option.Importance.ALWAYS) 108 private String mApkDir; 109 getApkDir()110 private String getApkDir() { 111 if (mApkDir == null) { 112 String out = System.getenv("ANDROID_PRODUCT_OUT"); 113 if (out != null) { 114 CLog.i("--apk-dir was not set, looking in ANDROID_PRODUCT_OUT(%s) for apk files.", 115 out); 116 mApkDir = out + "/data/app"; 117 } else { 118 CLog.i("--apk-dir and ANDROID_PRODUCT_OUT was not set, looking in current " 119 + "directory(%s) for apk files.", 120 new File(".").getAbsolutePath()); 121 mApkDir = "."; 122 } 123 } 124 return mApkDir; 125 } 126 127 @Override addIncludeFilter(String filter)128 public void addIncludeFilter(String filter) { 129 mIncludeFilters.add(filter); 130 } 131 132 @Override addAllIncludeFilters(Set<String> filters)133 public void addAllIncludeFilters(Set<String> filters) { 134 mIncludeFilters.addAll(filters); 135 } 136 137 @Override addExcludeFilter(String filter)138 public void addExcludeFilter(String filter) { 139 mExcludeFilters.add(filter); 140 } 141 142 @Override addAllExcludeFilters(Set<String> filters)143 public void addAllExcludeFilters(Set<String> filters) { 144 mExcludeFilters.addAll(filters); 145 } 146 147 @Override getIncludeFilters()148 public Set<String> getIncludeFilters() { 149 return mIncludeFilters; 150 } 151 152 @Override getExcludeFilters()153 public Set<String> getExcludeFilters() { 154 return mExcludeFilters; 155 } 156 157 @Override clearIncludeFilters()158 public void clearIncludeFilters() { 159 mIncludeFilters.clear(); 160 } 161 162 @Override clearExcludeFilters()163 public void clearExcludeFilters() { 164 mExcludeFilters.clear(); 165 } 166 167 @Override setMetricCollectors(List<IMetricCollector> list)168 public void setMetricCollectors(List<IMetricCollector> list) { 169 mCollectors = list; 170 mAGQMetricCollectors = new ArrayList<>(); 171 for (IMetricCollector collector : list) { 172 if (collector instanceof BaseGameQualificationMetricCollector) { 173 mAGQMetricCollectors.add((BaseGameQualificationMetricCollector) collector); 174 } 175 } 176 } 177 178 179 @Override setInvocationContext(IInvocationContext iInvocationContext)180 public void setInvocationContext(IInvocationContext iInvocationContext) { 181 mContext = iInvocationContext; 182 } 183 184 @Override setConfiguration(IConfiguration configuration)185 public void setConfiguration(IConfiguration configuration) { 186 mConfiguration = configuration; 187 } 188 189 @Override split(int shardCountHint)190 public Collection<IRemoteTest> split(int shardCountHint) { 191 initApkList(); 192 List<IRemoteTest> shards = new ArrayList<>(); 193 for(int i = 0; i < shardCountHint; i++) { 194 if (i >= mApks.size()) { 195 break; 196 } 197 List<ApkInfo> apkInfo = new ArrayList<>(); 198 for(int j = i; j < mApks.size(); j += shardCountHint) { 199 apkInfo.add(mApks.get(j)); 200 } 201 GameQualificationHostsideController shard = new GameQualificationHostsideController(); 202 shard.mApks = apkInfo; 203 shard.mApkDir = getApkDir(); 204 shard.mApkInfoFileName = mApkInfoFileName; 205 shard.mApkInfoFile = mApkInfoFile; 206 207 shards.add(shard); 208 } 209 return shards; 210 } 211 212 @Override run(ITestInvocationListener listener)213 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 214 // Find result reporter 215 if (mResultReporter == null) { 216 for (ITestInvocationListener testListener 217 : mConfiguration.getTestInvocationListeners()) { 218 if (testListener instanceof GameQualificationResultReporter) { 219 mResultReporter = (GameQualificationResultReporter) testListener; 220 } 221 } 222 } 223 224 assert !(mAGQMetricCollectors.isEmpty()); 225 for (IMetricCollector collector : mCollectors) { 226 listener = collector.init(mContext, listener); 227 } 228 229 for (BaseGameQualificationMetricCollector collector : mAGQMetricCollectors) { 230 collector.setDevice(getDevice()); 231 } 232 233 HashMap<String, MetricMeasurement.Metric> runMetrics = new HashMap<>(); 234 235 initApkList(); 236 getDevice().pushFile(mApkInfoFile, ApkInfo.APK_LIST_LOCATION); 237 238 long startTime = System.currentTimeMillis(); 239 listener.testRunStarted("gamequalification", mApks.size()); 240 241 for (ApkInfo apk : mApks) { 242 PerformanceTest test = 243 new PerformanceTest( 244 getDevice(), 245 listener, 246 mAGQMetricCollectors, 247 apk, 248 getApkDir(), 249 mApkInfoFile.getParentFile()); 250 for (BaseGameQualificationMetricCollector collector : mAGQMetricCollectors) { 251 collector.setApkInfo(apk); 252 collector.setCertificationRequirements( 253 mGameCoreConfiguration.findCertificationRequirements(apk.getName())); 254 } 255 for (PerformanceTest.Test t : PerformanceTest.Test.values()) { 256 TestDescription identifier = new TestDescription(CLASS, t.getName() + "[" + apk.getName() + "]"); 257 if (t.isEnableCollectors()) { 258 for (BaseGameQualificationMetricCollector collector : mAGQMetricCollectors) { 259 collector.enable(); 260 } 261 if (mResultReporter != null) { 262 CertificationRequirements req = 263 mGameCoreConfiguration.findCertificationRequirements(apk.getName()); 264 if (req != null) { 265 mResultReporter.putRequirements(identifier, req); 266 } 267 } 268 } 269 listener.testStarted(identifier); 270 try { 271 t.getMethod().run(test); 272 } catch(AssumptionViolatedException e) { 273 listener.testAssumptionFailure(identifier, e.getMessage()); 274 } catch (Error | Exception e) { 275 test.failed(); 276 listener.testFailed(identifier, e.getMessage()); 277 } 278 listener.testEnded(identifier, new HashMap<String, MetricMeasurement.Metric>()); 279 280 if (t.isEnableCollectors()) { 281 for (BaseGameQualificationMetricCollector collector : mAGQMetricCollectors) { 282 if (collector.hasError()) { 283 listener.testFailed(identifier, collector.getErrorMessage()); 284 } 285 collector.disable(); 286 } 287 } 288 } 289 } 290 listener.testRunEnded(System.currentTimeMillis() - startTime, runMetrics); 291 } 292 initApkList()293 private void initApkList() { 294 if (mApks != null) { 295 return; 296 } 297 298 // Find an apk info file. The priorities are: 299 // 1. Use the specified apk-info if available. 300 // 2. Use 'apk-info.xml' if there is one in the apk-dir directory. 301 // 3. Use the default apk-info.xml in res. 302 if (mApkInfoFileName != null) { 303 mApkInfoFile = new File(mApkInfoFileName); 304 } else { 305 mApkInfoFile = new File(getApkDir(), "apk-info.xml"); 306 307 if (!mApkInfoFile.exists()) { 308 String resource = "/com/android/game/qualification/apk-info.xml"; 309 try(InputStream inputStream = ApkInfo.class.getResourceAsStream(resource)) { 310 if (inputStream == null) { 311 throw new FileNotFoundException("Unable to find resource: " + resource); 312 } 313 mApkInfoFile = File.createTempFile("apk-info", ".xml"); 314 try (OutputStream ostream = new FileOutputStream(mApkInfoFile)) { 315 ByteStreams.copy(inputStream, ostream); 316 } 317 mApkInfoFile.deleteOnExit(); 318 } catch (IOException e) { 319 throw new RuntimeException(e); 320 } 321 } 322 } 323 GameCoreConfigurationXmlParser parser = new GameCoreConfigurationXmlParser(); 324 try { 325 mGameCoreConfiguration = parser.parse(mApkInfoFile); 326 mApks = mGameCoreConfiguration.getApkInfo(); 327 } catch (IOException | ParserConfigurationException | SAXException e) { 328 throw new RuntimeException(e); 329 } 330 } 331 } 332