1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.game.qualification.test; 17 18 import static org.junit.Assert.assertFalse; 19 import static org.junit.Assert.assertNotNull; 20 import static org.junit.Assert.fail; 21 22 import com.android.annotations.VisibleForTesting; 23 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 24 import com.android.game.qualification.ApkInfo; 25 import com.android.game.qualification.ResultData; 26 import com.android.game.qualification.metric.BaseGameQualificationMetricCollector; 27 import com.android.game.qualification.proto.ResultDataProto; 28 import com.android.game.qualification.testtype.GameQualificationHostsideController; 29 import com.android.tradefed.device.DeviceNotAvailableException; 30 import com.android.tradefed.device.ITestDevice; 31 import com.android.tradefed.log.LogUtil.CLog; 32 import com.android.tradefed.result.CollectingTestListener; 33 import com.android.tradefed.result.ITestInvocationListener; 34 import com.android.tradefed.result.InputStreamSource; 35 import com.android.tradefed.result.LogDataType; 36 37 import com.google.common.io.ByteStreams; 38 import com.google.common.io.Files; 39 40 import org.junit.Assume; 41 42 import java.awt.image.BufferedImage; 43 import java.io.ByteArrayOutputStream; 44 import java.io.File; 45 import java.io.FileInputStream; 46 import java.io.IOException; 47 import java.io.InputStream; 48 import java.nio.charset.StandardCharsets; 49 import java.util.Collection; 50 import java.util.concurrent.TimeUnit; 51 52 import javax.imageio.ImageIO; 53 54 /** 55 * Performance test designed to be used with {@link GameQualificationHostsideController} 56 * 57 * Tests must be enumerated with the {@link Test} enum and unlike junit tests, they will be run 58 * sequentially. 59 */ 60 public class PerformanceTest { 61 private static final String AJUR_RUNNER = "androidx.test.runner.AndroidJUnitRunner"; 62 private static final long DEFAULT_TEST_TIMEOUT_MS = 30 * 60 * 1000L; //30min 63 private static final long DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS = 30 * 60 * 1000L; //30min 64 65 private ApkInfo mApk; 66 private String mApkDir; 67 private ITestDevice mDevice; 68 private Collection<BaseGameQualificationMetricCollector> mCollectors; 69 private ITestInvocationListener mListener; 70 private File mWorkingDirectory; 71 private boolean allTestsPassed = true; 72 73 public interface TestMethod { run(PerformanceTest test)74 void run(PerformanceTest test) throws Exception; 75 } 76 77 public enum Test { 78 SETUP("setUp", PerformanceTest::setUp, false), 79 RUN("run", PerformanceTest::run, true), 80 SCREENSHOT("screenshotTest", PerformanceTest::testScreenshot, false), 81 TEARDOWN("tearDown", PerformanceTest::tearDown, false); 82 83 private String mName; 84 private TestMethod mMethod; 85 private boolean mEnableCollectors; 86 Test(String name, TestMethod method, boolean enableCollectors)87 Test(String name, TestMethod method, boolean enableCollectors) { 88 mName = name; 89 mMethod = method; 90 mEnableCollectors = enableCollectors; 91 } 92 getName()93 public String getName() { 94 return mName; 95 } 96 getMethod()97 public TestMethod getMethod() { 98 return mMethod; 99 } 100 isEnableCollectors()101 public boolean isEnableCollectors() { 102 return mEnableCollectors; 103 } 104 } 105 PerformanceTest( ITestDevice device, ITestInvocationListener listener, Collection<BaseGameQualificationMetricCollector> collectors, ApkInfo apk, String apkDir, File workingDirectory)106 public PerformanceTest( 107 ITestDevice device, 108 ITestInvocationListener listener, 109 Collection<BaseGameQualificationMetricCollector> collectors, 110 ApkInfo apk, 111 String apkDir, 112 File workingDirectory) { 113 mApk = apk; 114 mApkDir = apkDir; 115 mDevice = device; 116 mCollectors = collectors; 117 mListener = listener; 118 mWorkingDirectory = workingDirectory; 119 } 120 failed()121 public void failed() { 122 this.allTestsPassed = false; 123 } 124 125 // BEGIN TESTS 126 setUp()127 private void setUp() throws DeviceNotAvailableException, IOException, InterruptedException { 128 if (mApk.getScript() != null) { 129 String cmd = mApk.getScript(); 130 CLog.i( 131 "Executing command: " + cmd + "\n" 132 + "Working directory: " + mWorkingDirectory.getPath()); 133 ProcessBuilder pb = new ProcessBuilder("sh", "-c", cmd); 134 pb.environment().put("ANDROID_SERIAL", mDevice.getSerialNumber()); 135 pb.directory(mWorkingDirectory); 136 pb.redirectErrorStream(true); 137 138 Process p = pb.start(); 139 boolean finished = p.waitFor(30, TimeUnit.MINUTES); 140 if (!finished || p.exitValue() != 0) { 141 ByteArrayOutputStream os = new ByteArrayOutputStream(); 142 ByteStreams.copy(p.getInputStream(), os); 143 String output = os.toString(StandardCharsets.UTF_8.name()); 144 if (!finished) { 145 output += "\n***TIMEOUT waiting for script to complete.***"; 146 p.destroy(); 147 } 148 fail("Execution of setup script returned non-zero value:\n" + output); 149 } 150 } 151 152 File apkFile = findApk(mApk.getFileName()); 153 assertNotNull( 154 String.format( 155 "Missing APK. Unable to find %s in %s.\n", 156 mApk.getFileName(), 157 mApkDir), 158 apkFile); 159 CLog.i("Installing %s on %s.", apkFile.getName(), mDevice.getSerialNumber()); 160 mDevice.installPackage(apkFile, true); 161 } 162 run()163 private void run() throws DeviceNotAvailableException { 164 Assume.assumeTrue(allTestsPassed); 165 // APK Test. 166 assertFalse( 167 "Unable to unlock device: " + mDevice.getDeviceDescriptor(), 168 mDevice.getKeyguardState().isKeyguardShowing()); 169 170 File apkFile = findApk(mApk.getFileName()); 171 assertNotNull( 172 String.format( 173 "Missing APK. Unable to find %s in %s.\n", 174 mApk.getFileName(), 175 mApkDir), 176 apkFile); 177 178 179 CollectingTestListener listener = new CollectingTestListener(); 180 runDeviceTests( 181 GameQualificationHostsideController.PACKAGE, 182 GameQualificationHostsideController.CLASS, 183 "run[" + mApk.getName() + "]", 184 listener); 185 ResultDataProto.Result resultData = retrieveResultData(); 186 for (BaseGameQualificationMetricCollector collector : mCollectors) { 187 collector.setDeviceResultData(resultData); 188 } 189 assertFalse(listener.hasFailedTests()); 190 } 191 testScreenshot()192 private void testScreenshot() throws IOException, DeviceNotAvailableException { 193 Assume.assumeTrue(allTestsPassed); 194 try (InputStreamSource screenSource = mDevice.getScreenshot()) { 195 mListener.testLog( 196 String.format("screenshot-%s", mApk.getName()), 197 LogDataType.PNG, 198 screenSource); 199 try (InputStream stream = screenSource.createInputStream()) { 200 stream.reset(); 201 assertFalse( 202 "A screenshot was taken just after metric collection and it was black.", 203 isImageBlack(stream)); 204 } 205 } catch (IOException e) { 206 throw new IOException("Failed reading screenshot data:\n" + e.getMessage()); 207 } 208 } 209 tearDown()210 private void tearDown() throws DeviceNotAvailableException { 211 mDevice.uninstallPackage(mApk.getPackageName()); 212 } 213 214 // END TESTS 215 216 /** Find an apk in the apk-dir directory */ findApk(String filename)217 private File findApk(String filename) { 218 File file = new File(mApkDir, filename); 219 if (file.exists()) { 220 return file; 221 } 222 // If a default sample app is named Sample.apk, it is outputted to 223 // $ANDROID_PRODUCT_OUT/data/app/Sample/Sample.apk. 224 file = new File(mApkDir, Files.getNameWithoutExtension(filename) + "/" + filename); 225 if (file.exists()) { 226 return file; 227 } 228 return null; 229 } 230 231 /** Check if an image is black. */ 232 @VisibleForTesting isImageBlack(InputStream stream)233 static boolean isImageBlack(InputStream stream) throws IOException { 234 BufferedImage img = ImageIO.read(stream); 235 for (int i = 0; i < img.getWidth(); i++) { 236 // Only check the middle portion of the image to avoid status bar. 237 for (int j = img.getHeight() / 4; j < img.getHeight() * 3 / 4; j++) { 238 int color = img.getRGB(i, j); 239 // Check if pixel is non-black and not fully transparent. 240 if ((color & 0x00ffffff) != 0 && (color >> 24) != 0) { 241 return false; 242 } 243 } 244 } 245 return true; 246 } 247 retrieveResultData()248 private ResultDataProto.Result retrieveResultData() throws DeviceNotAvailableException { 249 File resultFile = mDevice.pullFileFromExternal(ResultData.RESULT_FILE_LOCATION); 250 251 if (resultFile != null) { 252 try (InputStream inputStream = new FileInputStream(resultFile)) { 253 return ResultDataProto.Result.parseFrom(inputStream); 254 } catch (IOException e) { 255 throw new RuntimeException(e); 256 } 257 } 258 return null; 259 } 260 261 /** 262 * Method to run an installed instrumentation package. 263 * 264 * @param pkgName the name of the package to run. 265 * @param testClassName the name of the test class to run. 266 * @param testMethodName the name of the method to run. 267 */ runDeviceTests(String pkgName, String testClassName, String testMethodName, CollectingTestListener listener)268 private void runDeviceTests(String pkgName, String testClassName, String testMethodName, CollectingTestListener listener) 269 throws DeviceNotAvailableException { 270 RemoteAndroidTestRunner testRunner = 271 new RemoteAndroidTestRunner(pkgName, AJUR_RUNNER, mDevice.getIDevice()); 272 273 testRunner.setMethodName(testClassName, testMethodName); 274 275 testRunner.addInstrumentationArg( 276 "timeout_msec", Long.toString(DEFAULT_TEST_TIMEOUT_MS)); 277 testRunner.setMaxTimeout(DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, TimeUnit.MILLISECONDS); 278 279 mDevice.runInstrumentationTests(testRunner, listener); 280 } 281 } 282