1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 * in compliance with the License. You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software distributed under the License 10 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 * or implied. See the License for the specific language governing permissions and limitations under 12 * the License. 13 */ 14 15 package com.android.performance.tests; 16 17 import com.android.tradefed.device.DeviceNotAvailableException; 18 import com.android.tradefed.device.ITestDevice; 19 import com.android.tradefed.log.LogUtil.CLog; 20 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 21 import com.android.tradefed.result.ITestInvocationListener; 22 import com.android.tradefed.result.InputStreamSource; 23 import com.android.tradefed.result.TestDescription; 24 import com.android.tradefed.testtype.IDeviceTest; 25 import com.android.tradefed.testtype.IRemoteTest; 26 import com.android.tradefed.util.RunUtil; 27 import com.android.tradefed.util.proto.TfMetricProtoUtil; 28 29 import java.io.BufferedReader; 30 import java.io.IOException; 31 import java.io.InputStreamReader; 32 import java.util.HashMap; 33 import java.util.Map; 34 35 /** A harness that launches VellamoBenchmark and reports result. Requires VellamoBenchmark apk. */ 36 public class VellamoBenchmark implements IDeviceTest, IRemoteTest { 37 38 private static final String LOGTAG = "VAUTOMATIC"; 39 private static final String RUN_KEY = "vellamobenchmark-3202"; 40 private static final String PACKAGE_NAME = "com.quicinc.vellamo"; 41 private static final long TIMEOUT_MS = 30 * 60 * 1000; 42 private static final long POLLING_INTERVAL_MS = 10 * 1000; 43 private static final int INDEX_NAME = 0; 44 private static final int INDEX_CODE = 4; 45 private static final int INDEX_SCORE = 5; 46 47 private ITestDevice mDevice; 48 49 /** {@inheritDoc} */ 50 @Override setDevice(ITestDevice device)51 public void setDevice(ITestDevice device) { 52 mDevice = device; 53 } 54 55 /** {@inheritDoc} */ 56 @Override getDevice()57 public ITestDevice getDevice() { 58 return mDevice; 59 } 60 61 /** {@inheritDoc} */ 62 @Override run(ITestInvocationListener listener)63 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 64 TestDescription testId = new TestDescription(getClass().getCanonicalName(), RUN_KEY); 65 ITestDevice device = getDevice(); 66 listener.testRunStarted(RUN_KEY, 0); 67 listener.testStarted(testId); 68 69 long testStartTime = System.currentTimeMillis(); 70 Map<String, String> metrics = new HashMap<>(); 71 String errMsg = null; 72 73 boolean isTimedOut = false; 74 boolean isResultGenerated = false; 75 boolean hasScore = false; 76 double sumScore = 0; 77 int errorCode = 0; 78 device.clearErrorDialogs(); 79 isTimedOut = false; 80 81 long benchmarkStartTime = System.currentTimeMillis(); 82 // start the vellamo benchmark app and run all the tests 83 // the documentation and binary for the Vellamo 3.2.2 for Automation APK 84 // can be found here: 85 // https://b.corp.google.com/issue?id=23107318 86 CLog.i("Starting Vellamo Benchmark"); 87 device.executeShellCommand( 88 "am start -a com.quicinc.vellamo.AUTOMATIC" 89 + " -e w com.quicinc.skunkworks.wvb" // use System WebView 90 + " -n com.quicinc.vellamo/.main.MainActivity"); 91 String line; 92 while (!isResultGenerated && !isTimedOut) { 93 RunUtil.getDefault().sleep(POLLING_INTERVAL_MS); 94 isTimedOut = (System.currentTimeMillis() - benchmarkStartTime >= TIMEOUT_MS); 95 96 // get the logcat and parse 97 try (InputStreamSource logcatSource = device.getLogcat(); 98 BufferedReader logcat = 99 new BufferedReader( 100 new InputStreamReader(logcatSource.createInputStream()))) { 101 while ((line = logcat.readLine()) != null) { 102 // filter only output from the Vellamo process 103 if (!line.contains(LOGTAG)) { 104 continue; 105 } 106 line = line.substring(line.indexOf(LOGTAG) + LOGTAG.length()); 107 // we need to see if the score is generated since there are some 108 // cases the result with </automatic> tag is generated but no score is included 109 if (line.contains("</automatic>")) { 110 if (hasScore) { 111 isResultGenerated = true; 112 break; 113 } 114 } 115 // get the score out 116 if (line.contains(" b: ")) { 117 hasScore = true; 118 String[] results = line.split(" b: ")[1].split(","); 119 errorCode = Integer.parseInt(results[INDEX_CODE]); 120 if (errorCode != 0) { 121 CLog.w( 122 "Non-zero error code: %d from becnhmark '%s'", 123 errorCode, results[INDEX_NAME]); 124 } else { 125 sumScore += Double.parseDouble(results[INDEX_SCORE]); 126 } 127 metrics.put(results[INDEX_NAME], results[INDEX_SCORE]); 128 CLog.i("%s :: %s", results[INDEX_NAME], results[INDEX_SCORE]); 129 } 130 } 131 } catch (IOException e) { 132 CLog.e(e); 133 } 134 135 if (null == device.getProcessByName(PACKAGE_NAME)) { 136 break; 137 } 138 } 139 140 if (isTimedOut) { 141 errMsg = "Vellamo Benchmark timed out."; 142 } else { 143 CLog.i("Done running Vellamo Benchmark"); 144 } 145 if (!hasScore) { 146 errMsg = "Test ended but no scores can be found."; 147 } 148 if (errMsg != null) { 149 CLog.e(errMsg); 150 listener.testFailed(testId, errMsg); 151 } 152 long durationMs = System.currentTimeMillis() - testStartTime; 153 metrics.put("total", Double.toString(sumScore)); 154 CLog.i("total :: %f", sumScore); 155 listener.testEnded(testId, new HashMap<String, Metric>()); 156 listener.testRunEnded(durationMs, TfMetricProtoUtil.upgradeConvert(metrics)); 157 } 158 } 159