1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.server.cts; 17 18 import android.service.GraphicsStatsHistogramBucketProto; 19 import android.service.GraphicsStatsJankSummaryProto; 20 import android.service.GraphicsStatsProto; 21 import android.service.GraphicsStatsServiceDumpProto; 22 23 import com.android.tradefed.log.LogUtil.CLog; 24 25 import java.util.ArrayList; 26 import java.util.Date; 27 import java.util.List; 28 29 public class GraphicsStatsValidationTest extends ProtoDumpTestCase { 30 private static final String TAG = "GraphicsStatsValidationTest"; 31 32 private static final String DEVICE_SIDE_TEST_APK = "CtsGraphicsStatsApp.apk"; 33 private static final String DEVICE_SIDE_TEST_PACKAGE 34 = "com.android.server.cts.device.graphicsstats"; 35 36 @Override tearDown()37 protected void tearDown() throws Exception { 38 getDevice().uninstallPackage(DEVICE_SIDE_TEST_PACKAGE); 39 super.tearDown(); 40 } 41 42 @Override setUp()43 protected void setUp() throws Exception { 44 super.setUp(); 45 installPackage(DEVICE_SIDE_TEST_APK, /* grantPermissions= */ true); 46 turnScreenOn(); 47 // Ensure that we have a starting point for our stats 48 runDeviceTests(DEVICE_SIDE_TEST_PACKAGE, ".SimpleDrawFrameTests", 49 "testDrawTenFrames"); 50 // Kill to ensure that stats persist/merge across process death 51 killTestApp(); 52 } 53 turnScreenOn()54 private void turnScreenOn() throws Exception { 55 getDevice().executeShellCommand("input keyevent KEYCODE_WAKEUP"); 56 getDevice().executeShellCommand("wm dismiss-keyguard"); 57 } 58 testBasicDrawFrame()59 public void testBasicDrawFrame() throws Exception { 60 System.out.println("--------------------------------- testBasicDrawFrame BEGIN"); 61 GraphicsStatsProto[] results = runDrawTest("testDrawTenFrames"); 62 GraphicsStatsProto statsBefore = results[0]; 63 GraphicsStatsProto statsAfter = results[1]; 64 GraphicsStatsJankSummaryProto summaryBefore = statsBefore.getSummary(); 65 GraphicsStatsJankSummaryProto summaryAfter = statsAfter.getSummary(); 66 assertTrue(summaryAfter.getTotalFrames() > summaryBefore.getTotalFrames()); 67 68 System.out.println("summaryBefore: {\n" + summaryBefore + "}"); 69 System.out.println("summaryAfter: {\n" + summaryAfter + "}"); 70 System.out.println("statsBefore: {\n" + statsBefore + "}"); 71 System.out.println("statsAfter: {\n" + statsAfter + "}"); 72 73 int frameDelta = summaryAfter.getTotalFrames() - summaryBefore.getTotalFrames(); 74 int jankyDelta = summaryAfter.getJankyFrames() - summaryBefore.getJankyFrames(); 75 // We expect 11 frames to have been drawn (first frame + the 10 more explicitly requested) 76 assertTrue(frameDelta >= 11); 77 assertTrue(jankyDelta >= 1); 78 int veryJankyDelta = countFramesAbove(statsAfter, 40) - countFramesAbove(statsBefore, 40); 79 System.out.println("--------------------------------- testBasicDrawFrame END"); 80 } 81 testJankyDrawFrame()82 public void testJankyDrawFrame() throws Exception { 83 GraphicsStatsProto[] results = runDrawTest("testDrawJankyFrames"); 84 GraphicsStatsProto statsBefore = results[0]; 85 GraphicsStatsProto statsAfter = results[1]; 86 GraphicsStatsJankSummaryProto summaryBefore = statsBefore.getSummary(); 87 GraphicsStatsJankSummaryProto summaryAfter = statsAfter.getSummary(); 88 assertTrue(summaryAfter.getTotalFrames() > summaryBefore.getTotalFrames()); 89 90 int frameDelta = summaryAfter.getTotalFrames() - summaryBefore.getTotalFrames(); 91 int jankyDelta = summaryAfter.getJankyFrames() - summaryBefore.getJankyFrames(); 92 // Test draws 50 frames + 1 initial frame. We expect 40 of them to be janky, 93 // 10 of each of ANIMATION, LAYOUT, RECORD_DRAW, and MISSED_VSYNC 94 assertTrue(frameDelta < 55); 95 assertTrue(jankyDelta >= 40); 96 assertTrue(jankyDelta < 45); 97 98 // Although our current stats don't distinguish between ANIMATION, LAYOUT, and RECORD_DRAW 99 // so this will just be slowUi +30 100 int slowUiDelta = summaryAfter.getSlowUiThreadCount() - summaryBefore.getSlowUiThreadCount(); 101 assertTrue(slowUiDelta >= 30); 102 int missedVsyncDelta = summaryAfter.getMissedVsyncCount() 103 - summaryBefore.getMissedVsyncCount(); 104 assertTrue(missedVsyncDelta >= 10); 105 assertTrue(missedVsyncDelta <= 11); 106 107 int veryJankyDelta = countFramesAbove(statsAfter, 60) - countFramesAbove(statsBefore, 60); 108 // The 1st frame could be >40ms, but nothing after that should be 109 assertTrue(veryJankyDelta <= 1); 110 } 111 testDaveyDrawFrame()112 public void testDaveyDrawFrame() throws Exception { 113 GraphicsStatsProto[] results = runDrawTest("testDrawDaveyFrames"); 114 GraphicsStatsProto statsBefore = results[0]; 115 GraphicsStatsProto statsAfter = results[1]; 116 GraphicsStatsJankSummaryProto summaryBefore = statsBefore.getSummary(); 117 GraphicsStatsJankSummaryProto summaryAfter = statsAfter.getSummary(); 118 assertTrue(summaryAfter.getTotalFrames() > summaryBefore.getTotalFrames()); 119 120 int frameDelta = summaryAfter.getTotalFrames() - summaryBefore.getTotalFrames(); 121 int jankyDelta = summaryAfter.getJankyFrames() - summaryBefore.getJankyFrames(); 122 // Test draws 40 frames + 1 initial frame. We expect 10 of them to be daveys, 123 // 10 of them to be daveyjrs, and 20 to jank from missed vsync (from the davey/daveyjr prior to it) 124 assertTrue(frameDelta < 45); 125 assertTrue(jankyDelta >= 20); 126 assertTrue(jankyDelta < 25); 127 128 int gt150msDelta = countFramesAbove(statsAfter, 150) - countFramesAbove(statsBefore, 150); 129 assertTrue(gt150msDelta >= 20); // 10 davey jrs + 10 daveys + maybe first frame 130 assertTrue(gt150msDelta <= 21); 131 int gt700msDelta = countFramesAbove(statsAfter, 700) - countFramesAbove(statsBefore, 700); 132 assertEquals(10, gt700msDelta); // 10 daveys 133 } 134 runDrawTest(String testName)135 private GraphicsStatsProto[] runDrawTest(String testName) throws Exception { 136 return doRunDrawTest(testName, true); 137 } 138 doRunDrawTest(String testName, boolean canRetry)139 private GraphicsStatsProto[] doRunDrawTest(String testName, boolean canRetry) throws Exception { 140 GraphicsStatsProto statsBefore = fetchStats(); 141 assertNotNull(statsBefore); 142 killTestApp(); 143 turnScreenOn(); 144 runDeviceTests(DEVICE_SIDE_TEST_PACKAGE, ".SimpleDrawFrameTests", testName); 145 killTestApp(); 146 GraphicsStatsProto statsAfter = fetchStats(); 147 assertNotNull(statsAfter); 148 // If we get extremely unlucky a log rotate might have happened. If so we retry, but only once 149 // It's a failure if this test takes >24 hours such that 2 rotates could happen while running 150 // this test case, or more likely if stats are not being merged/persisted properly 151 if (canRetry) { 152 if (statsBefore.getStatsStart() != statsAfter.getStatsStart()) { 153 return doRunDrawTest(testName, false); 154 } 155 } else { 156 assertEquals(statsBefore.getStatsStart(), statsAfter.getStatsStart()); 157 } 158 validate(statsBefore); 159 validate(statsAfter); 160 return new GraphicsStatsProto[] { statsBefore, statsAfter }; 161 } 162 validate(GraphicsStatsProto proto)163 private void validate(GraphicsStatsProto proto) { 164 assertNotNull(proto.getPackageName()); 165 assertFalse(proto.getPackageName().isEmpty()); 166 assertTrue(proto.getVersionCode() > 0); 167 assertTrue(proto.getStatsStart() > 0); 168 assertTrue(proto.getStatsEnd() > 0); 169 assertTrue(proto.hasSummary()); 170 GraphicsStatsJankSummaryProto summary = proto.getSummary(); 171 assertTrue(summary.getTotalFrames() > 0); 172 // Our test app won't produce that many frames, so we can assert this is a realistic 173 // number. We cap it at 1,000,000 in case the test is repeated many, many times in one day 174 assertTrue(summary.getTotalFrames() < 1000000); 175 // We can't generically assert things about the janky frames, so just assert they fall into 176 // valid ranges. 177 assertTrue(summary.getJankyFrames() <= summary.getTotalFrames()); 178 assertTrue(summary.getMissedVsyncCount() <= summary.getJankyFrames()); 179 assertTrue(summary.getHighInputLatencyCount() <= summary.getTotalFrames()); 180 assertTrue(summary.getSlowUiThreadCount() <= summary.getJankyFrames()); 181 assertTrue(summary.getSlowBitmapUploadCount() <= summary.getJankyFrames()); 182 assertTrue(summary.getSlowDrawCount() <= summary.getJankyFrames()); 183 assertTrue(proto.getHistogramCount() > 0); 184 185 int histogramTotal = countTotalFrames(proto); 186 assertEquals(summary.getTotalFrames(), histogramTotal); 187 } 188 189 private int countFramesAbove(GraphicsStatsProto proto, int thresholdMs) { 190 int totalFrames = 0; 191 for (GraphicsStatsHistogramBucketProto bucket : proto.getHistogramList()) { 192 if (bucket.getRenderMillis() >= thresholdMs) { 193 totalFrames += bucket.getFrameCount(); 194 } 195 } 196 return totalFrames; 197 } 198 199 private int countTotalFrames(GraphicsStatsProto proto) { 200 return countFramesAbove(proto, 0); 201 } 202 203 private void killTestApp() throws Exception { 204 getDevice().executeShellCommand("am kill " + DEVICE_SIDE_TEST_PACKAGE); 205 } 206 207 private GraphicsStatsProto fetchStats() throws Exception { 208 GraphicsStatsServiceDumpProto serviceDumpProto = getDump(GraphicsStatsServiceDumpProto.parser(), 209 "dumpsys graphicsstats --proto"); 210 List<GraphicsStatsProto> protos = filterPackage(serviceDumpProto, DEVICE_SIDE_TEST_PACKAGE); 211 return findLatest(protos); 212 } 213 214 private List<GraphicsStatsProto> filterPackage(GraphicsStatsServiceDumpProto dump, String pkgName) { 215 return filterPackage(dump.getStatsList(), pkgName); 216 } 217 218 private List<GraphicsStatsProto> filterPackage(List<GraphicsStatsProto> list, String pkgName) { 219 ArrayList<GraphicsStatsProto> filtered = new ArrayList<>(); 220 for (GraphicsStatsProto proto : list) { 221 if (pkgName.equals(proto.getPackageName())) { 222 filtered.add(proto); 223 } 224 } 225 return filtered; 226 } 227 228 private GraphicsStatsProto findLatest(List<GraphicsStatsProto> list) { 229 if (list.size() == 0) { return null; } 230 GraphicsStatsProto latest = list.get(0); 231 Date latestDate = new Date(); 232 Date compareTo = new Date(); 233 latestDate.setTime(latest.getStatsEnd()); 234 for (int i = 1; i < list.size(); i++) { 235 GraphicsStatsProto proto = list.get(i); 236 compareTo.setTime(proto.getStatsEnd()); 237 if (compareTo.after(latestDate)) { 238 latestDate.setTime(proto.getStatsEnd()); 239 latest = proto; 240 } 241 } 242 return latest; 243 } 244 } 245