1 /** 2 * Copyright 2016 Google Inc. All Rights Reserved. 3 * 4 * <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 5 * except in compliance with the License. You may obtain a copy of the License at 6 * 7 * <p>http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * <p>Unless required by applicable law or agreed to in writing, software distributed under the 10 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 * express or implied. See the License for the specific language governing permissions and 12 * limitations under the License. 13 */ 14 package com.android.vts.util; 15 16 import com.android.vts.entity.DeviceInfoEntity; 17 import com.android.vts.entity.ProfilingPointRunEntity; 18 import com.android.vts.entity.TestEntity; 19 import com.android.vts.entity.TestRunEntity; 20 import com.android.vts.proto.VtsReportMessage.VtsProfilingRegressionMode; 21 import com.google.appengine.api.datastore.DatastoreService; 22 import com.google.appengine.api.datastore.DatastoreServiceFactory; 23 import com.google.appengine.api.datastore.Entity; 24 import com.google.appengine.api.datastore.Key; 25 import com.google.appengine.api.datastore.KeyFactory; 26 import com.google.appengine.api.datastore.Query; 27 import com.google.appengine.api.datastore.Query.Filter; 28 import com.google.appengine.api.datastore.Query.FilterOperator; 29 import com.google.appengine.api.datastore.Query.FilterPredicate; 30 import java.io.IOException; 31 import java.math.RoundingMode; 32 import java.text.DecimalFormat; 33 import java.util.ArrayList; 34 import java.util.List; 35 import java.util.Set; 36 import java.util.concurrent.TimeUnit; 37 import java.util.logging.Logger; 38 import org.apache.commons.lang.StringUtils; 39 40 /** PerformanceUtil, a helper class for analyzing profiling and performance data. */ 41 public class PerformanceUtil { 42 protected static Logger logger = Logger.getLogger(PerformanceUtil.class.getName()); 43 44 private static final DecimalFormat FORMATTER; 45 private static final String NAME_DELIMITER = ", "; 46 47 /** 48 * Initialize the decimal formatter. 49 */ 50 static { 51 FORMATTER = new DecimalFormat("#.##"); 52 FORMATTER.setRoundingMode(RoundingMode.HALF_UP); 53 } 54 55 public static class TimeInterval { 56 public final long start; 57 public final long end; 58 public final String label; 59 TimeInterval(long start, long end, String label)60 public TimeInterval(long start, long end, String label) { 61 this.start = start; 62 this.end = end; 63 this.label = label; 64 } 65 TimeInterval(long start, long end)66 public TimeInterval(long start, long end) { 67 this(start, end, "<span class='date-label'>" 68 + Long.toString(TimeUnit.MICROSECONDS.toMillis(end)) + "</span>"); 69 } 70 } 71 72 /** 73 * Creates the HTML for a table cell representing the percent change between two numbers. 74 * 75 * <p>Computes the percent change (after - before)/before * 100 and inserts it into a table cell 76 * with the specified style. The color of the cell is white if 'after' is less than before. 77 * Otherwise, the cell is colored red with opacity according to the percent change (100%+ delta 78 * means 100% opacity). If the before value is 0 and the after value is positive, then the color 79 * of the cell is 100% red to indicate an increase of undefined magnitude. 80 * 81 * @param baseline The baseline value observed. 82 * @param test The value to compare against the baseline. 83 * @param classNames A string containing HTML classes to apply to the table cell. 84 * @param style A string containing additional CSS styles. 85 * @returns An HTML string for a colored table cell containing the percent change. 86 */ getPercentChangeHTML(double baseline, double test, String classNames, String style, VtsProfilingRegressionMode mode)87 public static String getPercentChangeHTML(double baseline, double test, String classNames, 88 String style, VtsProfilingRegressionMode mode) { 89 String pctChangeString = "0 %"; 90 double alpha = 0; 91 double delta = test - baseline; 92 if (baseline != 0) { 93 double pctChange = delta / baseline; 94 alpha = pctChange * 2; 95 pctChangeString = FORMATTER.format(pctChange * 100) + " %"; 96 } else if (delta != 0) { 97 // If the percent change is undefined, the cell will be solid red or white 98 alpha = (int) Math.signum(delta); // get the sign of the delta (+1, 0, -1) 99 pctChangeString = ""; 100 } 101 if (mode == VtsProfilingRegressionMode.VTS_REGRESSION_MODE_DECREASING) { 102 alpha = -alpha; 103 } 104 String color = "background-color: rgba(255, 0, 0, " + alpha + "); "; 105 String html = "<td class='" + classNames + "' style='" + color + style + "'>"; 106 html += pctChangeString + "</td>"; 107 return html; 108 } 109 110 /** 111 * Compares a test StatSummary to a baseline StatSummary using best-case performance. 112 * 113 * @param baseline The StatSummary object containing initial values to compare against 114 * @param test The StatSummary object containing test values to be compared against the baseline 115 * @param innerClasses Class names to apply to cells on the inside of the grid 116 * @param outerClasses Class names to apply to cells on the outside of the grid 117 * @param innerStyles CSS styles to apply to cells on the inside of the grid 118 * @param outerStyles CSS styles to apply to cells on the outside of the grid 119 * @return HTML string representing the performance of the test versus the baseline 120 */ getBestCasePerformanceComparisonHTML(StatSummary baseline, StatSummary test, String innerClasses, String outerClasses, String innerStyles, String outerStyles)121 public static String getBestCasePerformanceComparisonHTML(StatSummary baseline, 122 StatSummary test, String innerClasses, String outerClasses, String innerStyles, 123 String outerStyles) { 124 if (test == null || baseline == null) { 125 return "<td></td><td></td><td></td><td></td>"; 126 } 127 String row = ""; 128 // Intensity of red color is a function of the relative (percent) change 129 // in the new value compared to the previous day's. Intensity is a linear function 130 // of percentage change, reaching a ceiling at 100% change (e.g. a doubling). 131 row += getPercentChangeHTML(baseline.getBestCase(), test.getBestCase(), innerClasses, 132 innerStyles, test.getRegressionMode()); 133 row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>"; 134 row += FORMATTER.format(baseline.getBestCase()); 135 row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>"; 136 row += FORMATTER.format(baseline.getMean()); 137 row += "<td class='" + outerClasses + "' style='" + outerStyles + "'>"; 138 row += FORMATTER.format(baseline.getStd()) + "</td>"; 139 return row; 140 } 141 142 /** 143 * Compares a test StatSummary to a baseline StatSummary using average-case performance. 144 * 145 * @param baseline The StatSummary object containing initial values to compare against 146 * @param test The StatSummary object containing test values to be compared against the baseline 147 * @param innerClasses Class names to apply to cells on the inside of the grid 148 * @param outerClasses Class names to apply to cells on the outside of the grid 149 * @param innerStyles CSS styles to apply to cells on the inside of the grid 150 * @param outerStyles CSS styles to apply to cells on the outside of the grid 151 * @return HTML string representing the performance of the test versus the baseline 152 */ getAvgCasePerformanceComparisonHTML(StatSummary baseline, StatSummary test, String innerClasses, String outerClasses, String innerStyles, String outerStyles)153 public static String getAvgCasePerformanceComparisonHTML(StatSummary baseline, StatSummary test, 154 String innerClasses, String outerClasses, String innerStyles, String outerStyles) { 155 if (test == null || baseline == null) { 156 return "<td></td><td></td><td></td><td></td>"; 157 } 158 String row = ""; 159 // Intensity of red color is a function of the relative (percent) change 160 // in the new value compared to the previous day's. Intensity is a linear function 161 // of percentage change, reaching a ceiling at 100% change (e.g. a doubling). 162 row += getPercentChangeHTML(baseline.getMean(), test.getMean(), innerClasses, innerStyles, 163 test.getRegressionMode()); 164 row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>"; 165 row += FORMATTER.format(baseline.getBestCase()); 166 row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>"; 167 row += FORMATTER.format(baseline.getMean()); 168 row += "<td class='" + outerClasses + "' style='" + outerStyles + "'>"; 169 row += FORMATTER.format(baseline.getStd()) + "</td>"; 170 return row; 171 } 172 173 /** 174 * Updates a PerformanceSummary object with data in the specified window. 175 * 176 * @param testName The name of the table whose profiling vectors to retrieve. 177 * @param startTime The (inclusive) start time in milliseconds to scan from. 178 * @param endTime The (inclusive) end time in milliseconds at which to stop scanning. 179 * @param selectedDevice The name of the device whose data to query for, or null for unfiltered. 180 * @param perfSummary The PerformanceSummary object to update with data. 181 * @throws IOException 182 */ updatePerformanceSummary(String testName, long startTime, long endTime, String selectedDevice, PerformanceSummary perfSummary)183 public static void updatePerformanceSummary(String testName, long startTime, long endTime, 184 String selectedDevice, PerformanceSummary perfSummary) throws IOException { 185 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 186 Key testKey = KeyFactory.createKey(TestEntity.KIND, testName); 187 Filter testTypeFilter = FilterUtil.getTestTypeFilter(false, true, false); 188 Filter runFilter = FilterUtil.getTimeFilter(testKey, startTime, endTime, testTypeFilter); 189 190 Filter deviceFilter = null; 191 if (selectedDevice != null) { 192 deviceFilter = new FilterPredicate( 193 DeviceInfoEntity.PRODUCT, FilterOperator.EQUAL, selectedDevice); 194 } 195 Query testRunQuery = new Query(TestRunEntity.KIND) 196 .setAncestor(testKey) 197 .setFilter(runFilter) 198 .setKeysOnly(); 199 for (Entity testRun : datastore.prepare(testRunQuery).asIterable()) { 200 if (deviceFilter != null) { 201 Query deviceQuery = new Query(DeviceInfoEntity.KIND) 202 .setAncestor(testRun.getKey()) 203 .setFilter(deviceFilter) 204 .setKeysOnly(); 205 if (!DatastoreHelper.hasEntities(deviceQuery)) 206 continue; 207 } 208 Query q = new Query(ProfilingPointRunEntity.KIND).setAncestor(testRun.getKey()); 209 210 for (Entity profilingRun : datastore.prepare(q).asIterable()) { 211 perfSummary.addData(profilingRun); 212 } 213 } 214 } 215 216 /** 217 * Generates a string of the values in optionsList which have matches in the profiling entity. 218 * 219 * @param profilingRun The entity for a profiling point run. 220 * @param optionKeys A list of keys to match against the optionsList key value pairs. 221 * @return The values in optionsList whose key match a key in optionKeys. 222 */ getOptionAlias(Entity profilingRun, Set<String> optionKeys)223 public static String getOptionAlias(Entity profilingRun, Set<String> optionKeys) { 224 String name = ""; 225 List<String> nameSuffixes = new ArrayList<String>(); 226 for (String key : optionKeys) { 227 if (profilingRun.hasProperty(key)) { 228 try { 229 nameSuffixes.add((String) profilingRun.getProperty(key)); 230 } catch (ClassCastException e) { 231 continue; 232 } 233 } 234 } 235 if (nameSuffixes.size() > 0) { 236 StringUtils.join(nameSuffixes, NAME_DELIMITER); 237 name += StringUtils.join(nameSuffixes, NAME_DELIMITER); 238 } 239 return name; 240 } 241 } 242