1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you 5 * may not use this file except in compliance with the License. You may 6 * obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 * implied. See the License for the specific language governing 14 * permissions and limitations under the License. 15 */ 16 17 package com.android.vts.job; 18 19 import com.android.vts.entity.DeviceInfoEntity; 20 import com.android.vts.entity.ProfilingPointEntity; 21 import com.android.vts.entity.ProfilingPointRunEntity; 22 import com.android.vts.entity.ProfilingPointSummaryEntity; 23 import com.android.vts.util.DatastoreHelper; 24 import com.android.vts.util.PerformanceUtil; 25 import com.android.vts.util.TaskQueueHelper; 26 import com.android.vts.util.TimeUtil; 27 import com.google.appengine.api.datastore.DatastoreFailureException; 28 import com.google.appengine.api.datastore.DatastoreService; 29 import com.google.appengine.api.datastore.DatastoreServiceFactory; 30 import com.google.appengine.api.datastore.DatastoreTimeoutException; 31 import com.google.appengine.api.datastore.Entity; 32 import com.google.appengine.api.datastore.EntityNotFoundException; 33 import com.google.appengine.api.datastore.Key; 34 import com.google.appengine.api.datastore.KeyFactory; 35 import com.google.appengine.api.datastore.Query; 36 import com.google.appengine.api.datastore.Transaction; 37 import com.google.appengine.api.taskqueue.Queue; 38 import com.google.appengine.api.taskqueue.QueueFactory; 39 import com.google.appengine.api.taskqueue.TaskOptions; 40 import java.io.IOException; 41 import java.time.Instant; 42 import java.time.ZonedDateTime; 43 import java.util.ArrayList; 44 import java.util.Arrays; 45 import java.util.ConcurrentModificationException; 46 import java.util.HashMap; 47 import java.util.HashSet; 48 import java.util.List; 49 import java.util.Map; 50 import java.util.Set; 51 import java.util.concurrent.TimeUnit; 52 import java.util.logging.Level; 53 import java.util.logging.Logger; 54 import javax.servlet.http.HttpServlet; 55 import javax.servlet.http.HttpServletRequest; 56 import javax.servlet.http.HttpServletResponse; 57 58 /** Represents the notifications service which is automatically called on a fixed schedule. */ 59 public class VtsProfilingStatsJobServlet extends BaseJobServlet { 60 protected static final Logger logger = 61 Logger.getLogger(VtsProfilingStatsJobServlet.class.getName()); 62 private static final String HIDL_HAL_OPTION = "hidl_hal_mode"; 63 private static final String[] splitKeysArray = new String[] {HIDL_HAL_OPTION}; 64 private static final Set<String> splitKeySet = new HashSet<>(Arrays.asList(splitKeysArray)); 65 66 public static final String PROFILING_STATS_JOB_URL = "/task/vts_profiling_stats_job"; 67 public static final String PROFILING_POINT_KEY = "profilingPointKey"; 68 public static final String QUEUE = "profilingStatsQueue"; 69 70 /** 71 * Round the date down to the start of the day (PST). 72 * 73 * @param time The time in microseconds. 74 * @return 75 */ getCanonicalTime(long time)76 public static long getCanonicalTime(long time) { 77 long timeMillis = TimeUnit.MICROSECONDS.toMillis(time); 78 ZonedDateTime zdt = 79 ZonedDateTime.ofInstant(Instant.ofEpochMilli(timeMillis), TimeUtil.PT_ZONE); 80 return TimeUnit.SECONDS.toMicros( 81 zdt.withHour(0).withMinute(0).withSecond(0).toEpochSecond()); 82 } 83 84 /** 85 * Add tasks to process profiling run data 86 * 87 * @param profilingPointKeys The list of keys of the profiling point runs whose data process. 88 */ addTasks(List<Key> profilingPointKeys)89 public static void addTasks(List<Key> profilingPointKeys) { 90 Queue queue = QueueFactory.getQueue(QUEUE); 91 List<TaskOptions> tasks = new ArrayList<>(); 92 for (Key key : profilingPointKeys) { 93 String keyString = KeyFactory.keyToString(key); 94 tasks.add( 95 TaskOptions.Builder.withUrl(PROFILING_STATS_JOB_URL) 96 .param(PROFILING_POINT_KEY, keyString) 97 .method(TaskOptions.Method.POST)); 98 } 99 TaskQueueHelper.addToQueue(queue, tasks); 100 } 101 102 /** 103 * Update the profiling summaries with the information from a profiling point run. 104 * 105 * @param testKey The key to the TestEntity whose profiling data to analyze. 106 * @param profilingPointRun The profiling data to analyze. 107 * @param devices The list of devices used in the profiling run. 108 * @param time The canonical timestamp of the summary to update. 109 * @return true if the update succeeds, false otherwise. 110 */ updateSummaries( Key testKey, ProfilingPointRunEntity profilingPointRun, List<DeviceInfoEntity> devices, long time)111 public static boolean updateSummaries( 112 Key testKey, 113 ProfilingPointRunEntity profilingPointRun, 114 List<DeviceInfoEntity> devices, 115 long time) { 116 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 117 Transaction tx = datastore.beginTransaction(); 118 try { 119 List<Entity> puts = new ArrayList<>(); 120 121 ProfilingPointEntity profilingPoint = 122 new ProfilingPointEntity( 123 testKey.getName(), 124 profilingPointRun.getName(), 125 profilingPointRun.getType(), 126 profilingPointRun.getRegressionMode(), 127 profilingPointRun.getXLabel(), 128 profilingPointRun.getYLabel()); 129 puts.add(profilingPoint.toEntity()); 130 131 String option = PerformanceUtil.getOptionAlias(profilingPointRun, splitKeySet); 132 133 Set<String> branches = new HashSet<>(); 134 Set<String> deviceNames = new HashSet<>(); 135 136 branches.add(ProfilingPointSummaryEntity.ALL); 137 deviceNames.add(ProfilingPointSummaryEntity.ALL); 138 139 for (DeviceInfoEntity d : devices) { 140 branches.add(d.getBranch()); 141 deviceNames.add(d.getBuildFlavor()); 142 } 143 144 List<Key> summaryGets = new ArrayList<>(); 145 for (String branch : branches) { 146 for (String device : deviceNames) { 147 summaryGets.add( 148 ProfilingPointSummaryEntity.createKey( 149 profilingPoint.getKey(), branch, device, option, time)); 150 } 151 } 152 153 Map<Key, Entity> summaries = datastore.get(tx, summaryGets); 154 Map<String, Map<String, ProfilingPointSummaryEntity>> summaryMap = new HashMap<>(); 155 for (Key key : summaries.keySet()) { 156 Entity e = summaries.get(key); 157 ProfilingPointSummaryEntity profilingPointSummary = 158 ProfilingPointSummaryEntity.fromEntity(e); 159 if (profilingPointSummary == null) { 160 logger.log(Level.WARNING, "Invalid profiling point summary: " + e.getKey()); 161 continue; 162 } 163 if (!summaryMap.containsKey(profilingPointSummary.getBranch())) { 164 summaryMap.put(profilingPointSummary.getBranch(), new HashMap<>()); 165 } 166 Map<String, ProfilingPointSummaryEntity> deviceMap = 167 summaryMap.get(profilingPointSummary.getBranch()); 168 deviceMap.put(profilingPointSummary.getBuildFlavor(), profilingPointSummary); 169 } 170 171 Set<ProfilingPointSummaryEntity> modifiedEntities = new HashSet<>(); 172 173 for (String branch : branches) { 174 if (!summaryMap.containsKey(branch)) { 175 summaryMap.put(branch, new HashMap<>()); 176 } 177 Map<String, ProfilingPointSummaryEntity> deviceMap = summaryMap.get(branch); 178 179 for (String device : deviceNames) { 180 ProfilingPointSummaryEntity summary; 181 if (deviceMap.containsKey(device)) { 182 summary = deviceMap.get(device); 183 } else { 184 summary = 185 new ProfilingPointSummaryEntity( 186 profilingPoint.getKey(), branch, device, option, time); 187 deviceMap.put(device, summary); 188 } 189 summary.update(profilingPointRun); 190 modifiedEntities.add(summary); 191 } 192 } 193 194 for (ProfilingPointSummaryEntity profilingPointSummary : modifiedEntities) { 195 puts.add(profilingPointSummary.toEntity()); 196 } 197 datastore.put(tx, puts); 198 tx.commit(); 199 } catch (ConcurrentModificationException 200 | DatastoreFailureException 201 | DatastoreTimeoutException e) { 202 return false; 203 } finally { 204 if (tx.isActive()) { 205 tx.rollback(); 206 logger.log( 207 Level.WARNING, 208 "Profiling stats job transaction still active: " 209 + profilingPointRun.getKey()); 210 return false; 211 } 212 } 213 return true; 214 } 215 216 @Override doPost(HttpServletRequest request, HttpServletResponse response)217 public void doPost(HttpServletRequest request, HttpServletResponse response) 218 throws IOException { 219 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 220 String profilingPointKeyString = request.getParameter(PROFILING_POINT_KEY); 221 222 Key profilingPointRunKey; 223 try { 224 profilingPointRunKey = KeyFactory.stringToKey(profilingPointKeyString); 225 } catch (IllegalArgumentException e) { 226 logger.log(Level.WARNING, "Invalid key specified: " + profilingPointKeyString); 227 return; 228 } 229 Key testKey = profilingPointRunKey.getParent().getParent(); 230 231 ProfilingPointRunEntity profilingPointRun = null; 232 try { 233 Entity profilingPointRunEntity = datastore.get(profilingPointRunKey); 234 profilingPointRun = ProfilingPointRunEntity.fromEntity(profilingPointRunEntity); 235 } catch (EntityNotFoundException e) { 236 // no run found 237 } 238 if (profilingPointRun == null) { 239 return; 240 } 241 242 Query deviceQuery = 243 new Query(DeviceInfoEntity.KIND).setAncestor(profilingPointRunKey.getParent()); 244 245 List<DeviceInfoEntity> devices = new ArrayList<>(); 246 for (Entity e : datastore.prepare(deviceQuery).asIterable()) { 247 DeviceInfoEntity deviceInfoEntity = DeviceInfoEntity.fromEntity(e); 248 if (e == null) continue; 249 devices.add(deviceInfoEntity); 250 } 251 252 long canonicalTime = getCanonicalTime(profilingPointRunKey.getParent().getId()); 253 int retryCount = 0; 254 while (retryCount++ <= DatastoreHelper.MAX_WRITE_RETRIES) { 255 boolean result = updateSummaries(testKey, profilingPointRun, devices, canonicalTime); 256 if (!result) { 257 logger.log( 258 Level.WARNING, "Retrying profiling stats update: " + profilingPointRunKey); 259 continue; 260 } 261 break; 262 } 263 if (retryCount > DatastoreHelper.MAX_WRITE_RETRIES) { 264 logger.log(Level.SEVERE, "Could not update profiling stats: " + profilingPointRunKey); 265 } 266 } 267 } 268