1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you 5 * may not use this file except in compliance with the License. You may 6 * obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 * implied. See the License for the specific language governing 14 * permissions and limitations under the License. 15 */ 16 17 package com.android.vts.job; 18 19 import static org.junit.Assert.assertEquals; 20 import static org.junit.Assert.assertNotNull; 21 import static org.junit.Assert.assertTrue; 22 import static org.junit.Assert.fail; 23 24 import com.android.vts.entity.DeviceInfoEntity; 25 import com.android.vts.entity.ProfilingPointEntity; 26 import com.android.vts.entity.ProfilingPointRunEntity; 27 import com.android.vts.entity.ProfilingPointSummaryEntity; 28 import com.android.vts.entity.TestEntity; 29 import com.android.vts.entity.TestRunEntity; 30 import com.android.vts.proto.VtsReportMessage; 31 import com.android.vts.util.ObjectifyTestBase; 32 import com.android.vts.util.StatSummary; 33 import com.android.vts.util.TimeUtil; 34 import com.google.appengine.api.datastore.DatastoreService; 35 import com.google.appengine.api.datastore.DatastoreServiceFactory; 36 import com.google.appengine.api.datastore.Entity; 37 import com.google.appengine.api.datastore.EntityNotFoundException; 38 import com.google.appengine.api.datastore.Key; 39 import com.google.appengine.api.datastore.KeyFactory; 40 import com.google.appengine.api.datastore.Query; 41 import com.google.appengine.api.taskqueue.dev.LocalTaskQueue; 42 import com.google.appengine.api.taskqueue.dev.QueueStateInfo; 43 import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig; 44 import com.google.appengine.tools.development.testing.LocalServiceTestHelper; 45 import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; 46 import java.time.Instant; 47 import java.time.LocalDateTime; 48 import java.time.Month; 49 import java.time.ZonedDateTime; 50 import java.util.ArrayList; 51 import java.util.Arrays; 52 import java.util.Date; 53 import java.util.HashMap; 54 import java.util.HashSet; 55 import java.util.List; 56 import java.util.Set; 57 import java.util.concurrent.TimeUnit; 58 import org.apache.commons.math3.stat.descriptive.moment.Mean; 59 import org.junit.jupiter.api.Test; 60 import org.junit.jupiter.api.AfterEach; 61 import org.junit.jupiter.api.BeforeEach; 62 63 public class VtsProfilingStatsJobServletTest extends ObjectifyTestBase { 64 private final LocalServiceTestHelper helper = 65 new LocalServiceTestHelper( 66 new LocalDatastoreServiceTestConfig(), 67 new LocalTaskQueueTestConfig() 68 .setQueueXmlPath("src/main/webapp/WEB-INF/queue.xml")); 69 private static final double THRESHOLD = 1e-10; 70 71 @BeforeEach setUp()72 public void setUp() { 73 helper.setUp(); 74 } 75 76 @AfterEach tearDown()77 public void tearDown() { 78 helper.tearDown(); 79 } 80 createProfilingRun()81 private static void createProfilingRun() { 82 Date d = new Date(); 83 long time = TimeUnit.MILLISECONDS.toMicros(d.getTime()); 84 long canonicalTime = VtsProfilingStatsJobServlet.getCanonicalTime(time); 85 String test = "test"; 86 String profilingPointName = "profilingPoint"; 87 String xLabel = "xLabel"; 88 String yLabel = "yLabel"; 89 VtsReportMessage.VtsProfilingType type = 90 VtsReportMessage.VtsProfilingType.VTS_PROFILING_TYPE_UNLABELED_VECTOR; 91 VtsReportMessage.VtsProfilingRegressionMode mode = 92 VtsReportMessage.VtsProfilingRegressionMode.VTS_REGRESSION_MODE_INCREASING; 93 94 Key testKey = KeyFactory.createKey(TestEntity.KIND, test); 95 Key testRunKey = KeyFactory.createKey(testKey, TestRunEntity.KIND, time); 96 Long[] valueArray = new Long[] {1l, 2l, 3l, 4l, 5l}; 97 StatSummary stats = 98 new StatSummary( 99 "expected", 100 VtsReportMessage.VtsProfilingRegressionMode.UNKNOWN_REGRESSION_MODE); 101 for (long value : valueArray) { 102 stats.updateStats(value); 103 } 104 Mean mean = new Mean(); 105 List<Long> values = Arrays.asList(valueArray); 106 ProfilingPointRunEntity profilingPointRunEntity = 107 new ProfilingPointRunEntity( 108 testRunKey, 109 profilingPointName, 110 type.getNumber(), 111 mode.getNumber(), 112 null, 113 values, 114 xLabel, 115 yLabel, 116 null); 117 118 String branch = "master"; 119 String product = "product"; 120 String flavor = "flavor"; 121 String id = "12345"; 122 String bitness = "64"; 123 String abiName = "abi"; 124 DeviceInfoEntity device = 125 new DeviceInfoEntity(testRunKey, branch, product, flavor, id, bitness, abiName); 126 } 127 128 /** 129 * Test that tasks are correctly scheduled on the queue. 130 * 131 * @throws InterruptedException 132 */ 133 @Test testTasksScheduled()134 public void testTasksScheduled() throws InterruptedException { 135 String[] testNames = new String[] {"test1", "test2", "test3"}; 136 List<Key> testKeys = new ArrayList(); 137 Set<Key> testKeySet = new HashSet<>(); 138 String kind = "TEST"; 139 for (String testName : testNames) { 140 Key key = KeyFactory.createKey(kind, testName); 141 testKeys.add(key); 142 testKeySet.add(key); 143 } 144 VtsProfilingStatsJobServlet.addTasks(testKeys); 145 Thread.sleep(1000); // wait one second (tasks are scheduled asychronously), must wait. 146 LocalTaskQueue taskQueue = LocalTaskQueueTestConfig.getLocalTaskQueue(); 147 QueueStateInfo qsi = taskQueue.getQueueStateInfo().get(VtsProfilingStatsJobServlet.QUEUE); 148 assertNotNull(qsi); 149 assertEquals(testNames.length, qsi.getTaskInfo().size()); 150 151 int i = 0; 152 for (QueueStateInfo.TaskStateInfo taskStateInfo : qsi.getTaskInfo()) { 153 assertEquals( 154 VtsProfilingStatsJobServlet.PROFILING_STATS_JOB_URL, taskStateInfo.getUrl()); 155 assertEquals("POST", taskStateInfo.getMethod()); 156 String body = taskStateInfo.getBody(); 157 String[] parts = body.split("="); 158 assertEquals(2, parts.length); 159 assertEquals(VtsProfilingStatsJobServlet.PROFILING_POINT_KEY, parts[0]); 160 String keyString = parts[1]; 161 Key profilingPointRunKey; 162 try { 163 profilingPointRunKey = KeyFactory.stringToKey(keyString); 164 } catch (IllegalArgumentException e) { 165 fail(); 166 return; 167 } 168 assertTrue(testKeys.contains(profilingPointRunKey)); 169 } 170 } 171 172 /** Test that canonical time is correctly derived from a timestamp in the middle of the day. */ 173 @Test testCanonicalTimeMidday()174 public void testCanonicalTimeMidday() { 175 int year = 2017; 176 Month month = Month.MAY; 177 int day = 28; 178 int hour = 14; 179 int minute = 30; 180 LocalDateTime now = LocalDateTime.of(year, month.getValue(), day, hour, minute); 181 ZonedDateTime zdt = ZonedDateTime.of(now, TimeUtil.PT_ZONE); 182 long time = TimeUnit.SECONDS.toMicros(zdt.toEpochSecond()); 183 long canonicalTime = VtsProfilingStatsJobServlet.getCanonicalTime(time); 184 long canonicalTimeSec = TimeUnit.MICROSECONDS.toSeconds(canonicalTime); 185 ZonedDateTime canonical = 186 ZonedDateTime.ofInstant(Instant.ofEpochSecond(canonicalTimeSec), TimeUtil.PT_ZONE); 187 assertEquals(month, canonical.getMonth()); 188 assertEquals(day, canonical.getDayOfMonth()); 189 assertEquals(0, canonical.getHour()); 190 assertEquals(0, canonical.getMinute()); 191 } 192 193 /** Test that canonical time is correctly derived at the boundary of two days (midnight). */ 194 @Test testCanonicalTimeMidnight()195 public void testCanonicalTimeMidnight() { 196 int year = 2017; 197 Month month = Month.MAY; 198 int day = 28; 199 int hour = 0; 200 int minute = 0; 201 LocalDateTime now = LocalDateTime.of(year, month.getValue(), day, hour, minute); 202 ZonedDateTime zdt = ZonedDateTime.of(now, TimeUtil.PT_ZONE); 203 long time = TimeUnit.SECONDS.toMicros(zdt.toEpochSecond()); 204 long canonicalTime = VtsProfilingStatsJobServlet.getCanonicalTime(time); 205 long canonicalTimeSec = TimeUnit.MICROSECONDS.toSeconds(canonicalTime); 206 ZonedDateTime canonical = 207 ZonedDateTime.ofInstant(Instant.ofEpochSecond(canonicalTimeSec), TimeUtil.PT_ZONE); 208 assertEquals(zdt, canonical); 209 } 210 211 /** Test that new summaries are created with a clean database. */ 212 @Test testNewSummary()213 public void testNewSummary() { 214 Date d = new Date(); 215 long time = TimeUnit.MILLISECONDS.toMicros(d.getTime()); 216 String test = "test"; 217 218 Key testKey = KeyFactory.createKey(TestEntity.KIND, test); 219 Key testRunKey = KeyFactory.createKey(testKey, TestRunEntity.KIND, time); 220 Long[] valueArray = new Long[] {1l, 2l, 3l, 4l, 5l}; 221 StatSummary expected = 222 new StatSummary( 223 "expected", 224 VtsReportMessage.VtsProfilingRegressionMode.UNKNOWN_REGRESSION_MODE); 225 for (long value : valueArray) { 226 expected.updateStats(value); 227 } 228 Mean mean = new Mean(); 229 List<Long> values = Arrays.asList(valueArray); 230 ProfilingPointRunEntity profilingPointRunEntity = 231 new ProfilingPointRunEntity( 232 testRunKey, 233 "profilingPoint", 234 VtsReportMessage.VtsProfilingType.VTS_PROFILING_TYPE_UNLABELED_VECTOR_VALUE, 235 VtsReportMessage.VtsProfilingRegressionMode 236 .VTS_REGRESSION_MODE_INCREASING_VALUE, 237 null, 238 values, 239 "xLabel", 240 "yLabel", 241 null); 242 243 DeviceInfoEntity device = 244 new DeviceInfoEntity( 245 testRunKey, "master", "product", "flavor", "12345", "64", "abi"); 246 247 List<DeviceInfoEntity> devices = new ArrayList<>(); 248 devices.add(device); 249 250 boolean result = 251 VtsProfilingStatsJobServlet.updateSummaries( 252 testKey, profilingPointRunEntity, devices, time); 253 assertTrue(result); 254 255 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 256 257 // Check profiling point entity 258 Key profilingPointKey = ProfilingPointEntity.createKey(test, profilingPointRunEntity.getName()); 259 ProfilingPointEntity profilingPointEntity = null; 260 try { 261 Entity profilingPoint = datastore.get(profilingPointKey); 262 profilingPointEntity = ProfilingPointEntity.fromEntity(profilingPoint); 263 } catch (EntityNotFoundException exception) { 264 fail(); 265 } 266 assertNotNull(profilingPointEntity); 267 assertEquals(profilingPointRunEntity.getName(), profilingPointEntity.getProfilingPointName()); 268 assertEquals(profilingPointRunEntity.getXLabel(), profilingPointEntity.getXLabel()); 269 assertEquals(profilingPointRunEntity.getYLabel(), profilingPointEntity.getYLabel()); 270 assertEquals(profilingPointRunEntity.getType(), profilingPointEntity.getType()); 271 assertEquals(profilingPointRunEntity.getRegressionMode(), profilingPointEntity.getRegressionMode()); 272 273 // Check all summary entities 274 Query q = new Query(ProfilingPointSummaryEntity.KIND).setAncestor(profilingPointKey); 275 for (Entity e : datastore.prepare(q).asIterable()) { 276 ProfilingPointSummaryEntity pps = ProfilingPointSummaryEntity.fromEntity(e); 277 assertNotNull(pps); 278 assertTrue( 279 pps.getBranch().equals(device.getBranch()) 280 || pps.getBranch().equals(ProfilingPointSummaryEntity.ALL)); 281 assertTrue( 282 pps.getBuildFlavor().equals(ProfilingPointSummaryEntity.ALL) 283 || pps.getBuildFlavor().equals(device.getBuildFlavor())); 284 assertEquals(expected.getCount(), pps.getGlobalStats().getCount()); 285 assertEquals(expected.getMax(), pps.getGlobalStats().getMax(), THRESHOLD); 286 assertEquals(expected.getMin(), pps.getGlobalStats().getMin(), THRESHOLD); 287 assertEquals(expected.getMean(), pps.getGlobalStats().getMean(), THRESHOLD); 288 assertEquals(expected.getSumSq(), pps.getGlobalStats().getSumSq(), THRESHOLD); 289 } 290 } 291 292 /** Test that existing summaries are updated correctly when a job pushes new profiling data. */ 293 @Test testUpdateSummary()294 public void testUpdateSummary() { 295 Date d = new Date(); 296 long time = TimeUnit.MILLISECONDS.toMicros(d.getTime()); 297 String test = "test2"; 298 299 Key testKey = KeyFactory.createKey(TestEntity.KIND, test); 300 Key testRunKey = KeyFactory.createKey(testKey, TestRunEntity.KIND, time); 301 Long[] valueArray = new Long[] {0l}; 302 List<Long> values = Arrays.asList(valueArray); 303 304 // Create a new profiling point run 305 ProfilingPointRunEntity profilingPointRunEntity = 306 new ProfilingPointRunEntity( 307 testRunKey, 308 "profilingPoint2", 309 VtsReportMessage.VtsProfilingType.VTS_PROFILING_TYPE_UNLABELED_VECTOR_VALUE, 310 VtsReportMessage.VtsProfilingRegressionMode 311 .VTS_REGRESSION_MODE_INCREASING_VALUE, 312 null, 313 values, 314 "xLabel", 315 "yLabel", 316 null); 317 318 // Create a device for the run 319 String series = ""; 320 DeviceInfoEntity device = 321 new DeviceInfoEntity( 322 testRunKey, "master", "product", "flavor", "12345", "64", "abi"); 323 324 List<DeviceInfoEntity> devices = new ArrayList<>(); 325 devices.add(device); 326 327 // Create the existing stats 328 Key profilingPointKey = ProfilingPointEntity.createKey(test, profilingPointRunEntity.getName()); 329 StatSummary expected = 330 new StatSummary( 331 "label", 332 0, 333 10, 334 5, 335 100, 336 10, 337 VtsReportMessage.VtsProfilingRegressionMode.UNKNOWN_REGRESSION_MODE); 338 ProfilingPointSummaryEntity summary = 339 new ProfilingPointSummaryEntity( 340 profilingPointKey, 341 expected, 342 new ArrayList<>(), 343 new HashMap<>(), 344 device.getBranch(), 345 device.getBuildFlavor(), 346 series, 347 time); 348 349 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 350 datastore.put(summary.toEntity()); 351 352 // Update the summaries in the database 353 boolean result = 354 VtsProfilingStatsJobServlet.updateSummaries( 355 testKey, profilingPointRunEntity, devices, time); 356 assertTrue(result); 357 358 // Calculate the expected stats with the values from the new run 359 for (long value : values) expected.updateStats(value); 360 361 // Get the summary and check the values match what is expected 362 Key summaryKey = 363 ProfilingPointSummaryEntity.createKey( 364 profilingPointKey, device.getBranch(), device.getBuildFlavor(), series, time); 365 ProfilingPointSummaryEntity pps = null; 366 try { 367 Entity e = datastore.get(summaryKey); 368 pps = ProfilingPointSummaryEntity.fromEntity(e); 369 } catch (EntityNotFoundException e) { 370 fail(); 371 } 372 assertNotNull(pps); 373 assertTrue(pps.getBranch().equals(device.getBranch())); 374 assertTrue(pps.getBuildFlavor().equals(device.getBuildFlavor())); 375 assertEquals(expected.getCount(), pps.getGlobalStats().getCount()); 376 assertEquals(expected.getMax(), pps.getGlobalStats().getMax(), THRESHOLD); 377 assertEquals(expected.getMin(), pps.getGlobalStats().getMin(), THRESHOLD); 378 assertEquals(expected.getMean(), pps.getGlobalStats().getMean(), THRESHOLD); 379 assertEquals(expected.getSumSq(), pps.getGlobalStats().getSumSq(), THRESHOLD); 380 } 381 } 382