1 /* 2 * Copyright (c) 2016 Google Inc. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you 5 * may not use this file except in compliance with the License. You may 6 * obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 * implied. See the License for the specific language governing 14 * permissions and limitations under the License. 15 */ 16 17 package com.android.vts.servlet; 18 19 import com.android.vts.entity.DeviceInfoEntity; 20 import com.android.vts.entity.TestCaseRunEntity; 21 import com.android.vts.entity.TestEntity; 22 import com.android.vts.entity.TestRunEntity; 23 import com.android.vts.proto.VtsReportMessage.TestCaseResult; 24 import com.android.vts.util.EmailHelper; 25 import com.android.vts.util.FilterUtil; 26 import com.google.appengine.api.datastore.DatastoreService; 27 import com.google.appengine.api.datastore.DatastoreServiceFactory; 28 import com.google.appengine.api.datastore.Entity; 29 import com.google.appengine.api.datastore.EntityNotFoundException; 30 import com.google.appengine.api.datastore.Key; 31 import com.google.appengine.api.datastore.KeyFactory; 32 import com.google.appengine.api.datastore.Query; 33 import com.google.appengine.api.datastore.Query.Filter; 34 import com.google.appengine.api.datastore.Query.SortDirection; 35 import com.google.appengine.api.datastore.Transaction; 36 import java.io.IOException; 37 import java.io.UnsupportedEncodingException; 38 import java.text.SimpleDateFormat; 39 import java.util.ArrayList; 40 import java.util.Collections; 41 import java.util.Date; 42 import java.util.HashMap; 43 import java.util.HashSet; 44 import java.util.List; 45 import java.util.Map; 46 import java.util.Set; 47 import java.util.concurrent.TimeUnit; 48 import java.util.logging.Level; 49 import javax.mail.Message; 50 import javax.mail.MessagingException; 51 import javax.servlet.http.HttpServletRequest; 52 import javax.servlet.http.HttpServletResponse; 53 import org.apache.commons.lang.StringUtils; 54 55 /** Represents the notifications service which is automatically called on a fixed schedule. */ 56 public class VtsAlertJobServlet extends BaseServlet { 57 58 @Override getNavbarLinks(HttpServletRequest request)59 public List<String[]> getNavbarLinks(HttpServletRequest request) { 60 return null; 61 } 62 63 /** 64 * Creates an email footer with the provided link. 65 * 66 * @param link The (string) link to provide in the footer. 67 * @return The full HTML email footer. 68 */ getFooter(String link)69 private String getFooter(String link) { 70 return "<br><br>For details, visit the" 71 + " <a href='" + link + "'>" 72 + "VTS dashboard.</a>"; 73 } 74 75 /** 76 * Compose an email if the test is inactive. 77 * 78 * @param test The TestEntity document storing the test status. 79 * @param link Fully specified link to the test's status page. 80 * @param emails The list of email addresses to send the email. 81 * @param messages The message list in which to insert the inactivity notification email. 82 * @return True if the test is inactive, false otherwise. 83 */ notifyIfInactive( TestEntity test, String link, List<String> emails, List<Message> messages)84 private boolean notifyIfInactive( 85 TestEntity test, String link, List<String> emails, List<Message> messages) { 86 long now = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()); 87 long diff = now - test.timestamp; 88 // Send an email daily to notify that the test hasn't been running. 89 // After 7 full days have passed, notifications will no longer be sent (i.e. the 90 // test is assumed to be deprecated). 91 if (diff > TimeUnit.DAYS.toMicros(1) && diff < TimeUnit.DAYS.toMicros(8) 92 && diff % TimeUnit.DAYS.toMicros(1) < TimeUnit.MINUTES.toMicros(3)) { 93 Date lastUpload = new Date(TimeUnit.MICROSECONDS.toMillis(test.timestamp)); 94 String uploadTimeString = 95 new SimpleDateFormat("MM/dd/yyyy HH:mm:ss").format(lastUpload); 96 String subject = "Warning! Inactive test: " + test; 97 String body = "Hello,<br><br>Test \"" + test + "\" is inactive. " 98 + "No new data has been uploaded since " + uploadTimeString + "." 99 + getFooter(link); 100 try { 101 messages.add(EmailHelper.composeEmail(emails, subject, body)); 102 return true; 103 } catch (MessagingException | UnsupportedEncodingException e) { 104 logger.log(Level.WARNING, "Error composing email : ", e); 105 } 106 } 107 return false; 108 } 109 110 /** 111 * Checks whether any new failures have occurred beginning since (and including) startTime. 112 * 113 * @param test The TestEntity object for the test. 114 * @param link The string URL linking to the test's status table. 115 * @param failedTestcaseIdMap The map of test case names to ID for those failing in the 116 * last status update. 117 * @param emailAddresses The list of email addresses to send notifications to. 118 * @param messages The email Message queue. 119 * @returns latest TestStatusMessage or null if no update is available. 120 * @throws IOException 121 */ getTestStatus(TestEntity test, String link, Map<String, Long> failedTestcaseIdMap, List<String> emailAddresses, List<Message> messages)122 public TestEntity getTestStatus(TestEntity test, String link, 123 Map<String, Long> failedTestcaseIdMap, List<String> emailAddresses, 124 List<Message> messages) throws IOException { 125 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 126 String footer = getFooter(link); 127 128 TestRunEntity mostRecentRun = null; 129 Map<String, TestCaseRunEntity> mostRecentTestCaseResults = new HashMap<>(); 130 Map<String, Long> testBreakageIdMap = new HashMap<>(); 131 int passingTestcaseCount = 0; 132 List<Long> failingTestcaseIds = new ArrayList<>(); 133 Set<String> fixedTestcases = new HashSet<>(); 134 Set<String> newTestcaseFailures = new HashSet<>(); 135 Set<String> continuedTestcaseFailures = new HashSet<>(); 136 Set<String> skippedTestcaseFailures = new HashSet<>(); 137 Set<String> transientTestcaseFailures = new HashSet<>(); 138 139 String testName = test.testName; 140 Key testKey = KeyFactory.createKey(TestEntity.KIND, testName); 141 Filter testTypeFilter = FilterUtil.getTestTypeFilter(false, true, false); 142 Filter runFilter = 143 FilterUtil.getTimeFilter(testKey, test.timestamp + 1, null, testTypeFilter); 144 Query q = new Query(TestRunEntity.KIND) 145 .setAncestor(testKey) 146 .setFilter(runFilter) 147 .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.DESCENDING); 148 149 for (Entity testRun : datastore.prepare(q).asIterable()) { 150 TestRunEntity testRunEntity = TestRunEntity.fromEntity(testRun); 151 if (testRunEntity == null) { 152 logger.log(Level.WARNING, "Invalid test run detected: " + testRun.getKey()); 153 } 154 if (mostRecentRun == null) { 155 mostRecentRun = testRunEntity; 156 } 157 for (long testCaseId : testRunEntity.testCaseIds) { 158 Entity testCaseRun; 159 try { 160 testCaseRun = 161 datastore.get(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); 162 } catch (EntityNotFoundException e) { 163 logger.log(Level.WARNING, 164 "Test case \"" + testCaseId + "\" from test " + testName); 165 continue; 166 } 167 TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun); 168 if (testCaseRunEntity == null) { 169 logger.log(Level.WARNING, "Invalid test case run: " + testCaseRun.getKey()); 170 continue; 171 } 172 String testCaseName = testCaseRunEntity.testCaseName; 173 TestCaseResult result = TestCaseResult.valueOf(testCaseRunEntity.result); 174 175 if (mostRecentRun == testRunEntity) { 176 mostRecentTestCaseResults.put(testCaseName, testCaseRunEntity); 177 } else { 178 if (!mostRecentTestCaseResults.containsKey(testCaseName)) { 179 // Deprecate notifications for tests that are not present on newer runs 180 continue; 181 } 182 TestCaseResult mostRecentRes = TestCaseResult.valueOf( 183 mostRecentTestCaseResults.get(testCaseName).result); 184 if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_SKIP) { 185 mostRecentTestCaseResults.put(testCaseName, testCaseRunEntity); 186 } else if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_PASS) { 187 // Test is passing now, witnessed a transient failure 188 if (result != TestCaseResult.TEST_CASE_RESULT_PASS 189 && result != TestCaseResult.TEST_CASE_RESULT_SKIP) { 190 transientTestcaseFailures.add(testCaseName); 191 } 192 } 193 } 194 195 // Record test case breakages 196 if (result != TestCaseResult.TEST_CASE_RESULT_PASS 197 && result != TestCaseResult.TEST_CASE_RESULT_SKIP) { 198 testBreakageIdMap.put(testCaseName, testCaseRunEntity.key.getId()); 199 } 200 } 201 } 202 203 if (mostRecentRun == null) { 204 notifyIfInactive(test, link, emailAddresses, messages); 205 return null; 206 } 207 208 for (String testCaseName : mostRecentTestCaseResults.keySet()) { 209 TestCaseRunEntity testCaseRunEntity = mostRecentTestCaseResults.get(testCaseName); 210 TestCaseResult mostRecentResult = TestCaseResult.valueOf(testCaseRunEntity.result); 211 boolean previouslyFailed = failedTestcaseIdMap.containsKey(testCaseName); 212 if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_SKIP) { 213 // persist previous status 214 if (previouslyFailed) { 215 failingTestcaseIds.add(failedTestcaseIdMap.get(testCaseName)); 216 } else { 217 ++passingTestcaseCount; 218 } 219 } else if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_PASS) { 220 ++passingTestcaseCount; 221 if (previouslyFailed && !transientTestcaseFailures.contains(testCaseName)) { 222 fixedTestcases.add(testCaseName); 223 } 224 } else { 225 if (!previouslyFailed) { 226 newTestcaseFailures.add(testCaseName); 227 failingTestcaseIds.add(testBreakageIdMap.get(testCaseName)); 228 } else { 229 continuedTestcaseFailures.add(testCaseName); 230 failingTestcaseIds.add(failedTestcaseIdMap.get(testCaseName)); 231 } 232 } 233 } 234 235 Set<String> buildIdList = new HashSet<>(); 236 Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(mostRecentRun.key); 237 for (Entity device : datastore.prepare(deviceQuery).asIterable()) { 238 DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); 239 if (deviceEntity == null) { 240 continue; 241 } 242 buildIdList.add(deviceEntity.buildId); 243 } 244 String buildId = StringUtils.join(buildIdList, ","); 245 String summary = new String(); 246 if (newTestcaseFailures.size() + continuedTestcaseFailures.size() > 0) { 247 summary += "The following test cases failed in the latest test run:<br>"; 248 249 // Add new test case failures to top of summary in bold font. 250 List<String> sortedNewTestcaseFailures = new ArrayList<>(newTestcaseFailures); 251 Collections.sort(sortedNewTestcaseFailures); 252 for (String testcaseName : sortedNewTestcaseFailures) { 253 summary += "- " 254 + "<b>" + testcaseName + "</b><br>"; 255 } 256 257 // Add continued test case failures to summary. 258 List<String> sortedContinuedTestcaseFailures = 259 new ArrayList<>(continuedTestcaseFailures); 260 Collections.sort(sortedContinuedTestcaseFailures); 261 for (String testcaseName : sortedContinuedTestcaseFailures) { 262 summary += "- " + testcaseName + "<br>"; 263 } 264 } 265 if (fixedTestcases.size() > 0) { 266 // Add fixed test cases to summary. 267 summary += "<br><br>The following test cases were fixed in the latest test run:<br>"; 268 List<String> sortedFixedTestcases = new ArrayList<>(fixedTestcases); 269 Collections.sort(sortedFixedTestcases); 270 for (String testcaseName : sortedFixedTestcases) { 271 summary += "- <i>" + testcaseName + "</i><br>"; 272 } 273 } 274 if (transientTestcaseFailures.size() > 0) { 275 // Add transient test case failures to summary. 276 summary += "<br><br>The following transient test case failures occured:<br>"; 277 List<String> sortedTransientTestcaseFailures = 278 new ArrayList<>(transientTestcaseFailures); 279 Collections.sort(sortedTransientTestcaseFailures); 280 for (String testcaseName : sortedTransientTestcaseFailures) { 281 summary += "- " + testcaseName + "<br>"; 282 } 283 } 284 if (skippedTestcaseFailures.size() > 0) { 285 // Add skipped test case failures to summary. 286 summary += "<br><br>The following test cases have not been run since failing:<br>"; 287 List<String> sortedSkippedTestcaseFailures = new ArrayList<>(skippedTestcaseFailures); 288 Collections.sort(sortedSkippedTestcaseFailures); 289 for (String testcaseName : sortedSkippedTestcaseFailures) { 290 summary += "- " + testcaseName + "<br>"; 291 } 292 } 293 294 if (newTestcaseFailures.size() > 0) { 295 String subject = "New test failures in " + testName + " @ " + buildId; 296 String body = "Hello,<br><br>Test cases are failing in " + testName 297 + " for device build ID(s): " + buildId + ".<br><br>" + summary + footer; 298 try { 299 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 300 } catch (MessagingException | UnsupportedEncodingException e) { 301 logger.log(Level.WARNING, "Error composing email : ", e); 302 } 303 } else if (continuedTestcaseFailures.size() > 0) { 304 String subject = "Continued test failures in " + testName + " @ " + buildId; 305 String body = "Hello,<br><br>Test cases are failing in " + testName 306 + " for device build ID(s): " + buildId + ".<br><br>" + summary + footer; 307 try { 308 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 309 } catch (MessagingException | UnsupportedEncodingException e) { 310 logger.log(Level.WARNING, "Error composing email : ", e); 311 } 312 } else if (transientTestcaseFailures.size() > 0) { 313 String subject = "Transient test failure in " + testName + " @ " + buildId; 314 String body = "Hello,<br><br>Some test cases failed in " + testName + " but tests all " 315 + "are passing in the latest device build(s): " + buildId + ".<br><br>" 316 + summary + footer; 317 try { 318 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 319 } catch (MessagingException | UnsupportedEncodingException e) { 320 logger.log(Level.WARNING, "Error composing email : ", e); 321 } 322 } else if (fixedTestcases.size() > 0) { 323 String subject = "All test cases passing in " + testName + " @ " + buildId; 324 String body = "Hello,<br><br>All test cases passed in " + testName 325 + " for device build ID(s): " + buildId + "!<br><br>" + summary + footer; 326 try { 327 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 328 } catch (MessagingException | UnsupportedEncodingException e) { 329 logger.log(Level.WARNING, "Error composing email : ", e); 330 } 331 } 332 return new TestEntity(test.testName, mostRecentRun.startTimestamp, passingTestcaseCount, 333 failingTestcaseIds.size(), failingTestcaseIds); 334 } 335 336 /** 337 * Process the current test case failures for a test. 338 * 339 * @param testEntity The TestEntity object for the test. 340 * @returns a map from test case name to the test case run ID for which the test case failed. 341 */ getCurrentFailures(TestEntity testEntity)342 public static Map<String, Long> getCurrentFailures(TestEntity testEntity) { 343 if (testEntity.failingTestcaseIds == null) { 344 return new HashMap<>(); 345 } 346 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 347 Map<String, Long> failingTestcases = new HashMap<>(); 348 for (long testCaseId : testEntity.failingTestcaseIds) { 349 try { 350 Entity testCaseRun = 351 datastore.get(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); 352 TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun); 353 if (testCaseRunEntity != null) { 354 failingTestcases.put(testCaseRunEntity.testCaseName, testCaseId); 355 } 356 } catch (EntityNotFoundException e) { 357 // not found 358 } 359 } 360 return failingTestcases; 361 } 362 363 @Override doGet(HttpServletRequest request, HttpServletResponse response)364 public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { 365 doGetHandler(request, response); 366 } 367 368 @Override doGetHandler(HttpServletRequest request, HttpServletResponse response)369 public void doGetHandler(HttpServletRequest request, HttpServletResponse response) 370 throws IOException { 371 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 372 Query q = new Query(TestEntity.KIND); 373 for (Entity test : datastore.prepare(q).asIterable()) { 374 TestEntity testEntity = TestEntity.fromEntity(test); 375 if (testEntity == null) { 376 logger.log(Level.WARNING, "Corrupted test entity: " + test.getKey().getName()); 377 continue; 378 } 379 List<String> emails = EmailHelper.getSubscriberEmails(test.getKey()); 380 381 StringBuffer fullUrl = request.getRequestURL(); 382 String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI())); 383 String link = baseUrl + "/show_table?testName=" + testEntity.testName; 384 385 List<Message> messageQueue = new ArrayList<>(); 386 Map<String, Long> failedTestcaseMap = getCurrentFailures(testEntity); 387 388 TestEntity newTestEntity = 389 getTestStatus(testEntity, link, failedTestcaseMap, emails, messageQueue); 390 391 // Send any inactivity notifications 392 if (newTestEntity == null) { 393 if (messageQueue.size() > 0) { 394 EmailHelper.sendAll(messageQueue); 395 } 396 continue; 397 } 398 399 Transaction txn = datastore.beginTransaction(); 400 try { 401 try { 402 testEntity = TestEntity.fromEntity(datastore.get(test.getKey())); 403 404 // Another job updated the test entity 405 if (testEntity == null || testEntity.timestamp >= newTestEntity.timestamp) { 406 txn.rollback(); 407 } else { // This update is most recent. 408 datastore.put(newTestEntity.toEntity()); 409 txn.commit(); 410 EmailHelper.sendAll(messageQueue); 411 } 412 } catch (EntityNotFoundException e) { 413 logger.log(Level.INFO, 414 "Test disappeared during updated: " + newTestEntity.testName); 415 } 416 } finally { 417 if (txn.isActive()) { 418 txn.rollback(); 419 } 420 } 421 } 422 } 423 } 424