1 /* 2 * Copyright (c) 2016 Google Inc. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you 5 * may not use this file except in compliance with the License. You may 6 * obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 * implied. See the License for the specific language governing 14 * permissions and limitations under the License. 15 */ 16 17 package com.android.vts.job; 18 19 import com.android.vts.entity.DeviceInfoEntity; 20 import com.android.vts.entity.TestAcknowledgmentEntity; 21 import com.android.vts.entity.TestCaseRunEntity; 22 import com.android.vts.entity.TestCaseRunEntity.TestCase; 23 import com.android.vts.entity.TestRunEntity; 24 import com.android.vts.entity.TestStatusEntity; 25 import com.android.vts.entity.TestStatusEntity.TestCaseReference; 26 import com.android.vts.proto.VtsReportMessage.TestCaseResult; 27 import com.android.vts.util.DatastoreHelper; 28 import com.android.vts.util.EmailHelper; 29 import com.android.vts.util.FilterUtil; 30 import com.android.vts.util.TimeUtil; 31 import com.google.appengine.api.datastore.DatastoreFailureException; 32 import com.google.appengine.api.datastore.DatastoreService; 33 import com.google.appengine.api.datastore.DatastoreServiceFactory; 34 import com.google.appengine.api.datastore.DatastoreTimeoutException; 35 import com.google.appengine.api.datastore.Entity; 36 import com.google.appengine.api.datastore.EntityNotFoundException; 37 import com.google.appengine.api.datastore.FetchOptions; 38 import com.google.appengine.api.datastore.Key; 39 import com.google.appengine.api.datastore.KeyFactory; 40 import com.google.appengine.api.datastore.Query; 41 import com.google.appengine.api.datastore.Query.Filter; 42 import com.google.appengine.api.datastore.Query.SortDirection; 43 import com.google.appengine.api.datastore.Transaction; 44 import com.google.appengine.api.taskqueue.Queue; 45 import com.google.appengine.api.taskqueue.QueueFactory; 46 import com.google.appengine.api.taskqueue.TaskOptions; 47 import java.io.IOException; 48 import java.io.UnsupportedEncodingException; 49 import java.util.ArrayList; 50 import java.util.Comparator; 51 import java.util.ConcurrentModificationException; 52 import java.util.HashMap; 53 import java.util.HashSet; 54 import java.util.List; 55 import java.util.Map; 56 import java.util.Set; 57 import java.util.concurrent.TimeUnit; 58 import java.util.logging.Level; 59 import java.util.logging.Logger; 60 import javax.mail.Message; 61 import javax.mail.MessagingException; 62 import javax.servlet.http.HttpServletRequest; 63 import javax.servlet.http.HttpServletResponse; 64 import org.apache.commons.lang.StringUtils; 65 66 /** Represents the notifications service which is automatically called on a fixed schedule. */ 67 public class VtsAlertJobServlet extends BaseJobServlet { 68 private static final String ALERT_JOB_URL = "/task/vts_alert_job"; 69 protected static final Logger logger = Logger.getLogger(VtsAlertJobServlet.class.getName()); 70 protected static final int MAX_RUN_COUNT = 1000; // maximum number of runs to query for 71 72 /** 73 * Process the current test case failures for a test. 74 * 75 * @param status The TestStatusEntity object for the test. 76 * @returns a map from test case name to the test case run ID for which the test case failed. 77 */ getCurrentFailures(TestStatusEntity status)78 private static Map<String, TestCase> getCurrentFailures(TestStatusEntity status) { 79 if (status.getFailingTestCases() == null || status.getFailingTestCases().size() == 0) { 80 return new HashMap<>(); 81 } 82 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 83 Map<String, TestCase> failingTestcases = new HashMap<>(); 84 Set<Key> gets = new HashSet<>(); 85 for (TestCaseReference testCaseRef : status.getFailingTestCases()) { 86 gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId)); 87 } 88 if (gets.size() == 0) { 89 return failingTestcases; 90 } 91 Map<Key, Entity> testCaseMap = datastore.get(gets); 92 93 for (TestCaseReference testCaseRef : status.getFailingTestCases()) { 94 Key key = KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId); 95 if (!testCaseMap.containsKey(key)) { 96 continue; 97 } 98 Entity testCaseRun = testCaseMap.get(key); 99 TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun); 100 if (testCaseRunEntity.testCases.size() <= testCaseRef.offset) { 101 continue; 102 } 103 TestCase testCase = testCaseRunEntity.testCases.get(testCaseRef.offset); 104 failingTestcases.put(testCase.name, testCase); 105 } 106 return failingTestcases; 107 } 108 109 /** 110 * Get the test acknowledgments for a test key. 111 * 112 * @param testKey The key to the test whose acknowledgments to fetch. 113 * @return A list of test acknowledgments. 114 */ getTestCaseAcknowledgments(Key testKey)115 private static List<TestAcknowledgmentEntity> getTestCaseAcknowledgments(Key testKey) { 116 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 117 118 List<TestAcknowledgmentEntity> acks = new ArrayList<>(); 119 Filter testFilter = 120 new Query.FilterPredicate( 121 TestAcknowledgmentEntity.TEST_KEY, Query.FilterOperator.EQUAL, testKey); 122 Query q = new Query(TestAcknowledgmentEntity.KIND).setFilter(testFilter); 123 124 for (Entity ackEntity : datastore.prepare(q).asIterable()) { 125 TestAcknowledgmentEntity ack = TestAcknowledgmentEntity.fromEntity(ackEntity); 126 if (ack == null) continue; 127 acks.add(ack); 128 } 129 return acks; 130 } 131 132 /** 133 * Get the test runs for the test in the specified time window. 134 * 135 * <p>If the start and end time delta is greater than one day, the query will be truncated. 136 * 137 * @param testKey The key to the test whose runs to query. 138 * @param startTime The start time for the query. 139 * @param endTime The end time for the query. 140 * @return A list of test runs in the specified time window. 141 */ getTestRuns(Key testKey, long startTime, long endTime)142 private static List<TestRunEntity> getTestRuns(Key testKey, long startTime, long endTime) { 143 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 144 Filter testTypeFilter = FilterUtil.getTestTypeFilter(false, true, false); 145 long delta = endTime - startTime; 146 delta = Math.min(delta, TimeUnit.DAYS.toMicros(1)); 147 Filter runFilter = 148 FilterUtil.getTimeFilter( 149 testKey, TestRunEntity.KIND, endTime - delta + 1, endTime, testTypeFilter); 150 151 Query q = 152 new Query(TestRunEntity.KIND) 153 .setAncestor(testKey) 154 .setFilter(runFilter) 155 .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.DESCENDING); 156 157 List<TestRunEntity> testRuns = new ArrayList<>(); 158 for (Entity testRunEntity : 159 datastore.prepare(q).asIterable(FetchOptions.Builder.withLimit(MAX_RUN_COUNT))) { 160 TestRunEntity testRun = TestRunEntity.fromEntity(testRunEntity); 161 if (testRun == null) continue; 162 testRuns.add(testRun); 163 } 164 return testRuns; 165 } 166 167 /** 168 * Separate the test cases which are acknowledged by the provided acknowledgments. 169 * 170 * @param testCases The list of test case names. 171 * @param devices The list of devices for a test run. 172 * @param acks The list of acknowledgments for the test. 173 * @return A list of acknowledged test case names that have been removed from the input test 174 * cases. 175 */ separateAcknowledged( Set<String> testCases, List<DeviceInfoEntity> devices, List<TestAcknowledgmentEntity> acks)176 public static Set<String> separateAcknowledged( 177 Set<String> testCases, 178 List<DeviceInfoEntity> devices, 179 List<TestAcknowledgmentEntity> acks) { 180 Set<String> acknowledged = new HashSet<>(); 181 for (TestAcknowledgmentEntity ack : acks) { 182 boolean allDevices = ack.getDevices() == null || ack.getDevices().size() == 0; 183 boolean allBranches = ack.getBranches() == null || ack.getBranches().size() == 0; 184 boolean isRelevant = allDevices && allBranches; 185 186 // Determine if the acknowledgment is relevant to the devices. 187 if (!isRelevant) { 188 for (DeviceInfoEntity device : devices) { 189 boolean deviceAcknowledged = 190 allDevices || ack.getDevices().contains(device.getBuildFlavor()); 191 boolean branchAcknowledged = 192 allBranches || ack.getBranches().contains(device.getBranch()); 193 if (deviceAcknowledged && branchAcknowledged) isRelevant = true; 194 } 195 } 196 197 if (isRelevant) { 198 // Separate the test cases 199 boolean allTestCases = 200 ack.getTestCaseNames() == null || ack.getTestCaseNames().size() == 0; 201 if (allTestCases) { 202 acknowledged.addAll(testCases); 203 testCases.removeAll(acknowledged); 204 } else { 205 for (String testCase : ack.getTestCaseNames()) { 206 if (testCases.contains(testCase)) { 207 acknowledged.add(testCase); 208 testCases.remove(testCase); 209 } 210 } 211 } 212 } 213 } 214 return acknowledged; 215 } 216 217 /** 218 * Checks whether any new failures have occurred beginning since (and including) startTime. 219 * 220 * @param testRuns The list of test runs for which to update the status. 221 * @param link The string URL linking to the test's status table. 222 * @param failedTestCaseMap The map of test case names to TestCase for those failing in the last 223 * status update. 224 * @param emailAddresses The list of email addresses to send notifications to. 225 * @param messages The email Message queue. 226 * @returns latest TestStatusMessage or null if no update is available. 227 * @throws IOException 228 */ getTestStatus( List<TestRunEntity> testRuns, String link, Map<String, TestCase> failedTestCaseMap, List<TestAcknowledgmentEntity> testAcks, List<String> emailAddresses, List<Message> messages)229 public TestStatusEntity getTestStatus( 230 List<TestRunEntity> testRuns, 231 String link, 232 Map<String, TestCase> failedTestCaseMap, 233 List<TestAcknowledgmentEntity> testAcks, 234 List<String> emailAddresses, 235 List<Message> messages) 236 throws IOException { 237 if (testRuns.size() == 0) return null; 238 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 239 240 TestRunEntity mostRecentRun = null; 241 Map<String, TestCaseResult> mostRecentTestCaseResults = new HashMap<>(); 242 Map<String, TestCase> testCaseBreakageMap = new HashMap<>(); 243 int passingTestcaseCount = 0; 244 List<TestCaseReference> failingTestCases = new ArrayList<>(); 245 Set<String> fixedTestcases = new HashSet<>(); 246 Set<String> newTestcaseFailures = new HashSet<>(); 247 Set<String> continuedTestcaseFailures = new HashSet<>(); 248 Set<String> skippedTestcaseFailures = new HashSet<>(); 249 Set<String> transientTestcaseFailures = new HashSet<>(); 250 251 for (TestRunEntity testRun : testRuns) { 252 if (mostRecentRun == null) { 253 mostRecentRun = testRun; 254 } 255 List<Key> testCaseKeys = new ArrayList<>(); 256 for (long testCaseId : testRun.getTestCaseIds()) { 257 testCaseKeys.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); 258 } 259 Map<Key, Entity> entityMap = datastore.get(testCaseKeys); 260 for (Key testCaseKey : testCaseKeys) { 261 if (!entityMap.containsKey(testCaseKey)) { 262 logger.log(Level.WARNING, "Test case entity missing: " + testCaseKey); 263 continue; 264 } 265 Entity testCaseRun = entityMap.get(testCaseKey); 266 TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun); 267 if (testCaseRunEntity == null) { 268 logger.log(Level.WARNING, "Invalid test case run: " + testCaseRun.getKey()); 269 continue; 270 } 271 for (TestCase testCase : testCaseRunEntity.testCases) { 272 String testCaseName = testCase.name; 273 TestCaseResult result = TestCaseResult.valueOf(testCase.result); 274 275 if (mostRecentRun == testRun) { 276 mostRecentTestCaseResults.put(testCaseName, result); 277 } else { 278 if (!mostRecentTestCaseResults.containsKey(testCaseName)) { 279 // Deprecate notifications for tests that are not present on newer runs 280 continue; 281 } 282 TestCaseResult mostRecentRes = mostRecentTestCaseResults.get(testCaseName); 283 if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_SKIP) { 284 mostRecentTestCaseResults.put(testCaseName, result); 285 } else if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_PASS) { 286 // Test is passing now, witnessed a transient failure 287 if (result != TestCaseResult.TEST_CASE_RESULT_PASS 288 && result != TestCaseResult.TEST_CASE_RESULT_SKIP) { 289 transientTestcaseFailures.add(testCaseName); 290 } 291 } 292 } 293 294 // Record test case breakages 295 if (result != TestCaseResult.TEST_CASE_RESULT_PASS 296 && result != TestCaseResult.TEST_CASE_RESULT_SKIP) { 297 testCaseBreakageMap.put(testCaseName, testCase); 298 } 299 } 300 } 301 } 302 303 Set<String> buildIdList = new HashSet<>(); 304 List<DeviceInfoEntity> devices = new ArrayList<>(); 305 Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(mostRecentRun.getKey()); 306 for (Entity device : datastore.prepare(deviceQuery).asIterable()) { 307 DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); 308 if (deviceEntity == null) { 309 continue; 310 } 311 buildIdList.add(deviceEntity.getBuildId()); 312 devices.add(deviceEntity); 313 } 314 String footer = EmailHelper.getEmailFooter(mostRecentRun, devices, link); 315 String buildId = StringUtils.join(buildIdList, ","); 316 317 for (String testCaseName : mostRecentTestCaseResults.keySet()) { 318 TestCaseResult mostRecentResult = mostRecentTestCaseResults.get(testCaseName); 319 boolean previouslyFailed = failedTestCaseMap.containsKey(testCaseName); 320 if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_SKIP) { 321 // persist previous status 322 if (previouslyFailed) { 323 skippedTestcaseFailures.add(testCaseName); 324 failingTestCases.add( 325 new TestCaseReference(failedTestCaseMap.get(testCaseName))); 326 } else { 327 ++passingTestcaseCount; 328 } 329 } else if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_PASS) { 330 ++passingTestcaseCount; 331 if (previouslyFailed && !transientTestcaseFailures.contains(testCaseName)) { 332 fixedTestcases.add(testCaseName); 333 } 334 } else { 335 if (!previouslyFailed) { 336 newTestcaseFailures.add(testCaseName); 337 failingTestCases.add( 338 new TestCaseReference(testCaseBreakageMap.get(testCaseName))); 339 } else { 340 continuedTestcaseFailures.add(testCaseName); 341 failingTestCases.add( 342 new TestCaseReference(failedTestCaseMap.get(testCaseName))); 343 } 344 } 345 } 346 347 Set<String> acknowledgedFailures = 348 separateAcknowledged(newTestcaseFailures, devices, testAcks); 349 acknowledgedFailures.addAll( 350 separateAcknowledged(transientTestcaseFailures, devices, testAcks)); 351 acknowledgedFailures.addAll( 352 separateAcknowledged(continuedTestcaseFailures, devices, testAcks)); 353 354 String summary = new String(); 355 if (newTestcaseFailures.size() + continuedTestcaseFailures.size() > 0) { 356 summary += "The following test cases failed in the latest test run:<br>"; 357 358 // Add new test case failures to top of summary in bold font. 359 List<String> sortedNewTestcaseFailures = new ArrayList<>(newTestcaseFailures); 360 sortedNewTestcaseFailures.sort(Comparator.naturalOrder()); 361 for (String testcaseName : sortedNewTestcaseFailures) { 362 summary += "- " + "<b>" + testcaseName + "</b><br>"; 363 } 364 365 // Add continued test case failures to summary. 366 List<String> sortedContinuedTestcaseFailures = 367 new ArrayList<>(continuedTestcaseFailures); 368 sortedContinuedTestcaseFailures.sort(Comparator.naturalOrder()); 369 for (String testcaseName : sortedContinuedTestcaseFailures) { 370 summary += "- " + testcaseName + "<br>"; 371 } 372 } 373 if (fixedTestcases.size() > 0) { 374 // Add fixed test cases to summary. 375 summary += "<br><br>The following test cases were fixed in the latest test run:<br>"; 376 List<String> sortedFixedTestcases = new ArrayList<>(fixedTestcases); 377 sortedFixedTestcases.sort(Comparator.naturalOrder()); 378 for (String testcaseName : sortedFixedTestcases) { 379 summary += "- <i>" + testcaseName + "</i><br>"; 380 } 381 } 382 if (transientTestcaseFailures.size() > 0) { 383 // Add transient test case failures to summary. 384 summary += "<br><br>The following transient test case failures occured:<br>"; 385 List<String> sortedTransientTestcaseFailures = 386 new ArrayList<>(transientTestcaseFailures); 387 sortedTransientTestcaseFailures.sort(Comparator.naturalOrder()); 388 for (String testcaseName : sortedTransientTestcaseFailures) { 389 summary += "- " + testcaseName + "<br>"; 390 } 391 } 392 if (skippedTestcaseFailures.size() > 0) { 393 // Add skipped test case failures to summary. 394 summary += "<br><br>The following test cases have not been run since failing:<br>"; 395 List<String> sortedSkippedTestcaseFailures = new ArrayList<>(skippedTestcaseFailures); 396 sortedSkippedTestcaseFailures.sort(Comparator.naturalOrder()); 397 for (String testcaseName : sortedSkippedTestcaseFailures) { 398 summary += "- " + testcaseName + "<br>"; 399 } 400 } 401 if (acknowledgedFailures.size() > 0) { 402 // Add acknowledged test case failures to summary. 403 List<String> sortedAcknowledgedFailures = new ArrayList<>(acknowledgedFailures); 404 sortedAcknowledgedFailures.sort(Comparator.naturalOrder()); 405 if (acknowledgedFailures.size() > 0) { 406 summary += 407 "<br><br>The following acknowledged test case failures continued to fail:<br>"; 408 for (String testcaseName : sortedAcknowledgedFailures) { 409 summary += "- " + testcaseName + "<br>"; 410 } 411 } 412 } 413 414 String testName = mostRecentRun.getKey().getParent().getName(); 415 String uploadDateString = TimeUtil.getDateString(mostRecentRun.getStartTimestamp()); 416 String subject = "VTS Test Alert: " + testName + " @ " + uploadDateString; 417 if (newTestcaseFailures.size() > 0) { 418 String body = 419 "Hello,<br><br>New test case failure(s) in " 420 + testName 421 + " for device build ID(s): " 422 + buildId 423 + ".<br><br>" 424 + summary 425 + footer; 426 try { 427 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 428 } catch (MessagingException | UnsupportedEncodingException e) { 429 logger.log(Level.WARNING, "Error composing email : ", e); 430 } 431 } else if (continuedTestcaseFailures.size() > 0) { 432 String body = 433 "Hello,<br><br>Continuous test case failure(s) in " 434 + testName 435 + " for device build ID(s): " 436 + buildId 437 + ".<br><br>" 438 + summary 439 + footer; 440 try { 441 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 442 } catch (MessagingException | UnsupportedEncodingException e) { 443 logger.log(Level.WARNING, "Error composing email : ", e); 444 } 445 } else if (transientTestcaseFailures.size() > 0) { 446 String body = 447 "Hello,<br><br>Transient test case failure(s) in " 448 + testName 449 + " but tests all " 450 + "are passing in the latest device build(s): " 451 + buildId 452 + ".<br><br>" 453 + summary 454 + footer; 455 try { 456 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 457 } catch (MessagingException | UnsupportedEncodingException e) { 458 logger.log(Level.WARNING, "Error composing email : ", e); 459 } 460 } else if (fixedTestcases.size() > 0) { 461 String body = 462 "Hello,<br><br>All test cases passed in " 463 + testName 464 + " for device build ID(s): " 465 + buildId 466 + "!<br><br>" 467 + summary 468 + footer; 469 try { 470 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 471 } catch (MessagingException | UnsupportedEncodingException e) { 472 logger.log(Level.WARNING, "Error composing email : ", e); 473 } 474 } 475 return new TestStatusEntity( 476 testName, 477 mostRecentRun.getStartTimestamp(), 478 passingTestcaseCount, 479 failingTestCases.size(), 480 failingTestCases); 481 } 482 483 /** 484 * Add a task to process test run data 485 * 486 * @param testRunKey The key of the test run whose data process. 487 */ addTask(Key testRunKey)488 public static void addTask(Key testRunKey) { 489 Queue queue = QueueFactory.getDefaultQueue(); 490 String keyString = KeyFactory.keyToString(testRunKey); 491 queue.add( 492 TaskOptions.Builder.withUrl(ALERT_JOB_URL) 493 .param("runKey", keyString) 494 .method(TaskOptions.Method.POST)); 495 } 496 497 @Override doPost(HttpServletRequest request, HttpServletResponse response)498 public void doPost(HttpServletRequest request, HttpServletResponse response) 499 throws IOException { 500 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 501 String runKeyString = request.getParameter("runKey"); 502 503 Key testRunKey; 504 try { 505 testRunKey = KeyFactory.stringToKey(runKeyString); 506 } catch (IllegalArgumentException e) { 507 logger.log(Level.WARNING, "Invalid key specified: " + runKeyString); 508 return; 509 } 510 String testName = testRunKey.getParent().getName(); 511 512 TestStatusEntity status = null; 513 Key statusKey = KeyFactory.createKey(TestStatusEntity.KIND, testName); 514 try { 515 status = TestStatusEntity.fromEntity(datastore.get(statusKey)); 516 } catch (EntityNotFoundException e) { 517 // no existing status 518 } 519 if (status == null) { 520 status = new TestStatusEntity(testName); 521 } 522 if (status.getUpdatedTimestamp() >= testRunKey.getId()) { 523 // Another job has already updated the status first 524 return; 525 } 526 List<String> emails = EmailHelper.getSubscriberEmails(testRunKey.getParent()); 527 528 StringBuffer fullUrl = request.getRequestURL(); 529 String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI())); 530 String link = 531 baseUrl + "/show_tree?testName=" + testName + "&endTime=" + testRunKey.getId(); 532 533 List<Message> messageQueue = new ArrayList<>(); 534 Map<String, TestCase> failedTestcaseMap = getCurrentFailures(status); 535 List<TestAcknowledgmentEntity> testAcks = 536 getTestCaseAcknowledgments(testRunKey.getParent()); 537 List<TestRunEntity> testRuns = 538 getTestRuns( 539 testRunKey.getParent(), status.getUpdatedTimestamp(), testRunKey.getId()); 540 if (testRuns.size() == 0) return; 541 542 TestStatusEntity newStatus = 543 getTestStatus(testRuns, link, failedTestcaseMap, testAcks, emails, messageQueue); 544 if (newStatus == null) { 545 // No changes to status 546 return; 547 } 548 549 int retries = 0; 550 while (true) { 551 Transaction txn = datastore.beginTransaction(); 552 try { 553 try { 554 status = TestStatusEntity.fromEntity(datastore.get(statusKey)); 555 } catch (EntityNotFoundException e) { 556 // no status left 557 } 558 if (status == null 559 || status.getUpdatedTimestamp() >= newStatus.getUpdatedTimestamp()) { 560 txn.rollback(); 561 } else { // This update is most recent. 562 datastore.put(newStatus.toEntity()); 563 txn.commit(); 564 EmailHelper.sendAll(messageQueue); 565 } 566 break; 567 } catch (ConcurrentModificationException 568 | DatastoreFailureException 569 | DatastoreTimeoutException e) { 570 logger.log(Level.WARNING, "Retrying alert job insert: " + statusKey); 571 if (retries++ >= DatastoreHelper.MAX_WRITE_RETRIES) { 572 logger.log(Level.SEVERE, "Exceeded alert job retries: " + statusKey); 573 throw e; 574 } 575 } finally { 576 if (txn.isActive()) { 577 txn.rollback(); 578 } 579 } 580 } 581 } 582 } 583