1 /** 2 * Copyright 2016 Google Inc. All Rights Reserved. 3 * 4 * <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 5 * except in compliance with the License. You may obtain a copy of the License at 6 * 7 * <p>http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * <p>Unless required by applicable law or agreed to in writing, software distributed under the 10 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 * express or implied. See the License for the specific language governing permissions and 12 * limitations under the License. 13 */ 14 package com.android.vts.util; 15 16 17 import com.android.vts.entity.ApiCoverageEntity; 18 import com.android.vts.entity.BranchEntity; 19 import com.android.vts.entity.BuildTargetEntity; 20 import com.android.vts.entity.CodeCoverageEntity; 21 import com.android.vts.entity.CoverageEntity; 22 import com.android.vts.entity.DeviceInfoEntity; 23 import com.android.vts.entity.ProfilingPointRunEntity; 24 import com.android.vts.entity.TestCaseRunEntity; 25 import com.android.vts.entity.TestEntity; 26 import com.android.vts.entity.TestPlanEntity; 27 import com.android.vts.entity.TestPlanRunEntity; 28 import com.android.vts.entity.TestRunEntity; 29 import com.android.vts.entity.TestRunEntity.TestRunType; 30 import com.android.vts.job.VtsAlertJobServlet; 31 import com.android.vts.job.VtsCoverageAlertJobServlet; 32 import com.android.vts.job.VtsProfilingStatsJobServlet; 33 import com.android.vts.proto.VtsReportMessage.AndroidDeviceInfoMessage; 34 import com.android.vts.proto.VtsReportMessage.ApiCoverageReportMessage; 35 import com.android.vts.proto.VtsReportMessage.CoverageReportMessage; 36 import com.android.vts.proto.VtsReportMessage.HalInterfaceMessage; 37 import com.android.vts.proto.VtsReportMessage.LogMessage; 38 import com.android.vts.proto.VtsReportMessage.ProfilingReportMessage; 39 import com.android.vts.proto.VtsReportMessage.TestCaseReportMessage; 40 import com.android.vts.proto.VtsReportMessage.TestCaseResult; 41 import com.android.vts.proto.VtsReportMessage.TestPlanReportMessage; 42 import com.android.vts.proto.VtsReportMessage.TestReportMessage; 43 import com.android.vts.proto.VtsReportMessage.UrlResourceMessage; 44 import com.google.appengine.api.datastore.DatastoreFailureException; 45 import com.google.appengine.api.datastore.DatastoreService; 46 import com.google.appengine.api.datastore.DatastoreServiceFactory; 47 import com.google.appengine.api.datastore.DatastoreTimeoutException; 48 import com.google.appengine.api.datastore.Entity; 49 import com.google.appengine.api.datastore.EntityNotFoundException; 50 import com.google.appengine.api.datastore.FetchOptions; 51 import com.google.appengine.api.datastore.Key; 52 import com.google.appengine.api.datastore.KeyFactory; 53 import com.google.appengine.api.datastore.Query; 54 import com.google.appengine.api.datastore.Query.Filter; 55 import com.google.appengine.api.datastore.Query.FilterOperator; 56 import com.google.appengine.api.datastore.Query.FilterPredicate; 57 import com.google.appengine.api.datastore.Transaction; 58 import com.google.appengine.api.datastore.TransactionOptions; 59 import com.google.common.collect.Lists; 60 import java.util.ArrayList; 61 import java.util.Arrays; 62 import java.util.ConcurrentModificationException; 63 import java.util.HashSet; 64 import java.util.List; 65 import java.util.Map; 66 import java.util.Set; 67 import java.util.logging.Level; 68 import java.util.logging.Logger; 69 import java.util.stream.Collectors; 70 71 /** 72 * DatastoreHelper, a helper class for interacting with Cloud Datastore. 73 */ 74 public class DatastoreHelper { 75 76 /** 77 * The default kind name for datastore 78 */ 79 public static final String NULL_ENTITY_KIND = "nullEntity"; 80 81 public static final int MAX_WRITE_RETRIES = 5; 82 /** 83 * This variable is for maximum number of entities per transaction You can find the detail here 84 * (https://cloud.google.com/datastore/docs/concepts/limits) 85 */ 86 public static final int MAX_ENTITY_SIZE_PER_TRANSACTION = 300; 87 88 protected static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); 89 private static final DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 90 91 /** 92 * Get query fetch options for large batches of entities. 93 * 94 * @return FetchOptions with a large chunk and prefetch size. 95 */ getLargeBatchOptions()96 public static FetchOptions getLargeBatchOptions() { 97 return FetchOptions.Builder.withChunkSize(1000).prefetchSize(1000); 98 } 99 100 /** 101 * Returns true if there are data points newer than lowerBound in the results table. 102 * 103 * @param parentKey The parent key to use in the query. 104 * @param kind The query entity kind. 105 * @param lowerBound The (exclusive) lower time bound, long, microseconds. 106 * @return boolean True if there are newer data points. 107 */ hasNewer(Key parentKey, String kind, Long lowerBound)108 public static boolean hasNewer(Key parentKey, String kind, Long lowerBound) { 109 if (lowerBound == null || lowerBound <= 0) { 110 return false; 111 } 112 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 113 Key startKey = KeyFactory.createKey(parentKey, kind, lowerBound); 114 Filter startFilter = 115 new FilterPredicate( 116 Entity.KEY_RESERVED_PROPERTY, FilterOperator.GREATER_THAN, startKey); 117 Query q = new Query(kind).setAncestor(parentKey).setFilter(startFilter).setKeysOnly(); 118 return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0; 119 } 120 121 /** 122 * Returns true if there are data points older than upperBound in the table. 123 * 124 * @param parentKey The parent key to use in the query. 125 * @param kind The query entity kind. 126 * @param upperBound The (exclusive) upper time bound, long, microseconds. 127 * @return boolean True if there are older data points. 128 */ hasOlder(Key parentKey, String kind, Long upperBound)129 public static boolean hasOlder(Key parentKey, String kind, Long upperBound) { 130 if (upperBound == null || upperBound <= 0) { 131 return false; 132 } 133 Key endKey = KeyFactory.createKey(parentKey, kind, upperBound); 134 Filter endFilter = 135 new FilterPredicate(Entity.KEY_RESERVED_PROPERTY, FilterOperator.LESS_THAN, endKey); 136 Query q = new Query(kind).setAncestor(parentKey).setFilter(endFilter).setKeysOnly(); 137 return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0; 138 } 139 140 /** 141 * Get all of the devices branches. 142 * 143 * @return a list of all branches. 144 */ getAllBranches()145 public static List<String> getAllBranches() { 146 Query query = new Query(BranchEntity.KIND).setKeysOnly(); 147 List<String> branches = new ArrayList<>(); 148 for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) { 149 branches.add(e.getKey().getName()); 150 } 151 return branches; 152 } 153 154 /** 155 * Get all of the device build flavors. 156 * 157 * @return a list of all device build flavors. 158 */ getAllBuildFlavors()159 public static List<String> getAllBuildFlavors() { 160 Query query = new Query(BuildTargetEntity.KIND).setKeysOnly(); 161 List<String> devices = new ArrayList<>(); 162 for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) { 163 devices.add(e.getKey().getName()); 164 } 165 return devices; 166 } 167 168 /** 169 * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times and withXG of 170 * false value 171 * 172 * @param entity The entity that you want to insert to datastore. 173 * @param entityList The list of entity for using datastore put method. 174 */ datastoreTransactionalRetry(Entity entity, List<Entity> entityList)175 private static boolean datastoreTransactionalRetry(Entity entity, List<Entity> entityList) { 176 return datastoreTransactionalRetryWithXG(entity, entityList, false); 177 } 178 179 /** 180 * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times 181 * 182 * @param entity The entity that you want to insert to datastore. 183 * @param entityList The list of entity for using datastore put method. 184 */ datastoreTransactionalRetryWithXG( Entity entity, List<Entity> entityList, boolean withXG)185 private static boolean datastoreTransactionalRetryWithXG( 186 Entity entity, List<Entity> entityList, boolean withXG) { 187 int retries = 0; 188 while (true) { 189 Transaction txn; 190 if (withXG) { 191 TransactionOptions options = TransactionOptions.Builder.withXG(withXG); 192 txn = datastore.beginTransaction(options); 193 } else { 194 txn = datastore.beginTransaction(); 195 } 196 197 try { 198 // Check if test already exists in the database 199 if (!entity.getKind().equalsIgnoreCase(NULL_ENTITY_KIND)) { 200 try { 201 if (entity.getKind().equalsIgnoreCase("Test")) { 202 Entity datastoreEntity = datastore.get(entity.getKey()); 203 TestEntity datastoreTestEntity = TestEntity.fromEntity(datastoreEntity); 204 if (datastoreTestEntity == null 205 || !datastoreTestEntity.equals(entity)) { 206 entityList.add(entity); 207 } 208 } else if (entity.getKind().equalsIgnoreCase("TestPlan")) { 209 datastore.get(entity.getKey()); 210 } else { 211 datastore.get(entity.getKey()); 212 } 213 } catch (EntityNotFoundException e) { 214 entityList.add(entity); 215 } 216 } 217 datastore.put(txn, entityList); 218 txn.commit(); 219 break; 220 } catch (ConcurrentModificationException 221 | DatastoreFailureException 222 | DatastoreTimeoutException e) { 223 entityList.remove(entity); 224 logger.log( 225 Level.WARNING, 226 "Retrying insert kind: " + entity.getKind() + " key: " + entity.getKey()); 227 if (retries++ >= MAX_WRITE_RETRIES) { 228 logger.log( 229 Level.SEVERE, 230 "Exceeded maximum retries kind: " 231 + entity.getKind() 232 + " key: " 233 + entity.getKey()); 234 return false; 235 } 236 } finally { 237 if (txn.isActive()) { 238 logger.log( 239 Level.WARNING, "Transaction rollback forced for : " + entity.getKind()); 240 txn.rollback(); 241 } 242 } 243 } 244 return true; 245 } 246 } 247