1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.adservices.service.measurement.registration; 18 19 import android.annotation.NonNull; 20 import android.annotation.Nullable; 21 import android.content.ContentProviderClient; 22 import android.content.ContentResolver; 23 import android.content.Context; 24 import android.net.Uri; 25 import android.os.RemoteException; 26 import android.os.Trace; 27 28 import com.android.adservices.LoggerFactory; 29 import com.android.adservices.data.measurement.DatastoreException; 30 import com.android.adservices.data.measurement.DatastoreManager; 31 import com.android.adservices.data.measurement.DatastoreManagerFactory; 32 import com.android.adservices.data.measurement.IMeasurementDao; 33 import com.android.adservices.service.Flags; 34 import com.android.adservices.service.FlagsFactory; 35 import com.android.adservices.service.common.WebAddresses; 36 import com.android.adservices.service.measurement.Attribution; 37 import com.android.adservices.service.measurement.EventReport; 38 import com.android.adservices.service.measurement.EventSurfaceType; 39 import com.android.adservices.service.measurement.KeyValueData; 40 import com.android.adservices.service.measurement.KeyValueData.DataType; 41 import com.android.adservices.service.measurement.Source; 42 import com.android.adservices.service.measurement.Trigger; 43 import com.android.adservices.service.measurement.attribution.TriggerContentProvider; 44 import com.android.adservices.service.measurement.noising.SourceNoiseHandler; 45 import com.android.adservices.service.measurement.registration.SourceEligibilityChecker.InsertSourcePermission; 46 import com.android.adservices.service.measurement.reporting.AggregateDebugReportApi; 47 import com.android.adservices.service.measurement.reporting.DebugReportApi; 48 import com.android.adservices.service.measurement.util.Applications; 49 import com.android.adservices.service.measurement.util.BaseUriExtractor; 50 import com.android.adservices.service.measurement.util.UnsignedLong; 51 import com.android.adservices.service.stats.AdServicesLogger; 52 import com.android.adservices.service.stats.AdServicesLoggerImpl; 53 import com.android.adservices.shared.common.ApplicationContextSingleton; 54 import com.android.internal.annotations.VisibleForTesting; 55 56 import com.google.errorprone.annotations.concurrent.GuardedBy; 57 58 import org.json.JSONException; 59 60 import java.util.HashSet; 61 import java.util.List; 62 import java.util.Map; 63 import java.util.Objects; 64 import java.util.Optional; 65 import java.util.Set; 66 import java.util.UUID; 67 import java.util.stream.Collectors; 68 69 /** Runner for servicing queued registration requests */ 70 public final class AsyncRegistrationQueueRunner { 71 /** 72 * Single attribution entry is created for possibly multiple fake reports generated per source. 73 * Setting a value to such attributions will help identify them that they are associated to fake 74 * reports. 75 */ 76 @VisibleForTesting static final String ATTRIBUTION_FAKE_REPORT_ID = "-1"; 77 78 private static final Object LOCK = new Object(); 79 80 @GuardedBy("LOCK") 81 private static AsyncRegistrationQueueRunner sAsyncRegistrationQueueRunner; 82 83 private final DatastoreManager mDatastoreManager; 84 private final AsyncSourceFetcher mAsyncSourceFetcher; 85 private final AsyncTriggerFetcher mAsyncTriggerFetcher; 86 private final ContentResolver mContentResolver; 87 private final DebugReportApi mDebugReportApi; 88 private final AggregateDebugReportApi mAdrApi; 89 private final SourceNoiseHandler mSourceNoiseHandler; 90 private final Flags mFlags; 91 private final AdServicesLogger mLogger; 92 private final Context mContext; 93 private final SourceEligibilityChecker mSourceEligibilityChecker; 94 AsyncRegistrationQueueRunner(Context context)95 private AsyncRegistrationQueueRunner(Context context) { 96 mContext = context; 97 mDatastoreManager = DatastoreManagerFactory.getDatastoreManager(); 98 mAsyncSourceFetcher = new AsyncSourceFetcher(context); 99 mAsyncTriggerFetcher = new AsyncTriggerFetcher(context); 100 mContentResolver = context.getContentResolver(); 101 mFlags = FlagsFactory.getFlags(); 102 mDebugReportApi = new DebugReportApi(context, mFlags); 103 mAdrApi = new AggregateDebugReportApi(mFlags); 104 mSourceNoiseHandler = new SourceNoiseHandler(mFlags); 105 mLogger = AdServicesLoggerImpl.getInstance(); 106 mSourceEligibilityChecker = new SourceEligibilityChecker(mFlags, mDebugReportApi); 107 } 108 109 @VisibleForTesting AsyncRegistrationQueueRunner( Context context, ContentResolver contentResolver, AsyncSourceFetcher asyncSourceFetcher, AsyncTriggerFetcher asyncTriggerFetcher, DatastoreManager datastoreManager, DebugReportApi debugReportApi, AggregateDebugReportApi adrApi, SourceNoiseHandler sourceNoiseHandler, Flags flags)110 public AsyncRegistrationQueueRunner( 111 Context context, 112 ContentResolver contentResolver, 113 AsyncSourceFetcher asyncSourceFetcher, 114 AsyncTriggerFetcher asyncTriggerFetcher, 115 DatastoreManager datastoreManager, 116 DebugReportApi debugReportApi, 117 AggregateDebugReportApi adrApi, 118 SourceNoiseHandler sourceNoiseHandler, 119 Flags flags) { 120 this( 121 context, 122 contentResolver, 123 asyncSourceFetcher, 124 asyncTriggerFetcher, 125 datastoreManager, 126 debugReportApi, 127 adrApi, 128 sourceNoiseHandler, 129 flags, 130 AdServicesLoggerImpl.getInstance()); 131 } 132 133 @VisibleForTesting AsyncRegistrationQueueRunner( Context context, ContentResolver contentResolver, AsyncSourceFetcher asyncSourceFetcher, AsyncTriggerFetcher asyncTriggerFetcher, DatastoreManager datastoreManager, DebugReportApi debugReportApi, AggregateDebugReportApi adrApi, SourceNoiseHandler sourceNoiseHandler, Flags flags, AdServicesLogger logger)134 public AsyncRegistrationQueueRunner( 135 Context context, 136 ContentResolver contentResolver, 137 AsyncSourceFetcher asyncSourceFetcher, 138 AsyncTriggerFetcher asyncTriggerFetcher, 139 DatastoreManager datastoreManager, 140 DebugReportApi debugReportApi, 141 AggregateDebugReportApi adrApi, 142 SourceNoiseHandler sourceNoiseHandler, 143 Flags flags, 144 AdServicesLogger logger) { 145 mContext = context; 146 mAsyncSourceFetcher = asyncSourceFetcher; 147 mAsyncTriggerFetcher = asyncTriggerFetcher; 148 mDatastoreManager = datastoreManager; 149 mContentResolver = contentResolver; 150 mDebugReportApi = debugReportApi; 151 mAdrApi = adrApi; 152 mSourceNoiseHandler = sourceNoiseHandler; 153 mFlags = flags; 154 mLogger = logger; 155 mSourceEligibilityChecker = new SourceEligibilityChecker(flags, debugReportApi); 156 } 157 158 @VisibleForTesting AsyncRegistrationQueueRunner( Context context, ContentResolver contentResolver, AsyncSourceFetcher asyncSourceFetcher, AsyncTriggerFetcher asyncTriggerFetcher, DatastoreManager datastoreManager, DebugReportApi debugReportApi, AggregateDebugReportApi adrApi, SourceNoiseHandler sourceNoiseHandler, Flags flags, AdServicesLogger logger, SourceEligibilityChecker sourceEligibilityChecker)159 public AsyncRegistrationQueueRunner( 160 Context context, 161 ContentResolver contentResolver, 162 AsyncSourceFetcher asyncSourceFetcher, 163 AsyncTriggerFetcher asyncTriggerFetcher, 164 DatastoreManager datastoreManager, 165 DebugReportApi debugReportApi, 166 AggregateDebugReportApi adrApi, 167 SourceNoiseHandler sourceNoiseHandler, 168 Flags flags, 169 AdServicesLogger logger, 170 SourceEligibilityChecker sourceEligibilityChecker) { 171 mContext = context; 172 mAsyncSourceFetcher = asyncSourceFetcher; 173 mAsyncTriggerFetcher = asyncTriggerFetcher; 174 mDatastoreManager = datastoreManager; 175 mContentResolver = contentResolver; 176 mDebugReportApi = debugReportApi; 177 mAdrApi = adrApi; 178 mSourceNoiseHandler = sourceNoiseHandler; 179 mFlags = flags; 180 mLogger = logger; 181 mSourceEligibilityChecker = sourceEligibilityChecker; 182 } 183 184 enum ProcessingResult { 185 THREAD_INTERRUPTED, 186 SUCCESS_WITH_PENDING_RECORDS, 187 SUCCESS_ALL_RECORDS_PROCESSED 188 } 189 190 /** 191 * Returns an instance of AsyncRegistrationQueueRunner. 192 * 193 * @param context the current {@link Context}. 194 */ getInstance()195 public static synchronized AsyncRegistrationQueueRunner getInstance() { 196 synchronized (LOCK) { 197 if (sAsyncRegistrationQueueRunner == null) { 198 Context context = ApplicationContextSingleton.get(); 199 sAsyncRegistrationQueueRunner = new AsyncRegistrationQueueRunner(context); 200 } 201 return sAsyncRegistrationQueueRunner; 202 } 203 } 204 205 /** Processes records in the AsyncRegistration Queue table. */ runAsyncRegistrationQueueWorker()206 public ProcessingResult runAsyncRegistrationQueueWorker() { 207 Trace.beginSection("AsyncRegistrationQueueRunner#runAsyncRegistrationQueueWorker"); 208 int recordServiceLimit = mFlags.getMeasurementMaxRegistrationsPerJobInvocation(); 209 int retryLimit = mFlags.getMeasurementMaxRetriesPerRegistrationRequest(); 210 211 Set<Uri> failedOrigins = new HashSet<>(); 212 for (int i = 0; i < recordServiceLimit; i++) { 213 // If the job service's requirements specified at runtime are no longer met, the job 214 // service will interrupt this thread. If the thread has been interrupted, it will exit 215 // early. 216 if (Thread.currentThread().isInterrupted()) { 217 LoggerFactory.getMeasurementLogger() 218 .d( 219 "AsyncRegistrationQueueRunner runAsyncRegistrationQueueWorker " 220 + "thread interrupted, exiting early."); 221 return ProcessingResult.THREAD_INTERRUPTED; 222 } 223 224 AsyncRegistration asyncRegistration = fetchNext(retryLimit, failedOrigins); 225 if (null == asyncRegistration) { 226 LoggerFactory.getMeasurementLogger() 227 .d("AsyncRegistrationQueueRunner: no async registration fetched."); 228 return ProcessingResult.SUCCESS_ALL_RECORDS_PROCESSED; 229 } 230 231 processAsyncRecord(asyncRegistration, failedOrigins); 232 } 233 ProcessingResult processingResult = hasPendingRecords(retryLimit, failedOrigins); 234 Trace.endSection(); 235 return processingResult; 236 } 237 fetchNext(int retryLimit, Set<Uri> failedOrigins)238 private AsyncRegistration fetchNext(int retryLimit, Set<Uri> failedOrigins) { 239 return mDatastoreManager 240 .runInTransactionWithResult( 241 (dao) -> dao.fetchNextQueuedAsyncRegistration(retryLimit, failedOrigins)) 242 .orElse(null); 243 } 244 processAsyncRecord(AsyncRegistration asyncRegistration, Set<Uri> failedOrigins)245 private void processAsyncRecord(AsyncRegistration asyncRegistration, Set<Uri> failedOrigins) { 246 if (asyncRegistration.isSourceRequest()) { 247 LoggerFactory.getMeasurementLogger() 248 .d("AsyncRegistrationQueueRunner:" + " processing source"); 249 processSourceRegistration(asyncRegistration, failedOrigins); 250 } else { 251 LoggerFactory.getMeasurementLogger() 252 .d("AsyncRegistrationQueueRunner:" + " processing trigger"); 253 processTriggerRegistration(asyncRegistration, failedOrigins); 254 } 255 } 256 hasPendingRecords(int retryLimit, Set<Uri> failedOrigins)257 private ProcessingResult hasPendingRecords(int retryLimit, Set<Uri> failedOrigins) { 258 AsyncRegistration asyncRegistration = fetchNext(retryLimit, failedOrigins); 259 if (null == asyncRegistration) { 260 LoggerFactory.getMeasurementLogger() 261 .d("AsyncRegistrationQueueRunner: no more pending async records."); 262 return ProcessingResult.SUCCESS_ALL_RECORDS_PROCESSED; 263 } else { 264 return ProcessingResult.SUCCESS_WITH_PENDING_RECORDS; 265 } 266 } 267 processSourceRegistration( AsyncRegistration asyncRegistration, Set<Uri> failedOrigins)268 private void processSourceRegistration( 269 AsyncRegistration asyncRegistration, Set<Uri> failedOrigins) { 270 AsyncFetchStatus asyncFetchStatus = new AsyncFetchStatus(); 271 AsyncRedirects asyncRedirects = new AsyncRedirects(); 272 long startTime = asyncRegistration.getRequestTime(); 273 Optional<Source> resultSource = 274 mAsyncSourceFetcher.fetchSource( 275 asyncRegistration, asyncFetchStatus, asyncRedirects); 276 long endTime = System.currentTimeMillis(); 277 asyncFetchStatus.setRegistrationDelay(endTime - startTime); 278 279 boolean transactionResult = 280 mDatastoreManager.runInTransaction( 281 (dao) -> { 282 if (asyncFetchStatus.isRequestSuccess()) { 283 if (resultSource.isPresent()) { 284 LoggerFactory.getMeasurementLogger() 285 .d( 286 "AsyncRegistrationQueueRunner: Source fetched" 287 + " and parsed. Attempting to store it." 288 + " Enrollment ID: %s, Source ID: %s," 289 + " Source Event ID: %s", 290 resultSource.get().getEnrollmentId(), 291 resultSource.get().getId(), 292 resultSource.get().getEventId()); 293 storeSource( 294 resultSource.get(), 295 asyncRegistration, 296 dao, 297 asyncFetchStatus); 298 } 299 handleSuccess( 300 asyncRegistration, asyncFetchStatus, asyncRedirects, dao); 301 } else { 302 handleFailure( 303 asyncRegistration, asyncFetchStatus, failedOrigins, dao); 304 } 305 }); 306 307 if (!transactionResult) { 308 asyncFetchStatus.setEntityStatus(AsyncFetchStatus.EntityStatus.STORAGE_ERROR); 309 } 310 311 asyncFetchStatus.setRetryCount(Long.valueOf(asyncRegistration.getRetryCount()).intValue()); 312 FetcherUtil.emitHeaderMetrics( 313 mFlags.getMaxResponseBasedRegistrationPayloadSizeBytes(), 314 mLogger, 315 asyncRegistration, 316 asyncFetchStatus, 317 resultSource.map(Source::getEnrollmentId).orElse(null)); 318 } 319 320 /** Visible only for testing. */ 321 @VisibleForTesting storeSource( Source source, AsyncRegistration asyncRegistration, IMeasurementDao dao, AsyncFetchStatus asyncFetchStatus)322 public void storeSource( 323 Source source, 324 AsyncRegistration asyncRegistration, 325 IMeasurementDao dao, 326 AsyncFetchStatus asyncFetchStatus) 327 throws DatastoreException { 328 Uri topOrigin = 329 asyncRegistration.getType() == AsyncRegistration.RegistrationType.WEB_SOURCE 330 ? asyncRegistration.getTopOrigin() 331 : getPublisher(asyncRegistration); 332 @EventSurfaceType 333 int publisherType = 334 asyncRegistration.getType() == AsyncRegistration.RegistrationType.WEB_SOURCE 335 ? EventSurfaceType.WEB 336 : EventSurfaceType.APP; 337 Set<DebugReportApi.Type> adrTypes = new HashSet<>(); 338 // Build and validate privacy parameters before creating the source's noised status. 339 if (!areValidSourcePrivacyParameters(source, dao, mFlags, adrTypes)) { 340 mAdrApi.scheduleSourceRegistrationDebugReport(source, adrTypes, dao); 341 return; 342 } 343 // Create the source's noised status, so it's available to ascertain debug report types. 344 List<Source.FakeReport> fakeReports = 345 mSourceNoiseHandler.assignAttributionModeAndGenerateFakeReports(source); 346 347 InsertSourcePermission sourceAllowedToInsert = 348 mSourceEligibilityChecker.isAllowedToInsert( 349 source, topOrigin, publisherType, dao, asyncFetchStatus, adrTypes); 350 if (sourceAllowedToInsert.isAllowed()) { 351 // If preinstall check is enabled and any app destinations are already installed, 352 // mark the source for deletion. Note the source is persisted so that the fake event 353 // report generated can be cleaned up after the source is deleted by 354 // DeleteExpiredJobService. 355 if (mFlags.getMeasurementEnablePreinstallCheck() 356 && source.shouldDropSourceIfInstalled() 357 && Applications.anyAppsInstalled(mContext, source.getAppDestinations())) { 358 source.setStatus(Source.Status.MARKED_TO_DELETE); 359 } 360 Map<String, Object> additionalDebugReportParams = null; 361 if (mFlags.getMeasurementEnableSourceDestinationLimitPriority() 362 && InsertSourcePermission.ALLOWED_FIFO_SUCCESS.equals(sourceAllowedToInsert)) { 363 int limit = mFlags.getMeasurementMaxDistinctDestinationsInActiveSource(); 364 additionalDebugReportParams = 365 Map.of(DebugReportApi.Body.SOURCE_DESTINATION_LIMIT, String.valueOf(limit)); 366 } 367 insertSourceFromTransaction(source, fakeReports, dao, adrTypes); 368 DebugReportApi.Type type = 369 scheduleSourceSuccessOrNoisedDebugReport( 370 source, dao, additionalDebugReportParams); 371 adrTypes.add(type); 372 } 373 374 mAdrApi.scheduleSourceRegistrationDebugReport(source, adrTypes, dao); 375 } 376 377 @VisibleForTesting areValidSourcePrivacyParameters( Source source, IMeasurementDao dao, Flags flags, Set<DebugReportApi.Type> adrTypes)378 boolean areValidSourcePrivacyParameters( 379 Source source, IMeasurementDao dao, Flags flags, Set<DebugReportApi.Type> adrTypes) 380 throws DatastoreException { 381 try { 382 if (!source.validateAndSetNumReportStates(flags)) { 383 long maxTriggerStateCardinality = 384 flags.getMeasurementMaxReportStatesPerSourceRegistration(); 385 LoggerFactory.getMeasurementLogger() 386 .d( 387 "storeSource (FAILURE): Trigger state cardinality exceeded the" 388 + " limit of %s. Enrollment ID: %s, Source ID: %s, Source Event" 389 + " ID: %s", 390 maxTriggerStateCardinality, 391 source.getEnrollmentId(), 392 source.getId(), 393 source.getEventId()); 394 mDebugReportApi.scheduleSourceReport( 395 source, 396 DebugReportApi.Type.SOURCE_TRIGGER_STATE_CARDINALITY_LIMIT, 397 Map.of( 398 DebugReportApi.Body.LIMIT, 399 String.valueOf(maxTriggerStateCardinality)), 400 dao); 401 adrTypes.add(DebugReportApi.Type.SOURCE_TRIGGER_STATE_CARDINALITY_LIMIT); 402 return false; 403 } 404 if (!source.hasValidInformationGain(flags)) { 405 float maxEventLevelChannelCapacity = source.getInformationGainThreshold(flags); 406 LoggerFactory.getMeasurementLogger() 407 .d( 408 "storeSource (FAILURE): Event level channel capacity exceeded the" 409 + " limit of %s. Enrollment ID: %s, Source ID: %s, Source" 410 + " Event ID: %s", 411 maxEventLevelChannelCapacity, 412 source.getEnrollmentId(), 413 source.getId(), 414 source.getEventId()); 415 mDebugReportApi.scheduleSourceReport( 416 source, 417 DebugReportApi.Type.SOURCE_CHANNEL_CAPACITY_LIMIT, 418 Map.of(DebugReportApi.Body.LIMIT, maxEventLevelChannelCapacity), 419 dao); 420 adrTypes.add(DebugReportApi.Type.SOURCE_CHANNEL_CAPACITY_LIMIT); 421 return false; 422 } 423 424 if (flags.getMeasurementEnableAttributionScope()) { 425 Source.AttributionScopeValidationResult attributionScopeValidationResult = 426 source.validateAttributionScopeValues(flags); 427 if (!attributionScopeValidationResult.isValid()) { 428 LoggerFactory.getMeasurementLogger() 429 .d( 430 "storeSource (FAILURE): Invalid attribution scopes values - max" 431 + " event states limit or channel capacity limit." 432 + " Enrollment ID: %s, Source ID: %s, Source Event ID: %s", 433 source.getEnrollmentId(), source.getId(), source.getEventId()); 434 DebugReportApi.Type type = 435 mDebugReportApi.scheduleAttributionScopeDebugReport( 436 source, attributionScopeValidationResult, dao); 437 adrTypes.add(type); 438 return false; 439 } 440 441 if (source.getSourceType() == Source.SourceType.NAVIGATION) { 442 Optional<Set<String>> existingScopes = 443 dao.getAttributionScopesForRegistration( 444 source.getRegistrationId(), 445 source.getRegistrationOrigin().toString()); 446 Set<String> newScopes = 447 source.getAttributionScopes() != null 448 ? new HashSet<>(source.getAttributionScopes()) 449 : Set.of(); 450 if (existingScopes.isPresent() && !existingScopes.get().equals(newScopes)) { 451 LoggerFactory.getMeasurementLogger() 452 .d( 453 "storeSource (FAILURE): Source in registration sequence has" 454 + " mismatched attribution scopes. Enrollment ID: %s," 455 + " Source ID: %s, Source Event ID: %s", 456 source.getEnrollmentId(), 457 source.getId(), 458 source.getEventId()); 459 return false; 460 } 461 } 462 } 463 } catch (ArithmeticException e) { 464 LoggerFactory.getMeasurementLogger() 465 .d( 466 e, 467 "storeSource (FAILURE): Number of report states overflowed." 468 + " Enrollment ID: %s, Source ID: %s, Source Event ID: %s", 469 source.getEnrollmentId(), 470 source.getId(), 471 source.getEventId()); 472 mDebugReportApi.scheduleSourceReport( 473 source, 474 DebugReportApi.Type.SOURCE_FLEXIBLE_EVENT_REPORT_VALUE_ERROR, 475 /* additionalBodyParams= */ null, 476 dao); 477 adrTypes.add(DebugReportApi.Type.SOURCE_FLEXIBLE_EVENT_REPORT_VALUE_ERROR); 478 return false; 479 } 480 481 return true; 482 } 483 processTriggerRegistration( AsyncRegistration asyncRegistration, Set<Uri> failedOrigins)484 private void processTriggerRegistration( 485 AsyncRegistration asyncRegistration, Set<Uri> failedOrigins) { 486 AsyncFetchStatus asyncFetchStatus = new AsyncFetchStatus(); 487 AsyncRedirects asyncRedirects = new AsyncRedirects(); 488 long startTime = asyncRegistration.getRequestTime(); 489 Optional<Trigger> resultTrigger = 490 mAsyncTriggerFetcher.fetchTrigger( 491 asyncRegistration, asyncFetchStatus, asyncRedirects); 492 long endTime = System.currentTimeMillis(); 493 asyncFetchStatus.setRegistrationDelay(endTime - startTime); 494 495 boolean transactionResult = 496 mDatastoreManager.runInTransaction( 497 (dao) -> { 498 if (asyncFetchStatus.isRequestSuccess()) { 499 if (resultTrigger.isPresent()) { 500 LoggerFactory.getMeasurementLogger() 501 .d( 502 "AsyncRegistrationQueueRunner: Trigger fetched" 503 + " and parsed. Attempting to store it." 504 + " Enrollment ID: %s, Trigger ID: %s", 505 resultTrigger.get().getEnrollmentId(), 506 resultTrigger.get().getId()); 507 storeTrigger(resultTrigger.get(), dao); 508 } 509 handleSuccess( 510 asyncRegistration, asyncFetchStatus, asyncRedirects, dao); 511 } else { 512 handleFailure( 513 asyncRegistration, asyncFetchStatus, failedOrigins, dao); 514 } 515 }); 516 517 if (!transactionResult) { 518 asyncFetchStatus.setEntityStatus(AsyncFetchStatus.EntityStatus.STORAGE_ERROR); 519 } 520 521 asyncFetchStatus.setRetryCount(Long.valueOf(asyncRegistration.getRetryCount()).intValue()); 522 long headerSizeLimitBytes = 523 mFlags.getMeasurementEnableUpdateTriggerHeaderLimit() 524 ? mFlags.getMaxTriggerRegistrationHeaderSizeBytes() 525 : mFlags.getMaxResponseBasedRegistrationPayloadSizeBytes(); 526 FetcherUtil.emitHeaderMetrics( 527 headerSizeLimitBytes, 528 mLogger, 529 asyncRegistration, 530 asyncFetchStatus, 531 resultTrigger.map(Trigger::getEnrollmentId).orElse(null)); 532 } 533 534 /** Visible only for testing. */ 535 @VisibleForTesting storeTrigger(Trigger trigger, IMeasurementDao dao)536 public void storeTrigger(Trigger trigger, IMeasurementDao dao) throws DatastoreException { 537 if (isTriggerAllowedToInsert(dao, trigger)) { 538 try { 539 if (dao.insertTrigger(trigger) == null) { 540 // Trigger was not saved due to DB size restrictions 541 LoggerFactory.getMeasurementLogger() 542 .d( 543 "storeTrigger (FAILURE): Insertion prevented by the datastore" 544 + " manager. Enrollment ID: %s, Trigger ID: %s", 545 trigger.getEnrollmentId(), trigger.getId()); 546 return; 547 } 548 LoggerFactory.getMeasurementLogger() 549 .d( 550 "storeTrigger (SUCCESS): Trigger inserted to database. Enrollment" 551 + " ID: %s, Trigger ID: %s", 552 trigger.getEnrollmentId(), trigger.getId()); 553 } catch (DatastoreException e) { 554 LoggerFactory.getMeasurementLogger() 555 .e(e, "storeTrigger (FAILURE): Insert trigger to DB error."); 556 LoggerFactory.getMeasurementLogger() 557 .d( 558 "Enrollment ID: %s, Trigger ID: %s", 559 trigger.getEnrollmentId(), trigger.getId()); 560 if (isTriggerReportAllowed(trigger, mFlags)) { 561 mDebugReportApi.scheduleTriggerNoMatchingSourceDebugReport( 562 trigger, dao, DebugReportApi.Type.TRIGGER_UNKNOWN_ERROR); 563 } 564 throw new DatastoreException( 565 "Insert trigger to DB error, generate trigger-unknown-error report"); 566 } 567 568 notifyTriggerContentProvider(); 569 } 570 } 571 572 @VisibleForTesting isTriggerAllowedToInsert(IMeasurementDao dao, Trigger trigger)573 static boolean isTriggerAllowedToInsert(IMeasurementDao dao, Trigger trigger) { 574 long triggerInsertedPerDestination; 575 try { 576 triggerInsertedPerDestination = 577 dao.getNumTriggersPerDestination( 578 trigger.getAttributionDestination(), trigger.getDestinationType()); 579 } catch (DatastoreException e) { 580 LoggerFactory.getMeasurementLogger() 581 .e( 582 "storeTrigger (FAILURE): Unable to fetch number of triggers currently" 583 + " registered per destination. Enrollment ID: %s, Trigger ID: %s", 584 trigger.getEnrollmentId(), trigger.getId()); 585 return false; 586 } 587 return triggerInsertedPerDestination 588 < FlagsFactory.getFlags().getMeasurementMaxTriggersPerDestination(); 589 } 590 isTriggerReportAllowed(Trigger trigger, Flags flags)591 static boolean isTriggerReportAllowed(Trigger trigger, Flags flags) { 592 try { 593 return (trigger.hasAggregatableData(flags) 594 || !trigger.parseEventTriggers(flags).isEmpty()); 595 } catch (JSONException e) { 596 LoggerFactory.getMeasurementLogger() 597 .e( 598 e, 599 "JSONException when checking if trigger verbose debug report can be" 600 + " sent trigger with ID: " 601 + trigger.getId()); 602 } 603 return false; 604 } 605 createAsyncRegistrationFromRedirect( AsyncRegistration asyncRegistration, AsyncRedirect asyncRedirect)606 private AsyncRegistration createAsyncRegistrationFromRedirect( 607 AsyncRegistration asyncRegistration, AsyncRedirect asyncRedirect) { 608 return new AsyncRegistration.Builder() 609 .setId(UUID.randomUUID().toString()) 610 .setRegistrationUri(asyncRedirect.getUri()) 611 .setWebDestination(asyncRegistration.getWebDestination()) 612 .setOsDestination(asyncRegistration.getOsDestination()) 613 .setRegistrant(asyncRegistration.getRegistrant()) 614 .setVerifiedDestination(asyncRegistration.getVerifiedDestination()) 615 .setTopOrigin(asyncRegistration.getTopOrigin()) 616 .setType(asyncRegistration.getType()) 617 .setSourceType(asyncRegistration.getSourceType()) 618 .setRequestTime(asyncRegistration.getRequestTime()) 619 .setRetryCount(0) 620 .setDebugKeyAllowed(asyncRegistration.getDebugKeyAllowed()) 621 .setAdIdPermission(asyncRegistration.hasAdIdPermission()) 622 .setPlatformAdId(asyncRegistration.getPlatformAdId()) 623 .setRegistrationId(asyncRegistration.getRegistrationId()) 624 .setRedirectBehavior(asyncRedirect.getRedirectBehavior()) 625 .build(); 626 } 627 generateFakeEventReports( String sourceId, Source source, List<Source.FakeReport> fakeReports)628 private List<EventReport> generateFakeEventReports( 629 String sourceId, Source source, List<Source.FakeReport> fakeReports) { 630 return fakeReports.stream() 631 .map( 632 fakeReport -> 633 new EventReport.Builder() 634 .setId(UUID.randomUUID().toString()) 635 .setSourceId(sourceId) 636 .setSourceEventId(source.getEventId()) 637 .setReportTime(fakeReport.getReportingTime()) 638 .setTriggerData(fakeReport.getTriggerData()) 639 .setAttributionDestinations(fakeReport.getDestinations()) 640 .setEnrollmentId(source.getEnrollmentId()) 641 .setTriggerTime(fakeReport.getTriggerTime()) 642 .setTriggerPriority(0L) 643 .setTriggerDedupKey(null) 644 .setSourceType(source.getSourceType()) 645 .setStatus(EventReport.Status.PENDING) 646 .setRandomizedTriggerRate( 647 mSourceNoiseHandler.getRandomizedTriggerRate( 648 source)) 649 .setRegistrationOrigin(source.getRegistrationOrigin()) 650 .setTriggerSummaryBucket( 651 fakeReport.getTriggerSummaryBucket()) 652 .setSourceDebugKey(getSourceDebugKeyForNoisedReport(source)) 653 .build()) 654 .collect(Collectors.toList()); 655 } 656 657 @VisibleForTesting insertSourceFromTransaction( Source source, List<Source.FakeReport> fakeReports, IMeasurementDao dao, Set<DebugReportApi.Type> adrTypes)658 void insertSourceFromTransaction( 659 Source source, 660 List<Source.FakeReport> fakeReports, 661 IMeasurementDao dao, 662 Set<DebugReportApi.Type> adrTypes) 663 throws DatastoreException { 664 final String sourceId = insertSource(source, dao, adrTypes); 665 if (sourceId == null) { 666 // Source was not saved due to DB size restrictions 667 LoggerFactory.getMeasurementLogger() 668 .d( 669 "storeSource (FAILURE): Insertion prevented by the datastore manager." 670 + " Enrollment ID: %s, Source ID: %s, Source Event ID: %s", 671 source.getEnrollmentId(), source.getId(), source.getEventId()); 672 return; 673 } 674 LoggerFactory.getMeasurementLogger() 675 .d( 676 "storeSource (SUCCESS): Source inserted to database. Enrollment ID: %s," 677 + " Source ID: %s, Source Event ID: %s", 678 source.getEnrollmentId(), source.getId(), source.getEventId()); 679 680 if (mFlags.getMeasurementEnableAttributionScope()) { 681 dao.updateSourcesForAttributionScope(source); 682 } 683 684 if (fakeReports != null) { 685 for (EventReport report : generateFakeEventReports(sourceId, source, fakeReports)) { 686 dao.insertEventReport(report); 687 } 688 } 689 // We want to account for attribution if fake report generation was considered 690 // based on the probability. In that case the attribution mode will be NEVER 691 // (empty fake reports state) or FALSELY (non-empty fake reports). 692 if (source.getAttributionMode() != Source.AttributionMode.TRUTHFULLY) { 693 // Attribution rate limits for app and web destinations are counted 694 // separately, so add a fake report entry for each type of destination if 695 // non-null. 696 if (!Objects.isNull(source.getAppDestinations())) { 697 for (Uri destination : source.getAppDestinations()) { 698 dao.insertAttribution( 699 createFakeAttributionRateLimit(sourceId, source, destination)); 700 } 701 } 702 703 if (!Objects.isNull(source.getWebDestinations())) { 704 for (Uri destination : source.getWebDestinations()) { 705 dao.insertAttribution( 706 createFakeAttributionRateLimit(sourceId, source, destination)); 707 } 708 } 709 } 710 } 711 insertSource( Source source, IMeasurementDao dao, Set<DebugReportApi.Type> adrTypes)712 private String insertSource( 713 Source source, IMeasurementDao dao, Set<DebugReportApi.Type> adrTypes) 714 throws DatastoreException { 715 try { 716 return dao.insertSource(source); 717 } catch (DatastoreException e) { 718 mDebugReportApi.scheduleSourceReport( 719 source, 720 DebugReportApi.Type.SOURCE_UNKNOWN_ERROR, 721 /* additionalBodyParams= */ null, 722 dao); 723 adrTypes.add(DebugReportApi.Type.SOURCE_UNKNOWN_ERROR); 724 LoggerFactory.getMeasurementLogger() 725 .e(e, "storeSource (FAILURE): Insert source to DB error."); 726 LoggerFactory.getMeasurementLogger() 727 .d( 728 "Enrollment ID: %s, Source ID: %s, Source Event ID: %s", 729 source.getEnrollmentId(), source.getId(), source.getEventId()); 730 throw new DatastoreException( 731 "Insert source to DB error, generate source-unknown-error report"); 732 } 733 } 734 handleSuccess( AsyncRegistration asyncRegistration, AsyncFetchStatus asyncFetchStatus, AsyncRedirects asyncRedirects, IMeasurementDao dao)735 private void handleSuccess( 736 AsyncRegistration asyncRegistration, 737 AsyncFetchStatus asyncFetchStatus, 738 AsyncRedirects asyncRedirects, 739 IMeasurementDao dao) 740 throws DatastoreException { 741 // deleteAsyncRegistration will throw an exception & rollback the transaction if the record 742 // is already deleted. This can happen if both fallback & regular job are running at the 743 // same time or if deletion job deletes the records. 744 dao.deleteAsyncRegistration(asyncRegistration.getId()); 745 if (asyncRedirects.getRedirects().isEmpty()) { 746 return; 747 } 748 int maxRedirects = FlagsFactory.getFlags().getMeasurementMaxRegistrationRedirects(); 749 KeyValueData keyValueData = 750 dao.getKeyValueData( 751 asyncRegistration.getRegistrationId(), 752 DataType.REGISTRATION_REDIRECT_COUNT); 753 int currentCount = keyValueData.getRegistrationRedirectCount(); 754 if (currentCount >= maxRedirects) { 755 asyncFetchStatus.setRedirectError(true); 756 return; 757 } 758 759 for (AsyncRedirect asyncRedirect : asyncRedirects.getRedirects()) { 760 if (currentCount >= maxRedirects) { 761 break; 762 } 763 dao.insertAsyncRegistration( 764 createAsyncRegistrationFromRedirect(asyncRegistration, asyncRedirect)); 765 currentCount++; 766 } 767 keyValueData.setRegistrationRedirectCount(currentCount); 768 dao.insertOrUpdateKeyValueData(keyValueData); 769 } 770 handleFailure( AsyncRegistration asyncRegistration, AsyncFetchStatus asyncFetchStatus, Set<Uri> failedOrigins, IMeasurementDao dao)771 private void handleFailure( 772 AsyncRegistration asyncRegistration, 773 AsyncFetchStatus asyncFetchStatus, 774 Set<Uri> failedOrigins, 775 IMeasurementDao dao) 776 throws DatastoreException { 777 if (asyncFetchStatus.canRetry()) { 778 LoggerFactory.getMeasurementLogger() 779 .d( 780 "AsyncRegistrationQueueRunner: " 781 + "async " 782 + asyncRegistration.getType() 783 + " registration will be queued for retry " 784 + "Fetch Status : " 785 + asyncFetchStatus.getResponseStatus()); 786 failedOrigins.add(BaseUriExtractor.getBaseUri(asyncRegistration.getRegistrationUri())); 787 asyncRegistration.incrementRetryCount(); 788 dao.updateRetryCount(asyncRegistration); 789 } else { 790 LoggerFactory.getMeasurementLogger() 791 .d( 792 "AsyncRegistrationQueueRunner: " 793 + "async " 794 + asyncRegistration.getType() 795 + " registration will not be queued for retry. " 796 + "Fetch Status : " 797 + asyncFetchStatus.getResponseStatus()); 798 dao.deleteAsyncRegistration(asyncRegistration.getId()); 799 } 800 } 801 802 /** 803 * {@link Attribution} generated from here will only be used for fake report attribution. 804 * 805 * @param source source to derive parameters from 806 * @param destination destination for attribution 807 * @return a fake {@link Attribution} 808 */ createFakeAttributionRateLimit( String sourceId, Source source, Uri destination)809 private Attribution createFakeAttributionRateLimit( 810 String sourceId, Source source, Uri destination) { 811 Optional<Uri> topLevelPublisher = 812 getTopLevelPublisher(source.getPublisher(), source.getPublisherType()); 813 814 if (topLevelPublisher.isEmpty()) { 815 throw new IllegalArgumentException( 816 String.format( 817 "insertAttributionRateLimit: getSourceAndDestinationTopPrivateDomains" 818 + " failed. Publisher: %s; Attribution destination: %s", 819 source.getPublisher(), destination)); 820 } 821 822 return new Attribution.Builder() 823 .setSourceSite(topLevelPublisher.get().toString()) 824 .setSourceOrigin(source.getPublisher().toString()) 825 .setDestinationSite(destination.toString()) 826 .setDestinationOrigin(destination.toString()) 827 .setEnrollmentId(source.getEnrollmentId()) 828 .setTriggerTime(source.getEventTime()) 829 .setRegistrant(source.getRegistrant().toString()) 830 .setSourceId(sourceId) 831 // Intentionally kept it as null because it's a fake attribution 832 .setTriggerId(null) 833 // Intentionally using source here since trigger is not available 834 .setRegistrationOrigin(source.getRegistrationOrigin()) 835 .setReportId(ATTRIBUTION_FAKE_REPORT_ID) 836 .build(); 837 } 838 getTopLevelPublisher( Uri topOrigin, @EventSurfaceType int publisherType)839 private static Optional<Uri> getTopLevelPublisher( 840 Uri topOrigin, @EventSurfaceType int publisherType) { 841 return publisherType == EventSurfaceType.APP 842 ? Optional.of(topOrigin) 843 : WebAddresses.topPrivateDomainAndScheme(topOrigin); 844 } 845 getPublisher(AsyncRegistration request)846 private Uri getPublisher(AsyncRegistration request) { 847 return request.getRegistrant(); 848 } 849 scheduleSourceSuccessOrNoisedDebugReport( Source source, IMeasurementDao dao, Map<String, Object> additionalDebugReportParams)850 private DebugReportApi.Type scheduleSourceSuccessOrNoisedDebugReport( 851 Source source, IMeasurementDao dao, Map<String, Object> additionalDebugReportParams) { 852 DebugReportApi.Type type = DebugReportApi.Type.SOURCE_SUCCESS; 853 if (source.getAttributionMode() != Source.AttributionMode.TRUTHFULLY) { 854 type = DebugReportApi.Type.SOURCE_NOISED; 855 } 856 857 mDebugReportApi.scheduleSourceReport(source, type, additionalDebugReportParams, dao); 858 return type; 859 } 860 notifyTriggerContentProvider()861 private void notifyTriggerContentProvider() { 862 Uri triggerUri = TriggerContentProvider.getTriggerUri(); 863 try (ContentProviderClient contentProviderClient = 864 mContentResolver.acquireContentProviderClient(triggerUri)) { 865 if (contentProviderClient != null) { 866 contentProviderClient.insert(triggerUri, null); 867 } 868 } catch (RemoteException e) { 869 LoggerFactory.getMeasurementLogger() 870 .e(e, "Trigger Content Provider invocation failed."); 871 } 872 } 873 874 @Nullable getSourceDebugKeyForNoisedReport(@onNull Source source)875 private UnsignedLong getSourceDebugKeyForNoisedReport(@NonNull Source source) { 876 if (mFlags.getMeasurementEnableBothSideDebugKeysInReports()) { 877 return null; 878 } else if ((source.getPublisherType() == EventSurfaceType.APP && source.hasAdIdPermission()) 879 || (source.getPublisherType() == EventSurfaceType.WEB 880 && source.hasArDebugPermission())) { 881 return source.getDebugKey(); 882 } 883 return null; 884 } 885 } 886