• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.adservices.service.customaudience;
18 
19 import static com.android.adservices.service.PhFlagsFixture.EXTENDED_FLEDGE_BACKGROUND_FETCH_NETWORK_CONNECT_TIMEOUT_MS;
20 import static com.android.adservices.service.PhFlagsFixture.EXTENDED_FLEDGE_BACKGROUND_FETCH_NETWORK_READ_TIMEOUT_MS;
21 
22 import static com.google.common.truth.Truth.assertThat;
23 import static com.google.common.util.concurrent.Futures.immediateFuture;
24 import static com.google.common.util.concurrent.Futures.immediateVoidFuture;
25 
26 import static org.junit.Assert.assertThrows;
27 import static org.junit.Assert.assertTrue;
28 import static org.mockito.Mockito.any;
29 import static org.mockito.Mockito.anyLong;
30 import static org.mockito.Mockito.doAnswer;
31 import static org.mockito.Mockito.doReturn;
32 import static org.mockito.Mockito.never;
33 import static org.mockito.Mockito.times;
34 import static org.mockito.Mockito.verify;
35 import static org.mockito.Mockito.when;
36 
37 import android.adservices.common.CommonFixture;
38 import android.annotation.NonNull;
39 import android.content.Context;
40 import android.content.pm.PackageManager;
41 
42 import androidx.room.Room;
43 import androidx.test.core.app.ApplicationProvider;
44 
45 import com.android.adservices.LoggerFactory;
46 import com.android.adservices.concurrency.AdServicesExecutors;
47 import com.android.adservices.customaudience.DBCustomAudienceBackgroundFetchDataFixture;
48 import com.android.adservices.data.adselection.AppInstallDao;
49 import com.android.adservices.data.adselection.SharedStorageDatabase;
50 import com.android.adservices.data.customaudience.CustomAudienceDao;
51 import com.android.adservices.data.customaudience.CustomAudienceDatabase;
52 import com.android.adservices.data.customaudience.DBCustomAudience;
53 import com.android.adservices.data.customaudience.DBCustomAudienceBackgroundFetchData;
54 import com.android.adservices.data.enrollment.EnrollmentDao;
55 import com.android.adservices.service.Flags;
56 import com.android.adservices.service.FlagsFactory;
57 
58 import com.google.common.util.concurrent.FluentFuture;
59 import com.google.common.util.concurrent.ListenableFuture;
60 
61 import org.junit.Before;
62 import org.junit.Rule;
63 import org.junit.Test;
64 import org.mockito.Mock;
65 import org.mockito.Mockito;
66 import org.mockito.junit.MockitoJUnit;
67 import org.mockito.junit.MockitoRule;
68 
69 import java.time.Clock;
70 import java.time.Instant;
71 import java.util.ArrayList;
72 import java.util.Arrays;
73 import java.util.List;
74 import java.util.concurrent.CountDownLatch;
75 import java.util.concurrent.ExecutionException;
76 import java.util.concurrent.ExecutorService;
77 import java.util.concurrent.Executors;
78 import java.util.concurrent.TimeUnit;
79 import java.util.concurrent.TimeoutException;
80 import java.util.concurrent.atomic.AtomicInteger;
81 
82 public class BackgroundFetchWorkerTest {
83     private static final LoggerFactory.Logger sLogger = LoggerFactory.getFledgeLogger();
84     private static final Context CONTEXT = ApplicationProvider.getApplicationContext();
85 
86     private final Flags mFlags = new BackgroundFetchWorkerTestFlags(true);
87     private final ExecutorService mExecutorService = Executors.newFixedThreadPool(8);
88 
89     @Rule public MockitoRule mMockitoRule = MockitoJUnit.rule();
90 
91     @Mock private PackageManager mPackageManagerMock;
92     @Mock private EnrollmentDao mEnrollmentDaoMock;
93     @Mock private Clock mClock;
94 
95     private CustomAudienceDao mCustomAudienceDaoSpy;
96     private AppInstallDao mAppInstallDaoSpy;
97     private BackgroundFetchRunner mBackgroundFetchRunnerSpy;
98     private BackgroundFetchWorker mBackgroundFetchWorker;
99 
100     @Before
setup()101     public void setup() {
102         mCustomAudienceDaoSpy =
103                 Mockito.spy(
104                         Room.inMemoryDatabaseBuilder(CONTEXT, CustomAudienceDatabase.class)
105                                 .addTypeConverter(new DBCustomAudience.Converters(true))
106                                 .build()
107                                 .customAudienceDao());
108         mAppInstallDaoSpy =
109                 Mockito.spy(
110                         Room.inMemoryDatabaseBuilder(CONTEXT, SharedStorageDatabase.class)
111                                 .build()
112                                 .appInstallDao());
113 
114         mBackgroundFetchRunnerSpy =
115                 Mockito.spy(
116                         new BackgroundFetchRunner(
117                                 mCustomAudienceDaoSpy,
118                                 mAppInstallDaoSpy,
119                                 mPackageManagerMock,
120                                 mEnrollmentDaoMock,
121                                 mFlags));
122 
123         mBackgroundFetchWorker =
124                 new BackgroundFetchWorker(
125                         mCustomAudienceDaoSpy, mFlags, mBackgroundFetchRunnerSpy, mClock);
126     }
127 
128     @Test
testBackgroundFetchWorkerNullInputsCauseFailure()129     public void testBackgroundFetchWorkerNullInputsCauseFailure() {
130         assertThrows(
131                 NullPointerException.class,
132                 () ->
133                         new BackgroundFetchWorker(
134                                 null,
135                                 FlagsFactory.getFlagsForTest(),
136                                 mBackgroundFetchRunnerSpy,
137                                 mClock));
138 
139         assertThrows(
140                 NullPointerException.class,
141                 () ->
142                         new BackgroundFetchWorker(
143                                 mCustomAudienceDaoSpy, null, mBackgroundFetchRunnerSpy, mClock));
144 
145         assertThrows(
146                 NullPointerException.class,
147                 () ->
148                         new BackgroundFetchWorker(
149                                 mCustomAudienceDaoSpy,
150                                 FlagsFactory.getFlagsForTest(),
151                                 null,
152                                 mClock));
153 
154         assertThrows(
155                 NullPointerException.class,
156                 () ->
157                         new BackgroundFetchWorker(
158                                 mCustomAudienceDaoSpy,
159                                 FlagsFactory.getFlagsForTest(),
160                                 mBackgroundFetchRunnerSpy,
161                                 null));
162     }
163 
164     @Test
testRunBackgroundFetchThrowsTimeoutDuringUpdates()165     public void testRunBackgroundFetchThrowsTimeoutDuringUpdates() {
166         class FlagsWithSmallTimeout implements Flags {
167             @Override
168             public long getFledgeBackgroundFetchJobMaxRuntimeMs() {
169                 return 100L;
170             }
171         }
172 
173         class BackgroundFetchRunnerWithSleep extends BackgroundFetchRunner {
174             BackgroundFetchRunnerWithSleep(
175                     @NonNull CustomAudienceDao customAudienceDao, @NonNull Flags flags) {
176                 super(
177                         customAudienceDao,
178                         mAppInstallDaoSpy,
179                         mPackageManagerMock,
180                         mEnrollmentDaoMock,
181                         flags);
182             }
183 
184             @Override
185             public void deleteExpiredCustomAudiences(@NonNull Instant jobStartTime) {
186                 // Do nothing
187             }
188 
189             @Override
190             public FluentFuture<?> updateCustomAudience(
191                     @NonNull Instant jobStartTime,
192                     @NonNull DBCustomAudienceBackgroundFetchData fetchData) {
193 
194                 return FluentFuture.from(
195                         AdServicesExecutors.getBlockingExecutor()
196                                 .submit(
197                                         () -> {
198                                             try {
199                                                 Thread.sleep(500L);
200                                             } catch (InterruptedException e) {
201                                                 e.printStackTrace();
202                                             }
203                                             return null;
204                                         }));
205             }
206         }
207 
208         Flags flagsWithSmallTimeout = new FlagsWithSmallTimeout();
209         BackgroundFetchRunner backgroundFetchRunnerWithSleep =
210                 new BackgroundFetchRunnerWithSleep(mCustomAudienceDaoSpy, flagsWithSmallTimeout);
211         BackgroundFetchWorker backgroundFetchWorkerThatTimesOut =
212                 new BackgroundFetchWorker(
213                         mCustomAudienceDaoSpy,
214                         flagsWithSmallTimeout,
215                         backgroundFetchRunnerWithSleep,
216                         mClock);
217 
218         // Mock a custom audience eligible for update
219         DBCustomAudienceBackgroundFetchData fetchData =
220                 DBCustomAudienceBackgroundFetchDataFixture.getValidBuilderByBuyer(
221                                 CommonFixture.VALID_BUYER_1)
222                         .setEligibleUpdateTime(CommonFixture.FIXED_NOW)
223                         .build();
224         doReturn(Arrays.asList(fetchData))
225                 .when(mCustomAudienceDaoSpy)
226                 .getActiveEligibleCustomAudienceBackgroundFetchData(any(), anyLong());
227 
228         when(mClock.instant()).thenReturn(Instant.now());
229 
230         // Time out while updating custom audiences
231         ExecutionException expected =
232                 assertThrows(
233                         ExecutionException.class,
234                         () -> backgroundFetchWorkerThatTimesOut.runBackgroundFetch().get());
235         assertThat(expected.getCause()).isInstanceOf(TimeoutException.class);
236     }
237 
238     @Test
239     public void testRunBackgroundFetchNothingToUpdate()
240             throws ExecutionException, InterruptedException {
241         assertTrue(
242                 mCustomAudienceDaoSpy
243                         .getActiveEligibleCustomAudienceBackgroundFetchData(
244                                 CommonFixture.FIXED_NOW, 1)
245                         .isEmpty());
246 
247         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW);
248         mBackgroundFetchWorker.runBackgroundFetch().get();
249 
250         verify(mBackgroundFetchRunnerSpy).deleteExpiredCustomAudiences(any());
251         verify(mCustomAudienceDaoSpy).deleteAllExpiredCustomAudienceData(any());
252         verify(mBackgroundFetchRunnerSpy).deleteDisallowedOwnerCustomAudiences();
253         verify(mCustomAudienceDaoSpy).deleteAllDisallowedOwnerCustomAudienceData(any(), any());
254         verify(mBackgroundFetchRunnerSpy).deleteDisallowedBuyerCustomAudiences();
255         verify(mBackgroundFetchRunnerSpy).deleteDisallowedPackageAppInstallEntries();
256         verify(mCustomAudienceDaoSpy).deleteAllDisallowedBuyerCustomAudienceData(any(), any());
257         verify(mBackgroundFetchRunnerSpy, never()).updateCustomAudience(any(), any());
258     }
259 
260     @Test
261     public void testRunBackgroundFetchNothingToUpdateNoFilters()
262             throws ExecutionException, InterruptedException {
263         Flags flagsFilteringDisabled = new BackgroundFetchWorkerTestFlags(false);
264         mBackgroundFetchWorker =
265                 new BackgroundFetchWorker(
266                         mCustomAudienceDaoSpy,
267                         flagsFilteringDisabled,
268                         mBackgroundFetchRunnerSpy,
269                         mClock);
270         assertTrue(
271                 mCustomAudienceDaoSpy
272                         .getActiveEligibleCustomAudienceBackgroundFetchData(
273                                 CommonFixture.FIXED_NOW, 1)
274                         .isEmpty());
275 
276         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW);
277         mBackgroundFetchWorker.runBackgroundFetch().get();
278 
279         verify(mBackgroundFetchRunnerSpy).deleteExpiredCustomAudiences(any());
280         verify(mCustomAudienceDaoSpy).deleteAllExpiredCustomAudienceData(any());
281         verify(mBackgroundFetchRunnerSpy).deleteDisallowedOwnerCustomAudiences();
282         verify(mCustomAudienceDaoSpy).deleteAllDisallowedOwnerCustomAudienceData(any(), any());
283         verify(mBackgroundFetchRunnerSpy).deleteDisallowedBuyerCustomAudiences();
284         verify(mBackgroundFetchRunnerSpy, times(0)).deleteDisallowedPackageAppInstallEntries();
285         verify(mCustomAudienceDaoSpy).deleteAllDisallowedBuyerCustomAudienceData(any(), any());
286         verify(mBackgroundFetchRunnerSpy, never()).updateCustomAudience(any(), any());
287     }
288 
289     @Test
290     public void testRunBackgroundFetchUpdateOneCustomAudience()
291             throws ExecutionException, InterruptedException {
292         // Mock a single custom audience eligible for update
293         DBCustomAudienceBackgroundFetchData fetchData =
294                 DBCustomAudienceBackgroundFetchDataFixture.getValidBuilderByBuyer(
295                                 CommonFixture.VALID_BUYER_1)
296                         .setEligibleUpdateTime(CommonFixture.FIXED_NOW)
297                         .build();
298         doReturn(Arrays.asList(fetchData))
299                 .when(mCustomAudienceDaoSpy)
300                 .getActiveEligibleCustomAudienceBackgroundFetchData(any(), anyLong());
301         doReturn(FluentFuture.from(immediateFuture(null)))
302                 .when(mBackgroundFetchRunnerSpy)
303                 .updateCustomAudience(any(), any());
304 
305         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW);
306         mBackgroundFetchWorker.runBackgroundFetch().get();
307 
308         verify(mBackgroundFetchRunnerSpy).deleteExpiredCustomAudiences(any());
309         verify(mCustomAudienceDaoSpy).deleteAllExpiredCustomAudienceData(any());
310         verify(mBackgroundFetchRunnerSpy).deleteDisallowedOwnerCustomAudiences();
311         verify(mCustomAudienceDaoSpy).deleteAllDisallowedOwnerCustomAudienceData(any(), any());
312         verify(mBackgroundFetchRunnerSpy).deleteDisallowedBuyerCustomAudiences();
313         verify(mBackgroundFetchRunnerSpy).deleteDisallowedPackageAppInstallEntries();
314         verify(mCustomAudienceDaoSpy).deleteAllDisallowedBuyerCustomAudienceData(any(), any());
315         verify(mBackgroundFetchRunnerSpy).updateCustomAudience(any(), any());
316     }
317 
318     @Test
319     public void testRunBackgroundFetchUpdateCustomAudiences()
320             throws ExecutionException, InterruptedException {
321         int numEligibleCustomAudiences = 12;
322 
323         // Mock a list of custom audiences eligible for update
324         DBCustomAudienceBackgroundFetchData.Builder fetchDataBuilder =
325                 DBCustomAudienceBackgroundFetchDataFixture.getValidBuilderByBuyer(
326                                 CommonFixture.VALID_BUYER_1)
327                         .setEligibleUpdateTime(CommonFixture.FIXED_NOW);
328         List<DBCustomAudienceBackgroundFetchData> fetchDataList = new ArrayList<>();
329         for (int i = 0; i < numEligibleCustomAudiences; i++) {
330             fetchDataList.add(fetchDataBuilder.setName("ca" + i).build());
331         }
332 
333         doReturn(fetchDataList)
334                 .when(mCustomAudienceDaoSpy)
335                 .getActiveEligibleCustomAudienceBackgroundFetchData(any(), anyLong());
336         doReturn(FluentFuture.from(immediateFuture(null)))
337                 .when(mBackgroundFetchRunnerSpy)
338                 .updateCustomAudience(any(), any());
339 
340         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW);
341         mBackgroundFetchWorker.runBackgroundFetch().get();
342 
343         verify(mBackgroundFetchRunnerSpy).deleteExpiredCustomAudiences(any());
344         verify(mCustomAudienceDaoSpy).deleteAllExpiredCustomAudienceData(any());
345         verify(mBackgroundFetchRunnerSpy).deleteDisallowedOwnerCustomAudiences();
346         verify(mCustomAudienceDaoSpy).deleteAllDisallowedOwnerCustomAudienceData(any(), any());
347         verify(mBackgroundFetchRunnerSpy).deleteDisallowedBuyerCustomAudiences();
348         verify(mBackgroundFetchRunnerSpy).deleteDisallowedPackageAppInstallEntries();
349         verify(mCustomAudienceDaoSpy).deleteAllDisallowedBuyerCustomAudienceData(any(), any());
350         verify(mBackgroundFetchRunnerSpy, times(numEligibleCustomAudiences))
351                 .updateCustomAudience(any(), any());
352     }
353 
354     @Test
355     public void testRunBackgroundFetchChecksWorkInProgress()
356             throws InterruptedException, ExecutionException {
357         int numEligibleCustomAudiences = 16;
358         CountDownLatch partialCompletionLatch = new CountDownLatch(numEligibleCustomAudiences / 4);
359 
360         // Mock a list of custom audiences eligible for update
361         DBCustomAudienceBackgroundFetchData.Builder fetchDataBuilder =
362                 DBCustomAudienceBackgroundFetchDataFixture.getValidBuilderByBuyer(
363                                 CommonFixture.VALID_BUYER_1)
364                         .setEligibleUpdateTime(CommonFixture.FIXED_NOW);
365         List<DBCustomAudienceBackgroundFetchData> fetchDataList = new ArrayList<>();
366         for (int i = 0; i < numEligibleCustomAudiences; i++) {
367             fetchDataList.add(fetchDataBuilder.setName("ca" + i).build());
368         }
369 
370         doReturn(fetchDataList)
371                 .when(mCustomAudienceDaoSpy)
372                 .getActiveEligibleCustomAudienceBackgroundFetchData(any(), anyLong());
373         doAnswer(
374                         unusedInvocation -> {
375                             Thread.sleep(100);
376                             partialCompletionLatch.countDown();
377                             return FluentFuture.from(immediateFuture(null));
378                         })
379                 .when(mBackgroundFetchRunnerSpy)
380                 .updateCustomAudience(any(), any());
381 
382         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW);
383 
384         CountDownLatch bgfWorkStoppedLatch = new CountDownLatch(1);
385         mExecutorService.execute(
386                 () -> {
387                     try {
388                         mBackgroundFetchWorker.runBackgroundFetch().get();
389                     } catch (Exception exception) {
390                         sLogger.e(
391                                 exception, "Exception encountered while running background fetch");
392                     } finally {
393                         bgfWorkStoppedLatch.countDown();
394                     }
395                 });
396 
397         // Wait til updates are partially complete, then try running background fetch again and
398         // verify nothing is done
399         partialCompletionLatch.await();
400         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW.plusSeconds(1));
401         mBackgroundFetchWorker.runBackgroundFetch().get();
402 
403         bgfWorkStoppedLatch.await();
404         verify(mBackgroundFetchRunnerSpy).deleteExpiredCustomAudiences(any());
405         verify(mCustomAudienceDaoSpy).deleteAllExpiredCustomAudienceData(any());
406         verify(mBackgroundFetchRunnerSpy).deleteDisallowedOwnerCustomAudiences();
407         verify(mCustomAudienceDaoSpy).deleteAllDisallowedOwnerCustomAudienceData(any(), any());
408         verify(mBackgroundFetchRunnerSpy).deleteDisallowedBuyerCustomAudiences();
409         verify(mBackgroundFetchRunnerSpy).deleteDisallowedPackageAppInstallEntries();
410         verify(mCustomAudienceDaoSpy).deleteAllDisallowedBuyerCustomAudienceData(any(), any());
411         verify(mBackgroundFetchRunnerSpy, times(numEligibleCustomAudiences))
412                 .updateCustomAudience(any(), any());
413     }
414 
415     @Test
416     public void testStopWorkWithoutRunningFetchDoesNothing() {
417         // Verify no errors/exceptions thrown when no work in progress
418         mBackgroundFetchWorker.stopWork();
419     }
420 
421     @Test
422     public void testStopWorkGracefullyStopsBackgroundFetch() throws Exception {
423         int numEligibleCustomAudiences = 16;
424         CountDownLatch partialCompletionLatch = new CountDownLatch(numEligibleCustomAudiences / 4);
425 
426         // Mock a list of custom audiences eligible for update
427         DBCustomAudienceBackgroundFetchData.Builder fetchDataBuilder =
428                 DBCustomAudienceBackgroundFetchDataFixture.getValidBuilderByBuyer(
429                                 CommonFixture.VALID_BUYER_1)
430                         .setEligibleUpdateTime(CommonFixture.FIXED_NOW);
431         List<DBCustomAudienceBackgroundFetchData> fetchDataList = new ArrayList<>();
432         for (int i = 0; i < numEligibleCustomAudiences; i++) {
433             fetchDataList.add(fetchDataBuilder.setName("ca" + i).build());
434         }
435 
436         doReturn(fetchDataList)
437                 .when(mCustomAudienceDaoSpy)
438                 .getActiveEligibleCustomAudienceBackgroundFetchData(any(), anyLong());
439         doAnswer(
440                         unusedInvocation -> {
441                             Thread.sleep(100);
442                             partialCompletionLatch.countDown();
443                             return FluentFuture.from(immediateVoidFuture());
444                         })
445                 .when(mBackgroundFetchRunnerSpy)
446                 .updateCustomAudience(any(), any());
447 
448         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW);
449 
450         ListenableFuture<Void> backgrounFetchResult = mBackgroundFetchWorker.runBackgroundFetch();
451 
452         // Wait til updates are partially complete, then try stopping background fetch
453         partialCompletionLatch.await();
454         mBackgroundFetchWorker.stopWork();
455         // stopWork() should notify to the worker that the work should end so the future
456         // should complete within the time required to update the custom audiences
457         backgrounFetchResult.get(
458                 100 * (numEligibleCustomAudiences * 3 / 4) + 100, TimeUnit.SECONDS);
459     }
460 
461     @Test
462     public void testStopWorkPreemptsDataUpdates() throws Exception {
463         int numEligibleCustomAudiences = 16;
464         CountDownLatch beforeUpdatingCasLatch = new CountDownLatch(numEligibleCustomAudiences / 4);
465 
466         // Mock a list of custom audiences eligible for update
467         DBCustomAudienceBackgroundFetchData.Builder fetchDataBuilder =
468                 DBCustomAudienceBackgroundFetchDataFixture.getValidBuilderByBuyer(
469                                 CommonFixture.VALID_BUYER_1)
470                         .setEligibleUpdateTime(CommonFixture.FIXED_NOW);
471         List<DBCustomAudienceBackgroundFetchData> fetchDataList = new ArrayList<>();
472         for (int i = 0; i < numEligibleCustomAudiences; i++) {
473             fetchDataList.add(fetchDataBuilder.setName("ca" + i).build());
474         }
475 
476         // Ensuring that stopWork is called before the data update process
477         doAnswer(
478                         unusedInvocation -> {
479                             beforeUpdatingCasLatch.await();
480                             return fetchDataList;
481                         })
482                 .when(mCustomAudienceDaoSpy)
483                 .getActiveEligibleCustomAudienceBackgroundFetchData(any(), anyLong());
484         doAnswer(
485                         unusedInvocation -> {
486                             Thread.sleep(100);
487                             return null;
488                         })
489                 .when(mBackgroundFetchRunnerSpy)
490                 .updateCustomAudience(any(), any());
491 
492         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW);
493 
494         ListenableFuture<Void> backgrounFetchResult = mBackgroundFetchWorker.runBackgroundFetch();
495 
496         // Wait til updates are partially complete, then try stopping background fetch
497         mBackgroundFetchWorker.stopWork();
498         beforeUpdatingCasLatch.countDown();
499         // stopWork() called before updating the data should cause immediate termination
500         // waiting for 200ms to handle thread scheduling delays.
501         // The important check is that the time is less than the time of updating all CAs
502         backgrounFetchResult.get(200, TimeUnit.MILLISECONDS);
503     }
504 
505     @Test
506     public void testRunBackgroundFetchInSequence() throws InterruptedException, ExecutionException {
507         int numEligibleCustomAudiences = 16;
508         CountDownLatch completionLatch = new CountDownLatch(numEligibleCustomAudiences / 2);
509 
510         // Mock two lists of custom audiences eligible for update
511         DBCustomAudienceBackgroundFetchData.Builder fetchDataBuilder =
512                 DBCustomAudienceBackgroundFetchDataFixture.getValidBuilderByBuyer(
513                                 CommonFixture.VALID_BUYER_1)
514                         .setEligibleUpdateTime(CommonFixture.FIXED_NOW);
515         List<DBCustomAudienceBackgroundFetchData> fetchDataList1 = new ArrayList<>();
516         List<DBCustomAudienceBackgroundFetchData> fetchDataList2 = new ArrayList<>();
517         for (int i = 0; i < numEligibleCustomAudiences; i++) {
518             DBCustomAudienceBackgroundFetchData fetchData =
519                     fetchDataBuilder.setName("ca" + i).build();
520             if (i < numEligibleCustomAudiences / 2) {
521                 fetchDataList1.add(fetchData);
522             } else {
523                 fetchDataList2.add(fetchData);
524             }
525         }
526 
527         // Count the number of times updateCustomAudience is run
528         AtomicInteger completionCount = new AtomicInteger(0);
529 
530         // Return the first list the first time, and the second list in the second call
531         doReturn(fetchDataList1)
532                 .doReturn(fetchDataList2)
533                 .when(mCustomAudienceDaoSpy)
534                 .getActiveEligibleCustomAudienceBackgroundFetchData(any(), anyLong());
535         doAnswer(
536                         unusedInvocation -> {
537                             completionLatch.countDown();
538                             completionCount.getAndIncrement();
539                             return FluentFuture.from(immediateFuture(null));
540                         })
541                 .when(mBackgroundFetchRunnerSpy)
542                 .updateCustomAudience(any(), any());
543 
544         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW);
545 
546         CountDownLatch bgfWorkStoppedLatch = new CountDownLatch(1);
547         mExecutorService.execute(
548                 () -> {
549                     try {
550                         mBackgroundFetchWorker.runBackgroundFetch().get();
551                     } catch (Exception exception) {
552                         sLogger.e(
553                                 exception, "Exception encountered while running background fetch");
554                     } finally {
555                         bgfWorkStoppedLatch.countDown();
556                     }
557                 });
558 
559         // Wait til updates are complete, then try running background fetch again and
560         // verify the second run updates more custom audiences successfully
561         completionLatch.await();
562         bgfWorkStoppedLatch.await();
563         when(mClock.instant()).thenReturn(CommonFixture.FIXED_NOW.plusSeconds(1));
564         mBackgroundFetchWorker.runBackgroundFetch().get();
565 
566         verify(mBackgroundFetchRunnerSpy, times(2)).deleteExpiredCustomAudiences(any());
567         verify(mCustomAudienceDaoSpy, times(2)).deleteAllExpiredCustomAudienceData(any());
568         verify(mBackgroundFetchRunnerSpy, times(2)).deleteDisallowedOwnerCustomAudiences();
569         verify(mCustomAudienceDaoSpy, times(2))
570                 .deleteAllDisallowedOwnerCustomAudienceData(any(), any());
571         verify(mBackgroundFetchRunnerSpy, times(2)).deleteDisallowedBuyerCustomAudiences();
572         verify(mBackgroundFetchRunnerSpy, times(2)).deleteDisallowedPackageAppInstallEntries();
573         verify(mCustomAudienceDaoSpy, times(2))
574                 .deleteAllDisallowedBuyerCustomAudienceData(any(), any());
575         verify(mBackgroundFetchRunnerSpy, times(numEligibleCustomAudiences))
576                 .updateCustomAudience(any(), any());
577         assertThat(completionCount.get()).isEqualTo(numEligibleCustomAudiences);
578     }
579 
580     private static class BackgroundFetchWorkerTestFlags implements Flags {
581         private final boolean mFledgeAdSelectionFilteringEnabled;
582 
583         BackgroundFetchWorkerTestFlags(boolean fledgeAdSelectionFilteringEnabled) {
584             mFledgeAdSelectionFilteringEnabled = fledgeAdSelectionFilteringEnabled;
585         }
586 
587         @Override
588         public int getFledgeBackgroundFetchThreadPoolSize() {
589             return 4;
590         }
591 
592         @Override
593         public boolean getFledgeAdSelectionFilteringEnabled() {
594             return mFledgeAdSelectionFilteringEnabled;
595         }
596 
597         @Override
598         public int getFledgeBackgroundFetchNetworkConnectTimeoutMs() {
599             return EXTENDED_FLEDGE_BACKGROUND_FETCH_NETWORK_CONNECT_TIMEOUT_MS;
600         }
601 
602         @Override
603         public int getFledgeBackgroundFetchNetworkReadTimeoutMs() {
604             return EXTENDED_FLEDGE_BACKGROUND_FETCH_NETWORK_READ_TIMEOUT_MS;
605         }
606     }
607 }
608