1 /* 2 * Copyright 2022 Google LLC 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.android.libraries.mobiledatadownload; 17 18 import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_CHECKSUM; 19 import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_GROUP_NAME; 20 import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_ID; 21 import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_SIZE; 22 import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_URL; 23 import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; 24 import static com.google.common.truth.Truth.assertThat; 25 import static org.junit.Assert.assertThrows; 26 import static org.junit.Assert.fail; 27 28 import android.content.Context; 29 import android.net.Uri; 30 import android.os.Environment; 31 import androidx.test.core.app.ApplicationProvider; 32 import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; 33 import com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest; 34 import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; 35 import com.google.android.libraries.mobiledatadownload.downloader.MultiSchemeFileDownloader; 36 import com.google.android.libraries.mobiledatadownload.downloader.inline.InlineFileDownloader; 37 import com.google.android.libraries.mobiledatadownload.downloader.offroad.dagger.downloader2.BaseFileDownloaderModule; 38 import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; 39 import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; 40 import com.google.android.libraries.mobiledatadownload.file.backends.FileUri; 41 import com.google.android.libraries.mobiledatadownload.file.backends.JavaFileBackend; 42 import com.google.android.libraries.mobiledatadownload.file.common.testing.FakeFileBackend; 43 import com.google.android.libraries.mobiledatadownload.file.common.testing.FakeFileBackend.OperationType; 44 import com.google.android.libraries.mobiledatadownload.file.integration.downloader.SharedPreferencesDownloadMetadata; 45 import com.google.android.libraries.mobiledatadownload.file.openers.ReadStreamOpener; 46 import com.google.android.libraries.mobiledatadownload.file.transforms.CompressTransform; 47 import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; 48 import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; 49 import com.google.android.libraries.mobiledatadownload.testing.BlockingFileDownloader; 50 import com.google.android.libraries.mobiledatadownload.testing.TestFlags; 51 import com.google.common.base.Optional; 52 import com.google.common.base.Supplier; 53 import com.google.common.collect.ImmutableList; 54 import com.google.common.collect.ImmutableMap; 55 import com.google.common.truth.Correspondence; 56 import com.google.common.util.concurrent.FutureCallback; 57 import com.google.common.util.concurrent.Futures; 58 import com.google.common.util.concurrent.ListenableFuture; 59 import com.google.common.util.concurrent.ListeningExecutorService; 60 import com.google.common.util.concurrent.MoreExecutors; 61 import com.google.mobiledatadownload.ClientConfigProto.ClientFile; 62 import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; 63 import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup.Status; 64 import com.google.mobiledatadownload.DownloadConfigProto.DataFile; 65 import com.google.mobiledatadownload.DownloadConfigProto.DataFile.ChecksumType; 66 import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; 67 import com.google.protobuf.ByteString; 68 import com.google.testing.junit.testparameterinjector.TestParameter; 69 import com.google.testing.junit.testparameterinjector.TestParameterInjector; 70 import java.io.IOException; 71 import java.io.InputStream; 72 import java.util.concurrent.ExecutionException; 73 import java.util.concurrent.Executors; 74 import java.util.concurrent.ScheduledExecutorService; 75 import java.util.concurrent.atomic.AtomicInteger; 76 import org.junit.After; 77 import org.junit.Before; 78 import org.junit.Rule; 79 import org.junit.Test; 80 import org.junit.runner.RunWith; 81 import org.mockito.Mock; 82 import org.mockito.junit.MockitoJUnit; 83 import org.mockito.junit.MockitoRule; 84 85 @RunWith(TestParameterInjector.class) 86 public final class ImportFilesIntegrationTest { 87 88 @Rule public final MockitoRule mocks = MockitoJUnit.rule(); 89 90 private static final String TAG = "ImportFilesIntegrationTest"; 91 92 private static final String TEST_DATA_ABSOLUTE_PATH = 93 Environment.getExternalStorageDirectory() 94 + "/googletest/test_runfiles/third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/"; 95 96 private static final ScheduledExecutorService DOWNLOAD_EXECUTOR = 97 Executors.newScheduledThreadPool(2); 98 99 private static final String FILE_ID_1 = "test-file-1"; 100 private static final Uri FILE_URI_1 = 101 Uri.parse( 102 FileUri.builder() 103 .setPath(TEST_DATA_ABSOLUTE_PATH + "odws1_empty.jar") 104 .build() 105 .toString()); 106 private static final String FILE_CHECKSUM_1 = "a1cba9d87b1440f41ce9e7da38c43e1f6bd7d5df"; 107 private static final String FILE_URL_1 = "inlinefile:sha1:" + FILE_CHECKSUM_1; 108 private static final int FILE_SIZE_1 = 554; 109 private static final DataFile INLINE_DATA_FILE_1 = 110 DataFile.newBuilder() 111 .setFileId(FILE_ID_1) 112 .setByteSize(FILE_SIZE_1) 113 .setUrlToDownload(FILE_URL_1) 114 .setChecksum(FILE_CHECKSUM_1) 115 .build(); 116 117 private static final String FILE_ID_2 = "test-file-2"; 118 private static final Uri FILE_URI_2 = 119 Uri.parse( 120 FileUri.builder() 121 .setPath(TEST_DATA_ABSOLUTE_PATH + "zip_test_folder.zip") 122 .build() 123 .toString()); 124 private static final String FILE_CHECKSUM_2 = "7024b6bcddf2b2897656e9353f7fc715df5ea986"; 125 private static final String FILE_URL_2 = "inlinefile:sha2:" + FILE_CHECKSUM_2; 126 private static final int FILE_SIZE_2 = 373; 127 private static final DataFile INLINE_DATA_FILE_2 = 128 DataFile.newBuilder() 129 .setFileId(FILE_ID_2) 130 .setByteSize(FILE_SIZE_2) 131 .setUrlToDownload(FILE_URL_2) 132 .setChecksum(FILE_CHECKSUM_2) 133 .build(); 134 135 private static final long BUILD_ID = 10; 136 private static final String VARIANT_ID = "default"; 137 private static final String FILE_ID_3 = "empty-inline-file"; 138 private static final String FILE_URL_3 = 139 String.format("inlinefile:buildId:%s:variantId:%s", BUILD_ID, VARIANT_ID); 140 private static final DataFile EMPTY_INLINE_FILE = 141 DataFile.newBuilder() 142 .setFileId(FILE_ID_3) 143 .setChecksumType(ChecksumType.NONE) 144 .setUrlToDownload(FILE_URL_3) 145 .build(); 146 147 private static final Context context = ApplicationProvider.getApplicationContext(); 148 149 private final TestFlags flags = new TestFlags(); 150 151 @Mock private TaskScheduler mockTaskScheduler; 152 @Mock private NetworkUsageMonitor mockNetworkUsageMonitor; 153 @Mock private DownloadProgressMonitor mockDownloadProgressMonitor; 154 155 private FakeFileBackend fakeFileBackend; 156 private SynchronousFileStorage fileStorage; 157 158 private Supplier<FileDownloader> multiSchemeFileDownloaderSupplier; 159 private MobileDataDownload mobileDataDownload; 160 private ListeningExecutorService controlExecutor; 161 162 private FileSource inlineFileSource1; 163 private FileSource inlineFileSource2; 164 165 @TestParameter ExecutorType controlExecutorType; 166 167 @Before setUp()168 public void setUp() throws Exception { 169 170 fakeFileBackend = new FakeFileBackend(AndroidFileBackend.builder(context).build()); 171 fileStorage = 172 new SynchronousFileStorage( 173 /* backends= */ ImmutableList.of(fakeFileBackend, new JavaFileBackend()), 174 /* transforms= */ ImmutableList.of(new CompressTransform()), 175 /* monitors= */ ImmutableList.of(mockNetworkUsageMonitor, mockDownloadProgressMonitor)); 176 177 // Set up inline file sources 178 try (InputStream fileStream1 = fileStorage.open(FILE_URI_1, ReadStreamOpener.create()); 179 InputStream fileStream2 = fileStorage.open(FILE_URI_2, ReadStreamOpener.create())) { 180 inlineFileSource1 = FileSource.ofByteString(ByteString.readFrom(fileStream1)); 181 inlineFileSource2 = FileSource.ofByteString(ByteString.readFrom(fileStream2)); 182 } 183 184 controlExecutor = controlExecutorType.executor(); 185 186 Supplier<FileDownloader> httpsFileDownloaderSupplier = 187 () -> 188 BaseFileDownloaderModule.createOffroad2FileDownloader( 189 context, 190 DOWNLOAD_EXECUTOR, 191 controlExecutor, 192 fileStorage, 193 new SharedPreferencesDownloadMetadata( 194 context.getSharedPreferences("downloadmetadata", 0), controlExecutor), 195 Optional.of(mockDownloadProgressMonitor), 196 /* urlEngineOptional= */ Optional.absent(), 197 /* exceptionHandlerOptional= */ Optional.absent(), 198 /* authTokenProviderOptional= */ Optional.absent(), 199 // /* cookieJarSupplierOptional= */ Optional.absent(), 200 /* trafficTag= */ Optional.absent(), 201 flags); 202 203 Supplier<FileDownloader> inlineFileDownloaderSupplier = 204 () -> new InlineFileDownloader(fileStorage, DOWNLOAD_EXECUTOR); 205 206 multiSchemeFileDownloaderSupplier = 207 () -> 208 MultiSchemeFileDownloader.builder() 209 .addScheme("https", httpsFileDownloaderSupplier.get()) 210 .addScheme("inlinefile", inlineFileDownloaderSupplier.get()) 211 .build(); 212 flags.enableDownloadStageExperimentIdPropagation = Optional.of(true); 213 } 214 215 @After tearDown()216 public void tearDown() throws Exception { 217 // Clear file group to ensure there is not cross-test pollination 218 mobileDataDownload.clear().get(); 219 // Reset fake file backend 220 fakeFileBackend.clearFailure(OperationType.ALL); 221 } 222 223 @Test importFiles_performsImport()224 public void importFiles_performsImport() throws Exception { 225 mobileDataDownload = builderForTest().build(); 226 227 DataFileGroup fileGroupWithInlineFile = 228 DataFileGroup.newBuilder() 229 .setGroupName(FILE_GROUP_NAME) 230 .addFile(INLINE_DATA_FILE_1) 231 .build(); 232 233 // Ensure that we add the file group successfully 234 assertThat( 235 mobileDataDownload 236 .addFileGroup( 237 AddFileGroupRequest.newBuilder() 238 .setDataFileGroup(fileGroupWithInlineFile) 239 .build()) 240 .get()) 241 .isTrue(); 242 243 // Perform the import 244 mobileDataDownload 245 .importFiles( 246 ImportFilesRequest.newBuilder() 247 .setGroupName(FILE_GROUP_NAME) 248 .setBuildId(fileGroupWithInlineFile.getBuildId()) 249 .setVariantId(fileGroupWithInlineFile.getVariantId()) 250 .setInlineFileMap(ImmutableMap.of(FILE_ID_1, inlineFileSource1)) 251 .build()) 252 .get(); 253 254 // Assert that the resulting group is downloaded and contains a reference to on device file 255 ClientFileGroup importResult = 256 mobileDataDownload 257 .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 258 .get(); 259 260 assertThat(importResult).isNotNull(); 261 assertThat(importResult.getGroupName()).isEqualTo(FILE_GROUP_NAME); 262 assertThat(importResult.getFileCount()).isEqualTo(1); 263 assertThat(importResult.getStatus()).isEqualTo(Status.DOWNLOADED); 264 assertThat(importResult.getFile(0).getFileUri()).isNotEmpty(); 265 266 assertThat(fileStorage.exists(Uri.parse(importResult.getFile(0).getFileUri()))).isTrue(); 267 } 268 269 @Test importFiles_whenImportingMultipleFiles_performsImport()270 public void importFiles_whenImportingMultipleFiles_performsImport() throws Exception { 271 mobileDataDownload = builderForTest().build(); 272 273 DataFileGroup fileGroupWithInlineFile = 274 DataFileGroup.newBuilder() 275 .setGroupName(FILE_GROUP_NAME) 276 .addFile(INLINE_DATA_FILE_1) 277 .addFile(INLINE_DATA_FILE_2) 278 .build(); 279 280 // Ensure that we add the file group successfully 281 assertThat( 282 mobileDataDownload 283 .addFileGroup( 284 AddFileGroupRequest.newBuilder() 285 .setDataFileGroup(fileGroupWithInlineFile) 286 .build()) 287 .get()) 288 .isTrue(); 289 290 // Perform the import 291 mobileDataDownload 292 .importFiles( 293 ImportFilesRequest.newBuilder() 294 .setGroupName(FILE_GROUP_NAME) 295 .setBuildId(fileGroupWithInlineFile.getBuildId()) 296 .setVariantId(fileGroupWithInlineFile.getVariantId()) 297 .setInlineFileMap( 298 ImmutableMap.of(FILE_ID_1, inlineFileSource1, FILE_ID_2, inlineFileSource2)) 299 .build()) 300 .get(); 301 302 // Assert that the resulting group is downloaded and contains a reference to on device file 303 ClientFileGroup importResult = 304 mobileDataDownload 305 .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 306 .get(); 307 308 assertThat(importResult).isNotNull(); 309 assertThat(importResult.getGroupName()).isEqualTo(FILE_GROUP_NAME); 310 assertThat(importResult.getFileCount()).isEqualTo(2); 311 assertThat(importResult.getStatus()).isEqualTo(Status.DOWNLOADED); 312 assertThat(importResult.getFile(0).getFileUri()).isNotEmpty(); 313 assertThat(importResult.getFile(1).getFileUri()).isNotEmpty(); 314 315 assertThat(fileStorage.exists(Uri.parse(importResult.getFile(0).getFileUri()))).isTrue(); 316 assertThat(fileStorage.exists(Uri.parse(importResult.getFile(1).getFileUri()))).isTrue(); 317 } 318 319 @Test importFiles_supportsMultipleCallsConcurrently()320 public void importFiles_supportsMultipleCallsConcurrently() throws Exception { 321 // Use BlockingFileDownloader to ensure both imports start around the same time. 322 AtomicInteger fileDownloaderInvocationCount = new AtomicInteger(0); 323 BlockingFileDownloader blockingFileDownloader = 324 new BlockingFileDownloader( 325 MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR), 326 new FileDownloader() { 327 @Override 328 public ListenableFuture<Void> startDownloading(DownloadRequest request) { 329 fileDownloaderInvocationCount.addAndGet(1); 330 return multiSchemeFileDownloaderSupplier.get().startDownloading(request); 331 } 332 }); 333 mobileDataDownload = 334 builderForTest().setFileDownloaderSupplier(() -> blockingFileDownloader).build(); 335 336 DataFileGroup fileGroup1WithInlineFile = 337 DataFileGroup.newBuilder() 338 .setGroupName(FILE_GROUP_NAME) 339 .addFile(INLINE_DATA_FILE_1) 340 .build(); 341 342 DataFileGroup fileGroup2WithInlineFile = 343 DataFileGroup.newBuilder() 344 .setGroupName(FILE_GROUP_NAME + "2") 345 .addFile(INLINE_DATA_FILE_2) 346 .build(); 347 348 // Ensure that we add the file groups successfully 349 assertThat( 350 mobileDataDownload 351 .addFileGroup( 352 AddFileGroupRequest.newBuilder() 353 .setDataFileGroup(fileGroup1WithInlineFile) 354 .build()) 355 .get()) 356 .isTrue(); 357 assertThat( 358 mobileDataDownload 359 .addFileGroup( 360 AddFileGroupRequest.newBuilder() 361 .setDataFileGroup(fileGroup2WithInlineFile) 362 .build()) 363 .get()) 364 .isTrue(); 365 366 // Perform the imports 367 ListenableFuture<Void> importFuture1 = 368 mobileDataDownload.importFiles( 369 ImportFilesRequest.newBuilder() 370 .setGroupName(FILE_GROUP_NAME) 371 .setBuildId(fileGroup1WithInlineFile.getBuildId()) 372 .setVariantId(fileGroup1WithInlineFile.getVariantId()) 373 .setInlineFileMap(ImmutableMap.of(FILE_ID_1, inlineFileSource1)) 374 .build()); 375 376 ListenableFuture<Void> importFuture2 = 377 mobileDataDownload.importFiles( 378 ImportFilesRequest.newBuilder() 379 .setGroupName(FILE_GROUP_NAME + "2") 380 .setBuildId(fileGroup2WithInlineFile.getBuildId()) 381 .setVariantId(fileGroup2WithInlineFile.getVariantId()) 382 .setInlineFileMap(ImmutableMap.of(FILE_ID_2, inlineFileSource2)) 383 .build()); 384 385 // blocking file downloader should be waiting on the imports, block the test to ensure both 386 // imports have started. 387 blockingFileDownloader.waitForDownloadStarted(); 388 389 // unblock the imports so both happen concurrently. 390 blockingFileDownloader.finishDownloading(); 391 392 // Wait for both futures to complete 393 Futures.whenAllSucceed(importFuture1, importFuture2) 394 .call(() -> null, MoreExecutors.directExecutor()) 395 .get(); 396 397 // Assert that the resulting group is downloaded and contains a reference to on device file 398 ClientFileGroup importResult1 = 399 mobileDataDownload 400 .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 401 .get(); 402 ClientFileGroup importResult2 = 403 mobileDataDownload 404 .getFileGroup( 405 GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME + "2").build()) 406 .get(); 407 408 assertThat(importResult1).isNotNull(); 409 assertThat(importResult1.getFileCount()).isEqualTo(1); 410 assertThat(importResult1.getStatus()).isEqualTo(Status.DOWNLOADED); 411 assertThat(importResult1.getFile(0).getFileUri()).isNotEmpty(); 412 413 assertThat(importResult2).isNotNull(); 414 assertThat(importResult2.getFileCount()).isEqualTo(1); 415 assertThat(importResult2.getStatus()).isEqualTo(Status.DOWNLOADED); 416 assertThat(importResult2.getFile(0).getFileUri()).isNotEmpty(); 417 418 assertThat(fileStorage.exists(Uri.parse(importResult1.getFile(0).getFileUri()))).isTrue(); 419 assertThat(fileStorage.exists(Uri.parse(importResult2.getFile(0).getFileUri()))).isTrue(); 420 421 // assert that file downloader was called 2 times, 1 for each import. 422 assertThat(fileDownloaderInvocationCount.get()).isEqualTo(2); 423 } 424 425 @Test importFiles_whenNewInlineFileSpecified_importsAndStoresFile()426 public void importFiles_whenNewInlineFileSpecified_importsAndStoresFile() throws Exception { 427 mobileDataDownload = builderForTest().build(); 428 429 DataFileGroup fileGroupWithOneInlineFile = 430 DataFileGroup.newBuilder() 431 .setGroupName(FILE_GROUP_NAME) 432 .addFile(INLINE_DATA_FILE_1) 433 .build(); 434 ImmutableList<DataFile> updatedDataFileList = ImmutableList.of(INLINE_DATA_FILE_2); 435 436 // Ensure that we add the file group successfully 437 assertThat( 438 mobileDataDownload 439 .addFileGroup( 440 AddFileGroupRequest.newBuilder() 441 .setDataFileGroup(fileGroupWithOneInlineFile) 442 .build()) 443 .get()) 444 .isTrue(); 445 446 // Perform the import 447 mobileDataDownload 448 .importFiles( 449 ImportFilesRequest.newBuilder() 450 .setGroupName(FILE_GROUP_NAME) 451 .setBuildId(fileGroupWithOneInlineFile.getBuildId()) 452 .setVariantId(fileGroupWithOneInlineFile.getVariantId()) 453 .setUpdatedDataFileList(updatedDataFileList) 454 .setInlineFileMap( 455 ImmutableMap.of(FILE_ID_1, inlineFileSource1, FILE_ID_2, inlineFileSource2)) 456 .build()) 457 .get(); 458 459 // Assert that the resulting group is downloaded and contains both files 460 ClientFileGroup importResult = 461 mobileDataDownload 462 .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 463 .get(); 464 465 assertThat(importResult.getGroupName()).isEqualTo(FILE_GROUP_NAME); 466 assertThat(importResult.getFileCount()).isEqualTo(2); 467 assertThat(importResult.getStatus()).isEqualTo(Status.DOWNLOADED); 468 assertThat(importResult.getFileList()) 469 .comparingElementsUsing(Correspondence.transforming(ClientFile::getFileId, "using file id")) 470 .containsExactly(FILE_ID_1, FILE_ID_2); 471 assertThat(importResult.getFile(0).getFileUri()).isNotEmpty(); 472 assertThat(importResult.getFile(1).getFileUri()).isNotEmpty(); 473 } 474 475 @Test importFiles_whenNewInlineFileAddedToPendingGroup_importsAndStoresFile()476 public void importFiles_whenNewInlineFileAddedToPendingGroup_importsAndStoresFile() 477 throws Exception { 478 mobileDataDownload = builderForTest().build(); 479 480 DataFileGroup fileGroupWithStandardFile = 481 DataFileGroup.newBuilder() 482 .setGroupName(FILE_GROUP_NAME) 483 .addFile( 484 DataFile.newBuilder() 485 .setFileId(FILE_ID) 486 .setUrlToDownload(FILE_URL) 487 .setChecksum(FILE_CHECKSUM) 488 .setByteSize(FILE_SIZE) 489 .build()) 490 .build(); 491 ImmutableList<DataFile> updatedDataFileList = ImmutableList.of(INLINE_DATA_FILE_2); 492 493 // Ensure that we add the file group successfully 494 assertThat( 495 mobileDataDownload 496 .addFileGroup( 497 AddFileGroupRequest.newBuilder() 498 .setDataFileGroup(fileGroupWithStandardFile) 499 .build()) 500 .get()) 501 .isTrue(); 502 503 // Perform the import 504 mobileDataDownload 505 .importFiles( 506 ImportFilesRequest.newBuilder() 507 .setGroupName(FILE_GROUP_NAME) 508 .setBuildId(fileGroupWithStandardFile.getBuildId()) 509 .setVariantId(fileGroupWithStandardFile.getVariantId()) 510 .setUpdatedDataFileList(updatedDataFileList) 511 .setInlineFileMap(ImmutableMap.of(FILE_ID_2, inlineFileSource2)) 512 .build()) 513 .get(); 514 515 // Assert that the file is pending (does not return from getFileGroup) 516 ClientFileGroup getFileGroupResult = 517 mobileDataDownload 518 .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 519 .get(); 520 assertThat(getFileGroupResult).isNull(); 521 522 // Use getFileGroupsByFilter to get the file group 523 ImmutableList<ClientFileGroup> allFileGroups = 524 mobileDataDownload 525 .getFileGroupsByFilter( 526 GetFileGroupsByFilterRequest.newBuilder() 527 .setGroupNameOptional(Optional.of(FILE_GROUP_NAME)) 528 .build()) 529 .get(); 530 531 // GetFileGroupsByFilter returns both downloaded and pending, so find the pending group. 532 ClientFileGroup pendingInlineGroup = null; 533 for (ClientFileGroup group : allFileGroups) { 534 if (group.getStatus().equals(Status.PENDING)) { 535 pendingInlineGroup = group; 536 break; 537 } 538 } 539 540 // Assert that the resulting group is pending and but contains imported file 541 assertThat(pendingInlineGroup).isNotNull(); 542 assertThat(pendingInlineGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME); 543 assertThat(pendingInlineGroup.getFileCount()).isEqualTo(2); 544 assertThat(pendingInlineGroup.getStatus()).isEqualTo(Status.PENDING); 545 assertThat(pendingInlineGroup.getFileList()) 546 .comparingElementsUsing(Correspondence.transforming(ClientFile::getFileId, "using file id")) 547 .containsExactly(FILE_ID, FILE_ID_2); 548 } 549 550 @Test importFiles_toNonExistentDataFileGroup_fails()551 public void importFiles_toNonExistentDataFileGroup_fails() throws Exception { 552 mobileDataDownload = builderForTest().build(); 553 554 FileSource inlineFileSource = FileSource.ofByteString(ByteString.copyFromUtf8("TEST_CONTENT")); 555 556 // Perform the import 557 ExecutionException ex = 558 assertThrows( 559 ExecutionException.class, 560 () -> 561 mobileDataDownload 562 .importFiles( 563 ImportFilesRequest.newBuilder() 564 .setGroupName(FILE_GROUP_NAME) 565 .setBuildId(0) 566 .setVariantId("") 567 .setInlineFileMap(ImmutableMap.of(FILE_ID_1, inlineFileSource)) 568 .build()) 569 .get()); 570 assertThat(ex).hasCauseThat().isInstanceOf(DownloadException.class); 571 DownloadException dex = (DownloadException) ex.getCause(); 572 assertThat(dex.getDownloadResultCode()).isEqualTo(DownloadResultCode.GROUP_NOT_FOUND_ERROR); 573 } 574 575 @Test importFiles_whenMismatchedVersion_failToImport()576 public void importFiles_whenMismatchedVersion_failToImport() throws Exception { 577 mobileDataDownload = builderForTest().build(); 578 579 DataFileGroup fileGroupWithInlineFile = 580 DataFileGroup.newBuilder() 581 .setGroupName(FILE_GROUP_NAME) 582 .addFile(INLINE_DATA_FILE_1) 583 .build(); 584 585 // Ensure that we add the file group successfully 586 assertThat( 587 mobileDataDownload 588 .addFileGroup( 589 AddFileGroupRequest.newBuilder() 590 .setDataFileGroup(fileGroupWithInlineFile) 591 .build()) 592 .get()) 593 .isTrue(); 594 595 // Perform the import 596 ExecutionException ex = 597 assertThrows( 598 ExecutionException.class, 599 () -> 600 mobileDataDownload 601 .importFiles( 602 ImportFilesRequest.newBuilder() 603 .setGroupName(FILE_GROUP_NAME) 604 .setBuildId(10) 605 .setVariantId("") 606 .setInlineFileMap(ImmutableMap.of(FILE_ID_1, inlineFileSource1)) 607 .build()) 608 .get()); 609 assertThat(ex).hasCauseThat().isInstanceOf(DownloadException.class); 610 DownloadException dex = (DownloadException) ex.getCause(); 611 assertThat(dex.getDownloadResultCode()).isEqualTo(DownloadResultCode.GROUP_NOT_FOUND_ERROR); 612 } 613 614 @Test importFiles_whenImportFails_doesNotWriteUpdatedMetadata()615 public void importFiles_whenImportFails_doesNotWriteUpdatedMetadata() throws Exception { 616 mobileDataDownload = builderForTest().build(); 617 618 // Create initial file group to import 619 DataFileGroup initialFileGroup = 620 DataFileGroup.newBuilder() 621 .setGroupName(FILE_GROUP_NAME) 622 .addFile(INLINE_DATA_FILE_1) 623 .build(); 624 625 // Ensure that we add the file group successfully 626 assertThat( 627 mobileDataDownload 628 .addFileGroup( 629 AddFileGroupRequest.newBuilder().setDataFileGroup(initialFileGroup).build()) 630 .get()) 631 .isTrue(); 632 633 // Perform the initial import 634 mobileDataDownload 635 .importFiles( 636 ImportFilesRequest.newBuilder() 637 .setGroupName(FILE_GROUP_NAME) 638 .setBuildId(initialFileGroup.getBuildId()) 639 .setVariantId(initialFileGroup.getVariantId()) 640 .setInlineFileMap(ImmutableMap.of(FILE_ID_1, inlineFileSource1)) 641 .build()) 642 .get(); 643 644 // Assert that the resulting group is downloaded and contains a reference to on device file 645 ClientFileGroup currentFileGroup = 646 mobileDataDownload 647 .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 648 .get(); 649 650 assertThat(currentFileGroup).isNotNull(); 651 assertThat(currentFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME); 652 assertThat(currentFileGroup.getFileCount()).isEqualTo(1); 653 assertThat(currentFileGroup.getStatus()).isEqualTo(Status.DOWNLOADED); 654 assertThat(currentFileGroup.getFile(0).getFileUri()).isNotEmpty(); 655 656 // Use fake file backend to invoke a failure when importing another file 657 fakeFileBackend.setFailure(OperationType.WRITE_STREAM, new IOException("test failure")); 658 659 // Assert that importFiles fails due to failure importing file 660 ExecutionException ex = 661 assertThrows( 662 ExecutionException.class, 663 () -> 664 mobileDataDownload 665 .importFiles( 666 ImportFilesRequest.newBuilder() 667 .setGroupName(FILE_GROUP_NAME) 668 .setBuildId(initialFileGroup.getBuildId()) 669 .setVariantId(initialFileGroup.getVariantId()) 670 .setUpdatedDataFileList(ImmutableList.of(INLINE_DATA_FILE_2)) 671 .setInlineFileMap(ImmutableMap.of(FILE_ID_2, inlineFileSource2)) 672 .build()) 673 .get()); 674 assertThat(ex).hasCauseThat().isInstanceOf(AggregateException.class); 675 AggregateException aex = (AggregateException) ex.getCause(); 676 assertThat(aex.getFailures()).hasSize(1); 677 assertThat(aex.getFailures().get(0)).isInstanceOf(DownloadException.class); 678 DownloadException dex = (DownloadException) aex.getFailures().get(0); 679 assertThat(dex.getDownloadResultCode()).isEqualTo(DownloadResultCode.INLINE_FILE_IO_ERROR); 680 681 // Get the file group again after the second import fails 682 currentFileGroup = 683 mobileDataDownload 684 .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 685 .get(); 686 687 // Assert that file group remains unchanged (no metadata change) 688 assertThat(currentFileGroup).isNotNull(); 689 assertThat(currentFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME); 690 assertThat(currentFileGroup.getFileCount()).isEqualTo(1); 691 assertThat(currentFileGroup.getStatus()).isEqualTo(Status.DOWNLOADED); 692 assertThat(currentFileGroup.getFile(0).getFileUri()).isNotEmpty(); 693 } 694 695 @Test importFiles_supportsDedup()696 public void importFiles_supportsDedup() throws Exception { 697 // Use BlockingFileDownloader to block the import of a file indefinitely. This is used to ensure 698 // a file import is in-progress before starting another import 699 AtomicInteger fileDownloaderInvocationCount = new AtomicInteger(0); 700 BlockingFileDownloader blockingFileDownloader = 701 new BlockingFileDownloader( 702 MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR), 703 new FileDownloader() { 704 @Override 705 public ListenableFuture<Void> startDownloading(DownloadRequest request) { 706 fileDownloaderInvocationCount.addAndGet(1); 707 return multiSchemeFileDownloaderSupplier.get().startDownloading(request); 708 } 709 }); 710 711 mobileDataDownload = 712 builderForTest().setFileDownloaderSupplier(() -> blockingFileDownloader).build(); 713 714 DataFileGroup fileGroup1WithInlineFile = 715 DataFileGroup.newBuilder() 716 .setGroupName(FILE_GROUP_NAME) 717 .addFile(INLINE_DATA_FILE_1) 718 .build(); 719 720 DataFileGroup fileGroup2WithInlineFile = 721 DataFileGroup.newBuilder() 722 .setGroupName(FILE_GROUP_NAME + "2") 723 .addFile(INLINE_DATA_FILE_1) 724 .build(); 725 726 // Ensure that we add the file groups successfully 727 assertThat( 728 mobileDataDownload 729 .addFileGroup( 730 AddFileGroupRequest.newBuilder() 731 .setDataFileGroup(fileGroup1WithInlineFile) 732 .build()) 733 .get()) 734 .isTrue(); 735 assertThat( 736 mobileDataDownload 737 .addFileGroup( 738 AddFileGroupRequest.newBuilder() 739 .setDataFileGroup(fileGroup2WithInlineFile) 740 .build()) 741 .get()) 742 .isTrue(); 743 744 // Start the first import and keep it in progress 745 ListenableFuture<Void> importFilesFuture1 = 746 mobileDataDownload.importFiles( 747 ImportFilesRequest.newBuilder() 748 .setGroupName(FILE_GROUP_NAME) 749 .setBuildId(fileGroup1WithInlineFile.getBuildId()) 750 .setVariantId(fileGroup1WithInlineFile.getVariantId()) 751 .setInlineFileMap(ImmutableMap.of(FILE_ID_1, inlineFileSource1)) 752 .build()); 753 754 blockingFileDownloader.waitForDownloadStarted(); 755 756 // Start the second import after the first is already in-progress 757 ListenableFuture<Void> importFilesFuture2 = 758 mobileDataDownload.importFiles( 759 ImportFilesRequest.newBuilder() 760 .setGroupName(FILE_GROUP_NAME + "2") 761 .setBuildId(fileGroup2WithInlineFile.getBuildId()) 762 .setVariantId(fileGroup2WithInlineFile.getVariantId()) 763 .setInlineFileMap(ImmutableMap.of(FILE_ID_1, inlineFileSource1)) 764 .build()); 765 766 // Allow the download to continue and trigger our delegate FileDownloader. If the future isn't 767 // cancelled, the onSuccess callback should fail the test. 768 blockingFileDownloader.finishDownloading(); 769 blockingFileDownloader.waitForDownloadCompleted(); 770 771 // wait for importFilesFuture2 to complete, check that importFiles1 is also complete 772 importFilesFuture2.get(); 773 assertThat(importFilesFuture1.isDone()).isTrue(); 774 775 // Ensure that file downloader was only invoked once 776 assertThat(fileDownloaderInvocationCount.get()).isEqualTo(1); 777 778 mobileDataDownload.clear().get(); 779 } 780 781 @Test importFiles_supportsCancellation()782 public void importFiles_supportsCancellation() throws Exception { 783 // Use BlockingFileDownloader to block the import of a file indefinitely. Check that the future 784 // returned by importFiles fails with a cancellation exception 785 BlockingFileDownloader blockingFileDownloader = 786 new BlockingFileDownloader( 787 MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR), 788 new FileDownloader() { 789 @Override 790 public ListenableFuture<Void> startDownloading(DownloadRequest downloadRequest) { 791 ListenableFuture<Void> importTaskFuture = Futures.immediateVoidFuture(); 792 Futures.addCallback( 793 importTaskFuture, 794 new FutureCallback<Void>() { 795 @Override 796 public void onSuccess(Void result) { 797 // Should not get here since we will cancel the future. 798 fail(); 799 } 800 801 @Override 802 public void onFailure(Throwable t) { 803 // Even though importTaskFuture was just created, this method should be 804 // invoked in the future chain that gets cancelled -- Ensure that the 805 // cancellation propagates to this future. 806 assertThat(importTaskFuture.isCancelled()).isTrue(); 807 } 808 }, 809 DOWNLOAD_EXECUTOR); 810 return importTaskFuture; 811 } 812 }); 813 814 mobileDataDownload = 815 builderForTest().setFileDownloaderSupplier(() -> blockingFileDownloader).build(); 816 817 DataFileGroup fileGroupWithInlineFile = 818 DataFileGroup.newBuilder() 819 .setGroupName(FILE_GROUP_NAME) 820 .addFile(INLINE_DATA_FILE_1) 821 .build(); 822 823 // Ensure that we add the file group successfully 824 assertThat( 825 mobileDataDownload 826 .addFileGroup( 827 AddFileGroupRequest.newBuilder() 828 .setDataFileGroup(fileGroupWithInlineFile) 829 .build()) 830 .get()) 831 .isTrue(); 832 833 // Perform the import 834 ListenableFuture<Void> importFilesFuture = 835 mobileDataDownload.importFiles( 836 ImportFilesRequest.newBuilder() 837 .setGroupName(FILE_GROUP_NAME) 838 .setBuildId(fileGroupWithInlineFile.getBuildId()) 839 .setVariantId(fileGroupWithInlineFile.getVariantId()) 840 .setInlineFileMap(ImmutableMap.of(FILE_ID_1, inlineFileSource1)) 841 .build()); 842 843 // Note: We could have a race condition when we call cancel() on the future, since the 844 // FileDownloader's startDownloading() may not have been invoked yet. To prevent this, we first 845 // wait for the file downloader to be invoked before performing the cancel. 846 blockingFileDownloader.waitForDownloadStarted(); 847 848 importFilesFuture.cancel(/* mayInterruptIfRunning= */ true); 849 850 // Allow the download to continue and trigger our delegate FileDownloader. If the future isn't 851 // cancelled, the onSuccess callback should fail the test. 852 blockingFileDownloader.finishDownloading(); 853 blockingFileDownloader.waitForDownloadCompleted(); 854 855 assertThat(importFilesFuture.isCancelled()).isTrue(); 856 857 mobileDataDownload.clear().get(); 858 } 859 860 @Test importFiles_emptyInlineFileImport_withExperimentInfo()861 public void importFiles_emptyInlineFileImport_withExperimentInfo() throws Exception { 862 mobileDataDownload = builderForTest().build(); 863 864 DataFileGroup fileGroupWithInlineFile = 865 DataFileGroup.newBuilder() 866 .setBuildId(BUILD_ID) 867 .setStaleLifetimeSecs(0) 868 .setVariantId(VARIANT_ID) 869 .setGroupName(FILE_GROUP_NAME) 870 .addFile(EMPTY_INLINE_FILE) 871 .build(); 872 873 // Ensure that we add the file group successfully. 874 assertThat( 875 mobileDataDownload 876 .addFileGroup( 877 AddFileGroupRequest.newBuilder() 878 .setDataFileGroup(fileGroupWithInlineFile) 879 .build()) 880 .get()) 881 .isTrue(); 882 883 // Use getFileGroupsByFilter to get the file group. 884 ImmutableList<ClientFileGroup> allFileGroups = 885 mobileDataDownload 886 .getFileGroupsByFilter( 887 GetFileGroupsByFilterRequest.newBuilder() 888 .setGroupNameOptional(Optional.of(FILE_GROUP_NAME)) 889 .build()) 890 .get(); 891 892 // Assert that the resulting group is pending. 893 assertThat(allFileGroups.get(0).getStatus()).isEqualTo(Status.PENDING); 894 895 // Perform the import. 896 mobileDataDownload 897 .importFiles( 898 ImportFilesRequest.newBuilder() 899 .setGroupName(FILE_GROUP_NAME) 900 .setBuildId(BUILD_ID) 901 .setVariantId(VARIANT_ID) 902 .setInlineFileMap( 903 ImmutableMap.of(FILE_ID_3, FileSource.ofByteString(ByteString.EMPTY))) 904 .build()) 905 .get(); 906 907 // Assert that the resulting group is downloaded and contains a reference to on device file. 908 ClientFileGroup importResult = 909 mobileDataDownload 910 .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 911 .get(); 912 Uri importFileUri = Uri.parse(importResult.getFile(0).getFileUri()); 913 914 // Verify if correct DOWNLOADED stage experiment Ids are attached. 915 assertThat(importResult).isNotNull(); 916 assertThat(importResult.getGroupName()).isEqualTo(FILE_GROUP_NAME); 917 assertThat(importResult.getFileCount()).isEqualTo(1); 918 assertThat(importResult.getStatus()).isEqualTo(Status.DOWNLOADED); 919 assertThat(fileStorage.exists(importFileUri)).isTrue(); 920 921 // Remove the filegroup which has been downloaded. 922 mobileDataDownload 923 .removeFileGroup(RemoveFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 924 .get(); 925 926 importResult = 927 mobileDataDownload 928 .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) 929 .get(); 930 931 // Assert no active filegroup. 932 assertThat(importResult).isNull(); 933 934 // Run MDD maintenance task. 935 mobileDataDownload.handleTask(TaskScheduler.MAINTENANCE_PERIODIC_TASK).get(); 936 937 // Assert file removed from file storage. 938 assertThat(fileStorage.exists(importFileUri)).isFalse(); 939 } 940 941 /** 942 * Returns MDD Builder with common dependencies set -- additional dependencies are added in each 943 * test as needed. 944 */ builderForTest()945 private MobileDataDownloadBuilder builderForTest() { 946 return MobileDataDownloadBuilder.newBuilder() 947 .setContext(context) 948 .setControlExecutor(controlExecutor) 949 .setFileDownloaderSupplier(multiSchemeFileDownloaderSupplier) 950 .setTaskScheduler(Optional.of(mockTaskScheduler)) 951 .setDeltaDecoderOptional(Optional.absent()) 952 .setFileStorage(fileStorage) 953 .setNetworkUsageMonitor(mockNetworkUsageMonitor) 954 .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) 955 .setFlagsOptional(Optional.of(flags)); 956 } 957 } 958