/* * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.android.libraries.mobiledatadownload; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_CHECKSUM; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_GROUP_NAME; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_ID; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_SIZE; import static com.google.android.libraries.mobiledatadownload.TestFileGroupPopulator.FILE_URL; import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.DownloaderConfigurationType; import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertThrows; import android.accounts.Account; import android.content.Context; import android.net.Uri; import android.util.Log; import androidx.test.core.app.ApplicationProvider; import com.google.android.libraries.mobiledatadownload.account.AccountUtil; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend; import com.google.android.libraries.mobiledatadownload.file.backends.JavaFileBackend; import com.google.android.libraries.mobiledatadownload.file.transforms.CompressTransform; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; import com.google.android.libraries.mobiledatadownload.testing.BlockingFileDownloader; import com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies; import com.google.android.libraries.mobiledatadownload.testing.TestFileDownloader; import com.google.android.libraries.mobiledatadownload.testing.TestFlags; import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.mobiledatadownload.ClientConfigProto.ClientFile; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DataFile; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy; import com.google.mobiledatadownload.TransformProto; import com.google.mobiledatadownload.TransformProto.Transform; import com.google.mobiledatadownload.TransformProto.Transforms; import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; /** * Integration Tests that relate to {@link MobileDataDownload#downloadFileGroup}. * *

NOTE: Any tests related to cancellation should be added to {@link * DownloadFileGroupCancellationIntegrationTest} instead. */ @RunWith(TestParameterInjector.class) public class DownloadFileGroupIntegrationTest { private static final String TAG = "DownloadFileGroupIntegrationTest"; private static final int MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS = 60; private static final int MAX_MULTI_MDD_API_WAIT_TIME_SECS = 120; private static final long MAX_MDD_API_WAIT_TIME_SECS = 5L; private static final ListeningScheduledExecutorService DOWNLOAD_EXECUTOR = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(4)); private static final String FILE_GROUP_NAME_INSECURE_URL = "test-group-insecure-url"; private static final String FILE_GROUP_NAME_MULTIPLE_FILES = "test-group-multiple-files"; private static final String FILE_ID_1 = "test-file-1"; private static final String FILE_ID_2 = "test-file-2"; private static final String FILE_CHECKSUM_1 = "a1cba9d87b1440f41ce9e7da38c43e1f6bd7d5df"; private static final String FILE_CHECKSUM_2 = "cb2459d9f1b508993aba36a5ffd942a7e0d49ed6"; private static final String FILE_NOT_EXIST_URL = "https://www.gstatic.com/icing/idd/notexist/file.txt"; private static final String TEST_DATA_RELATIVE_PATH = "third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/"; private static final String TEST_DATA_URL = "https://test.url/full_file.txt"; private static final String TEST_DATA_CHECKSUM = "0c4f1e55c4ec28d0305c5cfde8610b7e6e9f7d9a"; private static final int TEST_DATA_BYTE_SIZE = 110; private static final String TEST_DATA_COMPRESS_URL = "https://test.url/full_file.zlib"; private static final String TEST_DATA_COMPRESS_CHECKSUM = "cbffcf480fd52a3c6bf9d21206d36f0a714bb97a"; private static final int TEST_DATA_COMPRESS_BYTE_SIZE = 92; private static final String VARIANT_1 = "test-variant-1"; private static final String VARIANT_2 = "test-variant-2"; private static final Account ACCOUNT_1 = AccountUtil.create("account-name-1", "account-type"); private static final Account ACCOUNT_2 = AccountUtil.create("account-name-2", "account-type"); private static final Context context = ApplicationProvider.getApplicationContext(); @Mock private TaskScheduler mockTaskScheduler; @Mock private NetworkUsageMonitor mockNetworkUsageMonitor; @Mock private DownloadProgressMonitor mockDownloadProgressMonitor; private SynchronousFileStorage fileStorage; private ListeningExecutorService controlExecutor; private final TestFlags flags = new TestFlags(); @Rule(order = 1) public final MockitoRule mocks = MockitoJUnit.rule(); @TestParameter ExecutorType controlExecutorType; @Before public void setUp() throws Exception { fileStorage = new SynchronousFileStorage( /* backends= */ ImmutableList.of( AndroidFileBackend.builder(context).build(), new JavaFileBackend()), /* transforms= */ ImmutableList.of(new CompressTransform()), /* monitors= */ ImmutableList.of(mockNetworkUsageMonitor, mockDownloadProgressMonitor)); controlExecutor = controlExecutorType.executor(); } @Test public void downloadAndRead( @TestParameter DownloaderConfigurationType downloaderConfigurationType) throws Exception { Optional instanceId = Optional.of(MddTestDependencies.randomInstanceId()); TestFileGroupPopulator testFileGroupPopulator = new TestFileGroupPopulator(context); MobileDataDownload mobileDataDownload = builderForTest() .setInstanceIdOptional(instanceId) .setFileDownloaderSupplier( downloaderConfigurationType.fileDownloaderSupplier( context, controlExecutor, DOWNLOAD_EXECUTOR, fileStorage, flags, Optional.of(mockDownloadProgressMonitor), instanceId)) .addFileGroupPopulator(testFileGroupPopulator) .build(); testFileGroupPopulator .refreshFileGroups(mobileDataDownload) .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); mobileDataDownload .downloadFileGroup( DownloadFileGroupRequest.newBuilder() .setGroupName(FILE_GROUP_NAME) .setListenerOptional( Optional.of( new DownloadListener() { @Override public void onProgress(long currentSize) { Log.i(TAG, "onProgress " + currentSize); } @Override public void onComplete(ClientFileGroup clientFileGroup) { Log.i(TAG, "onComplete " + clientFileGroup.getGroupName()); } })) .build()) .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS); ClientFileGroup clientFileGroup = mobileDataDownload .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); assertThat(clientFileGroup).isNotNull(); assertThat(clientFileGroup.getGroupName()).isEqualTo(FILE_GROUP_NAME); assertThat(clientFileGroup.getFileCount()).isEqualTo(1); ClientFile clientFile = clientFileGroup.getFileList().get(0); assertThat(clientFile.getFileId()).isEqualTo(FILE_ID); Uri androidUri = Uri.parse(clientFile.getFileUri()); assertThat(fileStorage.fileSize(androidUri)).isEqualTo(FILE_SIZE); mobileDataDownload.clear().get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); switch (downloaderConfigurationType) { case V2_PLATFORM: // No-op } } @Test public void downloadFailed() throws Exception { // NOTE: The test failures here are not network stack dependent, so there's // no need to parameterize this test for different network stacks. Optional instanceId = Optional.of(MddTestDependencies.randomInstanceId()); MobileDataDownload mobileDataDownload = builderForTest() .setInstanceIdOptional(instanceId) .setFileDownloaderSupplier( DownloaderConfigurationType.V2_PLATFORM.fileDownloaderSupplier( context, controlExecutor, DOWNLOAD_EXECUTOR, fileStorage, flags, Optional.of(mockDownloadProgressMonitor), instanceId)) .build(); // The data file group has a file with insecure url. DataFileGroup groupWithInsecureUrl = TestFileGroupPopulator.createDataFileGroup( FILE_GROUP_NAME_INSECURE_URL, context.getPackageName(), new String[] {FILE_ID}, new int[] {FILE_SIZE}, new String[] {FILE_CHECKSUM}, // Make the url insecure. This would lead to download failure. new String[] {FILE_URL.replace("https", "http")}, DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK); // The data file group has a file with non-existent url, and a file with insecure url. DataFileGroup groupWithMultipleFiles = TestFileGroupPopulator.createDataFileGroup( FILE_GROUP_NAME_MULTIPLE_FILES, context.getPackageName(), new String[] {FILE_ID_1, FILE_ID_2}, new int[] {FILE_SIZE, FILE_SIZE}, new String[] {FILE_CHECKSUM_1, FILE_CHECKSUM_2}, // The first file url doesn't exist and the second file url is insecure. new String[] {FILE_NOT_EXIST_URL, FILE_URL.replace("https", "http")}, DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK); assertThat( mobileDataDownload .addFileGroup( AddFileGroupRequest.newBuilder().setDataFileGroup(groupWithInsecureUrl).build()) .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) .isTrue(); assertThat( mobileDataDownload .addFileGroup( AddFileGroupRequest.newBuilder() .setDataFileGroup(groupWithMultipleFiles) .build()) .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS)) .isTrue(); ExecutionException exception = assertThrows( ExecutionException.class, () -> mobileDataDownload .downloadFileGroup( DownloadFileGroupRequest.newBuilder() .setGroupName(FILE_GROUP_NAME_INSECURE_URL) .build()) .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS)); assertThat(exception).hasCauseThat().isInstanceOf(AggregateException.class); AggregateException cause = (AggregateException) exception.getCause(); assertThat(cause).isNotNull(); ImmutableList failures = cause.getFailures(); assertThat(failures).hasSize(1); assertThat(failures.get(0)).isInstanceOf(DownloadException.class); assertThat(failures.get(0)).hasMessageThat().contains("INSECURE_URL_ERROR"); ExecutionException exception2 = assertThrows( ExecutionException.class, () -> mobileDataDownload .downloadFileGroup( DownloadFileGroupRequest.newBuilder() .setGroupName(FILE_GROUP_NAME_MULTIPLE_FILES) .build()) .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS)); assertThat(exception2).hasCauseThat().isInstanceOf(AggregateException.class); AggregateException cause2 = (AggregateException) exception2.getCause(); assertThat(cause2).isNotNull(); ImmutableList failures2 = cause2.getFailures(); assertThat(failures2).hasSize(2); assertThat(failures2.get(0)).isInstanceOf(DownloadException.class); assertThat(failures2.get(0)) .hasCauseThat() .hasMessageThat() .containsMatch("httpStatusCode=404"); assertThat(failures2.get(1)).isInstanceOf(DownloadException.class); assertThat(failures2.get(1)).hasMessageThat().contains("INSECURE_URL_ERROR"); AggregateException exception3 = assertThrows( AggregateException.class, () -> { try { ListenableFuture downloadFuture1 = mobileDataDownload.downloadFileGroup( DownloadFileGroupRequest.newBuilder() .setGroupName(FILE_GROUP_NAME_MULTIPLE_FILES) .build()); ListenableFuture downloadFuture2 = mobileDataDownload.downloadFileGroup( DownloadFileGroupRequest.newBuilder() .setGroupName(FILE_GROUP_NAME_INSECURE_URL) .build()); Futures.successfulAsList(downloadFuture1, downloadFuture2) .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS); AggregateException.throwIfFailed( ImmutableList.of(downloadFuture1, downloadFuture2), "Expected download failures"); } catch (ExecutionException e) { throw e; } }); assertThat(exception3.getFailures()).hasSize(2); } @Test public void removePartialDownloadThenDownloadAgain( @TestParameter DownloaderConfigurationType downloaderConfigurationType) throws Exception { Optional instanceId = Optional.of(MddTestDependencies.randomInstanceId()); Supplier fileDownloaderSupplier = downloaderConfigurationType.fileDownloaderSupplier( context, controlExecutor, DOWNLOAD_EXECUTOR, fileStorage, flags, Optional.of(mockDownloadProgressMonitor), instanceId); BlockingFileDownloader blockingFileDownloader = new BlockingFileDownloader(DOWNLOAD_EXECUTOR, fileDownloaderSupplier.get()); MobileDataDownload mobileDataDownload = builderForTest() .setInstanceIdOptional(instanceId) .setFileDownloaderSupplier(() -> blockingFileDownloader) .build(); mobileDataDownload.clear().get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); // Add the filegroup, start downloading, then cancel while in progress. TestFileGroupPopulator testFileGroupPopulator = new TestFileGroupPopulator(context); testFileGroupPopulator .refreshFileGroups(mobileDataDownload) .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); ListenableFuture downloadFuture = mobileDataDownload.downloadFileGroup( DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); blockingFileDownloader.finishDownloading(); // Unblocks blockingFileDownloader blockingFileDownloader.waitForDelegateStarted(); // Waits until offroadDownloader starts // NOTE: add a little wait to allow Downloader's listeners to run. Thread.sleep(/* millis= */ 200); downloadFuture.cancel(true /* may interrupt */); // NOTE: add a little wait to allow Downloader's listeners to run. Thread.sleep(/* millis= */ 200); // Remove the filegroup. ListenableFuture removeFuture = mobileDataDownload.removeFileGroup( RemoveFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); removeFuture.get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); // Add then try to download again. blockingFileDownloader.resetState(); blockingFileDownloader.finishDownloading(); // Unblocks blockingFileDownloader testFileGroupPopulator .refreshFileGroups(mobileDataDownload) .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); downloadFuture = mobileDataDownload.downloadFileGroup( DownloadFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()); downloadFuture.get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS); // The file should have downloaded as expected. ClientFileGroup clientFileGroup = mobileDataDownload .getFileGroup(GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_NAME).build()) .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); assertThat(clientFileGroup).isNotNull(); assertThat(clientFileGroup.getFileCount()).isEqualTo(1); Uri androidUri = Uri.parse(clientFileGroup.getFileList().get(0).getFileUri()); assertThat(fileStorage.fileSize(androidUri)).isEqualTo(FILE_SIZE); } @Test public void downloadDifferentGroupsWithSameFileTest() throws Exception { Optional instanceId = Optional.of(MddTestDependencies.randomInstanceId()); MobileDataDownload mobileDataDownload = builderForTest() .setInstanceIdOptional(instanceId) .setFileDownloaderSupplier( () -> new TestFileDownloader(TEST_DATA_RELATIVE_PATH, fileStorage, DOWNLOAD_EXECUTOR)) .build(); DataFile.Builder dataFileBuilder = DataFile.newBuilder() .setUrlToDownload(TEST_DATA_URL) .setChecksum(TEST_DATA_CHECKSUM) .setByteSize(TEST_DATA_BYTE_SIZE); DataFileGroup.Builder groupBuilder = DataFileGroup.newBuilder(); // Add all groups concurrently ArrayList> addFutures = new ArrayList<>(); for (int i = 0; i < 50; i++) { String groupName = String.format("group%d", i); String fileId = String.format("group%d_file", i); DataFile file = dataFileBuilder.setFileId(fileId).build(); DataFileGroup group = DataFileGroup.newBuilder().setGroupName(groupName).addFile(file).build(); addFutures.add( mobileDataDownload.addFileGroup( AddFileGroupRequest.newBuilder().setDataFileGroup(group).build())); } Futures.allAsList(addFutures).get(MAX_MULTI_MDD_API_WAIT_TIME_SECS, SECONDS); // Start all downloads concurrently ArrayList> downloadFutures = new ArrayList<>(); for (int i = 0; i < 50; i++) { String groupName = String.format("group%d", i); downloadFutures.add( mobileDataDownload.downloadFileGroup( DownloadFileGroupRequest.newBuilder().setGroupName(groupName).build())); } List groups = Futures.allAsList(downloadFutures).get(MAX_MULTI_MDD_API_WAIT_TIME_SECS, SECONDS); assertThat(groups).doesNotContain(null); } @Test public void concurrentDownloads_withSameFile_withDifferentDownloadTransforms_completes( @TestParameter boolean enableDedupByFileKey) throws Exception { flags.enableFileDownloadDedupByFileKey = Optional.of(enableDedupByFileKey); Optional instanceId = Optional.of(MddTestDependencies.randomInstanceId()); MobileDataDownload mobileDataDownload = builderForTest() .setInstanceIdOptional(instanceId) .setFileDownloaderSupplier( () -> new TestFileDownloader(TEST_DATA_RELATIVE_PATH, fileStorage, DOWNLOAD_EXECUTOR)) .build(); // Create two groups which share the same file, but have different download transforms DataFileGroup groupWithoutTransform = DataFileGroup.newBuilder() .setGroupName("groupWithoutTransform") .addFile( DataFile.newBuilder() .setFileId("file_no_transform") .setUrlToDownload(TEST_DATA_URL) .setChecksum(TEST_DATA_CHECKSUM) .setByteSize(TEST_DATA_BYTE_SIZE)) .build(); DataFileGroup groupWithTransform = DataFileGroup.newBuilder() .setGroupName("groupWithTransform") .addFile( DataFile.newBuilder() .setFileId("file_no_transform") .setUrlToDownload(TEST_DATA_COMPRESS_URL) .setChecksum(TEST_DATA_CHECKSUM) .setByteSize(TEST_DATA_BYTE_SIZE) .setDownloadedFileChecksum(TEST_DATA_COMPRESS_CHECKSUM) .setDownloadedFileByteSize(TEST_DATA_COMPRESS_BYTE_SIZE) .setDownloadTransforms( Transforms.newBuilder() .addTransform( Transform.newBuilder() .setCompress( TransformProto.CompressTransform.getDefaultInstance()) .build()) .build()) .build()) .build(); // Add both groups, then attempt to download both concurrently mobileDataDownload .addFileGroup( AddFileGroupRequest.newBuilder().setDataFileGroup(groupWithoutTransform).build()) .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); mobileDataDownload .addFileGroup(AddFileGroupRequest.newBuilder().setDataFileGroup(groupWithTransform).build()) .get(MAX_MDD_API_WAIT_TIME_SECS, SECONDS); ListenableFuture downloadWithoutTransform = mobileDataDownload.downloadFileGroup( DownloadFileGroupRequest.newBuilder().setGroupName("groupWithoutTransform").build()); ListenableFuture downloadWithTransform = mobileDataDownload.downloadFileGroup( DownloadFileGroupRequest.newBuilder().setGroupName("groupWithTransform").build()); List downloadedGroups = Futures.allAsList(ImmutableList.of(downloadWithoutTransform, downloadWithTransform)) .get(MAX_MULTI_MDD_API_WAIT_TIME_SECS, SECONDS); // Both groups are downloaded and both files point to the same on-device uri. assertThat(downloadedGroups).doesNotContain(null); assertThat(downloadedGroups.get(0).getFile(0).getFileUri()) .isEqualTo(downloadedGroups.get(1).getFile(0).getFileUri()); } /** * Returns MDD Builder with common dependencies set -- additional dependencies are added in each * test as needed. */ private MobileDataDownloadBuilder builderForTest() { return MobileDataDownloadBuilder.newBuilder() .setContext(context) .setControlExecutor(controlExecutor) .setFileStorage(fileStorage) .setTaskScheduler(Optional.of(mockTaskScheduler)) .setDeltaDecoderOptional(Optional.absent()) .setNetworkUsageMonitor(mockNetworkUsageMonitor) .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor)) .setFlagsOptional(Optional.of(flags)); } }