1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.adservices.download; 18 19 import static com.android.adservices.service.enrollment.EnrollmentUtil.BUILD_ID; 20 import static com.android.adservices.service.enrollment.EnrollmentUtil.ENROLLMENT_SHARED_PREF; 21 import static com.android.adservices.service.enrollment.EnrollmentUtil.FILE_GROUP_STATUS; 22 import static com.android.adservices.service.stats.AdServicesEncryptionKeyFetchedStats.FetchJobType.MDD_DOWNLOAD_JOB; 23 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__ERROR_CODE__ENROLLMENT_FAILED_PARSING; 24 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__ERROR_CODE__LOAD_MDD_FILE_GROUP_FAILURE; 25 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__ERROR_CODE__SHARED_PREF_UPDATE_FAILURE; 26 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__MEASUREMENT; 27 28 import android.content.Context; 29 import android.content.SharedPreferences; 30 import android.net.Uri; 31 import android.os.Build; 32 import android.util.Pair; 33 34 import androidx.annotation.RequiresApi; 35 36 import com.android.adservices.LogUtil; 37 import com.android.adservices.data.encryptionkey.EncryptionKeyDao; 38 import com.android.adservices.data.enrollment.EnrollmentDao; 39 import com.android.adservices.errorlogging.ErrorLogUtil; 40 import com.android.adservices.service.Flags; 41 import com.android.adservices.service.FlagsFactory; 42 import com.android.adservices.service.encryptionkey.EncryptionKey; 43 import com.android.adservices.service.encryptionkey.EncryptionKeyFetcher; 44 import com.android.adservices.service.enrollment.EnrollmentData; 45 import com.android.adservices.service.enrollment.EnrollmentUtil; 46 import com.android.adservices.service.proto.PrivacySandboxApi; 47 import com.android.adservices.service.proto.RbEnrollment; 48 import com.android.adservices.service.proto.RbEnrollmentList; 49 import com.android.adservices.service.stats.AdServicesLogger; 50 import com.android.adservices.service.stats.AdServicesLoggerImpl; 51 import com.android.adservices.shared.common.ApplicationContextSingleton; 52 import com.android.internal.annotations.VisibleForTesting; 53 54 import com.google.android.libraries.mobiledatadownload.GetFileGroupRequest; 55 import com.google.android.libraries.mobiledatadownload.MobileDataDownload; 56 import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; 57 import com.google.android.libraries.mobiledatadownload.file.openers.ReadStreamOpener; 58 import com.google.common.util.concurrent.Futures; 59 import com.google.common.util.concurrent.ListenableFuture; 60 import com.google.mobiledatadownload.ClientConfigProto.ClientFile; 61 import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; 62 63 import java.io.BufferedReader; 64 import java.io.IOException; 65 import java.io.InputStream; 66 import java.io.InputStreamReader; 67 import java.util.ArrayList; 68 import java.util.Arrays; 69 import java.util.List; 70 import java.util.Optional; 71 import java.util.concurrent.ExecutionException; 72 73 /** Handles EnrollmentData download from MDD server to device. */ 74 @RequiresApi(Build.VERSION_CODES.S) 75 public class EnrollmentDataDownloadManager { 76 private final Context mContext; 77 private static volatile EnrollmentDataDownloadManager sEnrollmentDataDownloadManager; 78 private final MobileDataDownload mMobileDataDownload; 79 private final SynchronousFileStorage mFileStorage; 80 private final Flags mFlags; 81 private final AdServicesLogger mLogger; 82 private final EnrollmentUtil mEnrollmentUtil; 83 private final EncryptionKeyFetcher mEncryptionKeyFetcher; 84 85 private static final String GROUP_NAME = "adtech_enrollment_data"; 86 private static final String PROTO_GROUP_NAME = "adtech_enrollment_proto_data"; 87 private static final String DOWNLOADED_ENROLLMENT_DATA_FILE_ID = "adtech_enrollment_data.csv"; 88 private static final String DOWNLOADED_ENROLLMENT_DATA_PROTO_FILE_ID = 89 "rb_prod_enrollment.binarypb"; 90 private static final String ENROLLMENT_FILE_READ_STATUS_SHARED_PREFERENCES = 91 "enrollment_data_read_status"; 92 93 @VisibleForTesting EnrollmentDataDownloadManager(Context context, Flags flags)94 EnrollmentDataDownloadManager(Context context, Flags flags) { 95 this( 96 context, 97 flags, 98 AdServicesLoggerImpl.getInstance(), 99 EnrollmentUtil.getInstance(), 100 new EncryptionKeyFetcher(MDD_DOWNLOAD_JOB)); 101 } 102 103 @VisibleForTesting EnrollmentDataDownloadManager( Context context, Flags flags, AdServicesLogger logger, EnrollmentUtil enrollmentUtil, EncryptionKeyFetcher encryptionKeyFetcher)104 EnrollmentDataDownloadManager( 105 Context context, 106 Flags flags, 107 AdServicesLogger logger, 108 EnrollmentUtil enrollmentUtil, 109 EncryptionKeyFetcher encryptionKeyFetcher) { 110 mContext = context.getApplicationContext(); 111 mMobileDataDownload = MobileDataDownloadFactory.getMdd(flags); 112 mFileStorage = MobileDataDownloadFactory.getFileStorage(); 113 mFlags = flags; 114 mLogger = logger; 115 mEnrollmentUtil = enrollmentUtil; 116 mEncryptionKeyFetcher = encryptionKeyFetcher; 117 } 118 119 /** Gets an instance of EnrollmentDataDownloadManager to be used. */ getInstance()120 public static EnrollmentDataDownloadManager getInstance() { 121 if (sEnrollmentDataDownloadManager == null) { 122 synchronized (EnrollmentDataDownloadManager.class) { 123 if (sEnrollmentDataDownloadManager == null) { 124 sEnrollmentDataDownloadManager = 125 new EnrollmentDataDownloadManager( 126 ApplicationContextSingleton.get(), 127 FlagsFactory.getFlags(), 128 AdServicesLoggerImpl.getInstance(), 129 EnrollmentUtil.getInstance(), 130 new EncryptionKeyFetcher(MDD_DOWNLOAD_JOB)); 131 } 132 } 133 } 134 return sEnrollmentDataDownloadManager; 135 } 136 137 /** 138 * Find, open and read the enrollment data file from MDD and only insert new data into the 139 * enrollment database. 140 */ readAndInsertEnrollmentDataFromMdd()141 public ListenableFuture<DownloadStatus> readAndInsertEnrollmentDataFromMdd() { 142 LogUtil.d("Reading MDD data from file."); 143 boolean protoFileFound = false; 144 Pair<ClientFile, String> FileGroupAndBuildIdPair = null; 145 if (mFlags.getEnrollmentProtoFileEnabled()) { 146 Pair<ClientFile, String> FileProtoGroupAndBuildIdPair = 147 getEnrollmentDataFile(/* getProto= */ true); 148 if (FileProtoGroupAndBuildIdPair == null 149 || FileProtoGroupAndBuildIdPair.first == null) { 150 // TODO (b/280579966): Add CEL Logging 151 LogUtil.d("Proto flag enabled, but no proto file found."); 152 } else { 153 protoFileFound = true; 154 FileGroupAndBuildIdPair = FileProtoGroupAndBuildIdPair; 155 } 156 } 157 158 if (!protoFileFound) { 159 FileGroupAndBuildIdPair = getEnrollmentDataFile(/* getProto= */ false); 160 if (FileGroupAndBuildIdPair == null || FileGroupAndBuildIdPair.first == null) { 161 return Futures.immediateFuture(DownloadStatus.NO_FILE_AVAILABLE); 162 } 163 } 164 165 ClientFile enrollmentDataFile = FileGroupAndBuildIdPair.first; 166 String fileGroupBuildId = FileGroupAndBuildIdPair.second; 167 168 @SuppressWarnings("AvoidSharedPreferences") // Legacy usage 169 SharedPreferences sharedPrefs = 170 mContext.getSharedPreferences( 171 ENROLLMENT_FILE_READ_STATUS_SHARED_PREFERENCES, Context.MODE_PRIVATE); 172 if (sharedPrefs.getBoolean(fileGroupBuildId, false)) { 173 LogUtil.d( 174 "Enrollment data build id = %s has been saved into DB. Skip adding same data.", 175 fileGroupBuildId); 176 return Futures.immediateFuture(DownloadStatus.SKIP); 177 } 178 boolean shouldTrimEnrollmentData = mFlags.getEnrollmentMddRecordDeletionEnabled(); 179 Optional<List<EnrollmentData>> enrollmentDataList; 180 if (protoFileFound) { 181 enrollmentDataList = 182 processDownloadedProtoFile(enrollmentDataFile, shouldTrimEnrollmentData); 183 } else { 184 enrollmentDataList = 185 processDownloadedFile(enrollmentDataFile, shouldTrimEnrollmentData); 186 } 187 188 if (enrollmentDataList.isPresent()) { 189 SharedPreferences.Editor editor = sharedPrefs.edit(); 190 editor.clear().putBoolean(fileGroupBuildId, true); 191 if (!editor.commit()) { 192 LogUtil.e("Saving to the enrollment file read status sharedpreference failed"); 193 ErrorLogUtil.e( 194 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__SHARED_PREF_UPDATE_FAILURE, 195 AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__MEASUREMENT); 196 } 197 LogUtil.d( 198 "Inserted new enrollment data build id = %s into DB. " 199 + "Enrollment Mdd Record Deletion Feature Enabled: %b", 200 fileGroupBuildId, shouldTrimEnrollmentData); 201 mEnrollmentUtil.logEnrollmentFileDownloadStats(mLogger, true, fileGroupBuildId); 202 203 if (!mFlags.getEncryptionKeyNewEnrollmentFetchKillSwitch()) { 204 // For new enrollment, fetch and save encryption/signing keys into DB. 205 LogUtil.i("Fetch and save encryption/signing keys for new enrollment."); 206 fetchEncryptionKeysForNewEnrollment(enrollmentDataList.get()); 207 } 208 return Futures.immediateFuture(DownloadStatus.SUCCESS); 209 } else { 210 mEnrollmentUtil.logEnrollmentFileDownloadStats(mLogger, false, fileGroupBuildId); 211 return Futures.immediateFuture(DownloadStatus.PARSING_FAILED); 212 } 213 } 214 fetchEncryptionKeysForNewEnrollment(List<EnrollmentData> enrollmentDataList)215 private void fetchEncryptionKeysForNewEnrollment(List<EnrollmentData> enrollmentDataList) { 216 EncryptionKeyDao encryptionKeyDao = EncryptionKeyDao.getInstance(); 217 for (EnrollmentData enrollmentData : enrollmentDataList) { 218 List<EncryptionKey> existingKeys = 219 encryptionKeyDao.getEncryptionKeyFromEnrollmentId( 220 enrollmentData.getEnrollmentId()); 221 // New enrollment which doesn't have any keys before, fetch keys for the first time. 222 if (existingKeys == null || existingKeys.size() == 0) { 223 Optional<List<EncryptionKey>> currentEncryptionKeys = 224 mEncryptionKeyFetcher.fetchEncryptionKeys(null, enrollmentData, true); 225 if (currentEncryptionKeys.isEmpty()) { 226 LogUtil.d("No encryption key is provided by this enrollment data."); 227 } else { 228 for (EncryptionKey encryptionKey : currentEncryptionKeys.get()) { 229 encryptionKeyDao.insert(encryptionKey); 230 } 231 } 232 } 233 } 234 } 235 processDownloadedFile( ClientFile enrollmentDataFile, boolean trimTable)236 private Optional<List<EnrollmentData>> processDownloadedFile( 237 ClientFile enrollmentDataFile, boolean trimTable) { 238 LogUtil.d("Inserting MDD data into DB."); 239 try { 240 InputStream inputStream = 241 mFileStorage.open( 242 Uri.parse(enrollmentDataFile.getFileUri()), ReadStreamOpener.create()); 243 BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); 244 bufferedReader.readLine(); 245 String line = null; 246 // While loop runs from the second line. 247 EnrollmentDao enrollmentDao = EnrollmentDao.getInstance(); 248 List<EnrollmentData> newEnrollments = new ArrayList<>(); 249 250 while ((line = bufferedReader.readLine()) != null) { 251 // Parses CSV into EnrollmentData list. 252 String[] data = line.split(","); 253 if (data.length == 8) { 254 String enrollmentId = data[0]; 255 LogUtil.d("Adding enrollmentId - %s", enrollmentId); 256 EnrollmentData enrollmentData = 257 new EnrollmentData.Builder() 258 .setEnrollmentId(enrollmentId) 259 .setEnrolledAPIs(data[1]) 260 .setSdkNames(data[2]) 261 .setAttributionSourceRegistrationUrl( 262 data[3].contains(" ") 263 ? Arrays.asList(data[3].split(" ")) 264 : List.of(data[3])) 265 .setAttributionTriggerRegistrationUrl( 266 data[4].contains(" ") 267 ? Arrays.asList(data[4].split(" ")) 268 : List.of(data[4])) 269 .setAttributionReportingUrl( 270 data[5].contains(" ") 271 ? Arrays.asList(data[5].split(" ")) 272 : List.of(data[5])) 273 .setRemarketingResponseBasedRegistrationUrl( 274 data[6].contains(" ") 275 ? Arrays.asList(data[6].split(" ")) 276 : List.of(data[6])) 277 .setEncryptionKeyUrl(data[7]) 278 .setEnrolledSite(data[7]) 279 .build(); 280 newEnrollments.add(enrollmentData); 281 } else { 282 LogUtil.e("Incorrect number of elements in row."); 283 ErrorLogUtil.e( 284 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__ENROLLMENT_FAILED_PARSING, 285 AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__MEASUREMENT); 286 } 287 } 288 if (trimTable) { 289 enrollmentDao.overwriteData(newEnrollments); 290 return Optional.of(newEnrollments); 291 } 292 for (EnrollmentData enrollmentData : newEnrollments) { 293 enrollmentDao.insert(enrollmentData); 294 } 295 return Optional.of(newEnrollments); 296 } catch (IOException e) { 297 ErrorLogUtil.e( 298 e, 299 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__ENROLLMENT_FAILED_PARSING, 300 AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__MEASUREMENT); 301 return Optional.empty(); 302 } 303 } 304 processDownloadedProtoFile( ClientFile enrollmentDataFile, boolean trimTable)305 private Optional<List<EnrollmentData>> processDownloadedProtoFile( 306 ClientFile enrollmentDataFile, boolean trimTable) { 307 LogUtil.d("Inserting MDD data from proto file into DB."); 308 try { 309 InputStream inputStream = 310 mFileStorage.open( 311 Uri.parse(enrollmentDataFile.getFileUri()), ReadStreamOpener.create()); 312 313 EnrollmentDao enrollmentDao = EnrollmentDao.getInstance(); 314 List<EnrollmentData> newEnrollments = new ArrayList<>(); 315 316 RbEnrollmentList rbProdEnrollmentList = 317 RbEnrollmentList.newBuilder().build().parseFrom(inputStream); 318 // Parses proto file into EnrollmentData list. 319 for (RbEnrollment rbEnrollment : rbProdEnrollmentList.getEntryList()) { 320 String enrollmentId = rbEnrollment.getEnrollmentId(); 321 LogUtil.d("Adding enrollmentId - %s", enrollmentId); 322 EnrollmentData enrollmentData = 323 new EnrollmentData.Builder() 324 .setEnrollmentId(enrollmentId) 325 .setEnrolledAPIs( 326 enrolledApiEnumListToString( 327 rbEnrollment.getEnrolledApisList())) 328 .setEnrolledSite(rbEnrollment.getEnrolledSite()) 329 .setSdkNames(rbEnrollment.getSdkNamesList()) 330 .build(); 331 newEnrollments.add(enrollmentData); 332 } 333 if (newEnrollments.isEmpty()) { 334 LogUtil.e("No new enrollments found."); 335 return Optional.empty(); 336 } 337 if (trimTable) { 338 enrollmentDao.overwriteData(newEnrollments); 339 return Optional.of(newEnrollments); 340 } 341 for (EnrollmentData enrollmentData : newEnrollments) { 342 enrollmentDao.insert(enrollmentData); 343 } 344 return Optional.of(newEnrollments); 345 } catch (IOException e) { 346 LogUtil.e("Failed to parse proto file"); 347 ErrorLogUtil.e( 348 e, 349 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__ENROLLMENT_FAILED_PARSING, 350 AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__MEASUREMENT); 351 return Optional.empty(); 352 } 353 } 354 355 public enum DownloadStatus { 356 SUCCESS, 357 NO_FILE_AVAILABLE, 358 PARSING_FAILED, 359 // Skip reading and inserting same enrollment data to DB if the data has been saved 360 // previously. 361 SKIP; 362 } 363 getEnrollmentDataFile(boolean getProto)364 private Pair<ClientFile, String> getEnrollmentDataFile(boolean getProto) { 365 String groupName = getProto ? PROTO_GROUP_NAME : GROUP_NAME; 366 367 GetFileGroupRequest getFileGroupRequest = 368 GetFileGroupRequest.newBuilder().setGroupName(groupName).build(); 369 try { 370 ListenableFuture<ClientFileGroup> fileGroupFuture = 371 mMobileDataDownload.getFileGroup(getFileGroupRequest); 372 ClientFileGroup fileGroup = fileGroupFuture.get(); 373 if (fileGroup == null) { 374 LogUtil.d("MDD has not downloaded the Enrollment Data Files yet."); 375 return null; 376 } 377 378 // store file group status and build id in shared preference for logging purposes 379 commitFileGroupDataToSharedPref(fileGroup); 380 String fileGroupBuildId = String.valueOf(fileGroup.getBuildId()); 381 ClientFile enrollmentDataFile = null; 382 String targetFileId = 383 getProto 384 ? DOWNLOADED_ENROLLMENT_DATA_PROTO_FILE_ID 385 : DOWNLOADED_ENROLLMENT_DATA_FILE_ID; 386 for (ClientFile file : fileGroup.getFileList()) { 387 if (file.getFileId().equals(targetFileId)) { 388 enrollmentDataFile = file; 389 break; 390 } 391 } 392 return Pair.create(enrollmentDataFile, fileGroupBuildId); 393 394 } catch (ExecutionException | InterruptedException e) { 395 LogUtil.e(e, "Unable to load MDD file group."); 396 ErrorLogUtil.e( 397 e, 398 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__LOAD_MDD_FILE_GROUP_FAILURE, 399 AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__MEASUREMENT); 400 return null; 401 } 402 } 403 commitFileGroupDataToSharedPref(ClientFileGroup fileGroup)404 private void commitFileGroupDataToSharedPref(ClientFileGroup fileGroup) { 405 Long buildId = fileGroup.getBuildId(); 406 ClientFileGroup.Status fileGroupStatus = fileGroup.getStatus(); 407 408 @SuppressWarnings("AvoidSharedPreferences") // Legacy usage 409 SharedPreferences prefs = 410 mContext.getSharedPreferences(ENROLLMENT_SHARED_PREF, Context.MODE_PRIVATE); 411 SharedPreferences.Editor edit = prefs.edit(); 412 edit.putInt(BUILD_ID, buildId.intValue()); 413 edit.putInt(FILE_GROUP_STATUS, fileGroupStatus.getNumber()); 414 if (!edit.commit()) { 415 LogUtil.e( 416 "Saving shared preferences - %s , %s and %s failed", 417 ENROLLMENT_SHARED_PREF, BUILD_ID, FILE_GROUP_STATUS); 418 ErrorLogUtil.e( 419 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__SHARED_PREF_UPDATE_FAILURE, 420 AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__MEASUREMENT); 421 } 422 } 423 enrolledApiEnumListToString(List<PrivacySandboxApi> enrolledAPIList)424 private static String enrolledApiEnumListToString(List<PrivacySandboxApi> enrolledAPIList) { 425 StringBuilder enrolledAPIs = new StringBuilder(); 426 for (PrivacySandboxApi enrolledAPI : enrolledAPIList) { 427 enrolledAPIs 428 .append( 429 EnrollmentData.ENROLLMENT_API_ENUM_STRING_MAP.getOrDefault( 430 enrolledAPI, /* defaultValue= */ "PRIVACY_SANDBOX_API_UNKNOWN")) 431 .append(" "); 432 } 433 return enrolledAPIs.toString().stripTrailing(); 434 } 435 } 436