1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #define MLOG_TAG "BackgroundCloudFileProcessor"
17
18 #include "background_cloud_file_processor.h"
19
20 #include <sys/statvfs.h>
21
22 #include "abs_rdb_predicates.h"
23 #include "cloud_sync_manager.h"
24 #include "common_timer_errors.h"
25 #include "media_column.h"
26 #include "media_file_utils.h"
27 #include "media_log.h"
28 #include "medialibrary_errno.h"
29 #include "medialibrary_db_const.h"
30 #include "medialibrary_rdb_utils.h"
31 #include "medialibrary_rdbstore.h"
32 #include "medialibrary_unistore_manager.h"
33 #include "metadata_extractor.h"
34 #include "mimetype_utils.h"
35 #include "rdb_store.h"
36 #include "rdb_utils.h"
37 #include "result_set_utils.h"
38 #include "userfile_manager_types.h"
39 #include "values_bucket.h"
40 #include "preferences.h"
41 #include "preferences_helper.h"
42 #include "cloud_sync_utils.h"
43 #include "photo_day_month_year_operation.h"
44
45 namespace OHOS {
46 namespace Media {
47 using namespace FileManagement::CloudSync;
48
49 static constexpr int32_t UPDATE_BATCH_CLOUD_SIZE = 2;
50 static constexpr int32_t UPDATE_BATCH_LOCAL_VIDEO_SIZE = 50;
51 static constexpr int32_t UPDATE_BATCH_LOCAL_IMAGE_SIZE = 200;
52 static constexpr int32_t MAX_RETRY_COUNT = 2;
53 static constexpr int32_t UPDATE_DAY_MONTH_YEAR_BATCH_SIZE = 200;
54
55 // The task can be performed only when the ratio of available storage capacity reaches this value
56 static constexpr double DEVICE_STORAGE_FREE_RATIO_HIGH = 0.15;
57 static constexpr double DEVICE_STORAGE_FREE_RATIO_LOW = 0.05;
58
59 static constexpr int64_t ONEDAY_TO_SEC = 60 * 60 * 24;
60 static constexpr int64_t SEC_TO_MSEC = 1e3;
61 static constexpr int64_t DOWNLOAD_NUM_FREE_RATIO_HIGH = 1000;
62 static constexpr int64_t DOWNLOAD_DAY_FREE_RATIO_HIGH = 30;
63 static constexpr int64_t DOWNLOAD_NUM_FREE_RATIO_LOW = 250;
64 static constexpr int64_t DOWNLOAD_DAY_FREE_RATIO_LOW = 7;
65
66 static const int64_t DOWNLOAD_ID_DEFAULT = -1;
67
68 int32_t BackgroundCloudFileProcessor::processInterval_ = PROCESS_INTERVAL; // 5 minute
69 int32_t BackgroundCloudFileProcessor::downloadInterval_ = DOWNLOAD_INTERVAL; // 1 minute
70 int32_t BackgroundCloudFileProcessor::downloadDuration_ = DOWNLOAD_DURATION; // 10 seconds
71 recursive_mutex BackgroundCloudFileProcessor::mutex_;
72 Utils::Timer BackgroundCloudFileProcessor::timer_("background_cloud_file_processor");
73 uint32_t BackgroundCloudFileProcessor::cloudDataTimerId_ = 0;
74 uint32_t BackgroundCloudFileProcessor::startTimerId_ = 0;
75 uint32_t BackgroundCloudFileProcessor::stopTimerId_ = 0;
76 bool BackgroundCloudFileProcessor::isUpdating_ = true;
77 int32_t BackgroundCloudFileProcessor::cloudUpdateOffset_ = 0;
78 int32_t BackgroundCloudFileProcessor::localImageUpdateOffset_ = 0;
79 int32_t BackgroundCloudFileProcessor::localVideoUpdateOffset_ = 0;
80 int32_t BackgroundCloudFileProcessor::cloudRetryCount_ = 0;
81 std::mutex BackgroundCloudFileProcessor::downloadResultMutex_;
82 std::unordered_map<std::string, BackgroundCloudFileProcessor::DownloadStatus>
83 BackgroundCloudFileProcessor::downloadResult_;
84 int64_t BackgroundCloudFileProcessor::downloadId_ = DOWNLOAD_ID_DEFAULT;
85
86 const std::string BACKGROUND_CLOUD_FILE_CONFIG = "/data/storage/el2/base/preferences/background_cloud_file_config.xml";
87 const std::string DOWNLOAD_CNT_CONFIG = "/data/storage/el2/base/preferences/download_count_config.xml";
88
89 const std::string DOWNLOAD_LATEST_FINISHED = "download_latest_finished";
90 const std::string LAST_DOWNLOAD_MILLISECOND = "last_download_millisecond";
91 // when kernel hibernates, the timer expires longer, and the number of download images needs to be compensated
92 const int32_t MIN_DOWNLOAD_NUM = 1;
93 const int32_t MAX_DOWNLOAD_NUM = 30;
94 const double HALF = 0.5;
95
SetDownloadLatestFinished(bool downloadLatestFinished)96 void BackgroundCloudFileProcessor::SetDownloadLatestFinished(bool downloadLatestFinished)
97 {
98 int32_t errCode;
99 shared_ptr<NativePreferences::Preferences> prefs =
100 NativePreferences::PreferencesHelper::GetPreferences(BACKGROUND_CLOUD_FILE_CONFIG, errCode);
101 CHECK_AND_RETURN_LOG(prefs, "get preferences error: %{public}d", errCode);
102
103 prefs->PutBool(DOWNLOAD_LATEST_FINISHED, downloadLatestFinished);
104 prefs->FlushSync();
105 MEDIA_INFO_LOG("set preferences %{public}d", downloadLatestFinished);
106 }
107
GetDownloadLatestFinished()108 bool BackgroundCloudFileProcessor::GetDownloadLatestFinished()
109 {
110 int32_t errCode;
111 bool downloadLatestFinished = false;
112 shared_ptr<NativePreferences::Preferences> prefs =
113 NativePreferences::PreferencesHelper::GetPreferences(BACKGROUND_CLOUD_FILE_CONFIG, errCode);
114 CHECK_AND_RETURN_RET_LOG(prefs, false,
115 "get preferences error: %{public}d", errCode);
116 return prefs->GetBool(DOWNLOAD_LATEST_FINISHED, downloadLatestFinished);
117 }
118
SetLastDownloadMilliSecond(int64_t lastDownloadMilliSecond)119 void BackgroundCloudFileProcessor::SetLastDownloadMilliSecond(int64_t lastDownloadMilliSecond)
120 {
121 int32_t errCode;
122 shared_ptr<NativePreferences::Preferences> prefs =
123 NativePreferences::PreferencesHelper::GetPreferences(BACKGROUND_CLOUD_FILE_CONFIG, errCode);
124 CHECK_AND_RETURN_LOG(prefs, "get preferences error: %{public}d", errCode);
125
126 prefs->PutLong(LAST_DOWNLOAD_MILLISECOND, lastDownloadMilliSecond);
127 prefs->FlushSync();
128 }
129
GetLastDownloadMilliSecond()130 int64_t BackgroundCloudFileProcessor::GetLastDownloadMilliSecond()
131 {
132 int32_t errCode;
133 int64_t lastDownloadMilliSecond = 0;
134 shared_ptr<NativePreferences::Preferences> prefs =
135 NativePreferences::PreferencesHelper::GetPreferences(BACKGROUND_CLOUD_FILE_CONFIG, errCode);
136 CHECK_AND_RETURN_RET_LOG(prefs, false,
137 "get preferences error: %{public}d", errCode);
138 return prefs->GetLong(LAST_DOWNLOAD_MILLISECOND, lastDownloadMilliSecond);
139 }
140
ClearDownloadCnt()141 void BackgroundCloudFileProcessor::ClearDownloadCnt()
142 {
143 int32_t errCode;
144 shared_ptr<NativePreferences::Preferences> prefs =
145 NativePreferences::PreferencesHelper::GetPreferences(DOWNLOAD_CNT_CONFIG, errCode);
146 CHECK_AND_RETURN_LOG(prefs, "get preferences error: %{public}d", errCode);
147 prefs->Clear();
148 prefs->FlushSync();
149 }
150
UpdateDownloadCnt(std::string uri,int64_t cnt)151 void BackgroundCloudFileProcessor::UpdateDownloadCnt(std::string uri, int64_t cnt)
152 {
153 int32_t errCode;
154 shared_ptr<NativePreferences::Preferences> prefs =
155 NativePreferences::PreferencesHelper::GetPreferences(DOWNLOAD_CNT_CONFIG, errCode);
156 CHECK_AND_RETURN_LOG(prefs, "get preferences error: %{public}d", errCode);
157 prefs->PutLong(uri, cnt);
158 prefs->FlushSync();
159 }
160
GetDownloadCnt(std::string uri)161 int64_t BackgroundCloudFileProcessor::GetDownloadCnt(std::string uri)
162 {
163 int32_t errCode;
164 int64_t defaultCnt = 0;
165 shared_ptr<NativePreferences::Preferences> prefs =
166 NativePreferences::PreferencesHelper::GetPreferences(DOWNLOAD_CNT_CONFIG, errCode);
167 CHECK_AND_RETURN_RET_LOG(prefs, defaultCnt, "get preferences error: %{public}d", errCode);
168 return prefs->GetLong(uri, defaultCnt);
169 }
170
DownloadCloudFiles()171 void BackgroundCloudFileProcessor::DownloadCloudFiles()
172 {
173 CHECK_AND_RETURN_LOG(CloudSyncUtils::IsCloudSyncSwitchOn(),
174 "Cloud sync switch off, skip DownloadCloudFiles");
175 MEDIA_DEBUG_LOG("Start downloading cloud files task");
176
177 double freeRatio = 0.0;
178 CHECK_AND_RETURN_LOG(GetStorageFreeRatio(freeRatio),
179 "GetStorageFreeRatio failed, stop downloading cloud files");
180 auto resultSet = QueryCloudFiles(freeRatio);
181 CHECK_AND_RETURN_LOG(resultSet != nullptr, "Failed to query cloud files!");
182
183 DownloadFiles downloadFiles;
184 ParseDownloadFiles(resultSet, downloadFiles);
185 CHECK_AND_RETURN_LOG(!downloadFiles.uris.empty(), "No cloud files need to be downloaded");
186 int32_t ret = AddDownloadTask(downloadFiles);
187 CHECK_AND_PRINT_LOG(ret == E_OK, "Failed to add download task! err: %{public}d", ret);
188 }
189
UpdateCloudData()190 void BackgroundCloudFileProcessor::UpdateCloudData()
191 {
192 MEDIA_DEBUG_LOG("Start update cloud data task");
193 std::vector<QueryOption> queryList = {{false, true}, {false, false}, {true, true}};
194 int32_t count = 0;
195 UpdateData updateData;
196 for (auto option : queryList) {
197 std::shared_ptr<NativeRdb::ResultSet> resultSet = QueryUpdateData(option.isCloud, option.isVideo);
198 if (resultSet == nullptr || resultSet->GetRowCount(count) != NativeRdb::E_OK) {
199 MEDIA_ERR_LOG("Failed to query data, %{public}d, %{public}d", option.isCloud, option.isVideo);
200 continue;
201 }
202 if (count == 0) {
203 MEDIA_DEBUG_LOG("no need to update, %{public}d, %{public}d", option.isCloud, option.isVideo);
204 continue;
205 }
206 ParseUpdateData(resultSet, updateData, option.isCloud, option.isVideo);
207 break;
208 }
209
210 if (updateData.abnormalData.empty()) {
211 MEDIA_DEBUG_LOG("No data need to update");
212 return;
213 }
214 int32_t ret = AddUpdateDataTask(updateData);
215 if (ret != E_OK) {
216 MEDIA_ERR_LOG("Failed to add update task! err: %{public}d", ret);
217 }
218 }
219
UpdateAbnormalDayMonthYearExecutor(AsyncTaskData * data)220 void BackgroundCloudFileProcessor::UpdateAbnormalDayMonthYearExecutor(AsyncTaskData *data)
221 {
222 auto *taskData = static_cast<UpdateAbnormalDayMonthYearData *>(data);
223 CHECK_AND_RETURN_LOG(taskData != nullptr, "taskData is nullptr!");
224
225 std::vector<std::string> fileIds = taskData->fileIds_;
226 auto ret = PhotoDayMonthYearOperation::UpdateAbnormalDayMonthYear(fileIds);
227 CHECK_AND_PRINT_LOG(ret == NativeRdb::E_OK,
228 "Failed to update abnormal day month year data task! err: %{public}d", ret);
229 }
230
UpdateAbnormalDayMonthYear()231 void BackgroundCloudFileProcessor::UpdateAbnormalDayMonthYear()
232 {
233 MEDIA_DEBUG_LOG("Start update abnormal day month year data task");
234
235 auto [ret, needUpdateFileIds] =
236 PhotoDayMonthYearOperation::QueryNeedUpdateFileIds(UPDATE_DAY_MONTH_YEAR_BATCH_SIZE);
237
238 CHECK_AND_RETURN_LOG(ret == NativeRdb::E_OK, "Failed to query abnormal day month year data! err: %{public}d", ret);
239 if (needUpdateFileIds.empty()) {
240 MEDIA_DEBUG_LOG("No abnormal day month year data need to update");
241 return;
242 }
243
244 auto asyncWorker = MediaLibraryAsyncWorker::GetInstance();
245 CHECK_AND_RETURN_LOG(asyncWorker != nullptr, "Failed to get async worker instance!");
246
247 auto *taskData = new (std::nothrow) UpdateAbnormalDayMonthYearData(needUpdateFileIds);
248 CHECK_AND_RETURN_LOG(taskData != nullptr, "Failed to alloc async data for update abnormal day month year data!");
249
250 auto asyncTask = std::make_shared<MediaLibraryAsyncTask>(UpdateAbnormalDayMonthYearExecutor, taskData);
251 asyncWorker->AddTask(asyncTask, false);
252 }
253
ProcessCloudData()254 void BackgroundCloudFileProcessor::ProcessCloudData()
255 {
256 UpdateCloudData();
257 UpdateAbnormalDayMonthYear();
258 }
259
GetStorageFreeRatio(double & freeRatio)260 bool BackgroundCloudFileProcessor::GetStorageFreeRatio(double &freeRatio)
261 {
262 struct statvfs diskInfo;
263 int ret = statvfs("/data/storage/el2/database", &diskInfo);
264 CHECK_AND_RETURN_RET_LOG(ret == 0, false, "Get file system status information failed, err: %{public}d", ret);
265
266 double totalSize = static_cast<double>(diskInfo.f_bsize) * static_cast<double>(diskInfo.f_blocks);
267 CHECK_AND_RETURN_RET_LOG(totalSize >= 1e-9, false,
268 "Get file system total size failed, totalSize=%{public}f", totalSize);
269
270 double freeSize = static_cast<double>(diskInfo.f_bsize) * static_cast<double>(diskInfo.f_bfree);
271 freeRatio = freeSize / totalSize;
272 return true;
273 }
274
QueryCloudFiles(double freeRatio)275 std::shared_ptr<NativeRdb::ResultSet> BackgroundCloudFileProcessor::QueryCloudFiles(double freeRatio)
276 {
277 auto uniStore = MediaLibraryUnistoreManager::GetInstance().GetRdbStore();
278 CHECK_AND_RETURN_RET_LOG(uniStore != nullptr, nullptr, "uniStore is nullptr!");
279
280 int64_t downloadNum;
281 int64_t downloadMilliSecond;
282
283 if (freeRatio >= DEVICE_STORAGE_FREE_RATIO_HIGH) {
284 downloadNum = DOWNLOAD_NUM_FREE_RATIO_HIGH;
285 downloadMilliSecond = DOWNLOAD_DAY_FREE_RATIO_HIGH * ONEDAY_TO_SEC * SEC_TO_MSEC;
286 } else if ((freeRatio >= DEVICE_STORAGE_FREE_RATIO_LOW) && (freeRatio < DEVICE_STORAGE_FREE_RATIO_HIGH)) {
287 downloadNum = DOWNLOAD_NUM_FREE_RATIO_LOW;
288 downloadMilliSecond = DOWNLOAD_DAY_FREE_RATIO_LOW * ONEDAY_TO_SEC * SEC_TO_MSEC;
289 MEDIA_INFO_LOG("freeRatio is %{public}.2f, available disk is low", freeRatio);
290 } else {
291 MEDIA_WARN_LOG("freeRatio is %{public}.2f less than %{public}.2f, stop downloading cloud files",
292 freeRatio, DEVICE_STORAGE_FREE_RATIO_LOW);
293 return nullptr;
294 }
295
296 auto currentMilliSecond = MediaFileUtils::UTCTimeMilliSeconds();
297
298 string sql = "SELECT " + PhotoColumn::MEDIA_FILE_PATH + ", " + PhotoColumn::PHOTO_POSITION + ", " +
299 PhotoColumn::MEDIA_ID + ", " + PhotoColumn::MEDIA_NAME +
300 " FROM " + PhotoColumn::PHOTOS_TABLE + " WHERE " +
301 PhotoColumn::PHOTO_CLEAN_FLAG + " = " + std::to_string(static_cast<int32_t>(CleanType::TYPE_NOT_CLEAN)) +
302 " AND " + PhotoColumn::MEDIA_FILE_PATH + " IS NOT NULL AND " + PhotoColumn::MEDIA_FILE_PATH + " != '' AND " +
303 MediaColumn::MEDIA_SIZE + " > 0 AND " + PhotoColumn::MEDIA_TYPE + " = " + std::to_string(MEDIA_TYPE_IMAGE) +
304 " AND " + MediaColumn::MEDIA_DATE_TAKEN + " > " + std::to_string(currentMilliSecond - downloadMilliSecond) +
305 " ORDER BY " + MediaColumn::MEDIA_DATE_TAKEN + " DESC LIMIT " + std::to_string(downloadNum);
306
307 return uniStore->QuerySql(sql);
308 }
309
CheckAndUpdateDownloadCnt(std::string uri,int64_t cnt)310 void BackgroundCloudFileProcessor::CheckAndUpdateDownloadCnt(std::string uri, int64_t cnt)
311 {
312 bool updateDownloadCntFlag = true;
313 unique_lock<mutex> downloadLock(downloadResultMutex_);
314 if (downloadResult_.find(uri) != downloadResult_.end()) {
315 if ((downloadResult_[uri] == DownloadStatus::NETWORK_UNAVAILABLE) ||
316 (downloadResult_[uri] == DownloadStatus::STORAGE_FULL)) {
317 updateDownloadCntFlag = false;
318 }
319 }
320 downloadLock.unlock();
321
322 if (updateDownloadCntFlag) {
323 UpdateDownloadCnt(uri, cnt + 1);
324 }
325 }
326
GetDownloadNum(int64_t & downloadNum)327 void BackgroundCloudFileProcessor::GetDownloadNum(int64_t &downloadNum)
328 {
329 int64_t currentMilliSecond = MediaFileUtils::UTCTimeMilliSeconds();
330 int64_t lastDownloadMilliSecond = GetLastDownloadMilliSecond();
331
332 int64_t minutes = (currentMilliSecond - lastDownloadMilliSecond) / downloadInterval_;
333 if (minutes < MIN_DOWNLOAD_NUM) {
334 downloadNum = MIN_DOWNLOAD_NUM;
335 } else if (minutes >= MAX_DOWNLOAD_NUM) {
336 downloadNum = MAX_DOWNLOAD_NUM;
337 } else {
338 downloadNum = minutes;
339 if ((currentMilliSecond - lastDownloadMilliSecond) - (minutes * downloadInterval_)
340 > (HALF * downloadInterval_)) {
341 downloadNum++;
342 }
343 }
344 }
345
DownloadLatestFinished()346 void BackgroundCloudFileProcessor::DownloadLatestFinished()
347 {
348 SetDownloadLatestFinished(true);
349 ClearDownloadCnt();
350
351 unique_lock<mutex> downloadLock(downloadResultMutex_);
352 downloadResult_.clear();
353 downloadLock.unlock();
354
355 lock_guard<recursive_mutex> lock(mutex_);
356 if (startTimerId_ > 0) {
357 timer_.Unregister(startTimerId_);
358 startTimerId_ = 0;
359 }
360 if (stopTimerId_ > 0) {
361 timer_.Unregister(stopTimerId_);
362 stopTimerId_ = 0;
363 }
364 }
365
ParseDownloadFiles(std::shared_ptr<NativeRdb::ResultSet> & resultSet,DownloadFiles & downloadFiles)366 void BackgroundCloudFileProcessor::ParseDownloadFiles(std::shared_ptr<NativeRdb::ResultSet> &resultSet,
367 DownloadFiles &downloadFiles)
368 {
369 int64_t downloadNum;
370 GetDownloadNum(downloadNum);
371 auto currentMilliSecond = MediaFileUtils::UTCTimeMilliSeconds();
372 SetLastDownloadMilliSecond(currentMilliSecond);
373
374 bool downloadLatestFinished = true;
375 downloadFiles.uris.clear();
376 while (resultSet->GoToNextRow() == NativeRdb::E_OK) {
377 std::string path =
378 get<std::string>(ResultSetUtils::GetValFromColumn(PhotoColumn::MEDIA_FILE_PATH, resultSet, TYPE_STRING));
379 if (path.empty()) {
380 MEDIA_WARN_LOG("Failed to get cloud file path!");
381 continue;
382 }
383
384 int32_t position =
385 get<int32_t>(ResultSetUtils::GetValFromColumn(PhotoColumn::PHOTO_POSITION, resultSet, TYPE_INT32));
386 if (position != static_cast<int32_t>(POSITION_CLOUD))
387 continue;
388
389 std::string uri = "";
390 int32_t fileId = get<int32_t>(ResultSetUtils::GetValFromColumn(PhotoColumn::MEDIA_ID, resultSet, TYPE_INT32));
391
392 std::string displayName =
393 get<std::string>(ResultSetUtils::GetValFromColumn(PhotoColumn::MEDIA_NAME, resultSet, TYPE_STRING));
394 if (displayName.empty()) {
395 MEDIA_WARN_LOG("Failed to get cloud file displayName!");
396 continue;
397 }
398 uri = MediaFileUri::GetPhotoUri(to_string(fileId), path, displayName);
399
400 int64_t cnt = GetDownloadCnt(uri);
401 if (cnt < DOWNLOAD_FAIL_MAX_TIMES) {
402 downloadLatestFinished = false;
403 downloadFiles.uris.push_back(uri);
404 downloadFiles.mediaType = MEDIA_TYPE_IMAGE;
405
406 CheckAndUpdateDownloadCnt(uri, cnt);
407
408 if ((int64_t)downloadFiles.uris.size() >= downloadNum) {
409 break;
410 }
411 }
412 }
413
414 if (downloadLatestFinished) {
415 DownloadLatestFinished();
416 }
417 }
418
removeFinishedResult(const std::vector<std::string> & downloadingPaths)419 void BackgroundCloudFileProcessor::removeFinishedResult(const std::vector<std::string>& downloadingPaths)
420 {
421 lock_guard<mutex> downloadLock(downloadResultMutex_);
422 for (auto it = downloadResult_.begin(); it != downloadResult_.end();) {
423 if (find(downloadingPaths.begin(), downloadingPaths.end(), it->first) == downloadingPaths.end()) {
424 it = downloadResult_.erase(it);
425 } else {
426 it++;
427 }
428 }
429 }
430
AddDownloadTask(const DownloadFiles & downloadFiles)431 int32_t BackgroundCloudFileProcessor::AddDownloadTask(const DownloadFiles &downloadFiles)
432 {
433 auto asyncWorker = MediaLibraryAsyncWorker::GetInstance();
434 CHECK_AND_RETURN_RET_LOG(asyncWorker != nullptr, E_FAIL, "Failed to get async worker instance!");
435
436 auto *taskData = new (std::nothrow) DownloadCloudFilesData(downloadFiles);
437 CHECK_AND_RETURN_RET_LOG(taskData != nullptr, E_NO_MEMORY,
438 "Failed to alloc async data for downloading cloud files!");
439
440 auto asyncTask = std::make_shared<MediaLibraryAsyncTask>(DownloadCloudFilesExecutor, taskData);
441 asyncWorker->AddTask(asyncTask, false);
442 return E_OK;
443 }
444
DownloadCloudFilesExecutor(AsyncTaskData * data)445 void BackgroundCloudFileProcessor::DownloadCloudFilesExecutor(AsyncTaskData *data)
446 {
447 auto *taskData = static_cast<DownloadCloudFilesData *>(data);
448 auto downloadFiles = taskData->downloadFiles_;
449
450 MEDIA_DEBUG_LOG("Try to download %{public}zu cloud files.", downloadFiles.uris.size());
451
452 unique_lock<mutex> downloadLock(downloadResultMutex_);
453 for (const auto &uri : downloadFiles.uris) {
454 downloadResult_[uri] = DownloadStatus::INIT;
455 MEDIA_INFO_LOG("Start to download cloud file, uri: %{public}s", MediaFileUtils::DesensitizePath(uri).c_str());
456 }
457 downloadLock.unlock();
458
459 removeFinishedResult(downloadFiles.uris);
460
461 std::shared_ptr<BackgroundCloudFileDownloadCallback> downloadCallback = nullptr;
462 downloadCallback = std::make_shared<BackgroundCloudFileDownloadCallback>();
463 CHECK_AND_RETURN_LOG(downloadCallback != nullptr, "downloadCallback is null.");
464 int32_t ret = CloudSyncManager::GetInstance().StartFileCache(downloadFiles.uris, downloadId_,
465 FieldKey::FIELDKEY_CONTENT, downloadCallback);
466 if (ret != E_OK || downloadId_ == DOWNLOAD_ID_DEFAULT) {
467 MEDIA_ERR_LOG("failed to StartFileCache, ret: %{public}d, downloadId_: %{public}s.",
468 ret, to_string(downloadId_).c_str());
469 downloadId_ = DOWNLOAD_ID_DEFAULT;
470 return;
471 }
472
473 MEDIA_INFO_LOG("Success, downloadId: %{public}d, downloadNum: %{public}d.",
474 static_cast<int32_t>(downloadId_), static_cast<int32_t>(downloadFiles.uris.size()));
475
476 lock_guard<recursive_mutex> lock(mutex_);
477
478 if (downloadFiles.mediaType == MEDIA_TYPE_VIDEO) {
479 if (stopTimerId_ > 0) {
480 timer_.Unregister(stopTimerId_);
481 }
482 stopTimerId_ = timer_.Register(StopDownloadFiles, downloadDuration_, true);
483 }
484 }
485
StopDownloadFiles()486 void BackgroundCloudFileProcessor::StopDownloadFiles()
487 {
488 if (downloadId_ != DOWNLOAD_ID_DEFAULT) {
489 int32_t ret = CloudSyncManager::GetInstance().StopFileCache(downloadId_);
490 MEDIA_INFO_LOG("Stop downloading cloud file, err: %{public}d, downloadId_: %{public}d",
491 ret, static_cast<int32_t>(downloadId_));
492 }
493 }
494
HandleSuccessCallback(const DownloadProgressObj & progress)495 void BackgroundCloudFileProcessor::HandleSuccessCallback(const DownloadProgressObj& progress)
496 {
497 lock_guard<mutex> downloadLock(downloadResultMutex_);
498 bool cond = (progress.downloadId != downloadId_ || downloadResult_.find(progress.path) == downloadResult_.end());
499 CHECK_AND_RETURN_WARN_LOG(!cond,
500 "downloadId or uri is err, uri: %{public}s, downloadId: %{public}s, downloadId_: %{public}s.",
501 MediaFileUtils::DesensitizePath(progress.path).c_str(), to_string(progress.downloadId).c_str(),
502 to_string(downloadId_).c_str());
503
504 downloadResult_[progress.path] = DownloadStatus::SUCCESS;
505 MEDIA_INFO_LOG("download success, uri: %{public}s.", MediaFileUtils::DesensitizePath(progress.path).c_str());
506 }
507
HandleFailedCallback(const DownloadProgressObj & progress)508 void BackgroundCloudFileProcessor::HandleFailedCallback(const DownloadProgressObj& progress)
509 {
510 lock_guard<mutex> downloadLock(downloadResultMutex_);
511 bool cond = (progress.downloadId != downloadId_ || downloadResult_.find(progress.path) == downloadResult_.end());
512 CHECK_AND_RETURN_WARN_LOG(!cond,
513 "downloadId or uri is err, uri: %{public}s, downloadId: %{public}s, downloadId_: %{public}s.",
514 MediaFileUtils::DesensitizePath(progress.path).c_str(), to_string(progress.downloadId).c_str(),
515 to_string(downloadId_).c_str());
516
517 MEDIA_ERR_LOG("download failed, error type: %{public}d, uri: %{public}s.", progress.downloadErrorType,
518 MediaFileUtils::DesensitizePath(progress.path).c_str());
519 switch (progress.downloadErrorType) {
520 case static_cast<int32_t>(DownloadProgressObj::DownloadErrorType::NETWORK_UNAVAILABLE): {
521 downloadResult_[progress.path] = DownloadStatus::NETWORK_UNAVAILABLE;
522 break;
523 }
524 case static_cast<int32_t>(DownloadProgressObj::DownloadErrorType::LOCAL_STORAGE_FULL): {
525 downloadResult_[progress.path] = DownloadStatus::STORAGE_FULL;
526 break;
527 }
528 default: {
529 downloadResult_[progress.path] = DownloadStatus::UNKNOWN;
530 MEDIA_WARN_LOG("download error type not exit.");
531 break;
532 }
533 }
534 }
535
HandleStoppedCallback(const DownloadProgressObj & progress)536 void BackgroundCloudFileProcessor::HandleStoppedCallback(const DownloadProgressObj& progress)
537 {
538 lock_guard<mutex> downloadLock(downloadResultMutex_);
539 bool cond = (progress.downloadId != downloadId_ || downloadResult_.find(progress.path) == downloadResult_.end());
540 CHECK_AND_RETURN_WARN_LOG(!cond,
541 "downloadId or uri is err, uri: %{public}s, downloadId: %{public}s, downloadId_: %{public}s.",
542 MediaFileUtils::DesensitizePath(progress.path).c_str(), to_string(progress.downloadId).c_str(),
543 to_string(downloadId_).c_str());
544
545 downloadResult_[progress.path] = DownloadStatus::STOPPED;
546 UpdateDownloadCnt(progress.path, 0);
547 MEDIA_ERR_LOG("download stopped, uri: %{public}s.", MediaFileUtils::DesensitizePath(progress.path).c_str());
548 }
549
SetPredicates(NativeRdb::RdbPredicates & predicates,bool isCloud,bool isVideo)550 void BackgroundCloudFileProcessor::SetPredicates(NativeRdb::RdbPredicates &predicates, bool isCloud, bool isVideo)
551 {
552 if (isCloud) {
553 predicates.EqualTo(PhotoColumn::PHOTO_POSITION, static_cast<int32_t>(PhotoPositionType::CLOUD))
554 ->OrderByAsc(MediaColumn::MEDIA_ID)
555 ->Limit(cloudUpdateOffset_, UPDATE_BATCH_CLOUD_SIZE);
556 } else {
557 if (isVideo) {
558 predicates.NotEqualTo(PhotoColumn::PHOTO_POSITION, static_cast<int32_t>(PhotoPositionType::CLOUD))->And()
559 ->BeginWrap()
560 ->EqualTo(MediaColumn::MEDIA_TYPE, static_cast<int32_t>(MEDIA_TYPE_VIDEO))->And()
561 ->BeginWrap()
562 ->EqualTo(MediaColumn::MEDIA_DURATION, 0)->Or()
563 ->IsNull(MediaColumn::MEDIA_DURATION)
564 ->EndWrap()
565 ->EndWrap()
566 ->OrderByAsc(MediaColumn::MEDIA_ID)
567 ->Limit(localVideoUpdateOffset_, UPDATE_BATCH_LOCAL_VIDEO_SIZE);
568 } else {
569 predicates.NotEqualTo(PhotoColumn::PHOTO_POSITION, static_cast<int32_t>(PhotoPositionType::CLOUD))->And()
570 ->NotEqualTo(MediaColumn::MEDIA_TYPE, static_cast<int32_t>(MEDIA_TYPE_VIDEO))
571 ->OrderByAsc(MediaColumn::MEDIA_ID)
572 ->Limit(localImageUpdateOffset_, UPDATE_BATCH_LOCAL_IMAGE_SIZE);
573 }
574 }
575 }
576
QueryUpdateData(bool isCloud,bool isVideo)577 std::shared_ptr<NativeRdb::ResultSet> BackgroundCloudFileProcessor::QueryUpdateData(bool isCloud, bool isVideo)
578 {
579 const std::vector<std::string> columns = { MediaColumn::MEDIA_ID, MediaColumn::MEDIA_FILE_PATH,
580 MediaColumn::MEDIA_TYPE, MediaColumn::MEDIA_SIZE, MediaColumn::MEDIA_NAME,
581 PhotoColumn::PHOTO_WIDTH, PhotoColumn::PHOTO_HEIGHT,
582 MediaColumn::MEDIA_MIME_TYPE, MediaColumn::MEDIA_DURATION, PhotoColumn::PHOTO_MEDIA_SUFFIX };
583
584 NativeRdb::RdbPredicates predicates(PhotoColumn::PHOTOS_TABLE);
585 predicates.BeginWrap()
586 ->EqualTo(MediaColumn::MEDIA_SIZE, 0)
587 ->Or()
588 ->IsNull(MediaColumn::MEDIA_SIZE)
589 ->Or()
590 ->EqualTo(PhotoColumn::PHOTO_WIDTH, 0)
591 ->Or()
592 ->IsNull(PhotoColumn::PHOTO_WIDTH)
593 ->Or()
594 ->EqualTo(PhotoColumn::PHOTO_HEIGHT, 0)
595 ->Or()
596 ->IsNull(PhotoColumn::PHOTO_HEIGHT)
597 ->Or()
598 ->EqualTo(MediaColumn::MEDIA_MIME_TYPE, "")
599 ->Or()
600 ->IsNull(MediaColumn::MEDIA_MIME_TYPE)
601 ->Or()
602 ->EqualTo(PhotoColumn::PHOTO_MEDIA_SUFFIX, "")
603 ->Or()
604 ->IsNull(PhotoColumn::PHOTO_MEDIA_SUFFIX)
605 ->Or()
606 ->BeginWrap()
607 ->EqualTo(MediaColumn::MEDIA_TYPE, static_cast<int32_t>(MEDIA_TYPE_VIDEO))
608 ->And()
609 ->BeginWrap()
610 ->EqualTo(MediaColumn::MEDIA_DURATION, 0)
611 ->Or()
612 ->IsNull(MediaColumn::MEDIA_DURATION)
613 ->EndWrap()
614 ->EndWrap()
615 ->EndWrap()
616 ->And()
617 ->EqualTo(MediaColumn::MEDIA_TIME_PENDING, 0)
618 ->And()
619 ->EqualTo(PhotoColumn::PHOTO_SYNC_STATUS, static_cast<int32_t>(SyncStatusType::TYPE_VISIBLE))
620 ->And()
621 ->EqualTo(PhotoColumn::PHOTO_CLEAN_FLAG, static_cast<int32_t>(CleanType::TYPE_NOT_CLEAN))
622 ->And();
623 SetPredicates(predicates, isCloud, isVideo);
624 return MediaLibraryRdbStore::QueryWithFilter(predicates, columns);
625 }
626
ParseUpdateData(std::shared_ptr<NativeRdb::ResultSet> & resultSet,UpdateData & updateData,bool isCloud,bool isVideo)627 void BackgroundCloudFileProcessor::ParseUpdateData(std::shared_ptr<NativeRdb::ResultSet> &resultSet,
628 UpdateData &updateData, bool isCloud, bool isVideo)
629 {
630 while (resultSet->GoToNextRow() == NativeRdb::E_OK) {
631 int32_t fileId =
632 get<int32_t>(ResultSetUtils::GetValFromColumn(MediaColumn::MEDIA_ID, resultSet, TYPE_INT32));
633 int64_t size =
634 get<int64_t>(ResultSetUtils::GetValFromColumn(MediaColumn::MEDIA_SIZE, resultSet, TYPE_INT64));
635 int32_t width =
636 get<int32_t>(ResultSetUtils::GetValFromColumn(PhotoColumn::PHOTO_WIDTH, resultSet, TYPE_INT32));
637 int32_t height =
638 get<int32_t>(ResultSetUtils::GetValFromColumn(PhotoColumn::PHOTO_HEIGHT, resultSet, TYPE_INT32));
639 int32_t duration =
640 get<int32_t>(ResultSetUtils::GetValFromColumn(MediaColumn::MEDIA_DURATION, resultSet, TYPE_INT32));
641 std::string path =
642 get<std::string>(ResultSetUtils::GetValFromColumn(MediaColumn::MEDIA_FILE_PATH, resultSet, TYPE_STRING));
643 std::string displayName =
644 get<std::string>(ResultSetUtils::GetValFromColumn(MediaColumn::MEDIA_NAME, resultSet, TYPE_STRING));
645 if (path.empty()) {
646 MEDIA_WARN_LOG("Failed to get data path");
647 continue;
648 }
649 std::string mimeType =
650 get<std::string>(ResultSetUtils::GetValFromColumn(MediaColumn::MEDIA_MIME_TYPE, resultSet, TYPE_STRING));
651 std::string suffix =
652 get<std::string>(ResultSetUtils::GetValFromColumn(PhotoColumn::PHOTO_MEDIA_SUFFIX, resultSet, TYPE_STRING));
653 int32_t mediaType =
654 get<int32_t>(ResultSetUtils::GetValFromColumn(MediaColumn::MEDIA_TYPE, resultSet, TYPE_INT32));
655
656 AbnormalData abnormalData;
657 abnormalData.fileId = fileId;
658 abnormalData.path = path;
659 abnormalData.displayName = displayName;
660 abnormalData.size = size;
661 abnormalData.width = width;
662 abnormalData.height = height;
663 abnormalData.duration = duration;
664 abnormalData.mimeType = mimeType;
665 abnormalData.mediaSuffix = suffix;
666 abnormalData.isCloud = isCloud;
667 abnormalData.isVideo = isVideo;
668
669 if (isCloud && mediaType == static_cast<int32_t>(MEDIA_TYPE_VIDEO)) {
670 updateData.abnormalData.clear();
671 abnormalData.mediaType = MEDIA_TYPE_VIDEO;
672 updateData.abnormalData.push_back(abnormalData);
673 return;
674 }
675 abnormalData.mediaType = static_cast<MediaType>(mediaType);
676 updateData.abnormalData.push_back(abnormalData);
677 }
678 }
679
AddUpdateDataTask(const UpdateData & updateData)680 int32_t BackgroundCloudFileProcessor::AddUpdateDataTask(const UpdateData &updateData)
681 {
682 auto asyncWorker = MediaLibraryAsyncWorker::GetInstance();
683 CHECK_AND_RETURN_RET_LOG(asyncWorker != nullptr, E_FAIL, "Failed to get async worker instance!");
684
685 auto *taskData = new (std::nothrow) UpdateAbnormalData(updateData);
686 CHECK_AND_RETURN_RET_LOG(taskData != nullptr, E_NO_MEMORY, "Failed to alloc async data for update cloud data!");
687
688 auto asyncTask = std::make_shared<MediaLibraryAsyncTask>(UpdateCloudDataExecutor, taskData);
689 asyncWorker->AddTask(asyncTask, false);
690 return E_OK;
691 }
692
UpdateCurrentOffset(bool isCloud,bool isVideo)693 void BackgroundCloudFileProcessor::UpdateCurrentOffset(bool isCloud, bool isVideo)
694 {
695 if (isCloud) {
696 if (cloudRetryCount_ >= MAX_RETRY_COUNT) {
697 cloudUpdateOffset_ += 1;
698 cloudRetryCount_ = 0;
699 } else {
700 cloudRetryCount_ += 1;
701 }
702 MEDIA_INFO_LOG("cloudUpdateOffset_ is %{public}d, cloudRetryCount_ is %{public}d",
703 cloudUpdateOffset_, cloudRetryCount_);
704 return;
705 }
706 if (isVideo) {
707 localVideoUpdateOffset_++;
708 MEDIA_INFO_LOG("localVideoUpdateOffset_ is %{public}d", localVideoUpdateOffset_);
709 } else {
710 localImageUpdateOffset_++;
711 MEDIA_INFO_LOG("localImageUpdateOffset_ is %{public}d", localImageUpdateOffset_);
712 }
713 }
714
UpdateCloudDataExecutor(AsyncTaskData * data)715 void BackgroundCloudFileProcessor::UpdateCloudDataExecutor(AsyncTaskData *data)
716 {
717 auto *taskData = static_cast<UpdateAbnormalData *>(data);
718 auto updateData = taskData->updateData_;
719
720 MEDIA_INFO_LOG("start update %{public}zu cloud files.", updateData.abnormalData.size());
721 for (const auto &abnormalData : updateData.abnormalData) {
722 CHECK_AND_RETURN_LOG(isUpdating_, "stop update data, isUpdating_ is %{public}d.", isUpdating_);
723 std::unique_ptr<Metadata> metadata = make_unique<Metadata>();
724 metadata->SetFilePath(abnormalData.path);
725 metadata->SetFileMediaType(abnormalData.mediaType);
726 metadata->SetFileId(abnormalData.fileId);
727 metadata->SetFileDuration(abnormalData.duration);
728 metadata->SetFileHeight(abnormalData.height);
729 metadata->SetFileWidth(abnormalData.width);
730 metadata->SetFileSize(abnormalData.size);
731 metadata->SetFileMimeType(abnormalData.mimeType);
732 GetSizeAndMimeType(metadata);
733 metadata->SetFileExtension(ScannerUtils::GetFileExtension(abnormalData.displayName));
734 if (abnormalData.size == 0 || abnormalData.mimeType.empty()) {
735 int64_t fileSize = metadata->GetFileSize();
736 string mimeType = metadata->GetFileMimeType();
737 metadata->SetFileSize(fileSize == 0 ? -1: fileSize);
738 metadata->SetFileMimeType(mimeType.empty() ? DEFAULT_IMAGE_MIME_TYPE : mimeType);
739 }
740 if (abnormalData.width == 0 || abnormalData.height == 0
741 || (abnormalData.duration == 0 && abnormalData.mediaType == MEDIA_TYPE_VIDEO)) {
742 int32_t ret = GetExtractMetadata(metadata);
743 if (ret != E_OK && MediaFileUtils::IsFileExists(abnormalData.path)) {
744 UpdateCurrentOffset(abnormalData.isCloud, abnormalData.isVideo);
745 MEDIA_ERR_LOG("failed to get extract metadata! err: %{public}d.", ret);
746 continue;
747 }
748 int32_t width = metadata->GetFileWidth();
749 int32_t height = metadata->GetFileHeight();
750 int32_t duration = metadata->GetFileDuration();
751 metadata->SetFileWidth(width == 0 ? -1: width);
752 metadata->SetFileHeight(height == 0 ? -1: height);
753 metadata->SetFileDuration((duration == 0 && abnormalData.mediaType == MEDIA_TYPE_VIDEO) ? -1: duration);
754 }
755 UpdateAbnormaldata(metadata, PhotoColumn::PHOTOS_TABLE);
756 if (abnormalData.isCloud) {
757 cloudRetryCount_ = 0;
758 }
759 }
760 }
761
SetAbnormalValuesFromMetaData(std::unique_ptr<Metadata> & metadata,NativeRdb::ValuesBucket & values)762 static void SetAbnormalValuesFromMetaData(std::unique_ptr<Metadata> &metadata, NativeRdb::ValuesBucket &values)
763 {
764 values.PutLong(MediaColumn::MEDIA_SIZE, metadata->GetFileSize());
765 values.PutInt(MediaColumn::MEDIA_DURATION, metadata->GetFileDuration());
766 values.PutInt(PhotoColumn::PHOTO_HEIGHT, metadata->GetFileHeight());
767 values.PutInt(PhotoColumn::PHOTO_WIDTH, metadata->GetFileWidth());
768 values.PutString(MediaColumn::MEDIA_MIME_TYPE, metadata->GetFileMimeType());
769 values.PutString(PhotoColumn::PHOTO_MEDIA_SUFFIX, metadata->GetFileExtension());
770 }
771
UpdateAbnormaldata(std::unique_ptr<Metadata> & metadata,const std::string & tableName)772 void BackgroundCloudFileProcessor::UpdateAbnormaldata(std::unique_ptr<Metadata> &metadata, const std::string &tableName)
773 {
774 int32_t updateCount(0);
775 NativeRdb::ValuesBucket values;
776 string whereClause = MediaColumn::MEDIA_ID + " = ?";
777 vector<string> whereArgs = { to_string(metadata->GetFileId()) };
778 SetAbnormalValuesFromMetaData(metadata, values);
779
780 auto rdbStore = MediaLibraryUnistoreManager::GetInstance().GetRdbStore();
781 CHECK_AND_RETURN_LOG(rdbStore != nullptr, "Update operation failed. rdbStore is null");
782 CHECK_AND_RETURN_LOG(isUpdating_, "stop update data,isUpdating_ is %{public}d.", isUpdating_);
783
784 int32_t result = rdbStore->Update(updateCount, tableName, values, whereClause, whereArgs);
785 bool cond = (result != NativeRdb::E_OK || updateCount <= 0);
786 CHECK_AND_RETURN_LOG(!cond, "Update operation failed. Result %{public}d. Updated %{public}d",
787 result, updateCount);
788 }
789
GetSizeAndMimeType(std::unique_ptr<Metadata> & metadata)790 void BackgroundCloudFileProcessor::GetSizeAndMimeType(std::unique_ptr<Metadata> &metadata)
791 {
792 std::string path = metadata->GetFilePath();
793 struct stat statInfo {};
794 if (stat(path.c_str(), &statInfo) != 0) {
795 MEDIA_ERR_LOG("stat syscall err %{public}d", errno);
796 metadata->SetFileSize(static_cast<int64_t>(0));
797 } else {
798 metadata->SetFileSize(statInfo.st_size);
799 }
800 string extension = ScannerUtils::GetFileExtension(path);
801 string mimeType = MimeTypeUtils::GetMimeTypeFromExtension(extension);
802 metadata->SetFileExtension(extension);
803 metadata->SetFileMimeType(mimeType);
804 }
805
GetExtractMetadata(std::unique_ptr<Metadata> & metadata)806 int32_t BackgroundCloudFileProcessor::GetExtractMetadata(std::unique_ptr<Metadata> &metadata)
807 {
808 int32_t err = 0;
809 if (metadata->GetFileMediaType() == MEDIA_TYPE_IMAGE) {
810 err = MetadataExtractor::ExtractImageMetadata(metadata);
811 } else {
812 err = MetadataExtractor::ExtractAVMetadata(metadata);
813 }
814
815 CHECK_AND_RETURN_RET_LOG(err == E_OK, err, "failed to extract data");
816 return E_OK;
817 }
818
StopUpdateData()819 void BackgroundCloudFileProcessor::StopUpdateData()
820 {
821 isUpdating_ = false;
822 }
823
StartTimer()824 void BackgroundCloudFileProcessor::StartTimer()
825 {
826 lock_guard<recursive_mutex> lock(mutex_);
827 MEDIA_INFO_LOG("Turn on the background download cloud file timer");
828
829 if (cloudDataTimerId_ > 0) {
830 timer_.Unregister(cloudDataTimerId_);
831 }
832 if (startTimerId_ > 0) {
833 timer_.Unregister(startTimerId_);
834 }
835
836 uint32_t ret = timer_.Setup();
837 CHECK_AND_PRINT_LOG(ret == Utils::TIMER_ERR_OK,
838 "Failed to start background download cloud files timer, err: %{public}d", ret);
839 isUpdating_ = true;
840 cloudDataTimerId_ = timer_.Register(ProcessCloudData, processInterval_);
841 if (!GetDownloadLatestFinished()) {
842 auto currentMilliSecond = MediaFileUtils::UTCTimeMilliSeconds();
843 SetLastDownloadMilliSecond(currentMilliSecond);
844 startTimerId_ = timer_.Register(DownloadCloudFiles, downloadInterval_);
845 }
846 }
847
StopTimer()848 void BackgroundCloudFileProcessor::StopTimer()
849 {
850 lock_guard<recursive_mutex> lock(mutex_);
851 MEDIA_INFO_LOG("Turn off the background download cloud file timer");
852
853 timer_.Unregister(cloudDataTimerId_);
854 timer_.Unregister(startTimerId_);
855 timer_.Unregister(stopTimerId_);
856 timer_.Shutdown();
857 cloudDataTimerId_ = 0;
858 startTimerId_ = 0;
859 stopTimerId_ = 0;
860 StopUpdateData();
861 StopDownloadFiles();
862 }
863 } // namespace Media
864 } // namespace OHOS
865