1 /*
2 * Copyright (C) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef OHOS_MEDIA_CLONE_RESTORE_H
17 #define OHOS_MEDIA_CLONE_RESTORE_H
18
19 #include <optional>
20 #include <type_traits>
21 #include <set>
22 #include <sstream>
23 #include <algorithm>
24 #include <iterator>
25 #include <vector>
26
27 #include "base_restore.h"
28 #include "backup_const.h"
29 #include "clone_restore_analysis_data.h"
30 #include "medialibrary_rdb_utils.h"
31 #include "medialibrary_errno.h"
32 #include "medialibrary_kvstore_manager.h"
33 #include "backup_database_utils.h"
34 #include "photo_album_clone.h"
35 #include "photos_clone.h"
36 #include "clone_restore_geo_dictionary.h"
37 #include "search_index_clone.h"
38 #include "video_face_clone.h"
39 #include "beauty_score_clone.h"
40 #include "ffrt.h"
41 #include "ffrt_inner.h"
42
43 namespace OHOS {
44 namespace Media {
45 struct CloudPhotoFileExistFlag {
46 bool isLcdExist {false};
47 bool isThmExist {false};
48 bool isDayAstcExist {false};
49 bool isYearAstcExist {false};
50 bool isExLcdExist {false};
51 bool isExThmExist {false};
52 };
53 class CloneRestore : public BaseRestore {
54 public:
55 CloneRestore();
56 virtual ~CloneRestore() = default;
57 // upgradePath is useless now
58 void StartRestore(const std::string &backupRestorePath, const std::string &upgradePath) override;
59 int32_t Init(const std::string &backupRestoreDir, const std::string &upgradeFilePath, bool isUpgrade) override;
60 NativeRdb::ValuesBucket GetInsertValue(const FileInfo &fileInfo, const std::string &newPath,
61 int32_t sourceType) override;
62 NativeRdb::ValuesBucket GetCloudInsertValue(const FileInfo &fileInfo, const std::string &newPath,
63 int32_t sourceType);
64 std::string GetBackupInfo() override;
65 void StartBackup() override;
66 using CoverUriInfo = std::pair<std::string, std::pair<std::string, int32_t>>;
67 void InheritManualCover();
68 void UpdatePhotoAlbumCoverUri(vector<AlbumCoverInfo>& albumCoverInfos);
69 protected:
70 void MoveMigrateCloudFile(std::vector<FileInfo> &fileInfos, int32_t &fileMoveCount, int32_t &videoFileMoveCount,
71 int32_t sceneCode) override;
72 void GetCloudPhotoFileExistFlag(const FileInfo &fileInfo, CloudPhotoFileExistFlag &resultExistFlag);
73 void CloudPhotoFilesVerify(const std::vector<FileInfo> &fileInfos, std::vector<FileInfo> &LCDNotFound,
74 std::vector<FileInfo> &THMNotFound, unordered_map<string, CloudPhotoFileExistFlag> &resultExistMap);
75 bool HasExThumbnail(const FileInfo &info) override;
76 void BackupRelease() override;
77
78 private:
79 void RestorePhoto(void) override;
80 void RestorePhotoForCloud(void);
81 void HandleRestData(void) override;
82 std::vector<FileInfo> QueryFileInfos(int32_t offset, int32_t isRelatedToPhotoMap = 0);
83 std::vector<FileInfo> QueryCloudFileInfos(int32_t offset, int32_t isRelatedToPhotoMap = 0);
84 bool ParseResultSet(const std::shared_ptr<NativeRdb::ResultSet> &resultSet, FileInfo &info,
85 std::string dbName = "") override;
86 bool ParseResultSetForAudio(const std::shared_ptr<NativeRdb::ResultSet> &resultSet, FileInfo &info) override;
87 void AnalyzeSource() override;
88 void RestoreAlbum(void);
89 void RestoreAudio(void) override;
90 int InsertPhoto(std::vector<FileInfo> &fileInfos);
91 std::vector<NativeRdb::ValuesBucket> GetInsertValues(int32_t sceneCode, std::vector<FileInfo> &fileInfos,
92 int32_t sourceType);
93 std::vector<NativeRdb::ValuesBucket> GetCloudInsertValues(int32_t sceneCode, std::vector<FileInfo> &fileInfos,
94 int32_t sourceType) override;
95 int InsertCloudPhoto(int32_t sceneCode, std::vector<FileInfo> &fileInfos, int32_t sourceType) override;
96 std::vector<NativeRdb::ValuesBucket> GetInsertValues(std::vector<AnalysisAlbumTbl> &analysisAlbumTbl);
97 int32_t MoveAsset(FileInfo &fileInfo);
98 bool IsFilePathExist(const std::string &filePath) const;
99 int32_t QueryTotalNumber(const std::string &tableName);
100 std::vector<AlbumInfo> QueryAlbumInfos(const std::string &tableName, int32_t offset);
101 bool ParseAlbumResultSet(const std::string &tableName, const std::shared_ptr<NativeRdb::ResultSet> &resultSet,
102 AlbumInfo &albumInfo);
103 bool PrepareCommonColumnInfoMap(const std::string &tableName,
104 const std::unordered_map<std::string, std::string> &srcColumnInfoMap,
105 const std::unordered_map<std::string, std::string> &dstColumnInfoMap);
106 bool HasSameColumn(const std::unordered_map<std::string, std::string> &columnInfoMap, const std::string &columnName,
107 const std::string &columnType);
108 void GetValFromResultSet(const std::shared_ptr<NativeRdb::ResultSet> &resultSet,
109 std::unordered_map<std::string, std::variant<int32_t, int64_t, double, std::string>> &valMap,
110 const std::string &columnName, const std::string &columnType);
111 void PrepareCommonColumnVal(NativeRdb::ValuesBucket &values, const std::string &columnName,
112 const std::variant<int32_t, int64_t, double, std::string> &columnVal,
113 const std::unordered_map<std::string, std::string> &commonColumnInfoMap) const;
114 void GetQueryWhereClause(const std::string &tableName,
115 const std::unordered_map<std::string, std::string> &columnInfoMap);
116 void BatchQueryPhoto(std::vector<FileInfo> &fileInfos);
117 void UpdateAlbumOrderColumns(const AlbumInfo &albumInfo, const string &tableName);
118 void UpdateSystemAlbumColumns(const string &tableName);
119 void InsertAlbum(std::vector<AlbumInfo> &albumInfos, const std::string &tableName);
120 std::vector<NativeRdb::ValuesBucket> GetInsertValues(std::vector<AlbumInfo> &albumInfos,
121 std::vector<std::string> &albumIds, const std::string &tableName);
122 bool HasSameAlbum(AlbumInfo &albumInfo, const std::string &tableName);
123 void BatchQueryAlbum(std::vector<AlbumInfo> &albumInfos, const std::string &tableName);
124 void BatchInsertMap(const std::vector<FileInfo> &fileInfos, int64_t &totalRowNum);
125 NativeRdb::ValuesBucket GetInsertValue(const MapInfo &mapInfo) const;
126 NativeRdb::ValuesBucket GetInsertValue(const AlbumInfo &albumInfo, const std::string &tableName) const;
127 void CheckTableColumnStatus(std::shared_ptr<NativeRdb::RdbStore> rdbStore,
128 const std::vector<std::vector<std::string>> &cloneTableList);
129 bool HasColumns(const std::unordered_map<std::string, std::string> &columnInfoMap,
130 const std::unordered_set<std::string> &columnSet);
131 bool HasColumn(const std::unordered_map<std::string, std::string> &columnInfoMap, const std::string &columnName);
132 void GetAlbumExtraQueryWhereClause(const std::string &tableName);
133 bool IsReadyForRestore(const std::string &tableName);
134 void PrepareEditTimeVal(NativeRdb::ValuesBucket &values, int64_t editTime, const FileInfo &fileInfo,
135 const std::unordered_map<std::string, std::string> &commonColumnInfoMap) const;
136 void RestoreGallery();
137 bool PrepareCloudPath(const std::string &tableName, FileInfo &fileInfo);
138 void RestoreMusic();
139 std::vector<FileInfo> QueryFileInfos(const std::string &tableName, int32_t offset);
140 bool ParseResultSet(const std::string &tableName, const std::shared_ptr<NativeRdb::ResultSet> &resultSet,
141 FileInfo &fileInfo);
142 void InsertAudio(std::vector<FileInfo> &fileInfos);
143 int32_t QueryTotalNumberByMediaType(std::shared_ptr<NativeRdb::RdbStore> rdbStore, const std::string &tableName,
144 MediaType mediaType);
145 size_t StatClonetotalSize(std::shared_ptr<NativeRdb::RdbStore> mediaRdb);
146 std::string GetBackupInfoByCount(int32_t photoCount, int32_t videoCount, int32_t audioCount, size_t totalSize);
147 void MoveMigrateFile(std::vector<FileInfo> &fileInfos, int64_t &fileMoveCount, int64_t &videoFileMoveCount);
148 void RestorePhotoBatch(int32_t offset, int32_t isRelatedToPhotoMap = 0);
149 void RestoreBatchForCloud(int32_t offset, int32_t isRelatedToPhotoMap = 0);
150 void RestoreAudioBatch(int32_t offset);
151 void InsertPhotoRelated(std::vector<FileInfo> &fileInfos, int32_t sourceType) override;
152 void SetFileIdReference(const std::vector<FileInfo> &fileInfos, std::string &selection,
153 std::unordered_map<int32_t, int32_t> &fileIdMap);
154 int32_t QueryMapTotalNumber(const std::string &baseQuerySql);
155 std::vector<MapInfo> QueryMapInfos(const std::string &tableName, const std::string &baseQuerySql, int32_t offset,
156 const std::unordered_map<int32_t, int32_t> &fileIdMap, const std::unordered_map<int32_t, int32_t> &albumIdMap);
157 int64_t InsertMapByTable(const std::string &tableName, const std::vector<MapInfo> &mapInfos,
158 std::unordered_set<int32_t> &albumSet);
159 std::vector<NativeRdb::ValuesBucket> GetInsertValues(const std::vector<MapInfo> &mapInfos);
160 std::string GetQueryWhereClauseByTable(const std::string &tableName);
161 void SetSpecialAttributes(const std::string &tableName, const std::shared_ptr<NativeRdb::ResultSet> &resultSet,
162 FileInfo &fileInfo);
163 bool IsSameFileForClone(const std::string &tableName, FileInfo &fileInfo);
164 NativeRdb::ValuesBucket GetInsertValue(const AnalysisAlbumTbl &portraitAlbumInfo);
165 int32_t InsertPortraitAlbumByTable(std::vector<AnalysisAlbumTbl> &analysisAlbumTbl);
166 void InsertPortraitAlbum(std::vector<AnalysisAlbumTbl> &analysisAlbumTbl, int64_t maxAlbumId);
167 void ParsePortraitAlbumResultSet(const std::shared_ptr<NativeRdb::ResultSet> &resultSet,
168 AnalysisAlbumTbl &analysisAlbumTbl);
169 std::vector<AnalysisAlbumTbl> QueryPortraitAlbumTbl(int32_t offset,
170 const std::vector<std::string>& commonColumns);
171 void RestoreFromGalleryPortraitAlbum();
172 int32_t QueryPortraitAlbumTotalNumber(std::shared_ptr<NativeRdb::RdbStore> rdbPtr, std::string query);
173 std::unordered_map<std::string, std::string> CreateImgFaceColumnFieldMap();
174 void ParseImageFaceResultSet(const std::shared_ptr<NativeRdb::ResultSet> &resultSet, ImageFaceTbl &imageFaceTbl);
175 void ParseFaceTagResultSet(const std::shared_ptr<NativeRdb::ResultSet> &resultSet, FaceTagTbl &faceTagTbl);
176 NativeRdb::ValuesBucket CreateValuesBucketFromImageFaceTbl(const ImageFaceTbl& imageFaceTbl);
177 void BatchInsertImageFaces(const std::vector<ImageFaceTbl>& imageFaceTbls);
178 std::vector<ImageFaceTbl> ProcessImageFaceTbls(const std::vector<ImageFaceTbl>& imageFaceTbls,
179 const std::vector<FileIdPair>& fileIdPairs);
180 std::vector<ImageFaceTbl> QueryImageFaceTbl(int32_t offset, std::string &fileIdClause,
181 const std::vector<std::string>& commonColumns);
182 std::vector<PortraitAlbumDfx> QueryAllPortraitAlbum(int32_t& offset, int32_t& rowCount);
183 void RecordOldPortraitAlbumDfx();
184 std::unordered_set<std::string> QueryAllPortraitAlbum();
185 void LogPortraitCloneDfx();
186 void RestoreImageFaceInfo(std::vector<FileInfo> &fileInfos);
187 NativeRdb::ValuesBucket CreateValuesBucketFromFaceTagTbl(const FaceTagTbl& faceTagTbl);
188 void BatchInsertFaceTags(const std::vector<FaceTagTbl>& faceTagTbls);
189 void DeleteExistingFaceTagData(const std::string& inClause);
190 std::vector<FaceTagTbl> QueryFaceTagTbl(int32_t offset, const std::string& inClause);
191 void RestorePortraitClusteringInfo();
192 void ReportPortraitCloneStat(int32_t sceneCode);
193 void AppendExtraWhereClause(std::string& whereClause, const std::string& tableName);
194 void GenNewCoverUris(const std::vector<CoverUriInfo>& coverUriInfo,
195 std::vector<FileInfo> &fileInfos);
196 bool GetFileInfoByFileId(int32_t fileId, const std::vector<FileInfo>& fileInfos, FileInfo& outFileInfo);
197 std::string GenCoverUriUpdateSql(const std::unordered_map<std::string, std::pair<std::string, int32_t>>&
198 tagIdToCoverInfo, const std::unordered_map<std::string, int32_t>& oldToNewFileId,
199 const std::vector<FileInfo>& fileInfos, std::vector<std::string>& tagIds);
200 std::string ProcessUriAndGenNew(const std::string& tagId, const std::string& oldCoverUri,
201 const std::unordered_map<std::string, int32_t>& oldToNewFileId, const std::vector<FileInfo>& fileInfos);
202 int32_t MovePicture(FileInfo &fileInfo);
203 int32_t MoveMovingPhotoVideo(FileInfo &fileInfo);
204 int32_t MoveEditedData(FileInfo &fileInfo);
205 int32_t MoveThumbnail(FileInfo &fileInfo);
206 int32_t MoveThumbnailDir(FileInfo &fileInfo);
207 int32_t MoveCloudThumbnailDir(FileInfo &fileInfo);
208 int32_t MoveAstc(FileInfo &fileInfo);
209 void InitThumbnailStatus();
210 bool InitAllKvStore();
211 void CloseAllKvStore();
212 bool BackupKvStore();
213 void GetThumbnailInsertValue(const FileInfo &fileInfo, NativeRdb::ValuesBucket &values);
214 void GetCloudThumbnailInsertValue(const FileInfo &fileInfo, NativeRdb::ValuesBucket &values);
215 int32_t GetNoNeedMigrateCount() override;
216 void GetAccountValid() override;
217 int32_t GetHighlightCloudMediaCnt();
218 void RestoreHighlightAlbums();
219 void AddToPhotosFailedOffsets(int32_t offset);
220 void ProcessPhotosBatchFailedOffsets(int32_t isRelatedToPhotoMap = 0);
221 void ProcessCloudPhotosFailedOffsets(int32_t isRelatedToPhotoMap = 0);
222 void RestoreAnalysisTablesData();
223 void RestoreAnalysisData();
224 void RestoreSearchIndexData();
225 void RestoreAnalysisClassify();
226 void RestoreAnalysisGeo();
227 void RestoreBeautyScoreData();
228 void RestoreVideoFaceData();
229 void PrepareShootingModeVal(const FileInfo &fileInfo, NativeRdb::ValuesBucket &values);
230 void GetInsertValueFromValMap(const FileInfo &fileInfo, NativeRdb::ValuesBucket &values);
231
232 template<typename T>
233 static void PutIfPresent(NativeRdb::ValuesBucket& values, const std::string& columnName,
234 const std::optional<T>& optionalValue);
235
236 template<typename T, typename U>
237 void PutWithDefault(NativeRdb::ValuesBucket& values, const std::string& columnName,
238 const std::optional<T>& optionalValue, const U& defaultValue);
239 std::string GetThumbnailLocalPath(const string path);
240 void BatchUpdateFileInfoData(std::vector<FileInfo> &fileInfos,
241 unordered_map<string, CloudPhotoFileExistFlag> &resultExistMap);
242 int32_t CheckThumbReady(const FileInfo &fileInfo,
243 const CloudPhotoFileExistFlag &cloudPhotoFileExistFlag);
244 int32_t CheckThumbStatus(const FileInfo &fileInfo,
245 const CloudPhotoFileExistFlag &cloudPhotoFileExistFlag);
246 int32_t CheckLcdVisitTime(const CloudPhotoFileExistFlag &cloudPhotoFileExistFlag);
247 void AddToPhotoInfoMaps(std::vector<FileInfo> &fileInfos);
248 void GetOrientationAndExifRotateValue(const shared_ptr<NativeRdb::ResultSet> &resultSet, FileInfo &fileInfo);
249 void UpdateExistNewAddColumnSet(const std::unordered_map<string, string> &srcColumnInfoMap);
250
251 private:
252 std::atomic<uint64_t> migrateDatabaseAlbumNumber_{0};
253 std::atomic<uint64_t> migrateDatabaseMapNumber_{0};
254 std::shared_ptr<NativeRdb::RdbStore> mediaRdb_;
255 std::string filePath_;
256 std::string dbPath_;
257 std::unordered_map<std::string, bool> tableColumnStatusMap_;
258 std::unordered_map<std::string, std::string> tableQueryWhereClauseMap_;
259 std::unordered_map<std::string, std::string> tableExtraQueryWhereClauseMap_;
260 std::unordered_map<std::string, std::unordered_map<int32_t, int32_t>> tableAlbumIdMap_;
261 std::unordered_map<std::string, std::unordered_map<std::string, std::string>> tableCommonColumnInfoMap_;
262 std::string garbagePath_;
263 std::vector<CoverUriInfo> coverUriInfo_;
264 std::vector<PortraitAlbumDfx> portraitAlbumDfx_;
265 PhotoAlbumClone photoAlbumClone_;
266 PhotosClone photosClone_;
267 static constexpr int32_t INVALID_COVER_SATISFIED_STATUS = 0;
268 bool hasCloneThumbnailDir_{false};
269 bool isInitKvstoreSuccess_{false};
270 std::shared_ptr<MediaLibraryKvStore> oldMonthKvStorePtr_ = nullptr;
271 std::shared_ptr<MediaLibraryKvStore> oldYearKvStorePtr_ = nullptr;
272 std::shared_ptr<MediaLibraryKvStore> newMonthKvStorePtr_ = nullptr;
273 std::shared_ptr<MediaLibraryKvStore> newYearKvStorePtr_ = nullptr;
274 std::vector<int> photosFailedOffsets_;
275 ffrt::mutex photosFailedMutex_;
276 std::atomic<uint64_t> lcdMigrateFileNumber_{0};
277 std::atomic<uint64_t> thumbMigrateFileNumber_{0};
278 std::atomic<uint64_t> migrateCloudSuccessNumber_{0};
279 CloneRestoreGeoDictionary cloneRestoreGeoDictionary_;
280 CloneRestoreAnalysisData cloneRestoreAnalysisData_;
281 int64_t maxSearchId_ {0};
282 int64_t maxAnalysisAlbumId_ {0};
283 int64_t maxBeautyFileId_ {0};
284 std::unordered_map<int32_t, PhotoInfo> photoInfoMap_;
285 std::unordered_set<std::string> existNewAddColumnSet_;
286 };
287
288 template<typename T>
PutIfPresent(NativeRdb::ValuesBucket & values,const std::string & columnName,const std::optional<T> & optionalValue)289 void CloneRestore::PutIfPresent(NativeRdb::ValuesBucket& values, const std::string& columnName,
290 const std::optional<T>& optionalValue)
291 {
292 if (optionalValue.has_value()) {
293 if constexpr (std::is_same_v<std::decay_t<T>, int32_t>) {
294 values.PutInt(columnName, optionalValue.value());
295 } else if constexpr (std::is_same_v<std::decay_t<T>, int64_t>) {
296 values.PutLong(columnName, optionalValue.value());
297 } else if constexpr (std::is_same_v<std::decay_t<T>, std::string>) {
298 values.PutString(columnName, optionalValue.value());
299 } else if constexpr (std::is_same_v<std::decay_t<T>, double>) {
300 values.PutDouble(columnName, optionalValue.value());
301 } else if constexpr (std::is_same_v<std::decay_t<T>, std::vector<uint8_t>>) {
302 values.PutBlob(columnName, optionalValue.value());
303 }
304 }
305 }
306
307 template<typename T, typename U>
PutWithDefault(NativeRdb::ValuesBucket & values,const std::string & columnName,const std::optional<T> & optionalValue,const U & defaultValue)308 void CloneRestore::PutWithDefault(NativeRdb::ValuesBucket& values, const std::string& columnName,
309 const std::optional<T>& optionalValue, const U& defaultValue)
310 {
311 if (optionalValue.has_value()) {
312 PutIfPresent(values, columnName, optionalValue);
313 } else {
314 PutIfPresent(values, columnName, std::optional<T>(static_cast<T>(defaultValue)));
315 }
316 }
317 } // namespace Media
318 } // namespace OHOS
319
320 #endif // OHOS_MEDIA_CLONE_RESTORE_H
321