1 /* 2 * Copyright (c) 2023 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef CLOUD_STORAGE_UTILS_H 17 #define CLOUD_STORAGE_UTILS_H 18 19 #include "cloud/asset_operation_utils.h" 20 #include "cloud/cloud_store_types.h" 21 #include "cloud/cloud_upload_recorder.h" 22 #include "icloud_sync_storage_interface.h" 23 #include "sqlite_single_ver_relational_storage_executor.h" 24 #include "sqlite_utils.h" 25 26 namespace DistributedDB { 27 class CloudStorageUtils final { 28 public: 29 static int BindInt64(int index, const VBucket &vBucket, const Field &field, sqlite3_stmt *upsertStmt); 30 static int BindBool(int index, const VBucket &vBucket, const Field &field, sqlite3_stmt *upsertStmt); 31 static int BindDouble(int index, const VBucket &vBucket, const Field &field, sqlite3_stmt *upsertStmt); 32 static int BindText(int index, const VBucket &vBucket, const Field &field, sqlite3_stmt *upsertStmt); 33 static int BindBlob(int index, const VBucket &vBucket, const Field &field, sqlite3_stmt *upsertStmt); 34 static int BindAsset(int index, const VBucket &vBucket, const Field &field, sqlite3_stmt *upsertStmt); 35 IsFieldValid(const Field & field,int errCode)36 static inline bool IsFieldValid(const Field &field, int errCode) 37 { 38 return (errCode == E_OK || (field.nullable && errCode == -E_NOT_FOUND)); 39 } 40 41 static int Int64ToVector(const VBucket &vBucket, const Field &field, CollateType collateType, 42 std::vector<uint8_t> &value); 43 static int BoolToVector(const VBucket &vBucket, const Field &field, CollateType collateType, 44 std::vector<uint8_t> &value); 45 static int DoubleToVector(const VBucket &vBucket, const Field &field, CollateType collateType, 46 std::vector<uint8_t> &value); 47 static int TextToVector(const VBucket &vBucket, const Field &field, CollateType collateType, 48 std::vector<uint8_t> &value); 49 static int BlobToVector(const VBucket &vBucket, const Field &field, CollateType collateType, 50 std::vector<uint8_t> &value); 51 52 static std::set<std::string> GetCloudPrimaryKey(const TableSchema &tableSchema); 53 static std::vector<Field> GetCloudPrimaryKeyField(const TableSchema &tableSchema, bool sortByName = false); 54 static std::map<std::string, Field> GetCloudPrimaryKeyFieldMap(const TableSchema &tableSchema, 55 bool sortByUpper = false); 56 static bool IsContainsPrimaryKey(const TableSchema &tableSchema); 57 static std::vector<Field> GetCloudAsset(const TableSchema &tableSchema); 58 static int GetAssetFieldsFromSchema(const TableSchema &tableSchema, const VBucket &vBucket, 59 std::vector<Field> &fields); 60 static void ObtainAssetFromVBucket(const VBucket &vBucket, VBucket &asset); 61 static AssetOpType StatusToFlag(AssetStatus status); 62 static AssetStatus FlagToStatus(AssetOpType opType); 63 static void ChangeAssetsOnVBucketToAsset(VBucket &vBucket, std::vector<Field> &fields); 64 static Type GetAssetFromAssets(Type &value); 65 static int FillAssetBeforeDownload(Asset &asset); 66 static int FillAssetAfterDownloadFail(Asset &asset, Asset &dbAsset, AssetOperationUtils::AssetOpType assetOpType); 67 static void FillAssetsAfterDownloadFail(Assets &assets, Assets &dbAssets, 68 const std::map<std::string, AssetOperationUtils::AssetOpType> &assetOpTypeMap); 69 static void MergeAssetWithFillFunc(Assets &assets, Assets &dbAssets, const std::map<std::string, 70 AssetOperationUtils::AssetOpType> &assetOpTypeMap, 71 std::function<int(Asset &, Asset &, AssetOperationUtils::AssetOpType)> fillAsset); 72 static int FillAssetAfterDownload(Asset &asset, Asset &dbAsset, AssetOperationUtils::AssetOpType assetOpType); 73 static void FillAssetsAfterDownload(Assets &assets, Assets &dbAssets, 74 const std::map<std::string, AssetOperationUtils::AssetOpType> &assetOpTypeMap); 75 static int FillAssetBeforeUpload(Asset &asset, Asset &dbAsset, AssetOperationUtils::AssetOpType assetOpType); 76 static void FillAssetsBeforeUpload(Assets &assets, Assets &dbAssets, 77 const std::map<std::string, AssetOperationUtils::AssetOpType> &assetOpTypeMap); 78 static int FillAssetForUpload(Asset &asset, Asset &dbAsset, AssetOperationUtils::AssetOpType assetOpType); 79 static void FillAssetsForUpload(Assets &assets, Assets &dbAssets, 80 const std::map<std::string, AssetOperationUtils::AssetOpType> &assetOpTypeMap); 81 static int FillAssetForUploadFailed(Asset &asset, Asset &dbAsset, AssetOperationUtils::AssetOpType assetOpType); 82 static void FillAssetsForUploadFailed(Assets &assets, Assets &dbAssets, 83 const std::map<std::string, AssetOperationUtils::AssetOpType> &assetOpTypeMap); 84 static void PrepareToFillAssetFromVBucket(VBucket &vBucket, std::function<int(Asset &)> fillAsset); 85 static void FillAssetFromVBucketFinish(const AssetOperationUtils::RecordAssetOpType &assetOpType, VBucket &vBucket, 86 VBucket &dbAssets, std::function<int(Asset &, Asset &, AssetOperationUtils::AssetOpType)> fillAsset, 87 std::function<void(Assets &, Assets &, 88 const std::map<std::string, AssetOperationUtils::AssetOpType> &)> fillAssets); 89 static bool IsAsset(const Type &type); 90 static bool IsAssets(const Type &type); 91 static bool IsAssetsContainDuplicateAsset(Assets &assets); 92 static void EraseNoChangeAsset(std::map<std::string, Assets> &assetsMap); 93 static void MergeDownloadAsset(std::map<std::string, Assets> &downloadAssets, 94 std::map<std::string, Assets> &mergeAssets); 95 static std::map<std::string, size_t> GenAssetsIndexMap(Assets &assets); 96 static bool IsVbucketContainsAllPK(const VBucket &vBucket, const std::set<std::string> &pkSet); 97 static bool CheckAssetStatus(const Assets &assets); 98 99 static int ConstraintsCheckForCloud(const TableInfo &table, const std::string &trimmedSql); 100 static std::string GetTableRefUpdateSql(const TableInfo &table, OpType opType); 101 static std::string GetLeftJoinLogSql(const std::string &tableName, bool logAsTableA = true); 102 static std::string GetUpdateLockChangedSql(); 103 static std::string GetDeleteLockChangedSql(); 104 static void AddUpdateColForShare(const TableSchema &tableSchema, std::string &updateLogSql, 105 std::vector<std::string> &updateColName); 106 static std::string GetSelectIncCursorSql(const std::string &tableName); 107 static std::string GetCursorIncSql(const std::string &tableName); 108 static std::string GetCursorIncSqlWhenAllow(const std::string &tableName); 109 static std::string GetCursorUpgradeSql(const std::string &tableName); 110 static std::string GetUpdateUploadFinishedSql(const std::string &tableName, bool isExistAssetsDownload); 111 112 static bool IsSharedTable(const TableSchema &tableSchema); 113 static bool ChkFillCloudAssetParam(const CloudSyncBatch &data, int errCode); 114 static void GetToBeRemoveAssets(const VBucket &vBucket, const AssetOperationUtils::RecordAssetOpType &assetOpType, 115 std::vector<Asset> &removeAssets); 116 static std::pair<int, std::vector<uint8_t>> GetHashValueWithPrimaryKeyMap(const VBucket &vBucket, 117 const TableSchema &tableSchema, const TableInfo &localTable, const std::map<std::string, Field> &pkMap, 118 bool allowEmpty); 119 static std::string GetUpdateRecordFlagSql(UpdateRecordFlagStruct updateRecordFlag, const LogInfo &logInfo, 120 const VBucket &uploadExtend = {}, const CloudWaterType &type = CloudWaterType::BUTT); 121 static std::string GetUpdateRecordFlagSqlUpload(const std::string &tableName, bool recordConflict, 122 const LogInfo &logInfo, const VBucket &uploadExtend = {}, const CloudWaterType &type = CloudWaterType::BUTT); 123 static int BindStepConsistentFlagStmt(sqlite3_stmt *stmt, const VBucket &data, 124 const std::set<std::string> &gidFilters); 125 static bool IsCloudGidMismatch(const std::string &downloadGid, const std::string &curGid); 126 127 template<typename T> GetValueFromOneField(Type & cloudValue,T & outVal)128 static int GetValueFromOneField(Type &cloudValue, T &outVal) 129 { 130 T *value = std::get_if<T>(&cloudValue); 131 if (value == nullptr) { 132 LOGE("Get cloud data fail because type mismatch."); 133 return -E_CLOUD_ERROR; 134 } 135 outVal = *value; 136 return E_OK; 137 } 138 139 template<typename T> GetValueFromVBucket(const std::string & fieldName,const VBucket & vBucket,T & outVal)140 static int GetValueFromVBucket(const std::string &fieldName, const VBucket &vBucket, T &outVal) 141 { 142 Type cloudValue; 143 bool isExisted = GetTypeCaseInsensitive(fieldName, vBucket, cloudValue); 144 if (!isExisted) { 145 return -E_NOT_FOUND; 146 } 147 return GetValueFromOneField(cloudValue, outVal); 148 } 149 150 template<typename T> GetValueFromType(Type & cloudValue,T & outVal)151 static int GetValueFromType(Type &cloudValue, T &outVal) 152 { 153 T *value = std::get_if<T>(&cloudValue); 154 if (value == nullptr) { 155 return -E_CLOUD_ERROR; 156 } 157 outVal = *value; 158 return E_OK; 159 } 160 161 static int CalculateHashKeyForOneField(const Field &field, const VBucket &vBucket, bool allowEmpty, 162 CollateType collateType, std::vector<uint8_t> &hashValue); 163 164 static bool CheckCloudSchemaFields(const TableSchema &tableSchema, const TableSchema &oldSchema); 165 166 static void TransferFieldToLower(VBucket &vBucket); 167 168 static bool GetTypeCaseInsensitive(const std::string &fieldName, const VBucket &vBucket, Type &data); 169 170 static int BindUpdateLogStmtFromVBucket(const VBucket &vBucket, const TableSchema &tableSchema, 171 const std::vector<std::string> &colNames, sqlite3_stmt *updateLogStmt); 172 173 static bool IsGetCloudDataContinue(uint32_t curNum, uint32_t curSize, uint32_t maxSize, uint32_t maxCount); 174 175 static int IdentifyCloudType(const CloudUploadRecorder &recorder, CloudSyncData &cloudSyncData, VBucket &data, 176 VBucket &log, VBucket &flags); 177 178 static int CheckAbnormalData(CloudSyncData &cloudSyncData, VBucket &data, bool isInsert, bool isAsyncDownloading); 179 180 static std::pair<int, DataItem> GetDataItemFromCloudData(VBucket &data); 181 182 static int GetBytesFromCloudData(const std::string &field, VBucket &data, Bytes &bytes); 183 184 static int GetStringFromCloudData(const std::string &field, VBucket &data, std::string &str); 185 186 static int GetUInt64FromCloudData(const std::string &field, VBucket &data, uint64_t &number); 187 188 static bool IsDataLocked(uint32_t status); 189 190 static std::pair<int, DataItem> GetDataItemFromCloudVersionData(VBucket &data); 191 192 static std::pair<int, DataItem> GetSystemRecordFromCloudData(VBucket &data); 193 194 static bool IsSystemRecord(const Key &key); 195 196 static int GetSyncQueryByPk(const std::string &tableName, const std::vector<VBucket> &data, bool isKv, 197 std::vector<QuerySyncObject> &syncQuery); 198 199 static bool IsAssetsContainDownloadRecord(const VBucket &dbAssets); 200 201 struct CloudSyncParam { 202 std::string tableName; 203 CloudWaterType type; 204 TableSchema tableSchema; 205 }; 206 207 static int UpdateRecordFlagAfterUpload(SQLiteSingleVerRelationalStorageExecutor *handle, 208 const CloudSyncParam ¶m, const CloudSyncBatch &updateData, CloudUploadRecorder &recorder, 209 bool isLock = false); 210 211 static int FillCloudQueryToExtend(QuerySyncObject &obj, VBucket &extend); 212 213 static void SaveChangedDataByType(const DataValue &dataValue, Type &value); 214 215 private: 216 static int IdentifyCloudTypeInner(CloudSyncData &cloudSyncData, VBucket &data, VBucket &log, VBucket &flags); 217 218 static int FillQueryByPK(const std::string &tableName, bool isKv, std::map<std::string, size_t> dataIndex, 219 std::vector<std::map<std::string, std::vector<Type>>> &syncPkVec, std::vector<QuerySyncObject> &syncQuery); 220 221 static void PutSyncPkVec(const std::string &col, std::map<std::string, std::vector<Type>> &syncPk, 222 std::vector<std::map<std::string, std::vector<Type>>> &syncPkVec); 223 224 static bool IsAssetNotDownload(const uint32_t &status); 225 226 static void CheckAbnormalDataInner(const bool isAsyncDownloading, VBucket &data, bool &isSyncAssetAbnormal, 227 bool &isAsyncAssetAbnormal); 228 }; 229 } 230 #endif // CLOUD_STORAGE_UTILS_H 231