1 /* 2 * Copyright (c) 2021 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 #ifndef DISTRIBUTED_DB_MODULE_TEST_TOOLS_H 16 #define DISTRIBUTED_DB_MODULE_TEST_TOOLS_H 17 #include <condition_variable> 18 #include <thread> 19 #include <vector> 20 21 #include "kv_store_delegate.h" 22 #include "kv_store_delegate_manager.h" 23 #include "kv_store_observer_impl.h" 24 #include "distributed_test_sysinfo.h" 25 #include "distributeddb_data_generator.h" 26 #include "log_print.h" 27 #ifdef RUN_MST_ON_TRUNCK // only need it in WAGNER env if run MST 28 #define HWTEST_F(test_case_name, test_name, level) TEST_F(test_case_name, test_name) 29 #endif 30 #define ULL(x) (static_cast<unsigned long long>(x)) 31 const int MAX_DIR_LENGTH = 4096; // the max length of directory 32 const int BUF_LEN = 8192; 33 const static std::string TAG = "DistributedTestTools"; // for log 34 const int AUTHORITY = 0755; 35 const int E_OK = 0; 36 const int E_ERROR = -1; 37 const std::string DIRECTOR = "/data/test/getstub/"; // default work dir. 38 static std::condition_variable g_conditionKvVar; 39 40 struct KvDBParameters { 41 std::string storeId; 42 std::string appId; 43 std::string userId; KvDBParametersKvDBParameters44 KvDBParameters(std::string storeIdStr, std::string appIdStr, std::string userIdStr) 45 : storeId(storeIdStr), appId(appIdStr), userId(userIdStr) 46 { 47 } 48 }; 49 50 struct KvOption { 51 bool createIfNecessary = true; 52 bool localOnly = false; 53 bool isEncryptedDb = false; // whether need encrypt 54 DistributedDB::CipherType cipher = DistributedDB::CipherType::DEFAULT; // cipher type 55 std::vector<uint8_t> passwd; // cipher password KvOptionKvOption56 KvOption(bool createIfNecessary1, bool isLocalOnly, 57 bool isEncryptedDb1, DistributedDB::CipherType cipher1, std::vector<uint8_t> passwd1) 58 : createIfNecessary(createIfNecessary1), 59 localOnly(isLocalOnly), 60 isEncryptedDb(isEncryptedDb1), 61 cipher(cipher1), passwd(passwd1) 62 { 63 } KvOptionKvOption64 KvOption() 65 {} 66 }; 67 68 struct EncrypteAttribute { 69 bool isEncryptedDb = false; 70 std::vector<uint8_t> passwd; 71 }; 72 73 const static KvDBParameters g_kvdbParameter1(DistributedDBDataGenerator::STORE_ID_1, 74 DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); 75 const static KvDBParameters g_kvdbParameter2(DistributedDBDataGenerator::STORE_ID_2, 76 DistributedDBDataGenerator::APP_ID_2, DistributedDBDataGenerator::USER_ID_2); 77 const static KvDBParameters g_kvdbParameter3(DistributedDBDataGenerator::STORE_ID_3, 78 DistributedDBDataGenerator::APP_ID_3, DistributedDBDataGenerator::USER_ID_3); 79 const static KvDBParameters g_kvdbParameter4(DistributedDBDataGenerator::STORE_ID_4, 80 DistributedDBDataGenerator::APP_ID_4, DistributedDBDataGenerator::USER_ID_4); 81 const static KvDBParameters g_kvdbParameter5(DistributedDBDataGenerator::STORE_ID_5, 82 DistributedDBDataGenerator::APP_ID_5, DistributedDBDataGenerator::USER_ID_5); 83 const static KvDBParameters g_kvdbParameter6(DistributedDBDataGenerator::STORE_ID_6, 84 DistributedDBDataGenerator::APP_ID_6, DistributedDBDataGenerator::USER_ID_6); 85 const static KvDBParameters g_kvdbParameter1_1_2(DistributedDBDataGenerator::STORE_ID_1, 86 DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_2); 87 const static KvDBParameters g_kvdbParameter1_2_1(DistributedDBDataGenerator::STORE_ID_1, 88 DistributedDBDataGenerator::APP_ID_2, DistributedDBDataGenerator::USER_ID_1); 89 const static KvDBParameters g_kvdbParameter1_2_2(DistributedDBDataGenerator::STORE_ID_1, 90 DistributedDBDataGenerator::APP_ID_2, DistributedDBDataGenerator::USER_ID_2); 91 const static KvDBParameters g_kvdbParameter2_1_1(DistributedDBDataGenerator::STORE_ID_2, 92 DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); 93 const static KvDBParameters g_kvdbParameter2_1_2(DistributedDBDataGenerator::STORE_ID_2, 94 DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_2); 95 const static KvDBParameters KVDB_PARAMETER_PERFORM(DistributedDBDataGenerator::STORE_ID_PERFORM, 96 DistributedDBDataGenerator::APP_ID_PERFORM, DistributedDBDataGenerator::USER_ID_PERFORM); 97 const static KvOption g_createKvDiskUnencrypted(true, false, false, DistributedDB::CipherType::DEFAULT, 98 DistributedDBDataGenerator::NULL_PASSWD_VECTOR); 99 const static KvOption g_createKvDiskEncrypted(true, false, true, DistributedDB::CipherType::DEFAULT, 100 DistributedDBDataGenerator::PASSWD_VECTOR_1); 101 const static KvOption g_createLocalDiskUnencrypted(true, true, false, DistributedDB::CipherType::DEFAULT, 102 DistributedDBDataGenerator::NULL_PASSWD_VECTOR); 103 const static KvOption g_createLocalDiskEncrypted(true, true, true, DistributedDB::CipherType::DEFAULT, 104 DistributedDBDataGenerator::PASSWD_VECTOR_1); 105 static KvOption g_kvOption = g_createKvDiskEncrypted; 106 bool CompareVector(const std::vector<uint8_t>& first, const std::vector<uint8_t>& second); 107 bool CompareEntriesVector(std::vector<DistributedDB::Entry>& retVec, 108 std::vector<DistributedDB::Entry>& expectVec); 109 void PutUniqueKey(std::vector<DistributedDB::Entry>& entryVec, 110 const DistributedDB::Key &putKey, const DistributedDB::Value &putValue); 111 int Uint8VecToString(std::vector<uint8_t>& vec, std::string& str); 112 int GetIntValue(DistributedDB::Value &value); 113 int RemoveDir(const std::string &directory); 114 int SetDir(const std::string &directory, const int authRight = AUTHORITY); 115 void CopyFile(const std::string &srcFile, const std::string &destFile); 116 void CopyDir(const std::string &srcDir, const std::string &destDir, const int authRight = AUTHORITY); 117 void CheckFileNumber(const std::string &filePath, int &fileCount); 118 DistributedDB::Value GetValueWithInt(int val); 119 std::vector<DistributedDB::Entry> GenRanKeyVal(int putGetTimes, int keyLength, int valueLength, char val); 120 std::vector<DistributedDB::Key> GetKeysFromEntries(std::vector<DistributedDB::Entry> entries, bool random); 121 bool GetRandBool(); 122 bool PutEntries(DistributedDB::KvStoreNbDelegate *&delegate, const std::vector<DistributedDB::Entry> &entries); 123 124 using SysTime = std::chrono::time_point<std::chrono::steady_clock, std::chrono::microseconds>; 125 using SysDurTime = std::chrono::duration<uint64_t, std::micro>; 126 127 struct PerformanceData { 128 int putGetTimes; 129 int keyLength; 130 int valueLength; 131 bool putBatch; 132 bool getBatch; 133 bool useClear; 134 bool getSysInfo; 135 bool isLocal; 136 double openDuration; 137 double putDuration; 138 double readPutDuration; 139 double updateDuration; 140 double readUpdateDuration; 141 double deleteDuration; 142 double closeDuration; PerformanceDataPerformanceData143 PerformanceData(int putGetTimes, int keyLength, int valueLength, 144 bool putBatch, bool getBatch, bool useClear, bool getSysInfo, bool isLocal) 145 : putGetTimes(putGetTimes), keyLength(keyLength), valueLength(valueLength), 146 putBatch(putBatch), getBatch(getBatch), useClear(useClear), 147 getSysInfo(getSysInfo), isLocal(isLocal), 148 openDuration(0.0), putDuration(0.0), readPutDuration(0.0), updateDuration(0.0), 149 readUpdateDuration(0.0), deleteDuration(0.0), closeDuration(0.0) 150 { 151 } 152 }; 153 154 struct Duration { 155 double putDuration = 0.0; 156 double readDuration = 0.0; 157 double updateDuration = 0.0; 158 double deleteDuration = 0.0; 159 Duration() = default; ClearDuration160 void Clear() 161 { 162 putDuration = readDuration = updateDuration = deleteDuration = 0.0; 163 } 164 }; 165 166 struct BackupDuration { 167 double exportDuration = 0.0; 168 double importDuration = 0.0; 169 BackupDuration() = default; BackupDurationBackupDuration170 BackupDuration(double exportDur, double importDur) 171 { 172 exportDuration = exportDur; 173 importDuration = importDur; 174 } 175 BackupDuration operator+(const BackupDuration &backupDur) const 176 { 177 return BackupDuration(exportDuration + backupDur.exportDuration, importDuration + backupDur.importDuration); 178 } ClearBackupDuration179 void Clear() 180 { 181 exportDuration = importDuration = 0; 182 } 183 }; 184 185 enum KvDbType { 186 ENCRYED = 0, 187 UNENCRYED = 1 188 }; 189 190 enum OperRecordNum { 191 SINGLE = 1, 192 SMALL_BATCH = 100, 193 BATCH = 128, 194 }; 195 196 enum class OperType { 197 PUT, 198 PUT_LOCAL, 199 UPDATE, 200 UPDATE_LOCAL, 201 DELETE, 202 DELETE_LOCAL 203 }; 204 205 struct KvPerfData { 206 KvDbType kvDbType; 207 int testCnt; 208 OperRecordNum operRecordNum; 209 int keyLength; 210 int valueLength; 211 bool isPresetRecords; 212 int presetRecordsCnt; 213 std::vector<Duration> allCrudDur; KvPerfDataKvPerfData214 KvPerfData(KvDbType kvDbType, int testCnt, OperRecordNum operRecordNum, int keyLength, int valueLength, 215 bool isPresetRecords, int presetRecordsCnt) 216 : kvDbType(kvDbType), testCnt(testCnt), operRecordNum(operRecordNum), 217 keyLength(keyLength), valueLength(valueLength), 218 isPresetRecords(isPresetRecords), presetRecordsCnt(presetRecordsCnt) 219 { 220 } 221 }; 222 223 struct RekeyTypesDur { 224 double nullPasswdToPasswd1 = 0.0; 225 double passwd1ToPasswd1 = 0.0; 226 double passwd1ToPasswd2 = 0.0; 227 double passwd2ToNullPasswd = 0.0; 228 RekeyTypesDur() = default; RekeyTypesDurRekeyTypesDur229 RekeyTypesDur(double nullPasswdToPasswd1, double passwd1ToPasswd1, 230 double passwd1ToPasswd2, double passwd2ToNullPasswd) 231 : nullPasswdToPasswd1(nullPasswdToPasswd1), passwd1ToPasswd1(passwd1ToPasswd1), 232 passwd1ToPasswd2(passwd1ToPasswd2), passwd2ToNullPasswd(passwd2ToNullPasswd) 233 { 234 } 235 RekeyTypesDur operator+(const RekeyTypesDur &rekeyTypesDur) const 236 { 237 return RekeyTypesDur(nullPasswdToPasswd1 + rekeyTypesDur.nullPasswdToPasswd1, 238 passwd1ToPasswd1 + rekeyTypesDur.passwd1ToPasswd1, 239 passwd1ToPasswd2 + rekeyTypesDur.passwd1ToPasswd2, 240 passwd2ToNullPasswd + rekeyTypesDur.passwd2ToNullPasswd); 241 } 242 }; 243 244 struct KvRekeyPerfData { 245 int testCnt; 246 int presetRecordsCnt; 247 int keyLength; 248 int valueLength; 249 std::vector<RekeyTypesDur> allRekeyDur; KvRekeyPerfDataKvRekeyPerfData250 KvRekeyPerfData(int testCnt, int presetRecordsCnt, int keyLength, int valueLength) 251 : testCnt(testCnt), presetRecordsCnt(presetRecordsCnt), keyLength(keyLength), valueLength(valueLength) 252 { 253 } 254 }; 255 256 struct KvBackupPerfData { 257 KvDbType kvDbType; 258 int testCnt; 259 int presetRecordsCnt; 260 int keyLength; 261 int valueLength; 262 std::vector<BackupDuration> allBackupDur; KvBackupPerfDataKvBackupPerfData263 KvBackupPerfData(KvDbType kvDbType, int testCnt, int presetRecordsCnt, int keyLength, int valueLength) 264 : kvDbType(kvDbType), testCnt(testCnt), presetRecordsCnt(presetRecordsCnt), 265 keyLength(keyLength), valueLength(valueLength) 266 { 267 } 268 }; 269 270 struct PerImageGallery { 271 uint64_t putDuration = 0; 272 uint64_t readDuration = 0; ClearPerImageGallery273 void Clear() 274 { 275 putDuration = 0; 276 readDuration = 0; 277 } 278 }; 279 struct NbGalleryPerfData { 280 int testCnt; 281 unsigned int keyLength; 282 unsigned int valueLength; 283 bool isLocal; 284 bool isPresetRecords; 285 int presetRecordsCnt; 286 std::vector<PerImageGallery> allCrudDur; NbGalleryPerfDataNbGalleryPerfData287 NbGalleryPerfData(int testCnt, int keyLength, int valueLength, 288 bool isLocal, bool isPresetRecords, int presetRecordsCnt) 289 : testCnt(testCnt), keyLength(keyLength), valueLength(valueLength), 290 isLocal(isLocal), isPresetRecords(isPresetRecords), presetRecordsCnt(presetRecordsCnt) 291 { 292 } 293 }; 294 struct NbLocalPerfData { 295 int testCnt; 296 unsigned int keyLength; 297 unsigned int valueLength; 298 bool isLocal; 299 bool isPresetRecords; 300 int presetRecordsCnt; 301 std::vector<Duration> allCrudDur; NbLocalPerfDataNbLocalPerfData302 NbLocalPerfData(int testCnt, int keyLength, int valueLength, 303 bool isLocal, bool isPresetRecords, int presetRecordsCnt) 304 : testCnt(testCnt), keyLength(keyLength), valueLength(valueLength), 305 isLocal(isLocal), isPresetRecords(isPresetRecords), presetRecordsCnt(presetRecordsCnt) 306 { 307 } 308 }; 309 310 struct QueryDur { 311 double getEntriesDuration = 0.0; 312 double getResultSetDuration = 0.0; 313 QueryDur() = default; ClearQueryDur314 void Clear() 315 { 316 getEntriesDuration = getResultSetDuration = 0; 317 } 318 }; 319 struct NbSchemaCRUDPerfData { 320 int testCnt; 321 OperRecordNum operRecordNum; 322 unsigned int keyLength; 323 unsigned int valueLength; 324 bool isLocal; 325 bool isPresetRecords; 326 unsigned int presetRecordsCnt; 327 bool isQueryNeeded; 328 bool isIndexSchema; 329 std::vector<Duration> allCrudDur; 330 std::vector<QueryDur> allQueryDur; NbSchemaCRUDPerfDataNbSchemaCRUDPerfData331 NbSchemaCRUDPerfData(int testCnt, OperRecordNum operRecordNum, int keyLength, int valueLength, 332 bool isLocal, bool isPresetRecords, int presetRecordsCnt, int isQueryNeeded, int isIndexSchema) 333 : testCnt(testCnt), operRecordNum(operRecordNum), keyLength(keyLength), valueLength(valueLength), 334 isLocal(isLocal), isPresetRecords(isPresetRecords), presetRecordsCnt(presetRecordsCnt), 335 isQueryNeeded(isQueryNeeded), isIndexSchema(isIndexSchema) 336 { 337 } 338 }; 339 340 // default kvStoreDelegateManager's config. 341 const DistributedDB::KvStoreConfig KV_CONFIG = { 342 .dataDir = DIRECTOR 343 }; 344 345 class DistributedTestTools final { 346 public: DistributedTestTools()347 DistributedTestTools() {} ~DistributedTestTools()348 ~DistributedTestTools() {} 349 350 // Delete the copy and assign constructors 351 DistributedTestTools(const DistributedTestTools &distributeDBTools) = delete; 352 DistributedTestTools& operator=(const DistributedTestTools &distributeDBTools) = delete; 353 DistributedTestTools(DistributedTestTools &&distributeDBTools) = delete; 354 DistributedTestTools& operator=(DistributedTestTools &&distributeDBTools) = delete; 355 356 // this static method is to compare if the two Value has the same data. 357 static bool IsValueEquals(const DistributedDB::Value &v1, const DistributedDB::Value &v2); 358 static DistributedDB::KvStoreDelegate::Option TransferKvOptionType(const KvOption &optionParam); 359 static DistributedDB::KvStoreDelegate* GetDelegateSuccess(DistributedDB::KvStoreDelegateManager *&outManager, 360 const KvDBParameters ¶m, const KvOption &optionParam); 361 static DistributedDB::KvStoreDelegate* GetDelegateStatus(DistributedDB::KvStoreDelegateManager *&outManager, 362 DistributedDB::DBStatus &status, const KvDBParameters ¶m, const KvOption &optionParam); 363 364 static DistributedDB::DBStatus GetDelegateNotGood( 365 DistributedDB::KvStoreDelegateManager *&outManager, DistributedDB::KvStoreDelegate *&outDelegate, 366 const std::string &storeId, const std::string &appId, const std::string &userId, const KvOption &optionParam); 367 368 static DistributedDB::DBStatus Put(DistributedDB::KvStoreDelegate &kvStoreDelegate, 369 const DistributedDB::Key &key, const DistributedDB::Value &value); 370 371 static DistributedDB::DBStatus PutBatch(DistributedDB::KvStoreDelegate &kvStoreDelegate, 372 const std::vector<DistributedDB::Entry> &entries); 373 374 static DistributedDB::DBStatus Delete(DistributedDB::KvStoreDelegate &kvStoreDelegate, 375 const DistributedDB::Key &key); 376 377 static DistributedDB::DBStatus DeleteBatch(DistributedDB::KvStoreDelegate &kvStoreDelegate, 378 const std::vector<DistributedDB::Key> &keys); 379 380 static DistributedDB::DBStatus Clear(DistributedDB::KvStoreDelegate &kvStoreDelegate); 381 382 static DistributedDB::KvStoreSnapshotDelegate *GetKvStoreSnapshot(DistributedDB::KvStoreDelegate &kvStoreDelegate); 383 static DistributedDB::Value Get(DistributedDB::KvStoreDelegate &kvStoreDelegate, const DistributedDB::Key &key); 384 static DistributedDB::Value Get(DistributedDB::KvStoreSnapshotDelegate &kvStoreSnapshotDelegate, 385 const DistributedDB::Key &key); 386 387 static std::vector<DistributedDB::Entry> GetEntries( 388 DistributedDB::KvStoreSnapshotDelegate &kvStoreSnapshotDelegate, const DistributedDB::Key &key); 389 static std::vector<DistributedDB::Entry> GetEntries(DistributedDB::KvStoreDelegate &kvStoreDelegate, 390 const DistributedDB::Key &keyPrefix); 391 392 static DistributedDB::KvStoreSnapshotDelegate *RegisterSnapObserver(DistributedDB::KvStoreDelegate *delegate, 393 DistributedDB::KvStoreObserver *observer); 394 static DistributedDB::DBStatus RegisterObserver(DistributedDB::KvStoreDelegate *delegate, 395 DistributedDB::KvStoreObserver *observer); 396 static DistributedDB::DBStatus UnRegisterObserver(DistributedDB::KvStoreDelegate *delegate, 397 DistributedDB::KvStoreObserver *observer); 398 static bool CalculateOpenPerformance(PerformanceData &performanceData); 399 static bool CalculateInsertPerformance(PerformanceData &performanceData); 400 static bool CalculateGetPutPerformance(PerformanceData &performanceData); 401 static bool CalculateUpdatePerformance(PerformanceData &performanceData); 402 static bool CalculateGetUpdatePerformance(PerformanceData &performanceData); 403 static bool CalculateUseClearPerformance(PerformanceData &performanceData); 404 static bool CalculateTransactionPerformance(PerformanceData &performanceData); 405 static bool CloseAndRelease(DistributedDB::KvStoreDelegateManager *&manager, 406 DistributedDB::KvStoreDelegate *&delegate); 407 static bool CloseAndRelease(DistributedDB::KvStoreDelegateManager *&manager, 408 DistributedDB::KvStoreNbDelegate *&delegate); 409 static bool VerifyDbRecordCnt(DistributedDB::KvStoreNbDelegate *&delegate, unsigned int recordCnt, 410 bool isLocal = false); 411 static bool VerifyRecordsInDb(DistributedDB::KvStoreNbDelegate *&delegate, 412 std::vector<DistributedDB::Entry> &entriesExpected, 413 const std::vector<uint8_t> &keyPrefix = DistributedDBDataGenerator::KEY_EMPTY, bool isLocal = false); 414 static bool GetRecordCntByKey(const std::string &dbName, 415 const std::string &strSql, std::vector<DistributedDB::Key> &sqlParam, KvOption &option, int &count); 416 static bool QuerySpecifiedData(const std::string &dbName, const std::string &strSql, 417 EncrypteAttribute &attribute, int &count); 418 static bool RepeatCheckAsyncResult(const std::function<bool(void)> &inPred, int repeatLimit, 419 uint32_t repeatInterval); 420 static bool CompareKey(const DistributedDB::Entry &entry1, const DistributedDB::Entry &entry2); 421 static void CopyFile(const std::string &srcFile, const std::string &destFile); 422 }; 423 424 std::string TransferStringToHashHexString(const std::string &origStr); 425 426 int RemoveDatabaseDirectory(const std::string &directory); 427 428 bool VerifyObserverResult(const KvStoreObserverImpl &pObserver, 429 int changedTimes, ListType type, const std::list<DistributedDB::Entry> &lst, 430 uint32_t timeout = DistributedDBDataGenerator::DistributedDBConstant::THIRTY_MINUTES); 431 bool VerifyObserverResult(const KvStoreObserverImpl &pObserver, 432 int changedTimes, ListType type, const std::vector<DistributedDB::Entry> &vec, 433 uint32_t timeout = DistributedDBDataGenerator::DistributedDBConstant::THIRTY_MINUTES); 434 bool VerifyObserverForSchema(const KvStoreObserverImpl &pObserver, 435 int changedTimes, ListType type, const std::vector<DistributedDB::Entry> &expectEntry, 436 uint32_t timeout = DistributedDBDataGenerator::DistributedDBConstant::THIRTY_MINUTES); 437 #endif // DISTRIBUTED_DB_MODULE_TEST_TOOLS_H 438