1From c5761b4ffbe2376f9f7ee1c67f139ecd214dd364 Mon Sep 17 00:00:00 2001 2From: MartinChoo <214582617@qq.com> 3Date: Thu, 14 Aug 2025 19:51:56 +0800 4Subject: [PATCH] Bugfix on current version 5 6--- 7 ext/misc/cksumvfs.c | 11 +- 8 src/compressvfs.c | 270 ++++++++++++++++++++++++------------ 9 src/sqlite3.c | 327 ++++++++++++++++++++++++++++++++++++-------- 10 3 files changed, 458 insertions(+), 150 deletions(-) 11 12diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c 13index 27b1028..e89edcd 100644 14--- a/ext/misc/cksumvfs.c 15+++ b/ext/misc/cksumvfs.c 16@@ -559,17 +559,17 @@ static int cksmRead( 17 ** (1) the size indicates that we are dealing with a complete 18 ** database page, only support pageSize:4K 19 ** (2) checksum verification is enabled 20- ** (3) we are not in the middle of checkpoint 21+ ** (3) Skip the WAL log write, it might write segment as iSyncPoint, do checksum at checkpoint 22 ** (4) magic number should be 0xff 23 ** (5) checksum type should be 1, 0 means without checksum 24 */ 25- if( iAmt==4096 /* (1) */ 26+ if( iAmt==4096 /* (1) */ 27 && p->verifyCksm /* (2) */ 28- && !p->inCkpt /* (3) */ 29+ && !p->isWal /* (3) */ 30 ){ 31 if( ((u8*)zBuf)[iAmt-CKSUMVFS_RESERVED_SIZE]!=CKSUMVFS_MAGIC_NUM ){ /* (4) */ 32 sqlite3_log(SQLITE_IOERR_DATA, "unrecognized format, offset %lld of \"%s\", amt:%d", iOfst, p->zFName, iAmt); 33- return SQLITE_IOERR_DATA; 34+ return rc; 35 } 36 if( ((u8*)zBuf)[iAmt-CKSUMVFS_RESERVED_SIZE+1]==CKSUMVFS_WITHOUT_CHECKSUM ){ /* (5) */ 37 return rc; 38@@ -584,7 +584,6 @@ static int cksmRead( 39 EncodeReservedBytesIntoBase16(cksum, CKSUMVFS_CHECKSUM_SIZE, actual, CKSUM_HEX_LEN); 40 sqlite3_log(SQLITE_IOERR_DATA, "checksum fault offset %lld of \"%s\", amt:%d, expect:%s, actual:%s", 41 iOfst, p->zFName, iAmt, expect, actual); 42- rc = SQLITE_IOERR_DATA; 43 } 44 } 45 } 46@@ -617,7 +616,7 @@ static int cksmWrite( 47 */ 48 if( iAmt==4096 49 && p->computeCksm 50- && !p->inCkpt 51+ && !p->isWal 52 ){ 53 ((u8*)zBuf)[iAmt-CKSUMVFS_RESERVED_SIZE]=CKSUMVFS_MAGIC_NUM; 54 ((u8*)zBuf)[iAmt-CKSUMVFS_RESERVED_SIZE+1]=p->verifyCksm ? CKSUMVFS_CALC_CHECKSUM : CKSUMVFS_WITHOUT_CHECKSUM; 55diff --git a/src/compressvfs.c b/src/compressvfs.c 56index f2fe169..5f5ec20 100644 57--- a/src/compressvfs.c 58+++ b/src/compressvfs.c 59@@ -151,6 +151,7 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ 60 typedef u32 Pgno; 61 /* VFS's name */ 62 #define COMPRESS_VFS_NAME "compressvfs" 63+#define CKSM_VFS_NAME "cksmvfs" 64 65 /* COMPRESSION OPTIONS */ 66 #define COMPRESSION_UNDEFINED 0 67@@ -162,6 +163,8 @@ typedef u32 Pgno; 68 #define SQLITE_SHMMAP_IS_WRITE 0x00000001 /* Flag for xShmMap, extend file if necessary */ 69 #define SQLITE_OPEN_COMPRESS_SHM 0x00010000 /* Flag for xShmMap, need to rename shm file */ 70 71+#define SQLITE_WARNING_DUMP (SQLITE_WARNING | (2<<8)) 72+ 73 /* An open file */ 74 typedef struct{ 75 sqlite3_file base; /* IO methods */ 76@@ -170,8 +173,10 @@ typedef struct{ 77 u8 bSubDbOpen; /* True to SubDB is opened */ 78 u8 bBegin; /* True to xSync() need commit */ 79 u8 compression; /* Compression options */ 80- int page_size; /* Uncompressed page size */ 81+ int pageSize; /* Uncompressed page size */ 82 int persistWalFlag; /* Flag to persist flag */ 83+ int openFlags; /* Flag to open file */ 84+ sqlite3_file *pLockFd; /* File handle to lock file */ 85 } CompressFile; 86 87 88@@ -505,14 +510,14 @@ static void getCompression(sqlite3 *db, CompressFile *pCompress){ 89 const char *sql = "SELECT count(*), compression, pagesize FROM vfs_compression;"; 90 int count = 0; 91 pCompress->compression = COMPRESSION_UNDEFINED; 92- pCompress->page_size = 0; 93+ pCompress->pageSize = 0; 94 95 if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){ 96 if( sqlite3_step(stmt)==SQLITE_ROW ){ 97 count = sqlite3_column_int(stmt, 0); 98 if( count==1 ){ 99 pCompress->compression = sqlite3_column_int(stmt, 1); 100- pCompress->page_size = sqlite3_column_int(stmt, 2); 101+ pCompress->pageSize = sqlite3_column_int(stmt, 2); 102 } 103 } 104 sqlite3_finalize(stmt); 105@@ -590,7 +595,9 @@ static int compressDeviceCharacteristics(sqlite3_file *pFile){ 106 */ 107 static int compressLock(sqlite3_file *pFile, int eFileLock){ 108 assert( pFile ); 109- return SQLITE_OK; 110+ CompressFile *pCompress = (CompressFile *)pFile; 111+ sqlite3_file *pSubFile = ORIGFILE(pFile); 112+ return pSubFile->pMethods->xLock(pCompress->pLockFd, eFileLock); 113 } 114 115 /* 116@@ -598,7 +605,9 @@ static int compressLock(sqlite3_file *pFile, int eFileLock){ 117 */ 118 static int compressUnlock(sqlite3_file *pFile, int eFileLock){ 119 assert( pFile ); 120- return SQLITE_OK; 121+ CompressFile *pCompress = (CompressFile *)pFile; 122+ sqlite3_file *pSubFile = ORIGFILE(pFile); 123+ return pSubFile->pMethods->xUnlock(pCompress->pLockFd, eFileLock); 124 } 125 126 /* 127@@ -608,11 +617,11 @@ static int compressFileSize(sqlite3_file *pFile, i64 *pSize){ 128 assert( pFile ); 129 CompressFile *pCompress = (CompressFile *)pFile; 130 sqlite3 *db = pCompress->pDb; 131- int rc = getCompressPgsize(db, &pCompress->page_size); 132+ int rc = getCompressPgsize(db, &pCompress->pageSize); 133 if( rc!=SQLITE_OK ){ 134 return SQLITE_IOERR_FSTAT; 135 } 136- int pgsize = pCompress->page_size; 137+ int pgsize = pCompress->pageSize; 138 int maxpgno = getMaxCompressPgno(db); 139 *pSize = (i64)maxpgno * pgsize; 140 return SQLITE_OK; 141@@ -625,11 +634,11 @@ static int compressTruncate(sqlite3_file *pFile, sqlite_int64 size){ 142 assert( pFile ); 143 CompressFile *pCompress = (CompressFile *)pFile; 144 sqlite3 *db = pCompress->pDb; 145- int rc = getCompressPgsize(db, &pCompress->page_size); 146- if( rc!=SQLITE_OK || pCompress->page_size==0 ){ 147+ int rc = getCompressPgsize(db, &pCompress->pageSize); 148+ if( rc!=SQLITE_OK || pCompress->pageSize==0 ){ 149 return SQLITE_IOERR_TRUNCATE; 150 } 151- int pgsize = pCompress->page_size; 152+ int pgsize = pCompress->pageSize; 153 int pgno = size / pgsize; 154 if( size % pgsize!=0 || getMaxCompressPgno(db) < pgno ){ 155 return SQLITE_IOERR_TRUNCATE; 156@@ -661,58 +670,70 @@ static int compressWrite(sqlite3_file *pFile, const void *pBuf, int iAmt, sqlite 157 assert( iAmt>0 ); 158 CompressFile *pCompress = (CompressFile *)pFile; 159 sqlite3 *db = pCompress->pDb; 160- int rc = getCompressPgsize(db, &pCompress->page_size); 161+ int rc = getCompressPgsize(db, &pCompress->pageSize); 162 if( rc!=SQLITE_OK ){ 163+ sqlite3_log(SQLITE_WARNING_DUMP, "Missing pgsz(%d), write ofst:%lld, iAmt:%d, flags:%d", pCompress->pageSize, 164+ iOfst, iAmt, pCompress->openFlags); 165 return SQLITE_IOERR_WRITE; 166 } 167 168- if( pCompress->page_size<=0 && iAmt >= 512 && iAmt <= 64*1024 && !(iAmt & (iAmt - 1)) ){ 169+ if( pCompress->pageSize<=0 && iAmt >= 512 && iAmt <= 64*1024 && !(iAmt & (iAmt - 1)) ){ 170 // new compress db need set orignal db's pagesize 171 rc = setCompressPgsize(db, iAmt); 172 if( rc!=SQLITE_OK ){ 173+ sqlite3_log(rc, "Save page size(%d) wrong, ofst:%lld, flags:%d", iAmt, iOfst, pCompress->openFlags); 174 return SQLITE_IOERR_WRITE; 175 } 176- pCompress->page_size = iAmt; 177+ pCompress->pageSize = iAmt; 178 } 179- int pgsize = pCompress->page_size; 180+ int pgsize = pCompress->pageSize; 181 int pgno = iOfst / pgsize + 1; 182 if( iAmt!=pgsize || iOfst % pgsize!=0 ){ 183+ sqlite3_log(SQLITE_IOERR_WRITE, "Mismatch info, iAmt(%d), pgsz(%d), offset:%ld", iAmt, pgsize, iOfst); 184 return SQLITE_IOERR_WRITE; 185 } 186 187- int max_compress_size = compressLen(iAmt, pCompress->compression); 188- if( max_compress_size<=0 ){ 189+ int maxSize = compressLen(iAmt, pCompress->compression); 190+ if( maxSize<=0 ){ 191+ sqlite3_log(SQLITE_IOERR_WRITE, "Get compress size(%d) wrong, compression(%d)", maxSize, pCompress->compression); 192 return SQLITE_IOERR_WRITE; 193 } 194- u8 *compressed_data = sqlite3_malloc(max_compress_size); 195- if( compressed_data==NULL ){ 196+ u8 *tmpData = sqlite3_malloc(maxSize); 197+ if( tmpData==NULL ){ 198+ sqlite3_log(SQLITE_NOMEM, "Malloc size(%d) wrong", maxSize); 199 return SQLITE_NOMEM; 200 } 201- int compress_data_len = 0; 202- if( compressBuf(compressed_data, max_compress_size, &compress_data_len, pBuf, iAmt, pCompress->compression) ){ 203- sqlite3_free(compressed_data); 204+ int len = 0; 205+ if( compressBuf(tmpData, maxSize, &len, pBuf, iAmt, pCompress->compression) ){ 206+ sqlite3_free(tmpData); 207+ sqlite3_log(SQLITE_IOERR_WRITE, "Compress buf wrong, pgno(%d), amt(%d), ofst(%lld)", pgno, iAmt, iOfst); 208 return SQLITE_IOERR_WRITE; 209 } 210 if( pCompress->bBegin!=1 ){ 211- if( sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL)!=SQLITE_OK ){ 212- sqlite3_free(compressed_data); 213+ if( (rc = sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL))!=SQLITE_OK ){ 214+ sqlite3_free(tmpData); 215+ sqlite3_log(rc, "Begin transaction to insert compressed page wrong, pgno(%d), ofst(%lld)", pgno, iOfst); 216 return SQLITE_IOERR_WRITE; 217 } 218 pCompress->bBegin = 1; 219 } 220 sqlite3_stmt *stmt = NULL; 221 const char *sql = "INSERT OR REPLACE INTO vfs_pages(data, pageno) VALUES (?,?);"; 222- if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){ 223- sqlite3_bind_blob(stmt, 1, compressed_data, compress_data_len, SQLITE_STATIC); 224- sqlite3_bind_int(stmt, 2, pgno); 225- if( sqlite3_step(stmt)!=SQLITE_DONE ){ 226- sqlite3_free(compressed_data); 227- sqlite3_finalize(stmt); 228- return SQLITE_IOERR_WRITE; 229- } 230- sqlite3_finalize(stmt); 231+ rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); 232+ if( rc!=SQLITE_OK ){ 233+ sqlite3_free(tmpData); 234+ sqlite3_log(rc, "Prepare stat to insert page wrong, pgno(%d), ofst(%lld)", pgno, iOfst); 235+ return SQLITE_IOERR_WRITE; 236+ } 237+ sqlite3_bind_blob(stmt, 1, tmpData, len, SQLITE_STATIC); 238+ sqlite3_bind_int(stmt, 2, pgno); 239+ rc = sqlite3_step(stmt); 240+ sqlite3_finalize(stmt); 241+ sqlite3_free(tmpData); 242+ if( rc!=SQLITE_DONE ){ 243+ sqlite3_log(rc, "Insert page wrong, pgno(%d), ofst(%lld)", pgno, iOfst); 244+ return SQLITE_IOERR_WRITE; 245 } 246- sqlite3_free(compressed_data); 247 return SQLITE_OK; 248 } 249 250@@ -731,71 +752,83 @@ static int compressRead(sqlite3_file *pFile, void *pBuf, int iAmt, sqlite_int64 251 } 252 (void)memset_s(pBuf, iAmt, 0, iAmt); 253 sqlite3 *db = pCompress->pDb; 254- int rc = getCompressPgsize(db, &pCompress->page_size); 255- if( rc!=SQLITE_OK || pCompress->page_size==0 ){ 256+ int rc = getCompressPgsize(db, &pCompress->pageSize); 257+ if( rc!=SQLITE_OK || pCompress->pageSize==0 ){ 258+ sqlite3_log(SQLITE_WARNING_DUMP, "Missing pgsz(%d), read ofst(%lld), amt(%d), flags(%d)", pCompress->pageSize, 259+ iOfst, iAmt, pCompress->openFlags); 260 return SQLITE_IOERR_SHORT_READ; 261 } 262- int pgsize = pCompress->page_size; 263+ int pgsize = pCompress->pageSize; 264 int pgno = iOfst / pgsize + 1; 265 int dataidx = iOfst % pgsize; 266 sqlite3_stmt *stmt = NULL; 267 const char *sql = "SELECT data, length(data) FROM vfs_pages WHERE pageno=?;"; 268 const void *data = NULL; 269- int data_len = 0; 270+ int dataLen = 0; 271 rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); 272 if( rc!=SQLITE_OK ){ 273+ sqlite3_log(rc, "Prepare to get compressed page(%d) wrong, ofst(%lld)", pgno, iOfst); 274 return SQLITE_CORRUPT; 275 } 276- u8 *decompressed_data = NULL; 277+ u8 *decompressedData = NULL; 278 if( pgsize!=iAmt ){ 279- decompressed_data = sqlite3_malloc(pgsize); 280- if( decompressed_data==NULL ){ 281+ decompressedData = sqlite3_malloc(pgsize); 282+ if( decompressedData==NULL ){ 283 sqlite3_finalize(stmt); 284+ sqlite3_log(SQLITE_NOMEM, "Malloc for decompress size(%d) wrong, amt(%d), ofst(%lld)", pgsize, iAmt, iOfst); 285 return SQLITE_NOMEM; 286 } 287 }else{ 288- decompressed_data = (u8*)pBuf; 289+ decompressedData = (u8*)pBuf; 290 } 291 292- int decompress_data_len = 0; 293+ int decompressLen = 0; 294 sqlite3_bind_int(stmt, 1, pgno); 295 rc = sqlite3_step(stmt); 296 if( rc==SQLITE_ROW ){ 297 data = sqlite3_column_blob(stmt, 0); 298 if( data==NULL ){ 299 rc = SQLITE_IOERR_SHORT_READ; 300- goto failed; 301+ sqlite3_log(rc, "Get compress page(%d) wrong, empty data, amt(%d), ofst(%lld)", pgno, iAmt, iOfst); 302+ goto END_OUT; 303 } 304- data_len = sqlite3_column_int(stmt, 1); 305- if( data_len==0 ){ 306+ dataLen = sqlite3_column_int(stmt, 1); 307+ if( dataLen==0 ){ 308 rc = SQLITE_IOERR_SHORT_READ; 309- goto failed; 310+ sqlite3_log(rc, "Get compress page(%d) wrong, short data, amt(%d), ofst(%lld)", pgno, iAmt, iOfst); 311+ goto END_OUT; 312 } 313- if( decompressBuf(decompressed_data, pgsize, &decompress_data_len, data, data_len, 314- pCompress->compression)!=SQLITE_OK ){ 315+ if( decompressBuf(decompressedData, pgsize, &decompressLen, data, dataLen, pCompress->compression)!=SQLITE_OK ){ 316 rc = SQLITE_IOERR_SHORT_READ; 317- goto failed; 318+ sqlite3_log(rc, "Decompress page(%d) wrong, compression(%d), len(%d), amt(%d), ofst(%lld)", pgno, 319+ (int)pCompress->compression, dataLen, iAmt, iOfst); 320+ goto END_OUT; 321 } 322- if( decompress_data_len!=pgsize ){ 323+ if( decompressLen!=pgsize ){ 324 rc = SQLITE_IOERR_SHORT_READ; 325- goto failed; 326+ sqlite3_log(rc, "Decompress page(%d) wrong, size(%d), pgsz(%d), amt(%d), ofst(%lld)", pgno, 327+ decompressLen, pgsize, iAmt, iOfst); 328+ goto END_OUT; 329 } 330 if( pgsize!=iAmt ){ 331- rc = memcpy_s(pBuf, iAmt, decompressed_data+dataidx, iAmt); 332+ rc = memcpy_s(pBuf, iAmt, decompressedData+dataidx, iAmt); 333 if( rc!=SQLITE_OK ){ 334 rc = SQLITE_IOERR_SHORT_READ; 335- goto failed; 336+ sqlite3_log(rc, "Copy decompress page(%d) wrong, size(%d), pgsz(%d), amt(%d), ofst(%lld)", pgno, 337+ decompressLen, pgsize, iAmt, iOfst); 338+ goto END_OUT; 339 } 340 } 341 rc = SQLITE_OK; 342 }else if( rc==SQLITE_DONE ){ 343 rc = SQLITE_IOERR_SHORT_READ; 344+ sqlite3_log(rc, "Missing page(%d) while try read", pgno); 345 } 346 347- failed: 348+END_OUT: 349 sqlite3_finalize(stmt); 350 if( pgsize!=iAmt ){ 351- sqlite3_free(decompressed_data); 352+ sqlite3_free(decompressedData); 353 } 354 355 return rc; 356@@ -808,6 +841,7 @@ static int compressClose(sqlite3_file *pFile){ 357 assert( pFile ); 358 CompressFile *pCompress = (CompressFile *)pFile; 359 sqlite3 *db = pCompress->pDb; 360+ sqlite3_file *pSubFile = ORIGFILE(pFile); 361 int rc = compressSync(pFile, 0); 362 if( rc!=SQLITE_OK ){ 363 return rc; 364@@ -821,9 +855,13 @@ static int compressClose(sqlite3_file *pFile){ 365 pCompress->pDb = NULL; 366 } 367 } 368+ if( pCompress->pLockFd ){ 369+ pSubFile->pMethods->xClose(pCompress->pLockFd); 370+ sqlite3_free(pCompress->pLockFd); 371+ pCompress->pLockFd = NULL; 372+ } 373 if( pCompress->bSubDbOpen ){ 374- pFile = ORIGFILE(pFile); 375- rc = pFile->pMethods->xClose(pFile); 376+ rc = pSubFile->pMethods->xClose(pSubFile); 377 } 378 return rc; 379 } 380@@ -888,6 +926,39 @@ static int compressUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage 381 return SQLITE_OK; 382 } 383 384+static int compressOpenLockFile(sqlite3_vfs *pVfs, const char *zName, int flags, sqlite3_file **pFile){ 385+ const char *walPath = sqlite3_filename_wal(zName); 386+ const char *lockPath = walPath + strlen(walPath) + 1; // For compress vfs, lock path place at thie position 387+ int isLockExist = 0; 388+ int rc = ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), lockPath, SQLITE_ACCESS_READWRITE, &isLockExist); 389+ if( rc!=SQLITE_OK ){ 390+ sqlite3_log(rc, "Access lock file failed, path:%s", lockPath); 391+ return rc; 392+ } 393+ if( isLockExist==0 && ((flags&SQLITE_OPEN_CREATE)==0) ){ 394+ sqlite3_log(SQLITE_WARNING_NOTCOMPRESSDB, "Check lock wrong, compress db should have lock file"); 395+ return SQLITE_WARNING_NOTCOMPRESSDB; 396+ } 397+ if( pVfs->szOsFile<=(int)sizeof(CompressFile) ){ 398+ sqlite3_log(SQLITE_CANTOPEN, "Unexpected file handle size(%d), vfs:%s", pVfs->szOsFile, pVfs->zName); 399+ return SQLITE_CANTOPEN; 400+ } 401+ sqlite3_file *pLockFd = sqlite3_malloc(pVfs->szOsFile - sizeof(CompressFile)); 402+ if( pLockFd==NULL ){ 403+ rc = SQLITE_NOMEM; 404+ sqlite3_log(rc, "Malloc file handle for lock wrong, size(%d)", (int)(pVfs->szOsFile - sizeof(CompressFile))); 405+ return rc; 406+ } 407+ memset(pLockFd, 0, pVfs->szOsFile - sizeof(CompressFile)); 408+ rc = ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), lockPath, pLockFd, flags, NULL); 409+ if( rc!=SQLITE_OK ){ 410+ sqlite3_log(rc, "Compress vfs lock file open wrong, path:%s, flag(%d)", lockPath, flags); 411+ sqlite3_free(pLockFd); 412+ } 413+ *pFile = pLockFd; 414+ return rc; 415+} 416+ 417 /* 418 ** Open a compress file.If this file is not a journal or wal file, 419 ** it will open a OutterDB and create vfs_pages and vfs_compression table 420@@ -902,87 +973,114 @@ static int compressOpen( 421 ){ 422 sqlite3_file *pSubFile = ORIGFILE(pFile); 423 CompressFile *pCompress = (CompressFile *)pFile; 424+ pCompress->openFlags = flags; 425 int rc = SQLITE_OK; 426 if( !(flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB)) ){ 427 return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, pOutFlags); 428 } 429 rc = ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pSubFile, flags, pOutFlags); 430 if( rc!=SQLITE_OK ){ 431+ sqlite3_log(rc, "Compress file open wrong, path:%s, flags:%d", zName, flags); 432 return rc; 433 } 434 pCompress->bSubDbOpen = 1; 435 sqlite3_int64 fileSize = 0; 436+ sqlite3 *db = NULL; 437 rc = pSubFile->pMethods->xFileSize(pSubFile, &fileSize); 438 if( rc!=SQLITE_OK ){ 439+ sqlite3_log(rc, "Calculate compress file size wrong, path:%s", zName); 440 rc = SQLITE_CANTOPEN; 441- goto open_end; 442+ goto END_OUT; 443 } 444 445+ rc = compressOpenLockFile(pVfs, zName, flags, &pCompress->pLockFd); 446+ if( rc!=SQLITE_OK ){ 447+ sqlite3_log(rc, "Compress vfs lock file open wrong, path:%s, flag(%d)", zName, flags); 448+ goto END_OUT; 449+ } 450 pFile->pMethods = &compress_io_methods; 451- rc = sqlite3_open_v2(zName, &pCompress->pDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, ORIGVFS(pVfs)->zName); 452+ rc = sqlite3_open_v2(zName, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, CKSM_VFS_NAME); 453 if( rc!=SQLITE_OK ){ 454+ sqlite3_log(rc, "Open compress db wrong, name:%s", zName); 455 rc = SQLITE_CANTOPEN; 456- goto open_end; 457+ goto END_OUT; 458 } 459 pCompress->bOutterDbOpen = 1; 460- sqlite3 *db = pCompress->pDb; 461- const char *pre_pragma = "PRAGMA page_size=4096;"\ 462- "PRAGMA auto_vacuum=INCREMENTAL;PRAGMA journal_mode=OFF;"; 463 sqlite3_busy_timeout(db, 2000); // Set time out:2s 464- rc = sqlite3_exec(db, pre_pragma, NULL, NULL, NULL); 465- if( rc!=SQLITE_OK ){ 466- rc = SQLITE_CANTOPEN; 467- goto open_end; 468- } 469 if( tableExists(db, "vfs_compression") ){ 470 getCompression(db, pCompress); 471 if( pCompress->compression!=COMPRESSION_BROTLI && pCompress->compression!=COMPRESSION_ZSTD ){ 472 rc = SQLITE_CANTOPEN; 473- goto open_end; 474+ sqlite3_log(rc, "Unrecognized compression(%d), name:%s", pCompress->compression, zName); 475+ goto END_OUT; 476 } 477- if( loadCompressAlgorithmExtension(pCompress->compression)==SQLITE_ERROR ){ 478+ if( loadCompressAlgorithmExtension(pCompress->compression)!=SQLITE_OK ){ 479 rc = SQLITE_CANTOPEN; 480- goto open_end; 481+ goto END_OUT; 482 } 483 }else if( flags&SQLITE_OPEN_MAIN_DB && fileSize!=0 ){ 484 rc = SQLITE_WARNING_NOTCOMPRESSDB; 485 sqlite3_log(rc, "open compress database go wrong, it should be a compressed db"); 486- goto open_end; 487+ goto END_OUT; 488 }else{ 489 if( loadCompressAlgorithmExtension(COMPRESSION_UNDEFINED)==SQLITE_ERROR ){ 490 rc = SQLITE_CANTOPEN; 491- goto open_end; 492+ goto END_OUT; 493+ } 494+ const char *pragmaStr = "PRAGMA checksum_persist_enable=ON;PRAGMA page_size=4096;"\ 495+ "PRAGMA auto_vacuum=INCREMENTAL;PRAGMA journal_mode=OFF;"; 496+ rc = sqlite3_exec(db, pragmaStr, NULL, NULL, NULL); 497+ if( rc!=SQLITE_OK ){ 498+ sqlite3_log(rc, "Config compress db wrong, name:%s", zName); 499+ rc = SQLITE_CANTOPEN; 500+ goto END_OUT; 501 } 502- const char *init_compression_sql = "CREATE TABLE vfs_compression (compression INTEGER, pagesize INTEGER);"; 503- rc = sqlite3_exec(db, init_compression_sql, NULL, NULL, NULL); 504+ const char *initCompressionStr = "CREATE TABLE vfs_compression (compression INTEGER, pagesize INTEGER);"; 505+ rc = sqlite3_exec(db, initCompressionStr, NULL, NULL, NULL); 506 if( rc!=SQLITE_OK ){ 507+ sqlite3_log(rc, "Configure compress db wrong, name:%s, create vfs_compression failed", zName); 508 rc = SQLITE_CANTOPEN; 509- goto open_end; 510+ goto END_OUT; 511 } 512- char set_compression_sql[COMPRESSION_SQL_MAX_LENGTH] = {0}; 513- if( sprintf_s(set_compression_sql, COMPRESSION_SQL_MAX_LENGTH, 514+ char compressionSql[COMPRESSION_SQL_MAX_LENGTH] = {0}; 515+ if( sprintf_s(compressionSql, COMPRESSION_SQL_MAX_LENGTH, 516 "INSERT INTO vfs_compression(compression, pagesize) VALUES (%u, 0);", g_compress_algo_load)<=0 ){ 517 rc = SQLITE_CANTOPEN; 518- goto open_end; 519+ sqlite3_log(rc, "Concatenate config stat wrong, name:%s, compression(%u)", zName, g_compress_algo_load); 520+ goto END_OUT; 521 } 522- rc = sqlite3_exec(db, set_compression_sql, NULL, NULL, NULL); 523+ rc = sqlite3_exec(db, compressionSql, NULL, NULL, NULL); 524 if( rc!=SQLITE_OK ){ 525+ sqlite3_log(rc, "Config compress db wrong, name:%s, insert compression(%u) failed", zName, g_compress_algo_load); 526 rc = SQLITE_CANTOPEN; 527- goto open_end; 528+ goto END_OUT; 529 } 530 pCompress->compression = g_compress_algo_load; 531- pCompress->page_size = 0; 532+ pCompress->pageSize = 0; 533 } 534 if( tableExists(db, "vfs_pages") ){ 535- goto open_end; 536+ goto END_OUT; 537 } 538- const char *create_sql = "CREATE TABLE vfs_pages (pageno INTEGER PRIMARY KEY, data BLOB NOT NULL);"; 539- rc = sqlite3_exec(db, create_sql, NULL, NULL, NULL); 540+ const char *pageDdlStr = "CREATE TABLE vfs_pages (pageno INTEGER PRIMARY KEY, data BLOB NOT NULL);"; 541+ rc = sqlite3_exec(db, pageDdlStr, NULL, NULL, NULL); 542 if( rc!=SQLITE_OK ){ 543+ sqlite3_log(rc, "Config compress db wrong, name:%s, create page table failed", zName); 544 rc = SQLITE_CANTOPEN; 545 } 546 547- open_end: 548+END_OUT: 549+ if( rc==SQLITE_OK ){ 550+ pCompress->pDb = db; 551+ }else{ 552+ if( db!=NULL ){ 553+ sqlite3_close_v2(db); 554+ } 555+ if( pCompress->pLockFd ){ 556+ pSubFile->pMethods->xClose(pCompress->pLockFd); 557+ sqlite3_free(pCompress->pLockFd); 558+ pCompress->pLockFd = NULL; 559+ } 560+ } 561 return rc; 562 } 563 564diff --git a/src/sqlite3.c b/src/sqlite3.c 565index 0cdb348..399c6b1 100644 566--- a/src/sqlite3.c 567+++ b/src/sqlite3.c 568@@ -5363,7 +5363,7 @@ SQLITE_API int sqlite3_set_droptable_handle(sqlite3*, void (*xFunc)(sqlite3*,con 569 #endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ 570 571 #ifdef SQLITE_ENABLE_BINLOG 572-SQLITE_API int sqlite3_is_support_binlog(void); 573+SQLITE_API int sqlite3_is_support_binlog(const char *notUsed); 574 575 SQLITE_API int sqlite3_replay_binlog(sqlite3 *srcDb, sqlite3 *destDb); 576 577@@ -15050,6 +15050,14 @@ typedef INT16_TYPE LogEst; 578 #define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32)) 579 #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) 580 581+/* 582+** Macro SMXV(n) return the maximum value that can be held in variable n, 583+** assuming n is a signed integer type. UMXV(n) is similar for unsigned 584+** integer types. 585+*/ 586+#define SMXV(n) ((((i64)1)<<(sizeof(n)*8-1))-1) 587+#define UMXV(n) ((((i64)1)<<(sizeof(n)*8))-1) 588+ 589 /* 590 ** Round up a number to the next larger multiple of 8. This is used 591 ** to force 8-byte alignment on 64-bit architectures. 592@@ -19061,7 +19069,7 @@ struct AggInfo { 593 ** from source tables rather than from accumulators */ 594 u8 useSortingIdx; /* In direct mode, reference the sorting index rather 595 ** than the source table */ 596- u16 nSortingColumn; /* Number of columns in the sorting index */ 597+ u32 nSortingColumn; /* Number of columns in the sorting index */ 598 int sortingIdx; /* Cursor number of the sorting index */ 599 int sortingIdxPTab; /* Cursor number of pseudo-table */ 600 int iFirstReg; /* First register in range for aCol[] and aFunc[] */ 601@@ -19070,8 +19078,8 @@ struct AggInfo { 602 Table *pTab; /* Source table */ 603 Expr *pCExpr; /* The original expression */ 604 int iTable; /* Cursor number of the source table */ 605- i16 iColumn; /* Column number within the source table */ 606- i16 iSorterColumn; /* Column number in the sorting index */ 607+ int iColumn; /* Column number within the source table */ 608+ int iSorterColumn; /* Column number in the sorting index */ 609 } *aCol; 610 int nColumn; /* Number of used entries in aCol[] */ 611 int nAccumulator; /* Number of columns that show through to the output. 612@@ -38805,8 +38813,8 @@ static void enableDbFileDelMonitor(int32_t fd) 613 } 614 flags |= HMFS_MONITOR_FL; 615 ret = ioctl(fd, HMFS_IOCTL_HW_SET_FLAGS, &flags); 616- if (ret < 0) { 617- sqlite3_log(SQLITE_WARNING, "Fd %d enable del monitor go wrong, errno = %d", fd, errno); 618+ if (ret < 0 && errno != EACCES) { 619+ sqlite3_log(SQLITE_WARNING, "Flags %u fd %d enable del monitor go wrong, errno = %d", flags, fd, errno); 620 } 621 } 622 623@@ -57976,6 +57984,8 @@ static void MetaDwrCheckVacuum(BtShared *pBt); 624 static int MetaDwrRecoverAndBeginTran(Btree *pBt, int wrflag, int *pSchemaVersion); 625 static int MetaDwrOpenAndCheck(Btree *pBt); 626 static void MetaDwrDisable(Btree *pBt); 627+static void MetaDwrDumpInfo(Pager *pPager); 628+static void MetaDwrCommitUpdate(Pager *pPager); 629 #define META_HEADER_CHANGED 1 630 #define META_SCHEMA_CHANGED 2 631 #define META_IN_RECOVERY 1 632@@ -58253,6 +58263,9 @@ struct Pager { 633 void *metaMapPage; 634 int (*xGetMethod)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ 635 #endif 636+#ifdef SQLITE_ENABLE_PAGE_COMPRESS 637+ char *zLock; /* Point to the lock filename, use to replace main db to lock */ 638+#endif 639 }; 640 641 /* 642@@ -59754,11 +59767,12 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ 643 sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); 644 } 645 #ifdef SQLITE_META_DWR 646- if (bCommit && pPager->metaChanged != 0) { 647- sqlite3BeginBenignMalloc(); 648- (void)MetaDwrWriteHeader(pPager, pPager->metaHdr); 649- sqlite3EndBenignMalloc(); 650- pPager->metaChanged = 0; 651+ if (bCommit) { 652+ MetaDwrCommitUpdate(pPager); 653+ if (pPager->metaChanged != 0) { 654+ (void)MetaDwrWriteHeader(pPager, pPager->metaHdr); 655+ pPager->metaChanged = 0; 656+ } 657 } 658 #endif 659 if( pagerUseWal(pPager) ){ 660@@ -62547,7 +62561,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( 661 ** specific formatting and order of the various filenames, so if the format 662 ** changes here, be sure to change it there as well. 663 ** 664- ** Addition, in case of enable page compression, journal&WAL filename will add a file extension:"compress" 665+ ** Addition, in case of enable page compression, journal&WAL filename will add a file extension:"compress", 666+ ** lock file need an other setting 667 ** So the final layout in memory is as follows: 668 ** 669 ** Pager object (sizeof(Pager) bytes) 670@@ -62561,13 +62576,16 @@ SQLITE_PRIVATE int sqlite3PagerOpen( 671 ** URI query parameters (nUriByte bytes) 672 ** Journal filename (nPathname+16+1 bytes) 673 ** WAL filename (nPathname+12+1 bytes) 674+ ** Lock filename (nPathname+13+1 bytes) 675 ** \0\0\0 terminator (3 bytes) 676 ** 677 */ 678 int fileExtSz = 0; 679+ int lockExtSz = 0; 680 #ifdef SQLITE_ENABLE_PAGE_COMPRESS 681 if( sqlite3_stricmp(pVfs->zName, "compressvfs")==0 ){ 682 fileExtSz = 8; /* 8 means the size of suffix:"compress" */ 683+ lockExtSz = 13; /* 13 means the size of suffix:"-lockcompress" */ 684 } 685 #endif 686 assert( SQLITE_PTRSIZE==sizeof(Pager*) ); 687@@ -62583,6 +62601,9 @@ SQLITE_PRIVATE int sqlite3PagerOpen( 688 nPathname + 8 + fileExtSz + 1 + /* Journal filename */ 689 #ifndef SQLITE_OMIT_WAL 690 nPathname + 4 + fileExtSz + 1 + /* WAL filename */ 691+#endif 692+#ifdef SQLITE_ENABLE_PAGE_COMPRESS 693+ nPathname + lockExtSz + 1 + /* Lock filename */ 694 #endif 695 3 /* Terminator */ 696 ); 697@@ -62651,6 +62672,19 @@ SQLITE_PRIVATE int sqlite3PagerOpen( 698 pPager->zWal = 0; 699 } 700 #endif 701+ 702+#ifdef SQLITE_ENABLE_PAGE_COMPRESS 703+ if( nPathname>0 ){ 704+ pPager->zLock = (char*)pPtr; 705+ memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; 706+ if( lockExtSz>0 ){ /* 13 means the size of string:"-lockcompress" */ 707+ memcpy(pPtr, "-lockcompress", 13); pPtr += lockExtSz; 708+ } 709+ pPtr += 1; /* Skip zero suffix */ 710+ }else{ 711+ pPager->zLock = 0; 712+ } 713+#endif 714 (void)pPtr; /* Suppress warning about unused pPtr value */ 715 716 if( nPathname ) sqlite3DbFree(0, zPathname); 717@@ -73773,7 +73807,13 @@ static void zeroPage(MemPage *pPage, int flags){ 718 data[hdr+7] = 0; 719 put2byte(&data[hdr+5], pBt->usableSize); 720 pPage->nFree = (u16)(pBt->usableSize - first); 721- decodeFlags(pPage, flags); 722+ /* 723+ ** pPage might not be a btree page, it might be ovf/ptrmap/free page 724+ ** and decodeFlags() return SQLITE_CORRUPT, but no harm is done by this 725+ */ 726+ if( decodeFlags(pPage, flags)==SQLITE_CORRUPT ){ 727+ sqlite3_log(SQLITE_WARNING_DUMP, "warn: zero page, might be ovf/ptrmap/free page:%d", pPage->pgno); 728+ } 729 pPage->cellOffset = first; 730 pPage->aDataEnd = &data[pBt->pageSize]; 731 pPage->aCellIdx = &data[first]; 732@@ -73982,7 +74022,9 @@ static void pageReinit(DbPage *pData){ 733 ** But no harm is done by this. And it is very important that 734 ** btreeInitPage() be called on every btree page so we make 735 ** the call for every page that comes in for re-initializing. */ 736- btreeInitPage(pPage); 737+ if( btreeInitPage(pPage)==SQLITE_CORRUPT ){ 738+ sqlite3_log(SQLITE_WARNING_DUMP, "warn: page reinit, might be ovf/ptrmap/free page:%d", pPage->pgno); 739+ } 740 } 741 } 742 } 743@@ -74898,6 +74940,11 @@ static int lockBtree(BtShared *pBt){ 744 } 745 if( nPage>nPageFile ){ 746 if( sqlite3WritableSchema(pBt->db)==0 ){ 747+ sqlite3_log(SQLITE_WARNING_DUMP, "sqlite3WritableSchema nPage %d nPageFile %d", nPage, nPageFile); 748+#ifdef SQLITE_META_DWR 749+ DumpLocksByPager(pBt->pPager); 750+ MetaDwrDumpInfo(pBt->pPager); 751+#endif 752 rc = SQLITE_CORRUPT_BKPT; 753 goto page1_init_failed; 754 }else{ 755@@ -92357,8 +92404,9 @@ SQLITE_API int sqlite3_set_droptable_handle(sqlite3 *db, void (*xFunc)(sqlite3*, 756 #endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ 757 758 #ifdef SQLITE_ENABLE_BINLOG 759-SQLITE_API int sqlite3_is_support_binlog(void) 760+SQLITE_API int sqlite3_is_support_binlog(const char *notUsed) 761 { 762+ (void)notUsed; 763 return SQLITE_ERROR; 764 } 765 766@@ -116854,7 +116902,9 @@ static void findOrCreateAggInfoColumn( 767 ){ 768 struct AggInfo_col *pCol; 769 int k; 770+ int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; 771 772+ assert( mxTerm <= SMXV(i16) ); 773 assert( pAggInfo->iFirstReg==0 ); 774 pCol = pAggInfo->aCol; 775 for(k=0; k<pAggInfo->nColumn; k++, pCol++){ 776@@ -116872,6 +116922,10 @@ static void findOrCreateAggInfoColumn( 777 assert( pParse->db->mallocFailed ); 778 return; 779 } 780+ if( k>mxTerm ){ 781+ sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); 782+ k = mxTerm; 783+ } 784 pCol = &pAggInfo->aCol[k]; 785 assert( ExprUseYTab(pExpr) ); 786 pCol->pTab = pExpr->y.pTab; 787@@ -116905,6 +116959,7 @@ fix_up_expr: 788 if( pExpr->op==TK_COLUMN ){ 789 pExpr->op = TK_AGG_COLUMN; 790 } 791+ assert( k <= SMXV(pExpr->iAgg) ); 792 pExpr->iAgg = (i16)k; 793 } 794 795@@ -116988,13 +117043,19 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ 796 ** function that is already in the pAggInfo structure 797 */ 798 struct AggInfo_func *pItem = pAggInfo->aFunc; 799+ int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; 800+ assert( mxTerm <= SMXV(i16) ); 801 for(i=0; i<pAggInfo->nFunc; i++, pItem++){ 802 if( pItem->pFExpr==pExpr ) break; 803 if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){ 804 break; 805 } 806 } 807- if( i>=pAggInfo->nFunc ){ 808+ if( i>mxTerm ){ 809+ sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); 810+ i = mxTerm; 811+ assert( i<pAggInfo->nFunc ); 812+ }else if( i>=pAggInfo->nFunc ){ 813 /* pExpr is original. Make a new entry in pAggInfo->aFunc[] 814 */ 815 u8 enc = ENC(pParse->db); 816@@ -117046,6 +117107,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ 817 */ 818 assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); 819 ExprSetVVAProperty(pExpr, EP_NoReduce); 820+ assert( i <= SMXV(pExpr->iAgg) ); 821 pExpr->iAgg = (i16)i; 822 pExpr->pAggInfo = pAggInfo; 823 return WRC_Prune; 824@@ -121758,8 +121820,8 @@ static void attachFunc( 825 if( rc==SQLITE_OK ){ 826 extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); 827 extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); 828- int nKey; 829- char *zKey; 830+ int nKey = 0; 831+ char *zKey = NULL; 832 int t = sqlite3_value_type(argv[2]); 833 switch( t ){ 834 case SQLITE_INTEGER: 835@@ -121776,14 +121838,7 @@ static void attachFunc( 836 break; 837 838 case SQLITE_NULL: 839- /* No key specified. Use the key from URI filename, or if none, 840- ** use the key from the main database. */ 841- if( sqlite3CodecQueryParameters(db, zName, zPath)==0 ){ 842- sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); 843- if( nKey || sqlite3BtreeGetRequestedReserve(db->aDb[0].pBt)>0 ){ 844- rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); 845- } 846- } 847+ /* No key specified. just use null key. */ 848 break; 849 } 850 } 851@@ -184151,10 +184206,12 @@ opendb_out: 852 db->eOpenState = SQLITE_STATE_SICK; 853 } 854 #ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK 855- db->isDropTable = 0; 856- db->mDropTableName = NULL; 857- db->mDropSchemaName = NULL; 858- db->xDropTableHandle = NULL; 859+ if( db!=0 ){ 860+ db->isDropTable = 0; 861+ db->mDropTableName = NULL; 862+ db->mDropSchemaName = NULL; 863+ db->xDropTableHandle = NULL; 864+ } 865 #endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ 866 #ifdef SQLITE_ENABLE_BINLOG 867 sqlite3BinlogReset(db); 868@@ -204600,6 +204657,39 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ 869 return rc; 870 } 871 872+/* 873+** Expression node pExpr is an MSR phrase. This function restarts pExpr 874+** so that it is a regular phrase query, not an MSR. SQLITE_OK is returned 875+** if successful, or an SQLite error code otherwise. 876+*/ 877+int sqlite3Fts3MsrCancel(Fts3Cursor *pCsr, Fts3Expr *pExpr){ 878+ int rc = SQLITE_OK; 879+ if( pExpr->bEof==0 ){ 880+ i64 iDocid = pExpr->iDocid; 881+ fts3EvalRestart(pCsr, pExpr, &rc); 882+ while( rc==SQLITE_OK && pExpr->iDocid!=iDocid ){ 883+ fts3EvalNextRow(pCsr, pExpr, &rc); 884+ if( pExpr->bEof ) rc = FTS_CORRUPT_VTAB; 885+ } 886+ } 887+ return rc; 888+} 889+ 890+/* 891+** If expression pExpr is a phrase expression that uses an MSR query, 892+** restart it as a regular, non-incremental query. Return SQLITE_OK 893+** if successful, or an SQLite error code otherwise. 894+*/ 895+static int fts3ExprRestartIfCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ 896+ TermOffsetCtx *p = (TermOffsetCtx*)ctx; 897+ int rc = SQLITE_OK; 898+ if( pExpr->pPhrase && pExpr->pPhrase->bIncr ){ 899+ rc = sqlite3Fts3MsrCancel(p->pCsr, pExpr); 900+ pExpr->pPhrase->bIncr = 0; 901+ } 902+ return rc; 903+} 904+ 905 /* 906 ** Implementation of offsets() function. 907 */ 908@@ -204636,6 +204726,12 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( 909 sCtx.iDocid = pCsr->iPrevId; 910 sCtx.pCsr = pCsr; 911 912+ /* If a query restart will be required, do it here, rather than later of 913+ ** after pointers to poslist buffers that may be invalidated by a restart 914+ ** have been saved. */ 915+ rc = sqlite3Fts3ExprIterate(pCsr->pExpr, fts3ExprRestartIfCb, (void*)&sCtx); 916+ if( rc!=SQLITE_OK ) goto offsets_out; 917+ 918 /* Loop through the table columns, appending offset information to 919 ** string-buffer res for each column. 920 */ 921@@ -254823,6 +254919,21 @@ SQLITE_API int sqlite3_stmt_init( 922 SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } 923 /************************** End of sqlite3.c ******************************/ 924 925+#if SQLITE_OS_UNIX 926+#include <unistd.h> 927+#include <sys/syscall.h> 928+static inline int OsGetTid(void) 929+{ 930+#if defined(__linux__) 931+ return (int)syscall(__NR_gettid); 932+#elif defined(__APPLE__) 933+ return (int)syscall(SYS_thread_selfid); 934+#else 935+ return 0; 936+#endif 937+} 938+#endif 939+ 940 #ifdef SQLITE_CKSUMVFS_STATIC 941 extern sqlite3_file *cksmvfsGetOrigFile(sqlite3_file *file); 942 #else 943@@ -255984,8 +256095,12 @@ void sqlite3CodecDetach(void *ctx){ 944 #if SQLITE_OS_UNIX 945 static int CodecFileLock(Pager *pPager, short lockType) 946 { 947- sqlite3_file *fd = pPager->fd; 948- unixFile *pFile = (unixFile *)fd; 949+ u8 checkFileId = Sqlite3GetCheckFileId(pPager->pVfs); 950+ if( checkFileId==0 ){ 951+ sqlite3_log(SQLITE_ERROR, "unrecognized vfs, mismatch info"); 952+ return SQLITE_ERROR; 953+ } 954+ unixFile *pFile = Sqlite3GetUnixFile(pPager->fd, checkFileId); 955 unixInodeInfo *pInode = pFile->pInode; 956 if (pInode == NULL) { 957 sqlite3_log(SQLITE_IOERR_RDLOCK, "Codec file lock %d go wrong", lockType); 958@@ -256060,6 +256175,7 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ 959 } 960 } 961 #endif 962+ p->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; 963 #ifdef SQLITE_CODEC_ATTACH_CHANGED 964 int rc = sqlite3CodecInitContext(ctx, p, pKey, nKey, nDb); 965 #else 966@@ -256636,7 +256752,12 @@ typedef struct MetaDwrHdr { 967 u32 pageSz; 968 u32 pageCnt; 969 u64 dbFileInode; 970- u32 reserved[12]; 971+ u64 walShmFileIno; 972+ u64 updateTimeStamp; 973+ int updateTid; 974+ int updatePid; 975+ u32 walFrameCksum[2]; 976+ u32 reserved[4]; 977 u32 checkSum; 978 u8 *zones; 979 Pgno *pages; 980@@ -256991,6 +257112,88 @@ static inline u64 CaculateMetaDwrWriteOffset(int pageSz, u32 idx, u8 zone) { 981 return META_DWR_HEADER_PAGE_SIZE + pageSz * (idx * 2 + zone); 982 } 983 984+static void MetaDwrDumpInfo(Pager *pPager) 985+{ 986+ MetaDwrHdr *hdr = pPager->metaHdr; 987+ if (hdr != NULL) { 988+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr dbSize %u mxFrameInWal %u dbIno %lu walShmIno %lu", 989+ hdr->dbSize, hdr->mxFrameInWal, hdr->dbFileInode, hdr->walShmFileIno); 990+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr updatestamp %ld updatetid %d updatepid %d", 991+ hdr->updateTimeStamp, hdr->updateTid, hdr->updatePid); 992+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr ckSum0 %u ckSum1 %u", hdr->walFrameCksum[0], hdr->walFrameCksum[1]); 993+ } 994+#ifndef SQLITE_OMIT_WAL 995+ if (!pagerUseWal(pPager)) { 996+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr ignore dump wal info"); 997+ return; 998+ } 999+ WalIndexHdr *pWalHdr = &pPager->pWal->hdr; 1000+ sqlite3_log(SQLITE_WARNING_DUMP, "wal ckSum0 %u ckSum1 %u maxFram %u nPage %u", pWalHdr->aFrameCksum[0], 1001+ pWalHdr->aFrameCksum[1], pWalHdr->mxFrame, pWalHdr->nPage); 1002+#if SQLITE_OS_UNIX 1003+ if (hdr == NULL || !hdr->checkFileId) { 1004+ return; 1005+ } 1006+ unixFile *fd = Sqlite3GetUnixFile(pPager->fd, hdr->checkFileId); 1007+ if (fd == NULL || fd->pInode == NULL) { 1008+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr dump invalid db fd"); 1009+ return; 1010+ } 1011+ if (fd->pShm == NULL || fd->pShm->pShmNode == NULL || fd->pShm->pShmNode->hShm < 0) { 1012+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr dump invalid shm fd"); 1013+ return; 1014+ } 1015+ struct stat sStat; 1016+ if (osFstat(fd->pShm->pShmNode->hShm, &sStat)) { 1017+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr dump shm stat go wrong %d", errno); 1018+ return; 1019+ } 1020+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr shm ino %lld", (i64)sStat.st_ino); 1021+#endif 1022+#endif 1023+} 1024+ 1025+static void MetaDwrCommitUpdate(Pager *pPager) 1026+{ 1027+#if SQLITE_OS_UNIX 1028+ MetaDwrHdr *hdr = pPager->metaHdr; 1029+ if (hdr == NULL) { 1030+ return; 1031+ } 1032+ hdr->updateTid = OsGetTid(); 1033+ hdr->updatePid = osGetpid(); 1034+ struct timeval time; 1035+ gettimeofday(&time, NULL); 1036+ hdr->updateTimeStamp = time.tv_sec * 1000 + time.tv_usec / 1000; 1037+#ifndef SQLITE_OMIT_WAL 1038+ if (pagerUseWal(pPager)) { 1039+ WalIndexHdr *pWalHdr = &pPager->pWal->hdr; 1040+ if (pWalHdr->isInit) { 1041+ hdr->walFrameCksum[0] = pWalHdr->aFrameCksum[0]; 1042+ hdr->walFrameCksum[1] = pWalHdr->aFrameCksum[1]; 1043+ } 1044+ } 1045+#endif 1046+#if !defined(NDEBUG) 1047+ if (!hdr->checkFileId) { 1048+ return; 1049+ } 1050+ unixFile *fd = Sqlite3GetUnixFile(pPager->fd, hdr->checkFileId); 1051+ if (fd == NULL || fd->pShm == NULL || fd->pShm->pShmNode == NULL || fd->pShm->pShmNode->hShm < 0) { 1052+ sqlite3_log(SQLITE_WARNING_DUMP, "update meta header invalid shm fd"); 1053+ return; 1054+ } 1055+ struct stat sStat; 1056+ if (osFstat(fd->pShm->pShmNode->hShm, &sStat)) { 1057+ hdr->walShmFileIno = 0; 1058+ sqlite3_log(SQLITE_WARNING_DUMP, "update meta header stat go wrong %d", errno); 1059+ return; 1060+ } 1061+ hdr->walShmFileIno = sStat.st_ino; 1062+#endif 1063+#endif 1064+} 1065+ 1066 static void MetaDwrUpdateHeaderDbInfo(BtShared *pBt) { 1067 MetaDwrHdr *hdr = pBt->pPager->metaHdr; 1068 // 28 offset: dbSize, freelist pageNo, freelist pages count, schema cookie 1069@@ -257146,7 +257349,10 @@ static int MetaDwrRestoreAllPages(Btree *pBt, const ScanPages *metaPages, MetaDw 1070 return rc; 1071 } 1072 } 1073- hdr->pageCnt = metaPages->pageCnt; 1074+ hdr->pageCnt = i; 1075+ if (metaPages->pageCnt > META_DWR_MAX_PAGES) { 1076+ sqlite3_log(SQLITE_WARNING_DUMP, "Meta dwr restore pages %u out of range", metaPages->pageCnt); 1077+ } 1078 MetaDwrUpdateHeaderDbInfo(pBt->pBt); 1079 return rc; 1080 } 1081@@ -257211,11 +257417,12 @@ static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { 1082 if (pPager->metaMapPage == NULL) { 1083 sqlite3_int64 sz = META_DWR_HEADER_PAGE_SIZE; 1084 sqlite3OsFileControlHint(metaFd, SQLITE_FCNTL_CHUNK_SIZE, &sz); 1085- sqlite3OsFileControlHint(metaFd, SQLITE_FCNTL_SIZE_HINT, &sz); 1086- void *page = osMmap(0, META_DWR_HEADER_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, 1087- ((unixFile *)metaFd)->h, 0); 1088- if (page != MAP_FAILED) { 1089- pPager->metaMapPage = page; 1090+ if (sqlite3OsFileControl(metaFd, SQLITE_FCNTL_SIZE_HINT, &sz) == SQLITE_OK) { 1091+ void *page = osMmap(0, META_DWR_HEADER_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, 1092+ ((unixFile *)metaFd)->h, 0); 1093+ if (page != MAP_FAILED) { 1094+ pPager->metaMapPage = page; 1095+ } 1096 } 1097 } 1098 #endif /* SQLITE_OS_UNIX */ 1099@@ -257589,7 +257796,7 @@ CHK_RESTORE_OUT: 1100 return rc; 1101 } 1102 1103-static inline u8 IsConnectionValidForCheck(Pager *pPager) 1104+static inline u8 IsOnlyOneConnection(Pager *pPager) 1105 { 1106 #if SQLITE_OS_UNIX 1107 u8 checkFileId = Sqlite3GetCheckFileId(pPager->pVfs); 1108@@ -257597,10 +257804,18 @@ static inline u8 IsConnectionValidForCheck(Pager *pPager) 1109 return 0; 1110 } 1111 unixFile *fd = Sqlite3GetUnixFile(pPager->fd, checkFileId); 1112+ if (fd == NULL || fd->pInode == NULL) { 1113+ sqlite3_log(SQLITE_WARNING_DUMP, "Check connection go wrong"); 1114+ return 0; 1115+ } 1116+ unixInodeInfo *pInode = fd->pInode; 1117+ sqlite3_mutex_enter(pInode->pLockMutex); 1118 // unix and only one connection exist 1119- if (fd == NULL || fd->pInode == NULL || fd->pInode->nRef != (checkFileId==SQLITE_CHECK_FILE_ID_COMPRESS ? 2 : 1)) { 1120+ if (pInode->nRef != (checkFileId==SQLITE_CHECK_FILE_ID_COMPRESS ? 2 : 1)) { 1121+ sqlite3_mutex_leave(pInode->pLockMutex); 1122 return 0; 1123 } 1124+ sqlite3_mutex_leave(pInode->pLockMutex); 1125 return 1; 1126 #else 1127 return 0; 1128@@ -257610,7 +257825,7 @@ static inline u8 IsConnectionValidForCheck(Pager *pPager) 1129 static int MetaDwrOpenAndCheck(Btree *pBt) 1130 { 1131 Pager *pPager = pBt->pBt->pPager; 1132- if (pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) { 1133+ if (pPager->memDb || pPager->readOnly || !IsOnlyOneConnection(pPager)) { 1134 return SQLITE_OK; 1135 } 1136 #ifdef SQLITE_HAS_CODEC 1137@@ -257655,7 +257870,7 @@ DWR_OPEN_OUT: 1138 static void MetaDwrDisable(Btree *pBt) 1139 { 1140 Pager *pPager = pBt->pBt->pPager; 1141- if (pPager->metaFd == NULL || pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) { 1142+ if (pPager->metaFd == NULL || pPager->memDb || pPager->readOnly || !IsOnlyOneConnection(pPager)) { 1143 return; 1144 } 1145 #ifdef SQLITE_HAS_CODEC 1146@@ -257681,19 +257896,6 @@ static void MetaDwrDisable(Btree *pBt) 1147 #endif /* SQLITE_META_DWR */ 1148 1149 #if SQLITE_OS_UNIX 1150-#include <unistd.h> 1151-#include <sys/syscall.h> 1152-static inline int OsGetTid(void) 1153-{ 1154-#if defined(__linux__) 1155- return (int)syscall(__NR_gettid); 1156-#elif defined(__APPLE__) 1157- return (int)syscall(SYS_thread_selfid); 1158-#else 1159- return 0; 1160-#endif 1161-} 1162- 1163 static void ResetLockStatus(void) 1164 { 1165 (void)memset(&g_lockStatus, 0, sizeof(g_lockStatus)); 1166@@ -257797,8 +257999,13 @@ static inline const char *FlockToName(int l_type) 1167 1168 static int DumpProcessLocks(int fd, struct flock *lock, const char *lockName, char *dumpBuf, int bufLen) 1169 { 1170+#ifdef F_OFD_GETLK 1171+ int lkType = F_OFD_GETLK; 1172+#else 1173+ int lkType = F_GETLK; 1174+#endif 1175 dumpBuf[0] = '\0'; 1176- if (osFcntl(fd, F_GETLK, lock) != SQLITE_OK) { 1177+ if (osFcntl(fd, lkType, lock) != SQLITE_OK) { 1178 sqlite3_log(SQLITE_ERROR, "[SQLite]Get wal file lock ofs %u failed, errno: %d", lock->l_start, errno); 1179 return 0; 1180 } 1181@@ -259039,9 +259246,13 @@ struct sqlite3_api_routines_extra { 1182 int (*key_v2)(sqlite3*,const char*,const void*,int); 1183 int (*rekey)(sqlite3*,const void*,int); 1184 int (*rekey_v2)(sqlite3*,const char*,const void*,int); 1185- int (*is_support_binlog)(void); 1186+ int (*is_support_binlog)(const char*); 1187 int (*replay_binlog)(sqlite3*, sqlite3*); 1188+#ifdef SQLITE_ENABLE_BINLOG 1189 int (*clean_binlog)(sqlite3*, BinlogFileCleanModeE); 1190+#else 1191+ void *dymmyFunc; 1192+#endif 1193 #ifdef SQLITE_ENABLE_PAGE_COMPRESS 1194 int (*compressdb_backup)(sqlite3*, const char*); 1195 #else 1196-- 11972.47.0.windows.2 1198 1199