1diff -ru ext-orig/fts2/fts2.c ext/fts2/fts2.c 2--- ext-orig/fts2/fts2.c 2009-09-04 13:37:41.000000000 -0700 3+++ ext/fts2/fts2.c 2009-09-30 14:48:14.000000000 -0700 4@@ -37,6 +37,20 @@ 5 ** This is an SQLite module implementing full-text search. 6 */ 7 8+/* TODO(shess): To make it easier to spot changes without groveling 9+** through changelogs, I've defined GEARS_FTS2_CHANGES to call them 10+** out, and I will document them here. On imports, these changes 11+** should be reviewed to make sure they are still present, or are 12+** dropped as appropriate. 13+** 14+** SQLite core adds the custom function fts2_tokenizer() to be used 15+** for defining new tokenizers. The second parameter is a vtable 16+** pointer encoded as a blob. Obviously this cannot be exposed to 17+** Gears callers for security reasons. It could be suppressed in the 18+** authorizer, but for now I have simply commented the definition out. 19+*/ 20+#define GEARS_FTS2_CHANGES 1 21+ 22 /* 23 ** The code in this file is only compiled if: 24 ** 25@@ -326,8 +326,10 @@ 26 #include "fts2_hash.h" 27 #include "fts2_tokenizer.h" 28 #include "sqlite3.h" 29-#include "sqlite3ext.h" 30-SQLITE_EXTENSION_INIT1 31+#ifndef SQLITE_CORE 32+# include "sqlite3ext.h" 33+ SQLITE_EXTENSION_INIT1 34+#endif 35 36 37 /* TODO(shess) MAN, this thing needs some refactoring. At minimum, it 38@@ -335,6 +349,16 @@ 39 # define TRACE(A) 40 #endif 41 42+#if 0 43+/* Useful to set breakpoints. See main.c sqlite3Corrupt(). */ 44+static int fts2Corrupt(void){ 45+ return SQLITE_CORRUPT; 46+} 47+# define SQLITE_CORRUPT_BKPT fts2Corrupt() 48+#else 49+# define SQLITE_CORRUPT_BKPT SQLITE_CORRUPT 50+#endif 51+ 52 /* It is not safe to call isspace(), tolower(), or isalnum() on 53 ** hi-bit-set characters. This is the same solution used in the 54 ** tokenizer. 55@@ -423,30 +447,41 @@ 56 /* Read a 64-bit variable-length integer from memory starting at p[0]. 57 * Return the number of bytes read, or 0 on error. 58 * The value is stored in *v. */ 59-static int getVarint(const char *p, sqlite_int64 *v){ 60+static int getVarintSafe(const char *p, sqlite_int64 *v, int max){ 61 const unsigned char *q = (const unsigned char *) p; 62 sqlite_uint64 x = 0, y = 1; 63- while( (*q & 0x80) == 0x80 ){ 64+ if( max>VARINT_MAX ) max = VARINT_MAX; 65+ while( max && (*q & 0x80) == 0x80 ){ 66+ max--; 67 x += y * (*q++ & 0x7f); 68 y <<= 7; 69- if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */ 70- assert( 0 ); 71- return 0; 72- } 73+ } 74+ if ( !max ){ 75+ assert( 0 ); 76+ return 0; /* tried to read too much; bad data */ 77 } 78 x += y * (*q++); 79 *v = (sqlite_int64) x; 80 return (int) (q - (unsigned char *)p); 81 } 82 83-static int getVarint32(const char *p, int *pi){ 84+static int getVarint(const char *p, sqlite_int64 *v){ 85+ return getVarintSafe(p, v, VARINT_MAX); 86+} 87+ 88+static int getVarint32Safe(const char *p, int *pi, int max){ 89 sqlite_int64 i; 90- int ret = getVarint(p, &i); 91+ int ret = getVarintSafe(p, &i, max); 92+ if( !ret ) return ret; 93 *pi = (int) i; 94 assert( *pi==i ); 95 return ret; 96 } 97 98+static int getVarint32(const char* p, int *pi){ 99+ return getVarint32Safe(p, pi, VARINT_MAX); 100+} 101+ 102 /*******************************************************************/ 103 /* DataBuffer is used to collect data into a buffer in piecemeal 104 ** fashion. It implements the usual distinction between amount of 105@@ -615,7 +650,7 @@ 106 107 static int dlrAtEnd(DLReader *pReader){ 108 assert( pReader->nData>=0 ); 109- return pReader->nData==0; 110+ return pReader->nData<=0; 111 } 112 static sqlite_int64 dlrDocid(DLReader *pReader){ 113 assert( !dlrAtEnd(pReader) ); 114@@ -639,7 +674,8 @@ 115 */ 116 static const char *dlrPosData(DLReader *pReader){ 117 sqlite_int64 iDummy; 118- int n = getVarint(pReader->pData, &iDummy); 119+ int n = getVarintSafe(pReader->pData, &iDummy, pReader->nElement); 120+ if( !n ) return NULL; 121 assert( !dlrAtEnd(pReader) ); 122 return pReader->pData+n; 123 } 124@@ -649,7 +685,7 @@ 125 assert( !dlrAtEnd(pReader) ); 126 return pReader->nElement-n; 127 } 128-static void dlrStep(DLReader *pReader){ 129+static int dlrStep(DLReader *pReader){ 130 assert( !dlrAtEnd(pReader) ); 131 132 /* Skip past current doclist element. */ 133@@ -658,32 +694,48 @@ 134 pReader->nData -= pReader->nElement; 135 136 /* If there is more data, read the next doclist element. */ 137- if( pReader->nData!=0 ){ 138+ if( pReader->nData>0 ){ 139 sqlite_int64 iDocidDelta; 140- int iDummy, n = getVarint(pReader->pData, &iDocidDelta); 141+ int nTotal = 0; 142+ int iDummy, n = getVarintSafe(pReader->pData, &iDocidDelta, pReader->nData); 143+ if( !n ) return SQLITE_CORRUPT_BKPT; 144+ nTotal += n; 145 pReader->iDocid += iDocidDelta; 146 if( pReader->iType>=DL_POSITIONS ){ 147- assert( n<pReader->nData ); 148 while( 1 ){ 149- n += getVarint32(pReader->pData+n, &iDummy); 150- assert( n<=pReader->nData ); 151+ n = getVarint32Safe(pReader->pData+nTotal, &iDummy, 152+ pReader->nData-nTotal); 153+ if( !n ) return SQLITE_CORRUPT_BKPT; 154+ nTotal += n; 155 if( iDummy==POS_END ) break; 156 if( iDummy==POS_COLUMN ){ 157- n += getVarint32(pReader->pData+n, &iDummy); 158- assert( n<pReader->nData ); 159+ n = getVarint32Safe(pReader->pData+nTotal, &iDummy, 160+ pReader->nData-nTotal); 161+ if( !n ) return SQLITE_CORRUPT_BKPT; 162+ nTotal += n; 163 }else if( pReader->iType==DL_POSITIONS_OFFSETS ){ 164- n += getVarint32(pReader->pData+n, &iDummy); 165- n += getVarint32(pReader->pData+n, &iDummy); 166- assert( n<pReader->nData ); 167+ n = getVarint32Safe(pReader->pData+nTotal, &iDummy, 168+ pReader->nData-nTotal); 169+ if( !n ) return SQLITE_CORRUPT_BKPT; 170+ nTotal += n; 171+ n = getVarint32Safe(pReader->pData+nTotal, &iDummy, 172+ pReader->nData-nTotal); 173+ if( !n ) return SQLITE_CORRUPT_BKPT; 174+ nTotal += n; 175 } 176 } 177 } 178- pReader->nElement = n; 179+ pReader->nElement = nTotal; 180 assert( pReader->nElement<=pReader->nData ); 181 } 182+ return SQLITE_OK; 183 } 184-static void dlrInit(DLReader *pReader, DocListType iType, 185- const char *pData, int nData){ 186+static void dlrDestroy(DLReader *pReader){ 187+ SCRAMBLE(pReader); 188+} 189+static int dlrInit(DLReader *pReader, DocListType iType, 190+ const char *pData, int nData){ 191+ int rc; 192 assert( pData!=NULL && nData!=0 ); 193 pReader->iType = iType; 194 pReader->pData = pData; 195@@ -692,10 +744,9 @@ 196 pReader->iDocid = 0; 197 198 /* Load the first element's data. There must be a first element. */ 199- dlrStep(pReader); 200-} 201-static void dlrDestroy(DLReader *pReader){ 202- SCRAMBLE(pReader); 203+ rc = dlrStep(pReader); 204+ if( rc!=SQLITE_OK ) dlrDestroy(pReader); 205+ return rc; 206 } 207 208 #ifndef NDEBUG 209@@ -782,9 +833,9 @@ 210 /* TODO(shess) This has become just a helper for docListMerge. 211 ** Consider a refactor to make this cleaner. 212 */ 213-static void dlwAppend(DLWriter *pWriter, 214- const char *pData, int nData, 215- sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){ 216+static int dlwAppend(DLWriter *pWriter, 217+ const char *pData, int nData, 218+ sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){ 219 sqlite_int64 iDocid = 0; 220 char c[VARINT_MAX]; 221 int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */ 222@@ -793,7 +844,8 @@ 223 #endif 224 225 /* Recode the initial docid as delta from iPrevDocid. */ 226- nFirstOld = getVarint(pData, &iDocid); 227+ nFirstOld = getVarintSafe(pData, &iDocid, nData); 228+ if( !nFirstOld ) return SQLITE_CORRUPT_BKPT; 229 assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) ); 230 nFirstNew = putVarint(c, iFirstDocid-pWriter->iPrevDocid); 231 232@@ -814,10 +866,11 @@ 233 dataBufferAppend(pWriter->b, c, nFirstNew); 234 } 235 pWriter->iPrevDocid = iLastDocid; 236+ return SQLITE_OK; 237 } 238-static void dlwCopy(DLWriter *pWriter, DLReader *pReader){ 239- dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader), 240- dlrDocid(pReader), dlrDocid(pReader)); 241+static int dlwCopy(DLWriter *pWriter, DLReader *pReader){ 242+ return dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader), 243+ dlrDocid(pReader), dlrDocid(pReader)); 244 } 245 static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){ 246 char c[VARINT_MAX]; 247@@ -878,45 +931,63 @@ 248 assert( !plrAtEnd(pReader) ); 249 return pReader->iEndOffset; 250 } 251-static void plrStep(PLReader *pReader){ 252- int i, n; 253+static int plrStep(PLReader *pReader){ 254+ int i, n, nTotal = 0; 255 256 assert( !plrAtEnd(pReader) ); 257 258- if( pReader->nData==0 ){ 259+ if( pReader->nData<=0 ){ 260 pReader->pData = NULL; 261- return; 262+ return SQLITE_OK; 263 } 264 265- n = getVarint32(pReader->pData, &i); 266+ n = getVarint32Safe(pReader->pData, &i, pReader->nData); 267+ if( !n ) return SQLITE_CORRUPT_BKPT; 268+ nTotal += n; 269 if( i==POS_COLUMN ){ 270- n += getVarint32(pReader->pData+n, &pReader->iColumn); 271+ n = getVarint32Safe(pReader->pData+nTotal, &pReader->iColumn, 272+ pReader->nData-nTotal); 273+ if( !n ) return SQLITE_CORRUPT_BKPT; 274+ nTotal += n; 275 pReader->iPosition = 0; 276 pReader->iStartOffset = 0; 277- n += getVarint32(pReader->pData+n, &i); 278+ n = getVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal); 279+ if( !n ) return SQLITE_CORRUPT_BKPT; 280+ nTotal += n; 281 } 282 /* Should never see adjacent column changes. */ 283 assert( i!=POS_COLUMN ); 284 285 if( i==POS_END ){ 286+ assert( nTotal<=pReader->nData ); 287 pReader->nData = 0; 288 pReader->pData = NULL; 289- return; 290+ return SQLITE_OK; 291 } 292 293 pReader->iPosition += i-POS_BASE; 294 if( pReader->iType==DL_POSITIONS_OFFSETS ){ 295- n += getVarint32(pReader->pData+n, &i); 296+ n = getVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal); 297+ if( !n ) return SQLITE_CORRUPT_BKPT; 298+ nTotal += n; 299 pReader->iStartOffset += i; 300- n += getVarint32(pReader->pData+n, &i); 301+ n = getVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal); 302+ if( !n ) return SQLITE_CORRUPT_BKPT; 303+ nTotal += n; 304 pReader->iEndOffset = pReader->iStartOffset+i; 305 } 306- assert( n<=pReader->nData ); 307- pReader->pData += n; 308- pReader->nData -= n; 309+ assert( nTotal<=pReader->nData ); 310+ pReader->pData += nTotal; 311+ pReader->nData -= nTotal; 312+ return SQLITE_OK; 313 } 314 315-static void plrInit(PLReader *pReader, DLReader *pDLReader){ 316+static void plrDestroy(PLReader *pReader){ 317+ SCRAMBLE(pReader); 318+} 319+ 320+static int plrInit(PLReader *pReader, DLReader *pDLReader){ 321+ int rc; 322 pReader->pData = dlrPosData(pDLReader); 323 pReader->nData = dlrPosDataLen(pDLReader); 324 pReader->iType = pDLReader->iType; 325@@ -924,10 +995,9 @@ 326 pReader->iPosition = 0; 327 pReader->iStartOffset = 0; 328 pReader->iEndOffset = 0; 329- plrStep(pReader); 330-} 331-static void plrDestroy(PLReader *pReader){ 332- SCRAMBLE(pReader); 333+ rc = plrStep(pReader); 334+ if( rc!=SQLITE_OK ) plrDestroy(pReader); 335+ return rc; 336 } 337 338 /*******************************************************************/ 339@@ -1113,14 +1183,16 @@ 340 ** deletion will be trimmed, and will thus not effect a deletion 341 ** during the merge. 342 */ 343-static void docListTrim(DocListType iType, const char *pData, int nData, 344- int iColumn, DocListType iOutType, DataBuffer *out){ 345+static int docListTrim(DocListType iType, const char *pData, int nData, 346+ int iColumn, DocListType iOutType, DataBuffer *out){ 347 DLReader dlReader; 348 DLWriter dlWriter; 349+ int rc; 350 351 assert( iOutType<=iType ); 352 353- dlrInit(&dlReader, iType, pData, nData); 354+ rc = dlrInit(&dlReader, iType, pData, nData); 355+ if( rc!=SQLITE_OK ) return rc; 356 dlwInit(&dlWriter, iOutType, out); 357 358 while( !dlrAtEnd(&dlReader) ){ 359@@ -1128,7 +1200,8 @@ 360 PLWriter plWriter; 361 int match = 0; 362 363- plrInit(&plReader, &dlReader); 364+ rc = plrInit(&plReader, &dlReader); 365+ if( rc!=SQLITE_OK ) break; 366 367 while( !plrAtEnd(&plReader) ){ 368 if( iColumn==-1 || plrColumn(&plReader)==iColumn ){ 369@@ -1139,7 +1212,11 @@ 370 plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader), 371 plrStartOffset(&plReader), plrEndOffset(&plReader)); 372 } 373- plrStep(&plReader); 374+ rc = plrStep(&plReader); 375+ if( rc!=SQLITE_OK ){ 376+ plrDestroy(&plReader); 377+ goto err; 378+ } 379 } 380 if( match ){ 381 plwTerminate(&plWriter); 382@@ -1147,10 +1224,13 @@ 383 } 384 385 plrDestroy(&plReader); 386- dlrStep(&dlReader); 387+ rc = dlrStep(&dlReader); 388+ if( rc!=SQLITE_OK ) break; 389 } 390+err: 391 dlwDestroy(&dlWriter); 392 dlrDestroy(&dlReader); 393+ return rc; 394 } 395 396 /* Used by docListMerge() to keep doclists in the ascending order by 397@@ -1207,19 +1287,20 @@ 398 /* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably 399 ** be fixed. 400 */ 401-static void docListMerge(DataBuffer *out, 402- DLReader *pReaders, int nReaders){ 403+static int docListMerge(DataBuffer *out, 404+ DLReader *pReaders, int nReaders){ 405 OrderedDLReader readers[MERGE_COUNT]; 406 DLWriter writer; 407 int i, n; 408 const char *pStart = 0; 409 int nStart = 0; 410 sqlite_int64 iFirstDocid = 0, iLastDocid = 0; 411+ int rc = SQLITE_OK; 412 413 assert( nReaders>0 ); 414 if( nReaders==1 ){ 415 dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders)); 416- return; 417+ return SQLITE_OK; 418 } 419 420 assert( nReaders<=MERGE_COUNT ); 421@@ -1252,20 +1333,23 @@ 422 nStart += dlrDocDataBytes(readers[0].pReader); 423 }else{ 424 if( pStart!=0 ){ 425- dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); 426+ rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); 427+ if( rc!=SQLITE_OK ) goto err; 428 } 429 pStart = dlrDocData(readers[0].pReader); 430 nStart = dlrDocDataBytes(readers[0].pReader); 431 iFirstDocid = iDocid; 432 } 433 iLastDocid = iDocid; 434- dlrStep(readers[0].pReader); 435+ rc = dlrStep(readers[0].pReader); 436+ if( rc!=SQLITE_OK ) goto err; 437 438 /* Drop all of the older elements with the same docid. */ 439 for(i=1; i<nReaders && 440 !dlrAtEnd(readers[i].pReader) && 441 dlrDocid(readers[i].pReader)==iDocid; i++){ 442- dlrStep(readers[i].pReader); 443+ rc = dlrStep(readers[i].pReader); 444+ if( rc!=SQLITE_OK ) goto err; 445 } 446 447 /* Get the readers back into order. */ 448@@ -1275,8 +1359,11 @@ 449 } 450 451 /* Copy over any remaining elements. */ 452- if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); 453+ if( nStart>0 ) 454+ rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); 455+err: 456 dlwDestroy(&writer); 457+ return rc; 458 } 459 460 /* Helper function for posListUnion(). Compares the current position 461@@ -1312,30 +1399,40 @@ 462 ** work with any doclist type, though both inputs and the output 463 ** should be the same type. 464 */ 465-static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){ 466+static int posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){ 467 PLReader left, right; 468 PLWriter writer; 469+ int rc; 470 471 assert( dlrDocid(pLeft)==dlrDocid(pRight) ); 472 assert( pLeft->iType==pRight->iType ); 473 assert( pLeft->iType==pOut->iType ); 474 475- plrInit(&left, pLeft); 476- plrInit(&right, pRight); 477+ rc = plrInit(&left, pLeft); 478+ if( rc != SQLITE_OK ) return rc; 479+ rc = plrInit(&right, pRight); 480+ if( rc != SQLITE_OK ){ 481+ plrDestroy(&left); 482+ return rc; 483+ } 484 plwInit(&writer, pOut, dlrDocid(pLeft)); 485 486 while( !plrAtEnd(&left) || !plrAtEnd(&right) ){ 487 int c = posListCmp(&left, &right); 488 if( c<0 ){ 489 plwCopy(&writer, &left); 490- plrStep(&left); 491+ rc = plrStep(&left); 492+ if( rc != SQLITE_OK ) break; 493 }else if( c>0 ){ 494 plwCopy(&writer, &right); 495- plrStep(&right); 496+ rc = plrStep(&right); 497+ if( rc != SQLITE_OK ) break; 498 }else{ 499 plwCopy(&writer, &left); 500- plrStep(&left); 501- plrStep(&right); 502+ rc = plrStep(&left); 503+ if( rc != SQLITE_OK ) break; 504+ rc = plrStep(&right); 505+ if( rc != SQLITE_OK ) break; 506 } 507 } 508 509@@ -1343,56 +1440,75 @@ 510 plwDestroy(&writer); 511 plrDestroy(&left); 512 plrDestroy(&right); 513+ return rc; 514 } 515 516 /* Write the union of doclists in pLeft and pRight to pOut. For 517 ** docids in common between the inputs, the union of the position 518 ** lists is written. Inputs and outputs are always type DL_DEFAULT. 519 */ 520-static void docListUnion( 521+static int docListUnion( 522 const char *pLeft, int nLeft, 523 const char *pRight, int nRight, 524 DataBuffer *pOut /* Write the combined doclist here */ 525 ){ 526 DLReader left, right; 527 DLWriter writer; 528+ int rc; 529 530 if( nLeft==0 ){ 531 if( nRight!=0) dataBufferAppend(pOut, pRight, nRight); 532- return; 533+ return SQLITE_OK; 534 } 535 if( nRight==0 ){ 536 dataBufferAppend(pOut, pLeft, nLeft); 537- return; 538+ return SQLITE_OK; 539 } 540 541- dlrInit(&left, DL_DEFAULT, pLeft, nLeft); 542- dlrInit(&right, DL_DEFAULT, pRight, nRight); 543+ rc = dlrInit(&left, DL_DEFAULT, pLeft, nLeft); 544+ if( rc!=SQLITE_OK ) return rc; 545+ rc = dlrInit(&right, DL_DEFAULT, pRight, nRight); 546+ if( rc!=SQLITE_OK ){ 547+ dlrDestroy(&left); 548+ return rc; 549+ } 550 dlwInit(&writer, DL_DEFAULT, pOut); 551 552 while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){ 553 if( dlrAtEnd(&right) ){ 554- dlwCopy(&writer, &left); 555- dlrStep(&left); 556+ rc = dlwCopy(&writer, &left); 557+ if( rc!=SQLITE_OK ) break; 558+ rc = dlrStep(&left); 559+ if( rc!=SQLITE_OK ) break; 560 }else if( dlrAtEnd(&left) ){ 561- dlwCopy(&writer, &right); 562- dlrStep(&right); 563+ rc = dlwCopy(&writer, &right); 564+ if( rc!=SQLITE_OK ) break; 565+ rc = dlrStep(&right); 566+ if( rc!=SQLITE_OK ) break; 567 }else if( dlrDocid(&left)<dlrDocid(&right) ){ 568- dlwCopy(&writer, &left); 569- dlrStep(&left); 570+ rc = dlwCopy(&writer, &left); 571+ if( rc!=SQLITE_OK ) break; 572+ rc = dlrStep(&left); 573+ if( rc!=SQLITE_OK ) break; 574 }else if( dlrDocid(&left)>dlrDocid(&right) ){ 575- dlwCopy(&writer, &right); 576- dlrStep(&right); 577+ rc = dlwCopy(&writer, &right); 578+ if( rc!=SQLITE_OK ) break; 579+ rc = dlrStep(&right); 580+ if( rc!=SQLITE_OK ) break; 581 }else{ 582- posListUnion(&left, &right, &writer); 583- dlrStep(&left); 584- dlrStep(&right); 585+ rc = posListUnion(&left, &right, &writer); 586+ if( rc!=SQLITE_OK ) break; 587+ rc = dlrStep(&left); 588+ if( rc!=SQLITE_OK ) break; 589+ rc = dlrStep(&right); 590+ if( rc!=SQLITE_OK ) break; 591 } 592 } 593 594 dlrDestroy(&left); 595 dlrDestroy(&right); 596 dlwDestroy(&writer); 597+ return rc; 598 } 599 600 /* pLeft and pRight are DLReaders positioned to the same docid. 601@@ -1407,35 +1523,47 @@ 602 ** include the positions from pRight that are one more than a 603 ** position in pLeft. In other words: pRight.iPos==pLeft.iPos+1. 604 */ 605-static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight, 606- DLWriter *pOut){ 607+static int posListPhraseMerge(DLReader *pLeft, DLReader *pRight, 608+ DLWriter *pOut){ 609 PLReader left, right; 610 PLWriter writer; 611 int match = 0; 612+ int rc; 613 614 assert( dlrDocid(pLeft)==dlrDocid(pRight) ); 615 assert( pOut->iType!=DL_POSITIONS_OFFSETS ); 616 617- plrInit(&left, pLeft); 618- plrInit(&right, pRight); 619+ rc = plrInit(&left, pLeft); 620+ if( rc!=SQLITE_OK ) return rc; 621+ rc = plrInit(&right, pRight); 622+ if( rc!=SQLITE_OK ){ 623+ plrDestroy(&left); 624+ return rc; 625+ } 626 627 while( !plrAtEnd(&left) && !plrAtEnd(&right) ){ 628 if( plrColumn(&left)<plrColumn(&right) ){ 629- plrStep(&left); 630+ rc = plrStep(&left); 631+ if( rc!=SQLITE_OK ) break; 632 }else if( plrColumn(&left)>plrColumn(&right) ){ 633- plrStep(&right); 634+ rc = plrStep(&right); 635+ if( rc!=SQLITE_OK ) break; 636 }else if( plrPosition(&left)+1<plrPosition(&right) ){ 637- plrStep(&left); 638+ rc = plrStep(&left); 639+ if( rc!=SQLITE_OK ) break; 640 }else if( plrPosition(&left)+1>plrPosition(&right) ){ 641- plrStep(&right); 642+ rc = plrStep(&right); 643+ if( rc!=SQLITE_OK ) break; 644 }else{ 645 if( !match ){ 646 plwInit(&writer, pOut, dlrDocid(pLeft)); 647 match = 1; 648 } 649 plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0); 650- plrStep(&left); 651- plrStep(&right); 652+ rc = plrStep(&left); 653+ if( rc!=SQLITE_OK ) break; 654+ rc = plrStep(&right); 655+ if( rc!=SQLITE_OK ) break; 656 } 657 } 658 659@@ -1446,6 +1574,7 @@ 660 661 plrDestroy(&left); 662 plrDestroy(&right); 663+ return rc; 664 } 665 666 /* We have two doclists with positions: pLeft and pRight. 667@@ -1457,7 +1586,7 @@ 668 ** iType controls the type of data written to pOut. If iType is 669 ** DL_POSITIONS, the positions are those from pRight. 670 */ 671-static void docListPhraseMerge( 672+static int docListPhraseMerge( 673 const char *pLeft, int nLeft, 674 const char *pRight, int nRight, 675 DocListType iType, 676@@ -1465,152 +1594,198 @@ 677 ){ 678 DLReader left, right; 679 DLWriter writer; 680+ int rc; 681 682- if( nLeft==0 || nRight==0 ) return; 683+ if( nLeft==0 || nRight==0 ) return SQLITE_OK; 684 685 assert( iType!=DL_POSITIONS_OFFSETS ); 686 687- dlrInit(&left, DL_POSITIONS, pLeft, nLeft); 688- dlrInit(&right, DL_POSITIONS, pRight, nRight); 689+ rc = dlrInit(&left, DL_POSITIONS, pLeft, nLeft); 690+ if( rc!=SQLITE_OK ) return rc; 691+ rc = dlrInit(&right, DL_POSITIONS, pRight, nRight); 692+ if( rc!=SQLITE_OK ){ 693+ dlrDestroy(&left); 694+ return rc; 695+ } 696 dlwInit(&writer, iType, pOut); 697 698 while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){ 699 if( dlrDocid(&left)<dlrDocid(&right) ){ 700- dlrStep(&left); 701+ rc = dlrStep(&left); 702+ if( rc!=SQLITE_OK ) break; 703 }else if( dlrDocid(&right)<dlrDocid(&left) ){ 704- dlrStep(&right); 705+ rc = dlrStep(&right); 706+ if( rc!=SQLITE_OK ) break; 707 }else{ 708- posListPhraseMerge(&left, &right, &writer); 709- dlrStep(&left); 710- dlrStep(&right); 711+ rc = posListPhraseMerge(&left, &right, &writer); 712+ if( rc!=SQLITE_OK ) break; 713+ rc = dlrStep(&left); 714+ if( rc!=SQLITE_OK ) break; 715+ rc = dlrStep(&right); 716+ if( rc!=SQLITE_OK ) break; 717 } 718 } 719 720 dlrDestroy(&left); 721 dlrDestroy(&right); 722 dlwDestroy(&writer); 723+ return rc; 724 } 725 726 /* We have two DL_DOCIDS doclists: pLeft and pRight. 727 ** Write the intersection of these two doclists into pOut as a 728 ** DL_DOCIDS doclist. 729 */ 730-static void docListAndMerge( 731+static int docListAndMerge( 732 const char *pLeft, int nLeft, 733 const char *pRight, int nRight, 734 DataBuffer *pOut /* Write the combined doclist here */ 735 ){ 736 DLReader left, right; 737 DLWriter writer; 738+ int rc; 739 740- if( nLeft==0 || nRight==0 ) return; 741+ if( nLeft==0 || nRight==0 ) return SQLITE_OK; 742 743- dlrInit(&left, DL_DOCIDS, pLeft, nLeft); 744- dlrInit(&right, DL_DOCIDS, pRight, nRight); 745+ rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft); 746+ if( rc!=SQLITE_OK ) return rc; 747+ rc = dlrInit(&right, DL_DOCIDS, pRight, nRight); 748+ if( rc!=SQLITE_OK ){ 749+ dlrDestroy(&left); 750+ return rc; 751+ } 752 dlwInit(&writer, DL_DOCIDS, pOut); 753 754 while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){ 755 if( dlrDocid(&left)<dlrDocid(&right) ){ 756- dlrStep(&left); 757+ rc = dlrStep(&left); 758+ if( rc!=SQLITE_OK ) break; 759 }else if( dlrDocid(&right)<dlrDocid(&left) ){ 760- dlrStep(&right); 761+ rc = dlrStep(&right); 762+ if( rc!=SQLITE_OK ) break; 763 }else{ 764 dlwAdd(&writer, dlrDocid(&left)); 765- dlrStep(&left); 766- dlrStep(&right); 767+ rc = dlrStep(&left); 768+ if( rc!=SQLITE_OK ) break; 769+ rc = dlrStep(&right); 770+ if( rc!=SQLITE_OK ) break; 771 } 772 } 773 774 dlrDestroy(&left); 775 dlrDestroy(&right); 776 dlwDestroy(&writer); 777+ return rc; 778 } 779 780 /* We have two DL_DOCIDS doclists: pLeft and pRight. 781 ** Write the union of these two doclists into pOut as a 782 ** DL_DOCIDS doclist. 783 */ 784-static void docListOrMerge( 785+static int docListOrMerge( 786 const char *pLeft, int nLeft, 787 const char *pRight, int nRight, 788 DataBuffer *pOut /* Write the combined doclist here */ 789 ){ 790 DLReader left, right; 791 DLWriter writer; 792+ int rc; 793 794 if( nLeft==0 ){ 795 if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight); 796- return; 797+ return SQLITE_OK; 798 } 799 if( nRight==0 ){ 800 dataBufferAppend(pOut, pLeft, nLeft); 801- return; 802+ return SQLITE_OK; 803 } 804 805- dlrInit(&left, DL_DOCIDS, pLeft, nLeft); 806- dlrInit(&right, DL_DOCIDS, pRight, nRight); 807+ rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft); 808+ if( rc!=SQLITE_OK ) return rc; 809+ rc = dlrInit(&right, DL_DOCIDS, pRight, nRight); 810+ if( rc!=SQLITE_OK ){ 811+ dlrDestroy(&left); 812+ return rc; 813+ } 814 dlwInit(&writer, DL_DOCIDS, pOut); 815 816 while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){ 817 if( dlrAtEnd(&right) ){ 818 dlwAdd(&writer, dlrDocid(&left)); 819- dlrStep(&left); 820+ rc = dlrStep(&left); 821+ if( rc!=SQLITE_OK ) break; 822 }else if( dlrAtEnd(&left) ){ 823 dlwAdd(&writer, dlrDocid(&right)); 824- dlrStep(&right); 825+ rc = dlrStep(&right); 826+ if( rc!=SQLITE_OK ) break; 827 }else if( dlrDocid(&left)<dlrDocid(&right) ){ 828 dlwAdd(&writer, dlrDocid(&left)); 829- dlrStep(&left); 830+ rc = dlrStep(&left); 831+ if( rc!=SQLITE_OK ) break; 832 }else if( dlrDocid(&right)<dlrDocid(&left) ){ 833 dlwAdd(&writer, dlrDocid(&right)); 834- dlrStep(&right); 835+ rc = dlrStep(&right); 836+ if( rc!=SQLITE_OK ) break; 837 }else{ 838 dlwAdd(&writer, dlrDocid(&left)); 839- dlrStep(&left); 840- dlrStep(&right); 841+ rc = dlrStep(&left); 842+ if( rc!=SQLITE_OK ) break; 843+ rc = dlrStep(&right); 844+ if( rc!=SQLITE_OK ) break; 845 } 846 } 847 848 dlrDestroy(&left); 849 dlrDestroy(&right); 850 dlwDestroy(&writer); 851+ return rc; 852 } 853 854 /* We have two DL_DOCIDS doclists: pLeft and pRight. 855 ** Write into pOut as DL_DOCIDS doclist containing all documents that 856 ** occur in pLeft but not in pRight. 857 */ 858-static void docListExceptMerge( 859+static int docListExceptMerge( 860 const char *pLeft, int nLeft, 861 const char *pRight, int nRight, 862 DataBuffer *pOut /* Write the combined doclist here */ 863 ){ 864 DLReader left, right; 865 DLWriter writer; 866+ int rc; 867 868- if( nLeft==0 ) return; 869+ if( nLeft==0 ) return SQLITE_OK; 870 if( nRight==0 ){ 871 dataBufferAppend(pOut, pLeft, nLeft); 872- return; 873+ return SQLITE_OK; 874 } 875 876- dlrInit(&left, DL_DOCIDS, pLeft, nLeft); 877- dlrInit(&right, DL_DOCIDS, pRight, nRight); 878+ rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft); 879+ if( rc!=SQLITE_OK ) return rc; 880+ rc = dlrInit(&right, DL_DOCIDS, pRight, nRight); 881+ if( rc!=SQLITE_OK ){ 882+ dlrDestroy(&left); 883+ return rc; 884+ } 885 dlwInit(&writer, DL_DOCIDS, pOut); 886 887 while( !dlrAtEnd(&left) ){ 888 while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){ 889- dlrStep(&right); 890+ rc = dlrStep(&right); 891+ if( rc!=SQLITE_OK ) goto err; 892 } 893 if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){ 894 dlwAdd(&writer, dlrDocid(&left)); 895 } 896- dlrStep(&left); 897+ rc = dlrStep(&left); 898+ if( rc!=SQLITE_OK ) break; 899 } 900 901+err: 902 dlrDestroy(&left); 903 dlrDestroy(&right); 904 dlwDestroy(&writer); 905+ return rc; 906 } 907 908 static char *string_dup_n(const char *s, int n){ 909@@ -1814,7 +1989,7 @@ 910 /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?", 911 /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)", 912 /* SEGDIR_SELECT_LEVEL */ 913- "select start_block, leaves_end_block, root from %_segdir " 914+ "select start_block, leaves_end_block, root, idx from %_segdir " 915 " where level = ? order by idx", 916 /* SEGDIR_SPAN */ 917 "select min(start_block), max(end_block) from %_segdir " 918@@ -3413,7 +3588,8 @@ 919 return SQLITE_OK; 920 } 921 rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader)); 922- dlrStep(&c->reader); 923+ if( rc!=SQLITE_OK ) return rc; 924+ rc = dlrStep(&c->reader); 925 if( rc!=SQLITE_OK ) return rc; 926 /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ 927 rc = sqlite3_step(c->pStmt); 928@@ -3421,8 +3597,11 @@ 929 c->eof = 0; 930 return SQLITE_OK; 931 } 932- /* an error occurred; abort */ 933- return rc==SQLITE_DONE ? SQLITE_ERROR : rc; 934+ 935+ /* Corrupt if the index refers to missing document. */ 936+ if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT; 937+ 938+ return rc; 939 } 940 } 941 942@@ -3470,14 +3649,18 @@ 943 return rc; 944 } 945 dataBufferInit(&new, 0); 946- docListPhraseMerge(left.pData, left.nData, right.pData, right.nData, 947- i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &new); 948+ rc = docListPhraseMerge(left.pData, left.nData, right.pData, right.nData, 949+ i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &new); 950 dataBufferDestroy(&left); 951 dataBufferDestroy(&right); 952+ if( rc!=SQLITE_OK ){ 953+ dataBufferDestroy(&new); 954+ return rc; 955+ } 956 left = new; 957 } 958 *pResult = left; 959- return SQLITE_OK; 960+ return rc; 961 } 962 963 /* Add a new term pTerm[0..nTerm-1] to the query *q. 964@@ -3544,6 +3727,7 @@ 965 int firstIndex = pQuery->nTerms; 966 int iCol; 967 int nTerm = 1; 968+ int iEndLast = -1; 969 970 int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor); 971 if( rc!=SQLITE_OK ) return rc; 972@@ -3568,6 +3752,20 @@ 973 pQuery->nextIsOr = 1; 974 continue; 975 } 976+ 977+ /* 978+ * The ICU tokenizer considers '*' a break character, so the code below 979+ * sets isPrefix correctly, but since that code doesn't eat the '*', the 980+ * ICU tokenizer returns it as the next token. So eat it here until a 981+ * better solution presents itself. 982+ */ 983+ if( pQuery->nTerms>0 && nToken==1 && pSegment[iBegin]=='*' && 984+ iEndLast==iBegin){ 985+ pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1; 986+ continue; 987+ } 988+ iEndLast = iEnd; 989+ 990 queryAdd(pQuery, pToken, nToken); 991 if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){ 992 pQuery->pTerms[pQuery->nTerms-1].isNot = 1; 993@@ -3707,18 +3905,30 @@ 994 return rc; 995 } 996 dataBufferInit(&new, 0); 997- docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new); 998+ rc = docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new); 999 dataBufferDestroy(&right); 1000 dataBufferDestroy(&or); 1001+ if( rc!=SQLITE_OK ){ 1002+ if( i!=nNot ) dataBufferDestroy(&left); 1003+ queryClear(pQuery); 1004+ dataBufferDestroy(&new); 1005+ return rc; 1006+ } 1007 right = new; 1008 } 1009 if( i==nNot ){ /* first term processed. */ 1010 left = right; 1011 }else{ 1012 dataBufferInit(&new, 0); 1013- docListAndMerge(left.pData, left.nData, right.pData, right.nData, &new); 1014+ rc = docListAndMerge(left.pData, left.nData, 1015+ right.pData, right.nData, &new); 1016 dataBufferDestroy(&right); 1017 dataBufferDestroy(&left); 1018+ if( rc!=SQLITE_OK ){ 1019+ queryClear(pQuery); 1020+ dataBufferDestroy(&new); 1021+ return rc; 1022+ } 1023 left = new; 1024 } 1025 } 1026@@ -3738,9 +3948,15 @@ 1027 return rc; 1028 } 1029 dataBufferInit(&new, 0); 1030- docListExceptMerge(left.pData, left.nData, right.pData, right.nData, &new); 1031+ rc = docListExceptMerge(left.pData, left.nData, 1032+ right.pData, right.nData, &new); 1033 dataBufferDestroy(&right); 1034 dataBufferDestroy(&left); 1035+ if( rc!=SQLITE_OK ){ 1036+ queryClear(pQuery); 1037+ dataBufferDestroy(&new); 1038+ return rc; 1039+ } 1040 left = new; 1041 } 1042 1043@@ -3834,7 +4050,8 @@ 1044 rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &c->result, &c->q); 1045 if( rc!=SQLITE_OK ) return rc; 1046 if( c->result.nData!=0 ){ 1047- dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData); 1048+ rc = dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData); 1049+ if( rc!=SQLITE_OK ) return rc; 1050 } 1051 break; 1052 } 1053@@ -4335,22 +4552,19 @@ 1054 SCRAMBLE(pReader); 1055 } 1056 1057-/* TODO(shess) The assertions are great, but what if we're in NDEBUG 1058-** and the blob is empty or otherwise contains suspect data? 1059-*/ 1060-static void interiorReaderInit(const char *pData, int nData, 1061- InteriorReader *pReader){ 1062+static int interiorReaderInit(const char *pData, int nData, 1063+ InteriorReader *pReader){ 1064 int n, nTerm; 1065 1066- /* Require at least the leading flag byte */ 1067+ /* These conditions are checked and met by the callers. */ 1068 assert( nData>0 ); 1069 assert( pData[0]!='\0' ); 1070 1071 CLEAR(pReader); 1072 1073 /* Decode the base blockid, and set the cursor to the first term. */ 1074- n = getVarint(pData+1, &pReader->iBlockid); 1075- assert( 1+n<=nData ); 1076+ n = getVarintSafe(pData+1, &pReader->iBlockid, nData-1); 1077+ if( !n ) return SQLITE_CORRUPT_BKPT; 1078 pReader->pData = pData+1+n; 1079 pReader->nData = nData-(1+n); 1080 1081@@ -4361,17 +4575,18 @@ 1082 if( pReader->nData==0 ){ 1083 dataBufferInit(&pReader->term, 0); 1084 }else{ 1085- n = getVarint32(pReader->pData, &nTerm); 1086+ n = getVarint32Safe(pReader->pData, &nTerm, pReader->nData); 1087+ if( !n || nTerm<0 || nTerm>pReader->nData-n) return SQLITE_CORRUPT_BKPT; 1088 dataBufferInit(&pReader->term, nTerm); 1089 dataBufferReplace(&pReader->term, pReader->pData+n, nTerm); 1090- assert( n+nTerm<=pReader->nData ); 1091 pReader->pData += n+nTerm; 1092 pReader->nData -= n+nTerm; 1093 } 1094+ return SQLITE_OK; 1095 } 1096 1097 static int interiorReaderAtEnd(InteriorReader *pReader){ 1098- return pReader->term.nData==0; 1099+ return pReader->term.nData<=0; 1100 } 1101 1102 static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){ 1103@@ -4388,7 +4603,7 @@ 1104 } 1105 1106 /* Step forward to the next term in the node. */ 1107-static void interiorReaderStep(InteriorReader *pReader){ 1108+static int interiorReaderStep(InteriorReader *pReader){ 1109 assert( !interiorReaderAtEnd(pReader) ); 1110 1111 /* If the last term has been read, signal eof, else construct the 1112@@ -4399,18 +4614,26 @@ 1113 }else{ 1114 int n, nPrefix, nSuffix; 1115 1116- n = getVarint32(pReader->pData, &nPrefix); 1117- n += getVarint32(pReader->pData+n, &nSuffix); 1118+ n = getVarint32Safe(pReader->pData, &nPrefix, pReader->nData); 1119+ if( !n ) return SQLITE_CORRUPT_BKPT; 1120+ pReader->nData -= n; 1121+ pReader->pData += n; 1122+ n = getVarint32Safe(pReader->pData, &nSuffix, pReader->nData); 1123+ if( !n ) return SQLITE_CORRUPT_BKPT; 1124+ pReader->nData -= n; 1125+ pReader->pData += n; 1126+ if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT; 1127+ if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT; 1128 1129 /* Truncate the current term and append suffix data. */ 1130 pReader->term.nData = nPrefix; 1131- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); 1132+ dataBufferAppend(&pReader->term, pReader->pData, nSuffix); 1133 1134- assert( n+nSuffix<=pReader->nData ); 1135- pReader->pData += n+nSuffix; 1136- pReader->nData -= n+nSuffix; 1137+ pReader->pData += nSuffix; 1138+ pReader->nData -= nSuffix; 1139 } 1140 pReader->iBlockid++; 1141+ return SQLITE_OK; 1142 } 1143 1144 /* Compare the current term to pTerm[nTerm], returning strcmp-style 1145@@ -4782,7 +5005,8 @@ 1146 n = putVarint(c, nData); 1147 dataBufferAppend(&pWriter->data, c, n); 1148 1149- docListMerge(&pWriter->data, pReaders, nReaders); 1150+ rc = docListMerge(&pWriter->data, pReaders, nReaders); 1151+ if( rc!= SQLITE_OK ) return rc; 1152 ASSERT_VALID_DOCLIST(DL_DEFAULT, 1153 pWriter->data.pData+iDoclistData+n, 1154 pWriter->data.nData-iDoclistData-n, NULL); 1155@@ -4892,7 +5116,8 @@ 1156 int rc; 1157 DLReader reader; 1158 1159- dlrInit(&reader, DL_DEFAULT, pData, nData); 1160+ rc = dlrInit(&reader, DL_DEFAULT, pData, nData); 1161+ if( rc!=SQLITE_OK ) return rc; 1162 rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1); 1163 dlrDestroy(&reader); 1164 1165@@ -4937,38 +5162,40 @@ 1166 static const char *leafReaderData(LeafReader *pReader){ 1167 int n, nData; 1168 assert( pReader->term.nData>0 ); 1169- n = getVarint32(pReader->pData, &nData); 1170+ n = getVarint32Safe(pReader->pData, &nData, pReader->nData); 1171+ if( !n || nData>pReader->nData-n ) return NULL; 1172 return pReader->pData+n; 1173 } 1174 1175-static void leafReaderInit(const char *pData, int nData, 1176- LeafReader *pReader){ 1177+static int leafReaderInit(const char *pData, int nData, LeafReader *pReader){ 1178 int nTerm, n; 1179 1180+ /* All callers check this precondition. */ 1181 assert( nData>0 ); 1182 assert( pData[0]=='\0' ); 1183 1184 CLEAR(pReader); 1185 1186 /* Read the first term, skipping the header byte. */ 1187- n = getVarint32(pData+1, &nTerm); 1188+ n = getVarint32Safe(pData+1, &nTerm, nData-1); 1189+ if( !n || nTerm<0 || nTerm>nData-1-n ) return SQLITE_CORRUPT_BKPT; 1190 dataBufferInit(&pReader->term, nTerm); 1191 dataBufferReplace(&pReader->term, pData+1+n, nTerm); 1192 1193 /* Position after the first term. */ 1194- assert( 1+n+nTerm<nData ); 1195 pReader->pData = pData+1+n+nTerm; 1196 pReader->nData = nData-1-n-nTerm; 1197+ return SQLITE_OK; 1198 } 1199 1200 /* Step the reader forward to the next term. */ 1201-static void leafReaderStep(LeafReader *pReader){ 1202+static int leafReaderStep(LeafReader *pReader){ 1203 int n, nData, nPrefix, nSuffix; 1204 assert( !leafReaderAtEnd(pReader) ); 1205 1206 /* Skip previous entry's data block. */ 1207- n = getVarint32(pReader->pData, &nData); 1208- assert( n+nData<=pReader->nData ); 1209+ n = getVarint32Safe(pReader->pData, &nData, pReader->nData); 1210+ if( !n || nData<0 || nData>pReader->nData-n ) return SQLITE_CORRUPT_BKPT; 1211 pReader->pData += n+nData; 1212 pReader->nData -= n+nData; 1213 1214@@ -4976,15 +5203,23 @@ 1215 /* Construct the new term using a prefix from the old term plus a 1216 ** suffix from the leaf data. 1217 */ 1218- n = getVarint32(pReader->pData, &nPrefix); 1219- n += getVarint32(pReader->pData+n, &nSuffix); 1220- assert( n+nSuffix<pReader->nData ); 1221+ n = getVarint32Safe(pReader->pData, &nPrefix, pReader->nData); 1222+ if( !n ) return SQLITE_CORRUPT_BKPT; 1223+ pReader->nData -= n; 1224+ pReader->pData += n; 1225+ n = getVarint32Safe(pReader->pData, &nSuffix, pReader->nData); 1226+ if( !n ) return SQLITE_CORRUPT_BKPT; 1227+ pReader->nData -= n; 1228+ pReader->pData += n; 1229+ if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT; 1230+ if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT; 1231 pReader->term.nData = nPrefix; 1232- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); 1233+ dataBufferAppend(&pReader->term, pReader->pData, nSuffix); 1234 1235- pReader->pData += n+nSuffix; 1236- pReader->nData -= n+nSuffix; 1237+ pReader->pData += nSuffix; 1238+ pReader->nData -= nSuffix; 1239 } 1240+ return SQLITE_OK; 1241 } 1242 1243 /* strcmp-style comparison of pReader's current term against pTerm. 1244@@ -5077,6 +5312,9 @@ 1245 ** the leaf data was entirely contained in the root), or from the 1246 ** stream of blocks between iStartBlockid and iEndBlockid, inclusive. 1247 */ 1248+/* TODO(shess): Figure out a means of indicating how many leaves are 1249+** expected, for purposes of detecting corruption. 1250+*/ 1251 static int leavesReaderInit(fulltext_vtab *v, 1252 int idx, 1253 sqlite_int64 iStartBlockid, 1254@@ -5088,32 +5326,67 @@ 1255 1256 dataBufferInit(&pReader->rootData, 0); 1257 if( iStartBlockid==0 ){ 1258+ int rc; 1259+ /* Corrupt if this can't be a leaf node. */ 1260+ if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){ 1261+ return SQLITE_CORRUPT_BKPT; 1262+ } 1263 /* Entire leaf level fit in root data. */ 1264 dataBufferReplace(&pReader->rootData, pRootData, nRootData); 1265- leafReaderInit(pReader->rootData.pData, pReader->rootData.nData, 1266- &pReader->leafReader); 1267+ rc = leafReaderInit(pReader->rootData.pData, pReader->rootData.nData, 1268+ &pReader->leafReader); 1269+ if( rc!=SQLITE_OK ){ 1270+ dataBufferDestroy(&pReader->rootData); 1271+ return rc; 1272+ } 1273 }else{ 1274 sqlite3_stmt *s; 1275 int rc = sql_get_leaf_statement(v, idx, &s); 1276 if( rc!=SQLITE_OK ) return rc; 1277 1278 rc = sqlite3_bind_int64(s, 1, iStartBlockid); 1279- if( rc!=SQLITE_OK ) return rc; 1280+ if( rc!=SQLITE_OK ) goto err; 1281 1282 rc = sqlite3_bind_int64(s, 2, iEndBlockid); 1283- if( rc!=SQLITE_OK ) return rc; 1284+ if( rc!=SQLITE_OK ) goto err; 1285 1286 rc = sqlite3_step(s); 1287+ 1288+ /* Corrupt if interior node referenced missing leaf node. */ 1289 if( rc==SQLITE_DONE ){ 1290- pReader->eof = 1; 1291- return SQLITE_OK; 1292+ rc = SQLITE_CORRUPT_BKPT; 1293+ goto err; 1294+ } 1295+ 1296+ if( rc!=SQLITE_ROW ) goto err; 1297+ rc = SQLITE_OK; 1298+ 1299+ /* Corrupt if leaf data isn't a blob. */ 1300+ if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){ 1301+ rc = SQLITE_CORRUPT_BKPT; 1302+ }else{ 1303+ const char *pLeafData = sqlite3_column_blob(s, 0); 1304+ int nLeafData = sqlite3_column_bytes(s, 0); 1305+ 1306+ /* Corrupt if this can't be a leaf node. */ 1307+ if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){ 1308+ rc = SQLITE_CORRUPT_BKPT; 1309+ }else{ 1310+ rc = leafReaderInit(pLeafData, nLeafData, &pReader->leafReader); 1311+ } 1312+ } 1313+ 1314+ err: 1315+ if( rc!=SQLITE_OK ){ 1316+ if( idx==-1 ){ 1317+ sqlite3_finalize(s); 1318+ }else{ 1319+ sqlite3_reset(s); 1320+ } 1321+ return rc; 1322 } 1323- if( rc!=SQLITE_ROW ) return rc; 1324 1325 pReader->pStmt = s; 1326- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), 1327- sqlite3_column_bytes(pReader->pStmt, 0), 1328- &pReader->leafReader); 1329 } 1330 return SQLITE_OK; 1331 } 1332@@ -5122,11 +5395,12 @@ 1333 ** end of the current leaf, step forward to the next leaf block. 1334 */ 1335 static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){ 1336+ int rc; 1337 assert( !leavesReaderAtEnd(pReader) ); 1338- leafReaderStep(&pReader->leafReader); 1339+ rc = leafReaderStep(&pReader->leafReader); 1340+ if( rc!=SQLITE_OK ) return rc; 1341 1342 if( leafReaderAtEnd(&pReader->leafReader) ){ 1343- int rc; 1344 if( pReader->rootData.pData ){ 1345 pReader->eof = 1; 1346 return SQLITE_OK; 1347@@ -5136,10 +5410,25 @@ 1348 pReader->eof = 1; 1349 return rc==SQLITE_DONE ? SQLITE_OK : rc; 1350 } 1351- leafReaderDestroy(&pReader->leafReader); 1352- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), 1353- sqlite3_column_bytes(pReader->pStmt, 0), 1354- &pReader->leafReader); 1355+ 1356+ /* Corrupt if leaf data isn't a blob. */ 1357+ if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){ 1358+ return SQLITE_CORRUPT_BKPT; 1359+ }else{ 1360+ LeafReader tmp; 1361+ const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0); 1362+ int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0); 1363+ 1364+ /* Corrupt if this can't be a leaf node. */ 1365+ if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){ 1366+ return SQLITE_CORRUPT_BKPT; 1367+ } 1368+ 1369+ rc = leafReaderInit(pLeafData, nLeafData, &tmp); 1370+ if( rc!=SQLITE_OK ) return rc; 1371+ leafReaderDestroy(&pReader->leafReader); 1372+ pReader->leafReader = tmp; 1373+ } 1374 } 1375 return SQLITE_OK; 1376 } 1377@@ -5200,8 +5489,19 @@ 1378 sqlite_int64 iEnd = sqlite3_column_int64(s, 1); 1379 const char *pRootData = sqlite3_column_blob(s, 2); 1380 int nRootData = sqlite3_column_bytes(s, 2); 1381+ sqlite_int64 iIndex = sqlite3_column_int64(s, 3); 1382+ 1383+ /* Corrupt if we get back different types than we stored. */ 1384+ /* Also corrupt if the index is not sequential starting at 0. */ 1385+ if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER || 1386+ sqlite3_column_type(s, 1)!=SQLITE_INTEGER || 1387+ sqlite3_column_type(s, 2)!=SQLITE_BLOB || 1388+ i!=iIndex || 1389+ i>=MERGE_COUNT ){ 1390+ rc = SQLITE_CORRUPT_BKPT; 1391+ break; 1392+ } 1393 1394- assert( i<MERGE_COUNT ); 1395 rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData, 1396 &pReaders[i]); 1397 if( rc!=SQLITE_OK ) break; 1398@@ -5212,6 +5512,7 @@ 1399 while( i-->0 ){ 1400 leavesReaderDestroy(&pReaders[i]); 1401 } 1402+ sqlite3_reset(s); /* So we don't leave a lock. */ 1403 return rc; 1404 } 1405 1406@@ -5235,13 +5536,26 @@ 1407 DLReader dlReaders[MERGE_COUNT]; 1408 const char *pTerm = leavesReaderTerm(pReaders); 1409 int i, nTerm = leavesReaderTermBytes(pReaders); 1410+ int rc; 1411 1412 assert( nReaders<=MERGE_COUNT ); 1413 1414 for(i=0; i<nReaders; i++){ 1415- dlrInit(&dlReaders[i], DL_DEFAULT, 1416- leavesReaderData(pReaders+i), 1417- leavesReaderDataBytes(pReaders+i)); 1418+ const char *pData = leavesReaderData(pReaders+i); 1419+ if( pData==NULL ){ 1420+ rc = SQLITE_CORRUPT_BKPT; 1421+ break; 1422+ } 1423+ rc = dlrInit(&dlReaders[i], DL_DEFAULT, 1424+ pData, 1425+ leavesReaderDataBytes(pReaders+i)); 1426+ if( rc!=SQLITE_OK ) break; 1427+ } 1428+ if( rc!=SQLITE_OK ){ 1429+ while( i-->0 ){ 1430+ dlrDestroy(&dlReaders[i]); 1431+ } 1432+ return rc; 1433 } 1434 1435 return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders); 1436@@ -5295,10 +5609,14 @@ 1437 memset(&lrs, '\0', sizeof(lrs)); 1438 rc = leavesReadersInit(v, iLevel, lrs, &i); 1439 if( rc!=SQLITE_OK ) return rc; 1440- assert( i==MERGE_COUNT ); 1441 1442 leafWriterInit(iLevel+1, idx, &writer); 1443 1444+ if( i!=MERGE_COUNT ){ 1445+ rc = SQLITE_CORRUPT_BKPT; 1446+ goto err; 1447+ } 1448+ 1449 /* Since leavesReaderReorder() pushes readers at eof to the end, 1450 ** when the first reader is empty, all will be empty. 1451 */ 1452@@ -5341,12 +5659,14 @@ 1453 } 1454 1455 /* Accumulate the union of *acc and *pData into *acc. */ 1456-static void docListAccumulateUnion(DataBuffer *acc, 1457- const char *pData, int nData) { 1458+static int docListAccumulateUnion(DataBuffer *acc, 1459+ const char *pData, int nData) { 1460 DataBuffer tmp = *acc; 1461+ int rc; 1462 dataBufferInit(acc, tmp.nData+nData); 1463- docListUnion(tmp.pData, tmp.nData, pData, nData, acc); 1464+ rc = docListUnion(tmp.pData, tmp.nData, pData, nData, acc); 1465 dataBufferDestroy(&tmp); 1466+ return rc; 1467 } 1468 1469 /* TODO(shess) It might be interesting to explore different merge 1470@@ -5388,8 +5708,13 @@ 1471 int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix); 1472 if( c>0 ) break; /* Past any possible matches. */ 1473 if( c==0 ){ 1474+ int iBuffer, nData; 1475 const char *pData = leavesReaderData(pReader); 1476- int iBuffer, nData = leavesReaderDataBytes(pReader); 1477+ if( pData==NULL ){ 1478+ rc = SQLITE_CORRUPT_BKPT; 1479+ break; 1480+ } 1481+ nData = leavesReaderDataBytes(pReader); 1482 1483 /* Find the first empty buffer. */ 1484 for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){ 1485@@ -5435,11 +5760,13 @@ 1486 ** with pData/nData. 1487 */ 1488 dataBufferSwap(p, pAcc); 1489- docListAccumulateUnion(pAcc, pData, nData); 1490+ rc = docListAccumulateUnion(pAcc, pData, nData); 1491+ if( rc!=SQLITE_OK ) goto err; 1492 1493 /* Accumulate remaining doclists into pAcc. */ 1494 for(++p; p<pAcc; ++p){ 1495- docListAccumulateUnion(pAcc, p->pData, p->nData); 1496+ rc = docListAccumulateUnion(pAcc, p->pData, p->nData); 1497+ if( rc!=SQLITE_OK ) goto err; 1498 1499 /* dataBufferReset() could allow a large doclist to blow up 1500 ** our memory requirements. 1501@@ -5464,13 +5791,15 @@ 1502 if( out->nData==0 ){ 1503 dataBufferSwap(out, &(pBuffers[iBuffer])); 1504 }else{ 1505- docListAccumulateUnion(out, pBuffers[iBuffer].pData, 1506- pBuffers[iBuffer].nData); 1507+ rc = docListAccumulateUnion(out, pBuffers[iBuffer].pData, 1508+ pBuffers[iBuffer].nData); 1509+ if( rc!=SQLITE_OK ) break; 1510 } 1511 } 1512 } 1513 } 1514 1515+err: 1516 while( nBuffers-- ){ 1517 dataBufferDestroy(&(pBuffers[nBuffers])); 1518 } 1519@@ -5529,20 +5858,26 @@ 1520 ** node. Consider whether breaking symmetry is worthwhile. I suspect 1521 ** it is not worthwhile. 1522 */ 1523-static void getChildrenContaining(const char *pData, int nData, 1524- const char *pTerm, int nTerm, int isPrefix, 1525- sqlite_int64 *piStartChild, 1526- sqlite_int64 *piEndChild){ 1527+static int getChildrenContaining(const char *pData, int nData, 1528+ const char *pTerm, int nTerm, int isPrefix, 1529+ sqlite_int64 *piStartChild, 1530+ sqlite_int64 *piEndChild){ 1531 InteriorReader reader; 1532+ int rc; 1533 1534 assert( nData>1 ); 1535 assert( *pData!='\0' ); 1536- interiorReaderInit(pData, nData, &reader); 1537+ rc = interiorReaderInit(pData, nData, &reader); 1538+ if( rc!=SQLITE_OK ) return rc; 1539 1540 /* Scan for the first child which could contain pTerm/nTerm. */ 1541 while( !interiorReaderAtEnd(&reader) ){ 1542 if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break; 1543- interiorReaderStep(&reader); 1544+ rc = interiorReaderStep(&reader); 1545+ if( rc!=SQLITE_OK ){ 1546+ interiorReaderDestroy(&reader); 1547+ return rc; 1548+ } 1549 } 1550 *piStartChild = interiorReaderCurrentBlockid(&reader); 1551 1552@@ -5552,7 +5887,11 @@ 1553 */ 1554 while( !interiorReaderAtEnd(&reader) ){ 1555 if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break; 1556- interiorReaderStep(&reader); 1557+ rc = interiorReaderStep(&reader); 1558+ if( rc!=SQLITE_OK ){ 1559+ interiorReaderDestroy(&reader); 1560+ return rc; 1561+ } 1562 } 1563 *piEndChild = interiorReaderCurrentBlockid(&reader); 1564 1565@@ -5561,6 +5900,7 @@ 1566 /* Children must ascend, and if !prefix, both must be the same. */ 1567 assert( *piEndChild>=*piStartChild ); 1568 assert( isPrefix || *piStartChild==*piEndChild ); 1569+ return rc; 1570 } 1571 1572 /* Read block at iBlockid and pass it with other params to 1573@@ -5588,11 +5928,31 @@ 1574 if( rc!=SQLITE_OK ) return rc; 1575 1576 rc = sqlite3_step(s); 1577- if( rc==SQLITE_DONE ) return SQLITE_ERROR; 1578+ /* Corrupt if interior node references missing child node. */ 1579+ if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT; 1580 if( rc!=SQLITE_ROW ) return rc; 1581 1582- getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0), 1583- pTerm, nTerm, isPrefix, piStartChild, piEndChild); 1584+ /* Corrupt if child node isn't a blob. */ 1585+ if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){ 1586+ sqlite3_reset(s); /* So we don't leave a lock. */ 1587+ return SQLITE_CORRUPT_BKPT; 1588+ }else{ 1589+ const char *pData = sqlite3_column_blob(s, 0); 1590+ int nData = sqlite3_column_bytes(s, 0); 1591+ 1592+ /* Corrupt if child is not a valid interior node. */ 1593+ if( pData==NULL || nData<1 || pData[0]=='\0' ){ 1594+ sqlite3_reset(s); /* So we don't leave a lock. */ 1595+ return SQLITE_CORRUPT_BKPT; 1596+ } 1597+ 1598+ rc = getChildrenContaining(pData, nData, pTerm, nTerm, 1599+ isPrefix, piStartChild, piEndChild); 1600+ if( rc!=SQLITE_OK ){ 1601+ sqlite3_reset(s); 1602+ return rc; 1603+ } 1604+ } 1605 1606 /* We expect only one row. We must execute another sqlite3_step() 1607 * to complete the iteration; otherwise the table will remain 1608@@ -5622,8 +5982,9 @@ 1609 /* Process pData as an interior node, then loop down the tree 1610 ** until we find the set of leaf nodes to scan for the term. 1611 */ 1612- getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix, 1613- &iStartChild, &iEndChild); 1614+ rc = getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix, 1615+ &iStartChild, &iEndChild); 1616+ if( rc!=SQLITE_OK ) return rc; 1617 while( iStartChild>iLeavesEnd ){ 1618 sqlite_int64 iNextStart, iNextEnd; 1619 rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix, 1620@@ -5675,7 +6036,8 @@ 1621 DataBuffer result; 1622 int rc; 1623 1624- assert( nData>1 ); 1625+ /* Corrupt if segment root can't be valid. */ 1626+ if( pData==NULL || nData<1 ) return SQLITE_CORRUPT_BKPT; 1627 1628 /* This code should never be called with buffered updates. */ 1629 assert( v->nPendingData<0 ); 1630@@ -5692,16 +6054,21 @@ 1631 DataBuffer merged; 1632 DLReader readers[2]; 1633 1634- dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData); 1635- dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData); 1636- dataBufferInit(&merged, out->nData+result.nData); 1637- docListMerge(&merged, readers, 2); 1638- dataBufferDestroy(out); 1639- *out = merged; 1640- dlrDestroy(&readers[0]); 1641- dlrDestroy(&readers[1]); 1642+ rc = dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData); 1643+ if( rc==SQLITE_OK ){ 1644+ rc = dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData); 1645+ if( rc==SQLITE_OK ){ 1646+ dataBufferInit(&merged, out->nData+result.nData); 1647+ rc = docListMerge(&merged, readers, 2); 1648+ dataBufferDestroy(out); 1649+ *out = merged; 1650+ dlrDestroy(&readers[1]); 1651+ } 1652+ dlrDestroy(&readers[0]); 1653+ } 1654 } 1655 } 1656+ 1657 dataBufferDestroy(&result); 1658 return rc; 1659 } 1660@@ -5729,11 +6096,20 @@ 1661 const char *pData = sqlite3_column_blob(s, 2); 1662 const int nData = sqlite3_column_bytes(s, 2); 1663 const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); 1664+ 1665+ /* Corrupt if we get back different types than we stored. */ 1666+ if( sqlite3_column_type(s, 1)!=SQLITE_INTEGER || 1667+ sqlite3_column_type(s, 2)!=SQLITE_BLOB ){ 1668+ rc = SQLITE_CORRUPT_BKPT; 1669+ goto err; 1670+ } 1671+ 1672 rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix, 1673 &doclist); 1674 if( rc!=SQLITE_OK ) goto err; 1675 } 1676 if( rc==SQLITE_DONE ){ 1677+ rc = SQLITE_OK; 1678 if( doclist.nData!=0 ){ 1679 /* TODO(shess) The old term_select_all() code applied the column 1680 ** restrict as we merged segments, leading to smaller buffers. 1681@@ -5741,13 +6117,13 @@ 1682 ** system is checked in. 1683 */ 1684 if( iColumn==v->nColumn) iColumn = -1; 1685- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, 1686- iColumn, iType, out); 1687+ rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, 1688+ iColumn, iType, out); 1689 } 1690- rc = SQLITE_OK; 1691 } 1692 1693 err: 1694+ sqlite3_reset(s); /* So we don't leave a lock. */ 1695 dataBufferDestroy(&doclist); 1696 return rc; 1697 } 1698@@ -6089,6 +6465,7 @@ 1699 LeafWriter *pWriter){ 1700 int i, rc = SQLITE_OK; 1701 DataBuffer doclist, merged, tmp; 1702+ const char *pData; 1703 1704 /* Order the readers. */ 1705 i = nReaders; 1706@@ -6109,14 +6486,21 @@ 1707 if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break; 1708 } 1709 1710+ pData = optLeavesReaderData(&readers[0]); 1711+ if( pData==NULL ){ 1712+ rc = SQLITE_CORRUPT_BKPT; 1713+ break; 1714+ } 1715+ 1716 /* Special-case for no merge. */ 1717 if( i==1 ){ 1718 /* Trim deletions from the doclist. */ 1719 dataBufferReset(&merged); 1720- docListTrim(DL_DEFAULT, 1721- optLeavesReaderData(&readers[0]), 1722- optLeavesReaderDataBytes(&readers[0]), 1723- -1, DL_DEFAULT, &merged); 1724+ rc = docListTrim(DL_DEFAULT, 1725+ pData, 1726+ optLeavesReaderDataBytes(&readers[0]), 1727+ -1, DL_DEFAULT, &merged); 1728+ if( rc!= SQLITE_OK ) break; 1729 }else{ 1730 DLReader dlReaders[MERGE_COUNT]; 1731 int iReader, nReaders; 1732@@ -6124,9 +6508,10 @@ 1733 /* Prime the pipeline with the first reader's doclist. After 1734 ** one pass index 0 will reference the accumulated doclist. 1735 */ 1736- dlrInit(&dlReaders[0], DL_DEFAULT, 1737- optLeavesReaderData(&readers[0]), 1738- optLeavesReaderDataBytes(&readers[0])); 1739+ rc = dlrInit(&dlReaders[0], DL_DEFAULT, 1740+ pData, 1741+ optLeavesReaderDataBytes(&readers[0])); 1742+ if( rc!=SQLITE_OK ) break; 1743 iReader = 1; 1744 1745 assert( iReader<i ); /* Must execute the loop at least once. */ 1746@@ -6134,24 +6519,35 @@ 1747 /* Merge 16 inputs per pass. */ 1748 for( nReaders=1; iReader<i && nReaders<MERGE_COUNT; 1749 iReader++, nReaders++ ){ 1750- dlrInit(&dlReaders[nReaders], DL_DEFAULT, 1751- optLeavesReaderData(&readers[iReader]), 1752- optLeavesReaderDataBytes(&readers[iReader])); 1753+ pData = optLeavesReaderData(&readers[iReader]); 1754+ if( pData == NULL ){ 1755+ rc = SQLITE_CORRUPT_BKPT; 1756+ break; 1757+ } 1758+ rc = dlrInit(&dlReaders[nReaders], DL_DEFAULT, 1759+ pData, 1760+ optLeavesReaderDataBytes(&readers[iReader])); 1761+ if( rc != SQLITE_OK ) break; 1762 } 1763 1764 /* Merge doclists and swap result into accumulator. */ 1765- dataBufferReset(&merged); 1766- docListMerge(&merged, dlReaders, nReaders); 1767- tmp = merged; 1768- merged = doclist; 1769- doclist = tmp; 1770+ if( rc==SQLITE_OK ){ 1771+ dataBufferReset(&merged); 1772+ rc = docListMerge(&merged, dlReaders, nReaders); 1773+ tmp = merged; 1774+ merged = doclist; 1775+ doclist = tmp; 1776+ } 1777 1778 while( nReaders-- > 0 ){ 1779 dlrDestroy(&dlReaders[nReaders]); 1780 } 1781 1782+ if( rc!=SQLITE_OK ) goto err; 1783+ 1784 /* Accumulated doclist to reader 0 for next pass. */ 1785- dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData); 1786+ rc = dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData); 1787+ if( rc!=SQLITE_OK ) goto err; 1788 } 1789 1790 /* Destroy reader that was left in the pipeline. */ 1791@@ -6159,8 +6555,9 @@ 1792 1793 /* Trim deletions from the doclist. */ 1794 dataBufferReset(&merged); 1795- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, 1796- -1, DL_DEFAULT, &merged); 1797+ rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, 1798+ -1, DL_DEFAULT, &merged); 1799+ if( rc!=SQLITE_OK ) goto err; 1800 } 1801 1802 /* Only pass doclists with hits (skip if all hits deleted). */ 1803@@ -6240,6 +6637,14 @@ 1804 const char *pRootData = sqlite3_column_blob(s, 2); 1805 int nRootData = sqlite3_column_bytes(s, 2); 1806 1807+ /* Corrupt if we get back different types than we stored. */ 1808+ if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER || 1809+ sqlite3_column_type(s, 1)!=SQLITE_INTEGER || 1810+ sqlite3_column_type(s, 2)!=SQLITE_BLOB ){ 1811+ rc = SQLITE_CORRUPT_BKPT; 1812+ break; 1813+ } 1814+ 1815 assert( i<nReaders ); 1816 rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData, 1817 &readers[i].reader); 1818@@ -6253,6 +6658,8 @@ 1819 if( rc==SQLITE_DONE ){ 1820 assert( i==nReaders ); 1821 rc = optimizeInternal(v, readers, nReaders, &writer); 1822+ }else{ 1823+ sqlite3_reset(s); /* So we don't leave a lock. */ 1824 } 1825 1826 while( i-- > 0 ){ 1827@@ -6316,9 +6723,18 @@ 1828 const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1); 1829 const char *pRootData = sqlite3_column_blob(s, 2); 1830 const int nRootData = sqlite3_column_bytes(s, 2); 1831+ int rc; 1832 LeavesReader reader; 1833- int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, 1834- pRootData, nRootData, &reader); 1835+ 1836+ /* Corrupt if we get back different types than we stored. */ 1837+ if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER || 1838+ sqlite3_column_type(s, 1)!=SQLITE_INTEGER || 1839+ sqlite3_column_type(s, 2)!=SQLITE_BLOB ){ 1840+ return SQLITE_CORRUPT_BKPT; 1841+ } 1842+ 1843+ rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, 1844+ pRootData, nRootData, &reader); 1845 if( rc!=SQLITE_OK ) return rc; 1846 1847 while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){ 1848@@ -6480,16 +6896,19 @@ 1849 const char *pData, int nData){ 1850 DataBuffer dump; 1851 DLReader dlReader; 1852+ int rc; 1853 1854 assert( pData!=NULL && nData>0 ); 1855 1856+ rc = dlrInit(&dlReader, DL_DEFAULT, pData, nData); 1857+ if( rc!=SQLITE_OK ) return rc; 1858 dataBufferInit(&dump, 0); 1859- dlrInit(&dlReader, DL_DEFAULT, pData, nData); 1860- for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){ 1861+ for( ; rc==SQLITE_OK && !dlrAtEnd(&dlReader); rc = dlrStep(&dlReader) ){ 1862 char buf[256]; 1863 PLReader plReader; 1864 1865- plrInit(&plReader, &dlReader); 1866+ rc = plrInit(&plReader, &dlReader); 1867+ if( rc!=SQLITE_OK ) break; 1868 if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){ 1869 sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader)); 1870 dataBufferAppend(&dump, buf, strlen(buf)); 1871@@ -6500,7 +6919,8 @@ 1872 dlrDocid(&dlReader), iColumn); 1873 dataBufferAppend(&dump, buf, strlen(buf)); 1874 1875- for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){ 1876+ for( ; !plrAtEnd(&plReader); rc = plrStep(&plReader) ){ 1877+ if( rc!=SQLITE_OK ) break; 1878 if( plrColumn(&plReader)!=iColumn ){ 1879 iColumn = plrColumn(&plReader); 1880 sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn); 1881@@ -6521,6 +6941,7 @@ 1882 dataBufferAppend(&dump, buf, strlen(buf)); 1883 } 1884 plrDestroy(&plReader); 1885+ if( rc!= SQLITE_OK ) break; 1886 1887 assert( dump.nData>0 ); 1888 dump.nData--; /* Overwrite trailing space. */ 1889@@ -6529,6 +6950,10 @@ 1890 } 1891 } 1892 dlrDestroy(&dlReader); 1893+ if( rc!=SQLITE_OK ){ 1894+ dataBufferDestroy(&dump); 1895+ return rc; 1896+ } 1897 1898 assert( dump.nData>0 ); 1899 dump.nData--; /* Overwrite trailing space. */ 1900@@ -6540,6 +6965,7 @@ 1901 sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free); 1902 dump.pData = NULL; 1903 dump.nData = dump.nCapacity = 0; 1904+ return SQLITE_OK; 1905 } 1906 1907 /* Implements dump_doclist() for use in inspecting the fts2 index from 1908@@ -6822,7 +7248,11 @@ 1909 ** module with sqlite. 1910 */ 1911 if( SQLITE_OK==rc 1912+#if GEARS_FTS2_CHANGES && !SQLITE_TEST 1913+ /* fts2_tokenizer() disabled for security reasons. */ 1914+#else 1915 && SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer")) 1916+#endif 1917 && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) 1918 && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1)) 1919 && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1)) 1920diff -ru ext-orig/fts2/fts2_icu.c ext/fts2/fts2_icu.c 1921--- ext-orig/fts2/fts2_icu.c 2009-09-03 13:32:06.000000000 -0700 1922+++ ext/fts2/fts2_icu.c 2009-09-18 14:39:41.000000000 -0700 1923@@ -198,7 +198,7 @@ 1924 1925 while( iStart<iEnd ){ 1926 int iWhite = iStart; 1927- U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c); 1928+ U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c); 1929 if( u_isspace(c) ){ 1930 iStart = iWhite; 1931 }else{ 1932diff -ru ext-orig/fts2/fts2_tokenizer.c ext/fts2/fts2_tokenizer.c 1933--- ext-orig/fts2/fts2_tokenizer.c 2009-09-03 13:32:06.000000000 -0700 1934+++ ext/fts2/fts2_tokenizer.c 2009-09-18 14:39:41.000000000 -0700 1935@@ -28,11 +28,14 @@ 1936 1937 #include "sqlite3.h" 1938 #include "sqlite3ext.h" 1939-SQLITE_EXTENSION_INIT1 1940+#ifndef SQLITE_CORE 1941+ SQLITE_EXTENSION_INIT1 1942+#endif 1943 1944 #include "fts2_hash.h" 1945 #include "fts2_tokenizer.h" 1946 #include <assert.h> 1947+#include <stddef.h> 1948 1949 /* 1950 ** Implementation of the SQL scalar function for accessing the underlying 1951