1 /* 2 ** 2001-09-15 3 ** 4 ** The author disclaims copyright to this source code. In place of 5 ** a legal notice, here is a blessing: 6 ** 7 ** May you do good and not evil. 8 ** May you find forgiveness for yourself and forgive others. 9 ** May you share freely, never taking more than you give. 10 ** 11 ************************************************************************* 12 */ 13 /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. 14 ** If tokenizers are to be allowed to call sqlite3_*() functions, then 15 ** we will need a way to register the API consistently. 16 */ 17 /* #include "sqlite3.h" */ 18 19 /* 20 ** Structures used by the tokenizer interface. When a new tokenizer 21 ** implementation is registered, the caller provides a pointer to 22 ** an sqlite3_tokenizer_module containing pointers to the callback 23 ** functions that make up an implementation. 24 ** 25 ** When an fts3 table is created, it passes any arguments passed to 26 ** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the 27 ** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer 28 ** implementation. The xCreate() function in turn returns an 29 ** sqlite3_tokenizer structure representing the specific tokenizer to 30 ** be used for the fts3 table (customized by the tokenizer clause arguments). 31 ** 32 ** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() 33 ** method is called. It returns an sqlite3_tokenizer_cursor object 34 ** that may be used to tokenize a specific input buffer based on 35 ** the tokenization rules supplied by a specific sqlite3_tokenizer 36 ** object. 37 */ 38 typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; 39 typedef struct sqlite3_tokenizer sqlite3_tokenizer; 40 typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; 41 42 struct sqlite3_tokenizer_module { 43 44 /* 45 ** Structure version. Should always be set to 0 or 1. 46 */ 47 int iVersion; 48 49 /* 50 ** Create a new tokenizer. The values in the argv[] array are the 51 ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL 52 ** TABLE statement that created the fts3 table. For example, if 53 ** the following SQL is executed: 54 ** 55 ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2) 56 ** 57 ** then argc is set to 2, and the argv[] array contains pointers 58 ** to the strings "arg1" and "arg2". 59 ** 60 ** This method should return either SQLITE_OK (0), or an SQLite error 61 ** code. If SQLITE_OK is returned, then *ppTokenizer should be set 62 ** to point at the newly created tokenizer structure. The generic 63 ** sqlite3_tokenizer.pModule variable should not be initialized by 64 ** this callback. The caller will do so. 65 */ 66 int (*xCreate)( 67 int argc, /* Size of argv array */ 68 const char *const*argv, /* Tokenizer argument strings */ 69 sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ 70 ); 71 72 /* 73 ** Destroy an existing tokenizer. The fts3 module calls this method 74 ** exactly once for each successful call to xCreate(). 75 */ 76 int (*xDestroy)(sqlite3_tokenizer *pTokenizer); 77 78 /* 79 ** Create a tokenizer cursor to tokenize an input buffer. The caller 80 ** is responsible for ensuring that the input buffer remains valid 81 ** until the cursor is closed (using the xClose() method). 82 */ 83 int (*xOpen)( 84 sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ 85 const char *pInput, int nBytes, /* Input buffer */ 86 sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ 87 ); 88 89 /* 90 ** Destroy an existing tokenizer cursor. The fts3 module calls this 91 ** method exactly once for each successful call to xOpen(). 92 */ 93 int (*xClose)(sqlite3_tokenizer_cursor *pCursor); 94 95 /* 96 ** Retrieve the next token from the tokenizer cursor pCursor. This 97 ** method should either return SQLITE_OK and set the values of the 98 ** "OUT" variables identified below, or SQLITE_DONE to indicate that 99 ** the end of the buffer has been reached, or an SQLite error code. 100 ** 101 ** *ppToken should be set to point at a buffer containing the 102 ** normalized version of the token (i.e. after any case-folding and/or 103 ** stemming has been performed). *pnBytes should be set to the length 104 ** of this buffer in bytes. The input text that generated the token is 105 ** identified by the byte offsets returned in *piStartOffset and 106 ** *piEndOffset. *piStartOffset should be set to the index of the first 107 ** byte of the token in the input buffer. *piEndOffset should be set 108 ** to the index of the first byte just past the end of the token in 109 ** the input buffer. 110 ** 111 ** The buffer *ppToken is set to point at is managed by the tokenizer 112 ** implementation. It is only required to be valid until the next call 113 ** to xNext() or xClose(). 114 */ 115 /* TODO(shess) current implementation requires pInput to be 116 ** nul-terminated. This should either be fixed, or pInput/nBytes 117 ** should be converted to zInput. 118 */ 119 int (*xNext)( 120 sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ 121 const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ 122 int *piStartOffset, /* OUT: Byte offset of token in input buffer */ 123 int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ 124 int *piPosition /* OUT: Number of tokens returned before this one */ 125 ); 126 127 /*********************************************************************** 128 ** Methods below this point are only available if iVersion>=1. 129 */ 130 131 /* 132 ** Configure the language id of a tokenizer cursor. 133 */ 134 int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); 135 }; 136 137 struct sqlite3_tokenizer { 138 const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ 139 /* Tokenizer implementations will typically add additional fields */ 140 }; 141 142 struct sqlite3_tokenizer_cursor { 143 sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ 144 /* Tokenizer implementations will typically add additional fields */ 145 };