1 /* 2 ** 2001-09-15 3 ** 4 ** The author disclaims copyright to this source code. In place of 5 ** a legal notice, here is a blessing: 6 ** 7 ** May you do good and not evil. 8 ** May you find forgiveness for yourself and forgive others. 9 ** May you share freely, never taking more than you give. 10 ** 11 ************************************************************************* 12 */ 13 /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. 14 ** If tokenizers are to be allowed to call sqlite3_*() functions, then 15 ** we will need a way to register the API consistently. 16 */ 17 /* #include "sqlite3.h" */ 18 19 /* 20 ** Structures used by the tokenizer interface. When a new tokenizer 21 ** implementation is registered, the caller provides a pointer to 22 ** an sqlite3_tokenizer_module containing pointers to the callback 23 ** functions that make up an implementation. 24 ** 25 ** When an fts3 table is created, it passes any arguments passed to 26 ** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the 27 ** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer 28 ** implementation. The xCreate() function in turn returns an 29 ** sqlite3_tokenizer structure representing the specific tokenizer to 30 ** be used for the fts3 table (customized by the tokenizer clause arguments). 31 ** 32 ** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() 33 ** method is called. It returns an sqlite3_tokenizer_cursor object 34 ** that may be used to tokenize a specific input buffer based on 35 ** the tokenization rules supplied by a specific sqlite3_tokenizer 36 ** object. 37 */ 38 #ifndef SQLITE3TOKENIZER_H 39 #define SQLITE3TOKENIZER_H 40 41 typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; 42 typedef struct sqlite3_tokenizer sqlite3_tokenizer; 43 typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; 44 45 struct sqlite3_tokenizer_module { 46 47 /* 48 ** Structure version. Should always be set to 0 or 1. 49 */ 50 int iVersion; 51 52 /* 53 ** Create a new tokenizer. The values in the argv[] array are the 54 ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL 55 ** TABLE statement that created the fts3 table. For example, if 56 ** the following SQL is executed: 57 ** 58 ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2) 59 ** 60 ** then argc is set to 2, and the argv[] array contains pointers 61 ** to the strings "arg1" and "arg2". 62 ** 63 ** This method should return either SQLITE_OK (0), or an SQLite error 64 ** code. If SQLITE_OK is returned, then *ppTokenizer should be set 65 ** to point at the newly created tokenizer structure. The generic 66 ** sqlite3_tokenizer.pModule variable should not be initialized by 67 ** this callback. The caller will do so. 68 */ 69 int (*xCreate)( 70 int argc, /* Size of argv array */ 71 const char *const*argv, /* Tokenizer argument strings */ 72 sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ 73 ); 74 75 /* 76 ** Destroy an existing tokenizer. The fts3 module calls this method 77 ** exactly once for each successful call to xCreate(). 78 */ 79 int (*xDestroy)(sqlite3_tokenizer *pTokenizer); 80 81 /* 82 ** Create a tokenizer cursor to tokenize an input buffer. The caller 83 ** is responsible for ensuring that the input buffer remains valid 84 ** until the cursor is closed (using the xClose() method). 85 */ 86 int (*xOpen)( 87 sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ 88 const char *pInput, int nBytes, /* Input buffer */ 89 sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ 90 ); 91 92 /* 93 ** Destroy an existing tokenizer cursor. The fts3 module calls this 94 ** method exactly once for each successful call to xOpen(). 95 */ 96 int (*xClose)(sqlite3_tokenizer_cursor *pCursor); 97 98 /* 99 ** Retrieve the next token from the tokenizer cursor pCursor. This 100 ** method should either return SQLITE_OK and set the values of the 101 ** "OUT" variables identified below, or SQLITE_DONE to indicate that 102 ** the end of the buffer has been reached, or an SQLite error code. 103 ** 104 ** *ppToken should be set to point at a buffer containing the 105 ** normalized version of the token (i.e. after any case-folding and/or 106 ** stemming has been performed). *pnBytes should be set to the length 107 ** of this buffer in bytes. The input text that generated the token is 108 ** identified by the byte offsets returned in *piStartOffset and 109 ** *piEndOffset. *piStartOffset should be set to the index of the first 110 ** byte of the token in the input buffer. *piEndOffset should be set 111 ** to the index of the first byte just past the end of the token in 112 ** the input buffer. 113 ** 114 ** The buffer *ppToken is set to point at is managed by the tokenizer 115 ** implementation. It is only required to be valid until the next call 116 ** to xNext() or xClose(). 117 */ 118 /* TODO(shess) current implementation requires pInput to be 119 ** nul-terminated. This should either be fixed, or pInput/nBytes 120 ** should be converted to zInput. 121 */ 122 int (*xNext)( 123 sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ 124 const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ 125 int *piStartOffset, /* OUT: Byte offset of token in input buffer */ 126 int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ 127 int *piPosition /* OUT: Number of tokens returned before this one */ 128 ); 129 130 /*********************************************************************** 131 ** Methods below this point are only available if iVersion>=1. 132 */ 133 134 /* 135 ** Configure the language id of a tokenizer cursor. 136 */ 137 int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); 138 }; 139 140 struct sqlite3_tokenizer { 141 const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ 142 /* Tokenizer implementations will typically add additional fields */ 143 }; 144 145 struct sqlite3_tokenizer_cursor { 146 sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ 147 /* Tokenizer implementations will typically add additional fields */ 148 }; 149 150 #endif