1 /* 2 ******************************************************************************* 3 * Copyright (C) 2014, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ******************************************************************************* 6 * dictionarydata.h 7 * 8 * created on: 2012may31 9 * created by: Markus W. Scherer & Maxime Serrano 10 */ 11 12 #ifndef __DICTIONARYDATA_H__ 13 #define __DICTIONARYDATA_H__ 14 15 #include "unicode/utypes.h" 16 17 #if !UCONFIG_NO_BREAK_ITERATION 18 19 #include "unicode/utext.h" 20 #include "unicode/udata.h" 21 #include "udataswp.h" 22 #include "unicode/uobject.h" 23 #include "unicode/ustringtrie.h" 24 25 U_NAMESPACE_BEGIN 26 27 class UCharsTrie; 28 class BytesTrie; 29 30 class U_COMMON_API DictionaryData : public UMemory { 31 public: 32 static const int32_t TRIE_TYPE_BYTES; // = 0; 33 static const int32_t TRIE_TYPE_UCHARS; // = 1; 34 static const int32_t TRIE_TYPE_MASK; // = 7; 35 static const int32_t TRIE_HAS_VALUES; // = 8; 36 37 static const int32_t TRANSFORM_NONE; // = 0; 38 static const int32_t TRANSFORM_TYPE_OFFSET; // = 0x1000000; 39 static const int32_t TRANSFORM_TYPE_MASK; // = 0x7f000000; 40 static const int32_t TRANSFORM_OFFSET_MASK; // = 0x1fffff; 41 42 enum { 43 // Byte offsets from the start of the data, after the generic header. 44 IX_STRING_TRIE_OFFSET, 45 IX_RESERVED1_OFFSET, 46 IX_RESERVED2_OFFSET, 47 IX_TOTAL_SIZE, 48 49 // Trie type: TRIE_HAS_VALUES | TRIE_TYPE_BYTES etc. 50 IX_TRIE_TYPE, 51 // Transform specification: TRANSFORM_TYPE_OFFSET | 0xe00 etc. 52 IX_TRANSFORM, 53 54 IX_RESERVED6, 55 IX_RESERVED7, 56 IX_COUNT 57 }; 58 }; 59 60 /** 61 * Wrapper class around generic dictionaries, implementing matches(). 62 * getType() should return a TRIE_TYPE_??? constant from DictionaryData. 63 * 64 * All implementations of this interface must be thread-safe if they are to be used inside of the 65 * dictionary-based break iteration code. 66 */ 67 class U_COMMON_API DictionaryMatcher : public UMemory { 68 public: DictionaryMatcher()69 DictionaryMatcher() {}; 70 virtual ~DictionaryMatcher(); 71 // this should emulate CompactTrieDictionary::matches() 72 /* @param text The text in which to look for matching words. Matching begins 73 * at the current position of the UText. 74 * @param maxLength The max length of match to consider. Units are the native indexing 75 * units of the UText. 76 * @param limit Capacity of output arrays, which is also the maximum number of 77 * matching words to be found. 78 * @param lengths output array, filled with the lengths of the matches, in order, 79 * from shortest to longest. Lengths are in native indexing units 80 * of the UText. May be NULL. 81 * @param cpLengths output array, filled with the lengths of the matches, in order, 82 * from shortest to longest. Lengths are the number of Unicode code points. 83 * May be NULL. 84 * @param values Output array, filled with the values associated with the words found. 85 * May be NULL. 86 * @param prefix Output parameter, the code point length of the prefix match, even if that 87 * prefix didn't lead to a complete word. Will always be >= the cpLength 88 * of the longest complete word matched. May be NULL. 89 * @return Number of matching words found. 90 */ 91 virtual int32_t matches(UText *text, int32_t maxLength, int32_t limit, 92 int32_t *lengths, int32_t *cpLengths, int32_t *values, 93 int32_t *prefix) const = 0; 94 95 /** @return DictionaryData::TRIE_TYPE_XYZ */ 96 virtual int32_t getType() const = 0; 97 }; 98 99 // Implementation of the DictionaryMatcher interface for a UCharsTrie dictionary 100 class U_COMMON_API UCharsDictionaryMatcher : public DictionaryMatcher { 101 public: 102 // constructs a new UCharsDictionaryMatcher. 103 // The UDataMemory * will be closed on this object's destruction. UCharsDictionaryMatcher(const UChar * c,UDataMemory * f)104 UCharsDictionaryMatcher(const UChar *c, UDataMemory *f) : characters(c), file(f) { } 105 virtual ~UCharsDictionaryMatcher(); 106 virtual int32_t matches(UText *text, int32_t maxLength, int32_t limit, 107 int32_t *lengths, int32_t *cpLengths, int32_t *values, 108 int32_t *prefix) const; 109 virtual int32_t getType() const; 110 private: 111 const UChar *characters; 112 UDataMemory *file; 113 }; 114 115 // Implementation of the DictionaryMatcher interface for a BytesTrie dictionary 116 class U_COMMON_API BytesDictionaryMatcher : public DictionaryMatcher { 117 public: 118 // constructs a new BytesTrieDictionaryMatcher 119 // the transform constant should be the constant read from the file, not a masked version! 120 // the UDataMemory * fed in here will be closed on this object's destruction BytesDictionaryMatcher(const char * c,int32_t t,UDataMemory * f)121 BytesDictionaryMatcher(const char *c, int32_t t, UDataMemory *f) 122 : characters(c), transformConstant(t), file(f) { } 123 virtual ~BytesDictionaryMatcher(); 124 virtual int32_t matches(UText *text, int32_t maxLength, int32_t limit, 125 int32_t *lengths, int32_t *cpLengths, int32_t *values, 126 int32_t *prefix) const; 127 virtual int32_t getType() const; 128 private: 129 UChar32 transform(UChar32 c) const; 130 131 const char *characters; 132 int32_t transformConstant; 133 UDataMemory *file; 134 }; 135 136 U_NAMESPACE_END 137 138 U_CAPI int32_t U_EXPORT2 139 udict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *pErrorCode); 140 141 /** 142 * Format of dictionary .dict data files. 143 * Format version 1.0. 144 * 145 * A dictionary .dict data file contains a byte-serialized BytesTrie or 146 * a UChars-serialized UCharsTrie. 147 * Such files are used in dictionary-based break iteration (DBBI). 148 * 149 * For a BytesTrie, a transformation type is specified for 150 * transforming Unicode strings into byte sequences. 151 * 152 * A .dict file begins with a standard ICU data file header 153 * (DataHeader, see ucmndata.h and unicode/udata.h). 154 * The UDataInfo.dataVersion field is currently unused (set to 0.0.0.0). 155 * 156 * After the header, the file contains the following parts. 157 * Constants are defined in the DictionaryData class. 158 * 159 * For the data structure of BytesTrie & UCharsTrie see 160 * http://site.icu-project.org/design/struct/tries 161 * and the bytestrie.h and ucharstrie.h header files. 162 * 163 * int32_t indexes[indexesLength]; -- indexesLength=indexes[IX_STRING_TRIE_OFFSET]/4; 164 * 165 * The first four indexes are byte offsets in ascending order. 166 * Each byte offset marks the start of the next part in the data file, 167 * and the end of the previous one. 168 * When two consecutive byte offsets are the same, then the corresponding part is empty. 169 * Byte offsets are offsets from after the header, 170 * that is, from the beginning of the indexes[]. 171 * Each part starts at an offset with proper alignment for its data. 172 * If necessary, the previous part may include padding bytes to achieve this alignment. 173 * 174 * trieType=indexes[IX_TRIE_TYPE] defines the trie type. 175 * transform=indexes[IX_TRANSFORM] defines the Unicode-to-bytes transformation. 176 * If the transformation type is TRANSFORM_TYPE_OFFSET, 177 * then the lower 21 bits contain the offset code point. 178 * Each code point c is mapped to byte b = (c - offset). 179 * Code points outside the range offset..(offset+0xff) cannot be mapped 180 * and do not occur in the dictionary. 181 * 182 * stringTrie; -- a serialized BytesTrie or UCharsTrie 183 * 184 * The dictionary maps strings to specific values (TRIE_HAS_VALUES bit set in trieType), 185 * or it maps all strings to 0 (TRIE_HAS_VALUES bit not set). 186 */ 187 188 #endif /* !UCONFIG_NO_BREAK_ITERATION */ 189 #endif /* __DICTIONARYDATA_H__ */ 190