1 /*
2 * Copyright (C) 2010, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef LATINIME_DEFINES_H
18 #define LATINIME_DEFINES_H
19
20 #include <stdint.h>
21
22 #if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG)
23 #include <android/log.h>
24 #ifndef LOG_TAG
25 #define LOG_TAG "LatinIME: "
26 #endif
27 #define AKLOGE(fmt, ...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, fmt, ##__VA_ARGS__)
28 #define AKLOGI(fmt, ...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, fmt, ##__VA_ARGS__)
29
30 #define DUMP_RESULT(words, frequencies, maxWordCount, maxWordLength) do { \
31 dumpResult(words, frequencies, maxWordCount, maxWordLength); } while (0)
32 #define DUMP_WORD(word, length) do { dumpWord(word, length); } while (0)
33 #define DUMP_WORD_INT(word, length) do { dumpWordInt(word, length); } while (0)
34 // TODO: INTS_TO_CHARS
35 #define SHORTS_TO_CHARS(input, length, output) do { \
36 shortArrayToCharArray(input, length, output); } while (0)
37
dumpWordInfo(const unsigned short * word,const int length,const int rank,const int frequency)38 static inline void dumpWordInfo(const unsigned short *word, const int length,
39 const int rank, const int frequency) {
40 static char charBuf[50];
41 int i = 0;
42 for (; i < length; ++i) {
43 const unsigned short c = word[i];
44 if (c == 0) {
45 break;
46 }
47 // static_cast only for debugging
48 charBuf[i] = static_cast<char>(c);
49 }
50 charBuf[i] = 0;
51 if (i > 1) {
52 AKLOGI("%2d [ %s ] (%d)", rank, charBuf, frequency);
53 }
54 }
55
dumpResult(const unsigned short * outWords,const int * frequencies,const int maxWordCounts,const int maxWordLength)56 static inline void dumpResult(
57 const unsigned short *outWords, const int *frequencies, const int maxWordCounts,
58 const int maxWordLength) {
59 AKLOGI("--- DUMP RESULT ---------");
60 for (int i = 0; i < maxWordCounts; ++i) {
61 dumpWordInfo(&outWords[i * maxWordLength], maxWordLength, i, frequencies[i]);
62 }
63 AKLOGI("-------------------------");
64 }
65
dumpWord(const unsigned short * word,const int length)66 static inline void dumpWord(const unsigned short *word, const int length) {
67 static char charBuf[50];
68 int i = 0;
69 for (; i < length; ++i) {
70 const unsigned short c = word[i];
71 if (c == 0) {
72 break;
73 }
74 // static_cast only for debugging
75 charBuf[i] = static_cast<char>(c);
76 }
77 charBuf[i] = 0;
78 if (i > 1) {
79 AKLOGI("[ %s ]", charBuf);
80 }
81 }
82
dumpWordInt(const int * word,const int length)83 static inline void dumpWordInt(const int *word, const int length) {
84 static char charBuf[50];
85
86 for (int i = 0; i < length; ++i) {
87 charBuf[i] = word[i];
88 }
89 charBuf[length] = 0;
90 AKLOGI("i[ %s ]", charBuf);
91 }
92
93 // TODO: Change this to intArrayToCharArray
shortArrayToCharArray(const unsigned short * input,const int length,char * output)94 static inline void shortArrayToCharArray(
95 const unsigned short *input, const int length, char *output) {
96 int i = 0;
97 for (;i < length; ++i) {
98 const unsigned short c = input[i];
99 if (c == 0) {
100 break;
101 }
102 // static_cast only for debugging
103 output[i] = static_cast<char>(c);
104 }
105 output[i] = 0;
106 }
107
108 #ifndef __ANDROID__
109 #include <cassert>
110 #include <execinfo.h>
111 #include <stdlib.h>
112
113 #define ASSERT(success) do { if (!(success)) { showStackTrace(); assert(success);} } while (0)
114 #define SHOW_STACK_TRACE do { showStackTrace(); } while (0)
115
showStackTrace()116 static inline void showStackTrace() {
117 void *callstack[128];
118 int i, frames = backtrace(callstack, 128);
119 char **strs = backtrace_symbols(callstack, frames);
120 for (i = 0; i < frames; ++i) {
121 if (i == 0) {
122 AKLOGI("=== Trace ===");
123 continue;
124 }
125 AKLOGI("%s", strs[i]);
126 }
127 free(strs);
128 }
129 #else
130 #include <cassert>
131 #define ASSERT(success) assert(success)
132 #define SHOW_STACK_TRACE
133 #endif
134
135 #else
136 #define AKLOGE(fmt, ...)
137 #define AKLOGI(fmt, ...)
138 #define DUMP_RESULT(words, frequencies, maxWordCount, maxWordLength)
139 #define DUMP_WORD(word, length)
140 #define DUMP_WORD_INT(word, length)
141 #define ASSERT(success)
142 #define SHOW_STACK_TRACE
143 // TODO: INTS_TO_CHARS
144 #define SHORTS_TO_CHARS(input, length, output)
145 #endif
146
147 #ifdef FLAG_DO_PROFILE
148 // Profiler
149 #include <time.h>
150
151 #define PROF_BUF_SIZE 100
152 static float profile_buf[PROF_BUF_SIZE];
153 static float profile_old[PROF_BUF_SIZE];
154 static unsigned int profile_counter[PROF_BUF_SIZE];
155
156 #define PROF_RESET prof_reset()
157 #define PROF_COUNT(prof_buf_id) ++profile_counter[prof_buf_id]
158 #define PROF_OPEN do { PROF_RESET; PROF_START(PROF_BUF_SIZE - 1); } while (0)
159 #define PROF_START(prof_buf_id) do { \
160 PROF_COUNT(prof_buf_id); profile_old[prof_buf_id] = (clock()); } while (0)
161 #define PROF_CLOSE do { PROF_END(PROF_BUF_SIZE - 1); PROF_OUTALL; } while (0)
162 #define PROF_END(prof_buf_id) profile_buf[prof_buf_id] += ((clock()) - profile_old[prof_buf_id])
163 #define PROF_CLOCKOUT(prof_buf_id) \
164 AKLOGI("%s : clock is %f", __FUNCTION__, (clock() - profile_old[prof_buf_id]))
165 #define PROF_OUTALL do { AKLOGI("--- %s ---", __FUNCTION__); prof_out(); } while (0)
166
prof_reset(void)167 static inline void prof_reset(void) {
168 for (int i = 0; i < PROF_BUF_SIZE; ++i) {
169 profile_buf[i] = 0;
170 profile_old[i] = 0;
171 profile_counter[i] = 0;
172 }
173 }
174
prof_out(void)175 static inline void prof_out(void) {
176 if (profile_counter[PROF_BUF_SIZE - 1] != 1) {
177 AKLOGI("Error: You must call PROF_OPEN before PROF_CLOSE.");
178 }
179 AKLOGI("Total time is %6.3f ms.",
180 profile_buf[PROF_BUF_SIZE - 1] * 1000.0f / static_cast<float>(CLOCKS_PER_SEC));
181 float all = 0;
182 for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) {
183 all += profile_buf[i];
184 }
185 if (all == 0) all = 1;
186 for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) {
187 if (profile_buf[i]) {
188 AKLOGI("(%d): Used %4.2f%%, %8.4f ms. Called %d times.",
189 i, (profile_buf[i] * 100 / all),
190 profile_buf[i] * 1000.0f / static_cast<float>(CLOCKS_PER_SEC),
191 profile_counter[i]);
192 }
193 }
194 }
195
196 #else // FLAG_DO_PROFILE
197 #define PROF_BUF_SIZE 0
198 #define PROF_RESET
199 #define PROF_COUNT(prof_buf_id)
200 #define PROF_OPEN
201 #define PROF_START(prof_buf_id)
202 #define PROF_CLOSE
203 #define PROF_END(prof_buf_id)
204 #define PROF_CLOCK_OUT(prof_buf_id)
205 #define PROF_CLOCKOUT(prof_buf_id)
206 #define PROF_OUTALL
207
208 #endif // FLAG_DO_PROFILE
209
210 #ifdef FLAG_DBG
211 #define DEBUG_DICT true
212 #define DEBUG_DICT_FULL false
213 #define DEBUG_EDIT_DISTANCE false
214 #define DEBUG_SHOW_FOUND_WORD false
215 #define DEBUG_NODE DEBUG_DICT_FULL
216 #define DEBUG_TRACE DEBUG_DICT_FULL
217 #define DEBUG_PROXIMITY_INFO false
218 #define DEBUG_PROXIMITY_CHARS false
219 #define DEBUG_CORRECTION false
220 #define DEBUG_CORRECTION_FREQ false
221 #define DEBUG_WORDS_PRIORITY_QUEUE false
222
223 #ifdef FLAG_FULL_DBG
224 #define DEBUG_GEO_FULL true
225 #else
226 #define DEBUG_GEO_FULL false
227 #endif
228
229 #else // FLAG_DBG
230
231 #define DEBUG_DICT false
232 #define DEBUG_DICT_FULL false
233 #define DEBUG_EDIT_DISTANCE false
234 #define DEBUG_SHOW_FOUND_WORD false
235 #define DEBUG_NODE false
236 #define DEBUG_TRACE false
237 #define DEBUG_PROXIMITY_INFO false
238 #define DEBUG_PROXIMITY_CHARS false
239 #define DEBUG_CORRECTION false
240 #define DEBUG_CORRECTION_FREQ false
241 #define DEBUG_WORDS_PRIORITY_QUEUE false
242
243 #define DEBUG_GEO_FULL false
244
245 #endif // FLAG_DBG
246
247 #ifndef U_SHORT_MAX
248 #define U_SHORT_MAX 65535 // ((1 << 16) - 1)
249 #endif
250 #ifndef S_INT_MAX
251 #define S_INT_MAX 2147483647 // ((1 << 31) - 1)
252 #endif
253 #ifndef S_INT_MIN
254 // The literal constant -2147483648 does not work in C prior C90, because
255 // the compiler tries to fit the positive number into an int and then negate it.
256 // GCC warns about this.
257 #define S_INT_MIN (-2147483647 - 1) // -(1 << 31)
258 #endif
259
260 // Define this to use mmap() for dictionary loading. Undefine to use malloc() instead of mmap().
261 // We measured and compared performance of both, and found mmap() is fairly good in terms of
262 // loading time, and acceptable even for several initial lookups which involve page faults.
263 #define USE_MMAP_FOR_DICTIONARY
264
265 // 22-bit address = ~4MB dictionary size limit, which on average would be about 200k-300k words
266 #define ADDRESS_MASK 0x3FFFFF
267
268 // The bit that decides if an address follows in the next 22 bits
269 #define FLAG_ADDRESS_MASK 0x40
270 // The bit that decides if this is a terminal node for a word. The node could still have children,
271 // if the word has other endings.
272 #define FLAG_TERMINAL_MASK 0x80
273
274 #define FLAG_BIGRAM_READ 0x80
275 #define FLAG_BIGRAM_CHILDEXIST 0x40
276 #define FLAG_BIGRAM_CONTINUED 0x80
277 #define FLAG_BIGRAM_FREQ 0x7F
278
279 #define DICTIONARY_VERSION_MIN 200
280 #define NOT_VALID_WORD (-99)
281 #define NOT_A_CODE_POINT (-1)
282 #define NOT_A_DISTANCE (-1)
283 #define NOT_A_COORDINATE (-1)
284 #define EQUIVALENT_CHAR_WITHOUT_DISTANCE_INFO (-2)
285 #define PROXIMITY_CHAR_WITHOUT_DISTANCE_INFO (-3)
286 #define ADDITIONAL_PROXIMITY_CHAR_DISTANCE_INFO (-4)
287 #define NOT_AN_INDEX (-1)
288 #define NOT_A_PROBABILITY (-1)
289
290 #define KEYCODE_SPACE ' '
291
292 #define CALIBRATE_SCORE_BY_TOUCH_COORDINATES true
293
294 #define SUGGEST_WORDS_WITH_MISSING_CHARACTER true
295 #define SUGGEST_WORDS_WITH_EXCESSIVE_CHARACTER true
296 #define SUGGEST_WORDS_WITH_TRANSPOSED_CHARACTERS true
297 #define SUGGEST_MULTIPLE_WORDS true
298
299 // The following "rate"s are used as a multiplier before dividing by 100, so they are in percent.
300 #define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 80
301 #define WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X 12
302 #define WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE 58
303 #define WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE 50
304 #define WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE 75
305 #define WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE 75
306 #define WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE 70
307 #define FULL_MATCHED_WORDS_PROMOTION_RATE 120
308 #define WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE 90
309 #define WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE 70
310 #define WORDS_WITH_MATCH_SKIP_PROMOTION_RATE 105
311 #define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE 148
312 #define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER 3
313 #define CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE 45
314 #define INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE 70
315 #define FIRST_CHAR_DIFFERENT_DEMOTION_RATE 96
316 #define TWO_WORDS_CAPITALIZED_DEMOTION_RATE 50
317 #define TWO_WORDS_CORRECTION_DEMOTION_BASE 80
318 #define TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER 1
319 #define ZERO_DISTANCE_PROMOTION_RATE 110
320 #define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f
321 #define HALF_SCORE_SQUARED_RADIUS 32.0f
322 #define MAX_FREQ 255
323 #define MAX_BIGRAM_FREQ 15
324
325 // This must be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java
326 // This is only used for the size of array. Not to be used in c functions.
327 #define MAX_WORD_LENGTH_INTERNAL 48
328
329 // This must be the same as ProximityInfo#MAX_PROXIMITY_CHARS_SIZE, currently it's 16.
330 #define MAX_PROXIMITY_CHARS_SIZE_INTERNAL 16
331
332 // This must be equal to ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE in KeyDetector.java
333 #define ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE 2
334
335 // Assuming locale strings such as en_US, sr-Latn etc.
336 #define MAX_LOCALE_STRING_LENGTH 10
337
338 // Word limit for sub queues used in WordsPriorityQueuePool. Sub queues are temporary queues used
339 // for better performance.
340 // Holds up to 1 candidate for each word
341 #define SUB_QUEUE_MAX_WORDS 1
342 #define SUB_QUEUE_MAX_COUNT 10
343 #define SUB_QUEUE_MIN_WORD_LENGTH 4
344 // TODO: Extend this limitation
345 #define MULTIPLE_WORDS_SUGGESTION_MAX_WORDS 5
346 // TODO: Remove this limitation
347 #define MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH 12
348 // TODO: Remove this limitation
349 #define MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT 45
350 #define MULTIPLE_WORDS_DEMOTION_RATE 80
351 #define MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION 6
352
353 #define TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD 0.35
354 #define START_TWO_WORDS_CORRECTION_THRESHOLD 0.185
355 /* heuristic... This should be changed if we change the unit of the frequency. */
356 #define SUPPRESS_SHORT_MULTIPLE_WORDS_THRESHOLD_FREQ (MAX_FREQ * 58 / 100)
357
358 #define MAX_DEPTH_MULTIPLIER 3
359
360 #define FIRST_WORD_INDEX 0
361
362 #define MAX_SPACES_INTERNAL 16
363
364 // Max Distance between point to key
365 #define MAX_POINT_TO_KEY_LENGTH 10000000
366
367 // The max number of the keys in one keyboard layout
368 #define MAX_KEY_COUNT_IN_A_KEYBOARD 64
369
370 // TODO: Reduce this constant if possible; check the maximum number of digraphs in the same
371 // word in the dictionary for languages with digraphs, like German and French
372 #define DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH 5
373
374 #define MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION 3
375 #define MIN_USER_TYPED_LENGTH_FOR_EXCESSIVE_CHARACTER_SUGGESTION 3
376
377 // TODO: Remove
378 #define MAX_POINTER_COUNT_FOR_G 2
379
380 // Size, in bytes, of the bloom filter index for bigrams
381 // 128 gives us 1024 buckets. The probability of false positive is (1 - e ** (-kn/m))**k,
382 // where k is the number of hash functions, n the number of bigrams, and m the number of
383 // bits we can test.
384 // At the moment 100 is the maximum number of bigrams for a word with the current
385 // dictionaries, so n = 100. 1024 buckets give us m = 1024.
386 // With 1 hash function, our false positive rate is about 9.3%, which should be enough for
387 // our uses since we are only using this to increase average performance. For the record,
388 // k = 2 gives 3.1% and k = 3 gives 1.6%. With k = 1, making m = 2048 gives 4.8%,
389 // and m = 4096 gives 2.4%.
390 #define BIGRAM_FILTER_BYTE_SIZE 128
391 // Must be smaller than BIGRAM_FILTER_BYTE_SIZE * 8, and preferably prime. 1021 is the largest
392 // prime under 128 * 8.
393 #define BIGRAM_FILTER_MODULO 1021
394 #if BIGRAM_FILTER_BYTE_SIZE * 8 < BIGRAM_FILTER_MODULO
395 #error "BIGRAM_FILTER_MODULO is larger than BIGRAM_FILTER_BYTE_SIZE"
396 #endif
397
min(T a,T b)398 template<typename T> inline T min(T a, T b) { return a < b ? a : b; }
max(T a,T b)399 template<typename T> inline T max(T a, T b) { return a > b ? a : b; }
400
401 // The ratio of neutral area radius to sweet spot radius.
402 #define NEUTRAL_AREA_RADIUS_RATIO 1.3f
403
404 // DEBUG
405 #define INPUTLENGTH_FOR_DEBUG (-1)
406 #define MIN_OUTPUT_INDEX_FOR_DEBUG (-1)
407
408 #define DISALLOW_COPY_AND_ASSIGN(TypeName) \
409 TypeName(const TypeName&); \
410 void operator=(const TypeName&)
411
412 #define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \
413 TypeName(); \
414 DISALLOW_COPY_AND_ASSIGN(TypeName)
415
416 // Used as a return value for character comparison
417 typedef enum {
418 // Same char, possibly with different case or accent
419 EQUIVALENT_CHAR,
420 // It is a char located nearby on the keyboard
421 NEAR_PROXIMITY_CHAR,
422 // It is an unrelated char
423 UNRELATED_CHAR,
424 // Additional proximity char which can differ by language.
425 ADDITIONAL_PROXIMITY_CHAR
426 } ProximityType;
427 #endif // LATINIME_DEFINES_H
428