1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef NLP_SAFT_COMPONENTS_LANG_ID_MOBILE_SCRIPT_APPROX_SCRIPT_DATA_H_ 18 #define NLP_SAFT_COMPONENTS_LANG_ID_MOBILE_SCRIPT_APPROX_SCRIPT_DATA_H_ 19 20 #include "lang_id/common/lite_base/integral-types.h" 21 22 namespace libtextclassifier3 { 23 namespace mobile { 24 namespace approx_script_internal { 25 26 // Number of contiguous ranges of same-script codepoints (see below). 27 extern const int kNumRanges; 28 29 // Non-overlapping ranges of unicode characters. Characters from each range has 30 // the same script (see kRangeScripts below). Multiple ranges may have the same 31 // script. Note: we represent the kNumRanges ranges as an array with their 32 // first codepoints, and a separate array with their sizes (see kRangeSize 33 // below). This leads to better memory locality during the binary search (which 34 // uses only the first codepoints, up until the very end). 35 // 36 // kRangeFirst[i] = first codepoint from range #i, \forall 0 <= i < kNumRanges. 37 extern const uint32 kRangeFirst[]; 38 39 // kRangeSize[i] > 0 is the number of consecutive codepoints in range #i *minus* 40 // 1, \forall 0 <= i < kNumRanges. I.e., 0 means that the range contains 1 41 // codepoints. Since we don't have empty ranges, this "minus one" convention 42 // allows us to use all 2^16 values here. 43 extern const uint16 kRangeSizeMinusOne[]; 44 45 // Scripts for the ranges from kRanges. For each i such that 0 <= i < 46 // kNumRanges, the range #i has the script kRangeScript[i]. Each uint8 element 47 // can be casted to an UScriptCode enum value (see 48 // unicode/uscript.h). 49 // 50 // NOTE: we don't use directly UScriptCode here, as that requires a full int 51 // (due to USCRIPT_INVALID_CODE = -1). uint8 is enough for us (and shorter!) 52 extern const uint8 kRangeScript[]; 53 54 // Max value from kRangeScript[]. Scripts are guaranteed to be in the interval 55 // [0, kMaxScript] (inclusive on both sides). Can be used to e.g., set the 56 // number of rows in an embedding table for a script-based feature. 57 extern const uint8 kMaxScript; 58 59 } // namespace approx_script_internal 60 } // namespace mobile 61 } // namespace nlp_saft 62 63 #endif // NLP_SAFT_COMPONENTS_LANG_ID_MOBILE_SCRIPT_APPROX_SCRIPT_DATA_H_ 64