Searched refs:CodepointIndex (Results 1 – 6 of 6) sorted by relevance
| /external/libtextclassifier/native/utils/grammar/parsing/ |
| D | parser.cc | 43 const std::unordered_map<CodepointIndex, CodepointIndex>& token_alignment, in MapCodepointToTokenPaddingIfPresent() argument
90 std::unordered_map<CodepointIndex, CodepointIndex> token_match_start; in SortedSymbolsForInput()
93 const CodepointIndex token_start = input.tokens[i].start; in SortedSymbolsForInput()
94 const CodepointIndex prev_token_end = input.tokens[i - 1].end; in SortedSymbolsForInput()
101 CodepointIndex match_offset = input.tokens[input.context_span.first].start; in SortedSymbolsForInput()
144 const CodepointIndex context_start = in SortedSymbolsForInput()
146 const CodepointIndex context_end = in SortedSymbolsForInput()
|
| D | matcher.h | 89 void AddTerminal(const CodepointIndex begin, const CodepointIndex end, in AddTerminal()
|
| D | lexer.cc | 41 CodepointIndex sub_token_start = codepoint_span.first; in AppendTokenSymbols()
|
| /external/libtextclassifier/native/annotator/ |
| D | types.h | 55 using CodepointIndex = int; variable
65 CodepointSpan(CodepointIndex start, CodepointIndex end) in CodepointSpan()
92 CodepointIndex first;
93 CodepointIndex second;
203 CodepointIndex start;
204 CodepointIndex end;
218 Token(const std::string& arg_value, CodepointIndex arg_start, in Token()
219 CodepointIndex arg_end) in Token()
224 Token(const std::string& arg_value, CodepointIndex arg_start, in Token()
225 CodepointIndex arg_end, bool is_padding, bool is_whitespace) in Token()
|
| D | feature-processor.cc | 161 const CodepointIndex last_line_begin_index = in StripTokensFromOtherLines()
163 const CodepointIndex last_line_end_index = in StripTokensFromOtherLines()
289 const CodepointIndex tokens_start = tokens[click_position - span_left].start; in SpanToLabel()
290 const CodepointIndex tokens_end = tokens[click_position + span_right].end; in SpanToLabel()
|
| /external/libtextclassifier/native/annotator/duration/ |
| D | duration.cc | 177 CodepointIndex start_index = kInvalidIndex; in FindDurationStartingAt()
178 CodepointIndex end_index = kInvalidIndex; in FindDurationStartingAt()
|