Searched defs:TokenSpan (Results 1 – 1 of 1) sorted by relevance
144 struct TokenSpan { struct145 static const TokenSpan kInvalid; argument147 TokenSpan() : first(kInvalidIndex), second(kInvalidIndex) {} in TokenSpan() argument149 TokenSpan(TokenIndex start, TokenIndex end) : first(start), second(end) {} in TokenSpan() argument152 explicit TokenSpan(int token_index) in TokenSpan() function155 TokenSpan& operator=(const TokenSpan& other) = default; argument179 TokenSpan Expand(int num_tokens_left, int num_tokens_right) const { in Expand() argument187 // Pretty-printing function for TokenSpan. argument