1 // Copyright (C) 2019 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #ifndef ICING_TOKENIZATION_TOKEN_H_ 16 #define ICING_TOKENIZATION_TOKEN_H_ 17 18 #include <string_view> 19 20 namespace icing { 21 namespace lib { 22 23 struct Token { 24 enum Type { 25 // Common types 26 REGULAR, // A token without special meanings, the value of it will be 27 // indexed or searched directly 28 29 // Types only used in raw query 30 QUERY_OR, // Indicates OR logic between its left and right tokens 31 QUERY_EXCLUSION, // Indicates exclusion operation on next token 32 QUERY_PROPERTY, // Indicates property restrict on next token 33 QUERY_LEFT_PARENTHESES, // Left parentheses 34 QUERY_RIGHT_PARENTHESES, // Right parentheses 35 36 // Indicates errors 37 INVALID, 38 }; 39 40 // The input text should outlive the Token instance. 41 explicit Token(Type type_in, std::string_view text_in = "") typeToken42 : type(type_in), text(text_in) {} 43 44 // The type of token 45 const Type type; 46 47 // The content of token 48 const std::string_view text; 49 }; 50 51 } // namespace lib 52 } // namespace icing 53 54 #endif // ICING_TOKENIZATION_TOKEN_H_ 55