1 // Copyright (C) 2015 The Android Open Source Project 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #ifndef _INIT_PARSER_TOKENIZER_H 16 #define _INIT_PARSER_TOKENIZER_H 17 18 #include <string> 19 20 namespace init { 21 22 // Used to tokenize a std::string. 23 // Call Next() to advance through each token until it returns false, 24 // indicating there are no more tokens left in the string. 25 // The current token can be accessed with current(), which returns 26 // a Token. 27 // Supported tokens are: 28 // TOK_START - Next() has yet to be called 29 // TOK_END - At the end of string 30 // TOK_NEWLINE - The end of a line denoted by \n. 31 // TOK_TEXT - A word. 32 // Comments are denoted with '#' and the tokenizer will ignore 33 // the rest of the line. 34 // Double quotes can be used to insert whitespace into words. 35 // A backslash at the end of a line denotes continuation and 36 // a TOK_NEWLINE will not be generated for that line. 37 class Tokenizer { 38 public: 39 explicit Tokenizer(const std::string& data); 40 ~Tokenizer(); 41 42 enum TokenType { TOK_START, TOK_END, TOK_NEWLINE, TOK_TEXT }; 43 struct Token { 44 TokenType type; 45 std::string text; 46 }; 47 48 // Returns the curret token. 49 const Token& current(); 50 51 // Move to the next token, returns false at the end of input. 52 bool Next(); 53 54 private: 55 void GetData(); 56 void AdvChar(); 57 void AdvText(); 58 void AdvUntil(char x); 59 void AdvWhiteSpace(); 60 void StartText(); 61 void EndText(); 62 63 const std::string& data_; 64 Token current_; 65 66 bool eof_; 67 size_t pos_; 68 char cur_char_; 69 size_t tok_start_; 70 }; 71 72 } // namespace init 73 74 #endif 75