1 // 2 // Copyright (c) 2012-2014 The ANGLE Project Authors. All rights reserved. 3 // Use of this source code is governed by a BSD-style license that can be 4 // found in the LICENSE file. 5 // 6 7 #ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_ 8 #define COMPILER_PREPROCESSOR_TOKENIZER_H_ 9 10 #include "Input.h" 11 #include "Lexer.h" 12 #include "pp_utils.h" 13 14 namespace pp 15 { 16 17 class Diagnostics; 18 19 class Tokenizer : public Lexer 20 { 21 public: 22 struct Context 23 { 24 Diagnostics *diagnostics; 25 26 Input input; 27 // The location where yytext points to. Token location should track 28 // scanLoc instead of Input::mReadLoc because they may not be the same 29 // if text is buffered up in the scanner input buffer. 30 Input::Location scanLoc; 31 32 bool leadingSpace; 33 bool lineStart; 34 }; 35 36 Tokenizer(Diagnostics *diagnostics); 37 ~Tokenizer(); 38 39 bool init(size_t count, const char * const string[], const int length[]); 40 41 void setFileNumber(int file); 42 void setLineNumber(int line); 43 void setMaxTokenSize(size_t maxTokenSize); 44 45 virtual void lex(Token *token); 46 47 private: 48 PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer); 49 bool initScanner(); 50 void destroyScanner(); 51 52 void *mHandle; // Scanner handle. 53 Context mContext; // Scanner extra. 54 size_t mMaxTokenSize; // Maximum token size 55 }; 56 57 } // namespace pp 58 #endif // COMPILER_PREPROCESSOR_TOKENIZER_H_ 59 60