1 //===-- lib/Parser/token-sequence.h -----------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef FORTRAN_PARSER_TOKEN_SEQUENCE_H_ 10 #define FORTRAN_PARSER_TOKEN_SEQUENCE_H_ 11 12 // A buffer class capable of holding a contiguous sequence of characters 13 // and a partitioning thereof into preprocessing tokens, along with their 14 // associated provenances. 15 16 #include "flang/Parser/char-block.h" 17 #include "flang/Parser/provenance.h" 18 #include <cstddef> 19 #include <cstring> 20 #include <string> 21 #include <utility> 22 #include <vector> 23 24 namespace llvm { 25 class raw_ostream; 26 } 27 28 namespace Fortran::parser { 29 30 class Messages; 31 32 // Buffers a contiguous sequence of characters that has been partitioned into 33 // a sequence of preprocessing tokens with provenances. 34 class TokenSequence { 35 public: TokenSequence()36 TokenSequence() {} TokenSequence(const TokenSequence & that)37 TokenSequence(const TokenSequence &that) { Put(that); } 38 TokenSequence( 39 const TokenSequence &that, std::size_t at, std::size_t count = 1) { 40 Put(that, at, count); 41 } TokenSequence(TokenSequence && that)42 TokenSequence(TokenSequence &&that) 43 : start_{std::move(that.start_)}, nextStart_{that.nextStart_}, 44 char_{std::move(that.char_)}, provenances_{ 45 std::move(that.provenances_)} {} TokenSequence(const std::string & s,Provenance p)46 TokenSequence(const std::string &s, Provenance p) { Put(s, p); } 47 48 TokenSequence &operator=(const TokenSequence &that) { 49 clear(); 50 Put(that); 51 return *this; 52 } 53 TokenSequence &operator=(TokenSequence &&that); empty()54 bool empty() const { return start_.empty(); } 55 void clear(); 56 void pop_back(); 57 void shrink_to_fit(); 58 void swap(TokenSequence &); 59 SizeInTokens()60 std::size_t SizeInTokens() const { return start_.size(); } SizeInChars()61 std::size_t SizeInChars() const { return char_.size(); } 62 ToCharBlock()63 CharBlock ToCharBlock() const { return {&char_[0], char_.size()}; } ToString()64 std::string ToString() const { return ToCharBlock().ToString(); } 65 TokenAt(std::size_t token)66 CharBlock TokenAt(std::size_t token) const { 67 return {&char_[start_.at(token)], TokenBytes(token)}; 68 } CharAt(std::size_t j)69 char CharAt(std::size_t j) const { return char_.at(j); } CurrentOpenToken()70 CharBlock CurrentOpenToken() const { 71 return {&char_[nextStart_], char_.size() - nextStart_}; 72 } 73 74 std::size_t SkipBlanks(std::size_t) const; 75 76 // True if anything remains in the sequence at & after the given offset 77 // except blanks and line-ending C++ and Fortran free-form comments. 78 bool IsAnythingLeft(std::size_t) const; 79 PutNextTokenChar(char ch,Provenance provenance)80 void PutNextTokenChar(char ch, Provenance provenance) { 81 char_.emplace_back(ch); 82 provenances_.Put({provenance, 1}); 83 } 84 CloseToken()85 void CloseToken() { 86 start_.emplace_back(nextStart_); 87 nextStart_ = char_.size(); 88 } 89 ReopenLastToken()90 void ReopenLastToken() { 91 nextStart_ = start_.back(); 92 start_.pop_back(); 93 } 94 95 void RemoveLastToken(); 96 97 void Put(const TokenSequence &); 98 void Put(const TokenSequence &, ProvenanceRange); 99 void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1); 100 void Put(const char *, std::size_t, Provenance); 101 void Put(const CharBlock &, Provenance); 102 void Put(const std::string &, Provenance); 103 void Put(llvm::raw_string_ostream &, Provenance); 104 105 Provenance GetTokenProvenance( 106 std::size_t token, std::size_t offset = 0) const; 107 ProvenanceRange GetTokenProvenanceRange( 108 std::size_t token, std::size_t offset = 0) const; 109 ProvenanceRange GetIntervalProvenanceRange( 110 std::size_t token, std::size_t tokens = 1) const; 111 ProvenanceRange GetProvenanceRange() const; 112 GetMutableCharData()113 char *GetMutableCharData() { return &char_[0]; } 114 TokenSequence &ToLowerCase(); 115 bool HasBlanks(std::size_t firstChar = 0) const; 116 bool HasRedundantBlanks(std::size_t firstChar = 0) const; 117 TokenSequence &RemoveBlanks(std::size_t firstChar = 0); 118 TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0); 119 TokenSequence &ClipComment(bool skipFirst = false); 120 const TokenSequence &CheckBadFortranCharacters(Messages &) const; 121 void Emit(CookedSource &) const; 122 void Dump(llvm::raw_ostream &) const; 123 124 private: TokenBytes(std::size_t token)125 std::size_t TokenBytes(std::size_t token) const { 126 return (token + 1 >= start_.size() ? char_.size() : start_[token + 1]) - 127 start_[token]; 128 } 129 130 std::vector<std::size_t> start_; 131 std::size_t nextStart_{0}; 132 std::vector<char> char_; 133 OffsetToProvenanceMappings provenances_; 134 }; 135 } // namespace Fortran::parser 136 #endif // FORTRAN_PARSER_TOKEN_SEQUENCE_H_ 137