1 //===--- YAMLParser.cpp - Simple YAML parser ------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements a YAML parser.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Support/YAMLParser.h"
15
16 #include "llvm/ADT/ilist.h"
17 #include "llvm/ADT/ilist_node.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringExtras.h"
20 #include "llvm/ADT/Twine.h"
21 #include "llvm/Support/ErrorHandling.h"
22 #include "llvm/Support/MemoryBuffer.h"
23 #include "llvm/Support/raw_ostream.h"
24 #include "llvm/Support/SourceMgr.h"
25
26 using namespace llvm;
27 using namespace yaml;
28
29 enum UnicodeEncodingForm {
30 UEF_UTF32_LE, ///< UTF-32 Little Endian
31 UEF_UTF32_BE, ///< UTF-32 Big Endian
32 UEF_UTF16_LE, ///< UTF-16 Little Endian
33 UEF_UTF16_BE, ///< UTF-16 Big Endian
34 UEF_UTF8, ///< UTF-8 or ascii.
35 UEF_Unknown ///< Not a valid Unicode encoding.
36 };
37
38 /// EncodingInfo - Holds the encoding type and length of the byte order mark if
39 /// it exists. Length is in {0, 2, 3, 4}.
40 typedef std::pair<UnicodeEncodingForm, unsigned> EncodingInfo;
41
42 /// getUnicodeEncoding - Reads up to the first 4 bytes to determine the Unicode
43 /// encoding form of \a Input.
44 ///
45 /// @param Input A string of length 0 or more.
46 /// @returns An EncodingInfo indicating the Unicode encoding form of the input
47 /// and how long the byte order mark is if one exists.
getUnicodeEncoding(StringRef Input)48 static EncodingInfo getUnicodeEncoding(StringRef Input) {
49 if (Input.size() == 0)
50 return std::make_pair(UEF_Unknown, 0);
51
52 switch (uint8_t(Input[0])) {
53 case 0x00:
54 if (Input.size() >= 4) {
55 if ( Input[1] == 0
56 && uint8_t(Input[2]) == 0xFE
57 && uint8_t(Input[3]) == 0xFF)
58 return std::make_pair(UEF_UTF32_BE, 4);
59 if (Input[1] == 0 && Input[2] == 0 && Input[3] != 0)
60 return std::make_pair(UEF_UTF32_BE, 0);
61 }
62
63 if (Input.size() >= 2 && Input[1] != 0)
64 return std::make_pair(UEF_UTF16_BE, 0);
65 return std::make_pair(UEF_Unknown, 0);
66 case 0xFF:
67 if ( Input.size() >= 4
68 && uint8_t(Input[1]) == 0xFE
69 && Input[2] == 0
70 && Input[3] == 0)
71 return std::make_pair(UEF_UTF32_LE, 4);
72
73 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFE)
74 return std::make_pair(UEF_UTF16_LE, 2);
75 return std::make_pair(UEF_Unknown, 0);
76 case 0xFE:
77 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFF)
78 return std::make_pair(UEF_UTF16_BE, 2);
79 return std::make_pair(UEF_Unknown, 0);
80 case 0xEF:
81 if ( Input.size() >= 3
82 && uint8_t(Input[1]) == 0xBB
83 && uint8_t(Input[2]) == 0xBF)
84 return std::make_pair(UEF_UTF8, 3);
85 return std::make_pair(UEF_Unknown, 0);
86 }
87
88 // It could still be utf-32 or utf-16.
89 if (Input.size() >= 4 && Input[1] == 0 && Input[2] == 0 && Input[3] == 0)
90 return std::make_pair(UEF_UTF32_LE, 0);
91
92 if (Input.size() >= 2 && Input[1] == 0)
93 return std::make_pair(UEF_UTF16_LE, 0);
94
95 return std::make_pair(UEF_UTF8, 0);
96 }
97
98 namespace llvm {
99 namespace yaml {
100 /// Token - A single YAML token.
101 struct Token : ilist_node<Token> {
102 enum TokenKind {
103 TK_Error, // Uninitialized token.
104 TK_StreamStart,
105 TK_StreamEnd,
106 TK_VersionDirective,
107 TK_TagDirective,
108 TK_DocumentStart,
109 TK_DocumentEnd,
110 TK_BlockEntry,
111 TK_BlockEnd,
112 TK_BlockSequenceStart,
113 TK_BlockMappingStart,
114 TK_FlowEntry,
115 TK_FlowSequenceStart,
116 TK_FlowSequenceEnd,
117 TK_FlowMappingStart,
118 TK_FlowMappingEnd,
119 TK_Key,
120 TK_Value,
121 TK_Scalar,
122 TK_Alias,
123 TK_Anchor,
124 TK_Tag
125 } Kind;
126
127 /// A string of length 0 or more whose begin() points to the logical location
128 /// of the token in the input.
129 StringRef Range;
130
Tokenllvm::yaml::Token131 Token() : Kind(TK_Error) {}
132 };
133 }
134 }
135
136 namespace llvm {
137 template<>
138 struct ilist_sentinel_traits<Token> {
createSentinelllvm::ilist_sentinel_traits139 Token *createSentinel() const {
140 return &Sentinel;
141 }
destroySentinelllvm::ilist_sentinel_traits142 static void destroySentinel(Token*) {}
143
provideInitialHeadllvm::ilist_sentinel_traits144 Token *provideInitialHead() const { return createSentinel(); }
ensureHeadllvm::ilist_sentinel_traits145 Token *ensureHead(Token*) const { return createSentinel(); }
noteHeadllvm::ilist_sentinel_traits146 static void noteHead(Token*, Token*) {}
147
148 private:
149 mutable Token Sentinel;
150 };
151
152 template<>
153 struct ilist_node_traits<Token> {
createNodellvm::ilist_node_traits154 Token *createNode(const Token &V) {
155 return new (Alloc.Allocate<Token>()) Token(V);
156 }
deleteNodellvm::ilist_node_traits157 static void deleteNode(Token *V) {}
158
addNodeToListllvm::ilist_node_traits159 void addNodeToList(Token *) {}
removeNodeFromListllvm::ilist_node_traits160 void removeNodeFromList(Token *) {}
transferNodesFromListllvm::ilist_node_traits161 void transferNodesFromList(ilist_node_traits & /*SrcTraits*/,
162 ilist_iterator<Token> /*first*/,
163 ilist_iterator<Token> /*last*/) {}
164
165 BumpPtrAllocator Alloc;
166 };
167 }
168
169 typedef ilist<Token> TokenQueueT;
170
171 namespace {
172 /// @brief This struct is used to track simple keys.
173 ///
174 /// Simple keys are handled by creating an entry in SimpleKeys for each Token
175 /// which could legally be the start of a simple key. When peekNext is called,
176 /// if the Token To be returned is referenced by a SimpleKey, we continue
177 /// tokenizing until that potential simple key has either been found to not be
178 /// a simple key (we moved on to the next line or went further than 1024 chars).
179 /// Or when we run into a Value, and then insert a Key token (and possibly
180 /// others) before the SimpleKey's Tok.
181 struct SimpleKey {
182 TokenQueueT::iterator Tok;
183 unsigned Column;
184 unsigned Line;
185 unsigned FlowLevel;
186 bool IsRequired;
187
operator ==__anon1ad258c40111::SimpleKey188 bool operator ==(const SimpleKey &Other) {
189 return Tok == Other.Tok;
190 }
191 };
192 }
193
194 /// @brief The Unicode scalar value of a UTF-8 minimal well-formed code unit
195 /// subsequence and the subsequence's length in code units (uint8_t).
196 /// A length of 0 represents an error.
197 typedef std::pair<uint32_t, unsigned> UTF8Decoded;
198
decodeUTF8(StringRef Range)199 static UTF8Decoded decodeUTF8(StringRef Range) {
200 StringRef::iterator Position= Range.begin();
201 StringRef::iterator End = Range.end();
202 // 1 byte: [0x00, 0x7f]
203 // Bit pattern: 0xxxxxxx
204 if ((*Position & 0x80) == 0) {
205 return std::make_pair(*Position, 1);
206 }
207 // 2 bytes: [0x80, 0x7ff]
208 // Bit pattern: 110xxxxx 10xxxxxx
209 if (Position + 1 != End &&
210 ((*Position & 0xE0) == 0xC0) &&
211 ((*(Position + 1) & 0xC0) == 0x80)) {
212 uint32_t codepoint = ((*Position & 0x1F) << 6) |
213 (*(Position + 1) & 0x3F);
214 if (codepoint >= 0x80)
215 return std::make_pair(codepoint, 2);
216 }
217 // 3 bytes: [0x8000, 0xffff]
218 // Bit pattern: 1110xxxx 10xxxxxx 10xxxxxx
219 if (Position + 2 != End &&
220 ((*Position & 0xF0) == 0xE0) &&
221 ((*(Position + 1) & 0xC0) == 0x80) &&
222 ((*(Position + 2) & 0xC0) == 0x80)) {
223 uint32_t codepoint = ((*Position & 0x0F) << 12) |
224 ((*(Position + 1) & 0x3F) << 6) |
225 (*(Position + 2) & 0x3F);
226 // Codepoints between 0xD800 and 0xDFFF are invalid, as
227 // they are high / low surrogate halves used by UTF-16.
228 if (codepoint >= 0x800 &&
229 (codepoint < 0xD800 || codepoint > 0xDFFF))
230 return std::make_pair(codepoint, 3);
231 }
232 // 4 bytes: [0x10000, 0x10FFFF]
233 // Bit pattern: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
234 if (Position + 3 != End &&
235 ((*Position & 0xF8) == 0xF0) &&
236 ((*(Position + 1) & 0xC0) == 0x80) &&
237 ((*(Position + 2) & 0xC0) == 0x80) &&
238 ((*(Position + 3) & 0xC0) == 0x80)) {
239 uint32_t codepoint = ((*Position & 0x07) << 18) |
240 ((*(Position + 1) & 0x3F) << 12) |
241 ((*(Position + 2) & 0x3F) << 6) |
242 (*(Position + 3) & 0x3F);
243 if (codepoint >= 0x10000 && codepoint <= 0x10FFFF)
244 return std::make_pair(codepoint, 4);
245 }
246 return std::make_pair(0, 0);
247 }
248
249 namespace llvm {
250 namespace yaml {
251 /// @brief Scans YAML tokens from a MemoryBuffer.
252 class Scanner {
253 public:
254 Scanner(const StringRef Input, SourceMgr &SM);
255
256 /// @brief Parse the next token and return it without popping it.
257 Token &peekNext();
258
259 /// @brief Parse the next token and pop it from the queue.
260 Token getNext();
261
printError(SMLoc Loc,SourceMgr::DiagKind Kind,const Twine & Message,ArrayRef<SMRange> Ranges=ArrayRef<SMRange> ())262 void printError(SMLoc Loc, SourceMgr::DiagKind Kind, const Twine &Message,
263 ArrayRef<SMRange> Ranges = ArrayRef<SMRange>()) {
264 SM.PrintMessage(Loc, Kind, Message, Ranges);
265 }
266
setError(const Twine & Message,StringRef::iterator Position)267 void setError(const Twine &Message, StringRef::iterator Position) {
268 if (Current >= End)
269 Current = End - 1;
270
271 // Don't print out more errors after the first one we encounter. The rest
272 // are just the result of the first, and have no meaning.
273 if (!Failed)
274 printError(SMLoc::getFromPointer(Current), SourceMgr::DK_Error, Message);
275 Failed = true;
276 }
277
setError(const Twine & Message)278 void setError(const Twine &Message) {
279 setError(Message, Current);
280 }
281
282 /// @brief Returns true if an error occurred while parsing.
failed()283 bool failed() {
284 return Failed;
285 }
286
287 private:
currentInput()288 StringRef currentInput() {
289 return StringRef(Current, End - Current);
290 }
291
292 /// @brief Decode a UTF-8 minimal well-formed code unit subsequence starting
293 /// at \a Position.
294 ///
295 /// If the UTF-8 code units starting at Position do not form a well-formed
296 /// code unit subsequence, then the Unicode scalar value is 0, and the length
297 /// is 0.
decodeUTF8(StringRef::iterator Position)298 UTF8Decoded decodeUTF8(StringRef::iterator Position) {
299 return ::decodeUTF8(StringRef(Position, End - Position));
300 }
301
302 // The following functions are based on the gramar rules in the YAML spec. The
303 // style of the function names it meant to closely match how they are written
304 // in the spec. The number within the [] is the number of the grammar rule in
305 // the spec.
306 //
307 // See 4.2 [Production Naming Conventions] for the meaning of the prefixes.
308 //
309 // c-
310 // A production starting and ending with a special character.
311 // b-
312 // A production matching a single line break.
313 // nb-
314 // A production starting and ending with a non-break character.
315 // s-
316 // A production starting and ending with a white space character.
317 // ns-
318 // A production starting and ending with a non-space character.
319 // l-
320 // A production matching complete line(s).
321
322 /// @brief Skip a single nb-char[27] starting at Position.
323 ///
324 /// A nb-char is 0x9 | [0x20-0x7E] | 0x85 | [0xA0-0xD7FF] | [0xE000-0xFEFE]
325 /// | [0xFF00-0xFFFD] | [0x10000-0x10FFFF]
326 ///
327 /// @returns The code unit after the nb-char, or Position if it's not an
328 /// nb-char.
329 StringRef::iterator skip_nb_char(StringRef::iterator Position);
330
331 /// @brief Skip a single b-break[28] starting at Position.
332 ///
333 /// A b-break is 0xD 0xA | 0xD | 0xA
334 ///
335 /// @returns The code unit after the b-break, or Position if it's not a
336 /// b-break.
337 StringRef::iterator skip_b_break(StringRef::iterator Position);
338
339 /// @brief Skip a single s-white[33] starting at Position.
340 ///
341 /// A s-white is 0x20 | 0x9
342 ///
343 /// @returns The code unit after the s-white, or Position if it's not a
344 /// s-white.
345 StringRef::iterator skip_s_white(StringRef::iterator Position);
346
347 /// @brief Skip a single ns-char[34] starting at Position.
348 ///
349 /// A ns-char is nb-char - s-white
350 ///
351 /// @returns The code unit after the ns-char, or Position if it's not a
352 /// ns-char.
353 StringRef::iterator skip_ns_char(StringRef::iterator Position);
354
355 typedef StringRef::iterator (Scanner::*SkipWhileFunc)(StringRef::iterator);
356 /// @brief Skip minimal well-formed code unit subsequences until Func
357 /// returns its input.
358 ///
359 /// @returns The code unit after the last minimal well-formed code unit
360 /// subsequence that Func accepted.
361 StringRef::iterator skip_while( SkipWhileFunc Func
362 , StringRef::iterator Position);
363
364 /// @brief Scan ns-uri-char[39]s starting at Cur.
365 ///
366 /// This updates Cur and Column while scanning.
367 ///
368 /// @returns A StringRef starting at Cur which covers the longest contiguous
369 /// sequence of ns-uri-char.
370 StringRef scan_ns_uri_char();
371
372 /// @brief Scan ns-plain-one-line[133] starting at \a Cur.
373 StringRef scan_ns_plain_one_line();
374
375 /// @brief Consume a minimal well-formed code unit subsequence starting at
376 /// \a Cur. Return false if it is not the same Unicode scalar value as
377 /// \a Expected. This updates \a Column.
378 bool consume(uint32_t Expected);
379
380 /// @brief Skip \a Distance UTF-8 code units. Updates \a Cur and \a Column.
381 void skip(uint32_t Distance);
382
383 /// @brief Return true if the minimal well-formed code unit subsequence at
384 /// Pos is whitespace or a new line
385 bool isBlankOrBreak(StringRef::iterator Position);
386
387 /// @brief If IsSimpleKeyAllowed, create and push_back a new SimpleKey.
388 void saveSimpleKeyCandidate( TokenQueueT::iterator Tok
389 , unsigned AtColumn
390 , bool IsRequired);
391
392 /// @brief Remove simple keys that can no longer be valid simple keys.
393 ///
394 /// Invalid simple keys are not on the current line or are further than 1024
395 /// columns back.
396 void removeStaleSimpleKeyCandidates();
397
398 /// @brief Remove all simple keys on FlowLevel \a Level.
399 void removeSimpleKeyCandidatesOnFlowLevel(unsigned Level);
400
401 /// @brief Unroll indentation in \a Indents back to \a Col. Creates BlockEnd
402 /// tokens if needed.
403 bool unrollIndent(int ToColumn);
404
405 /// @brief Increase indent to \a Col. Creates \a Kind token at \a InsertPoint
406 /// if needed.
407 bool rollIndent( int ToColumn
408 , Token::TokenKind Kind
409 , TokenQueueT::iterator InsertPoint);
410
411 /// @brief Skip whitespace and comments until the start of the next token.
412 void scanToNextToken();
413
414 /// @brief Must be the first token generated.
415 bool scanStreamStart();
416
417 /// @brief Generate tokens needed to close out the stream.
418 bool scanStreamEnd();
419
420 /// @brief Scan a %BLAH directive.
421 bool scanDirective();
422
423 /// @brief Scan a ... or ---.
424 bool scanDocumentIndicator(bool IsStart);
425
426 /// @brief Scan a [ or { and generate the proper flow collection start token.
427 bool scanFlowCollectionStart(bool IsSequence);
428
429 /// @brief Scan a ] or } and generate the proper flow collection end token.
430 bool scanFlowCollectionEnd(bool IsSequence);
431
432 /// @brief Scan the , that separates entries in a flow collection.
433 bool scanFlowEntry();
434
435 /// @brief Scan the - that starts block sequence entries.
436 bool scanBlockEntry();
437
438 /// @brief Scan an explicit ? indicating a key.
439 bool scanKey();
440
441 /// @brief Scan an explicit : indicating a value.
442 bool scanValue();
443
444 /// @brief Scan a quoted scalar.
445 bool scanFlowScalar(bool IsDoubleQuoted);
446
447 /// @brief Scan an unquoted scalar.
448 bool scanPlainScalar();
449
450 /// @brief Scan an Alias or Anchor starting with * or &.
451 bool scanAliasOrAnchor(bool IsAlias);
452
453 /// @brief Scan a block scalar starting with | or >.
454 bool scanBlockScalar(bool IsLiteral);
455
456 /// @brief Scan a tag of the form !stuff.
457 bool scanTag();
458
459 /// @brief Dispatch to the next scanning function based on \a *Cur.
460 bool fetchMoreTokens();
461
462 /// @brief The SourceMgr used for diagnostics and buffer management.
463 SourceMgr &SM;
464
465 /// @brief The original input.
466 MemoryBuffer *InputBuffer;
467
468 /// @brief The current position of the scanner.
469 StringRef::iterator Current;
470
471 /// @brief The end of the input (one past the last character).
472 StringRef::iterator End;
473
474 /// @brief Current YAML indentation level in spaces.
475 int Indent;
476
477 /// @brief Current column number in Unicode code points.
478 unsigned Column;
479
480 /// @brief Current line number.
481 unsigned Line;
482
483 /// @brief How deep we are in flow style containers. 0 Means at block level.
484 unsigned FlowLevel;
485
486 /// @brief Are we at the start of the stream?
487 bool IsStartOfStream;
488
489 /// @brief Can the next token be the start of a simple key?
490 bool IsSimpleKeyAllowed;
491
492 /// @brief True if an error has occurred.
493 bool Failed;
494
495 /// @brief Queue of tokens. This is required to queue up tokens while looking
496 /// for the end of a simple key. And for cases where a single character
497 /// can produce multiple tokens (e.g. BlockEnd).
498 TokenQueueT TokenQueue;
499
500 /// @brief Indentation levels.
501 SmallVector<int, 4> Indents;
502
503 /// @brief Potential simple keys.
504 SmallVector<SimpleKey, 4> SimpleKeys;
505 };
506
507 } // end namespace yaml
508 } // end namespace llvm
509
510 /// encodeUTF8 - Encode \a UnicodeScalarValue in UTF-8 and append it to result.
encodeUTF8(uint32_t UnicodeScalarValue,SmallVectorImpl<char> & Result)511 static void encodeUTF8( uint32_t UnicodeScalarValue
512 , SmallVectorImpl<char> &Result) {
513 if (UnicodeScalarValue <= 0x7F) {
514 Result.push_back(UnicodeScalarValue & 0x7F);
515 } else if (UnicodeScalarValue <= 0x7FF) {
516 uint8_t FirstByte = 0xC0 | ((UnicodeScalarValue & 0x7C0) >> 6);
517 uint8_t SecondByte = 0x80 | (UnicodeScalarValue & 0x3F);
518 Result.push_back(FirstByte);
519 Result.push_back(SecondByte);
520 } else if (UnicodeScalarValue <= 0xFFFF) {
521 uint8_t FirstByte = 0xE0 | ((UnicodeScalarValue & 0xF000) >> 12);
522 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6);
523 uint8_t ThirdByte = 0x80 | (UnicodeScalarValue & 0x3F);
524 Result.push_back(FirstByte);
525 Result.push_back(SecondByte);
526 Result.push_back(ThirdByte);
527 } else if (UnicodeScalarValue <= 0x10FFFF) {
528 uint8_t FirstByte = 0xF0 | ((UnicodeScalarValue & 0x1F0000) >> 18);
529 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0x3F000) >> 12);
530 uint8_t ThirdByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6);
531 uint8_t FourthByte = 0x80 | (UnicodeScalarValue & 0x3F);
532 Result.push_back(FirstByte);
533 Result.push_back(SecondByte);
534 Result.push_back(ThirdByte);
535 Result.push_back(FourthByte);
536 }
537 }
538
dumpTokens(StringRef Input,raw_ostream & OS)539 bool yaml::dumpTokens(StringRef Input, raw_ostream &OS) {
540 SourceMgr SM;
541 Scanner scanner(Input, SM);
542 while (true) {
543 Token T = scanner.getNext();
544 switch (T.Kind) {
545 case Token::TK_StreamStart:
546 OS << "Stream-Start: ";
547 break;
548 case Token::TK_StreamEnd:
549 OS << "Stream-End: ";
550 break;
551 case Token::TK_VersionDirective:
552 OS << "Version-Directive: ";
553 break;
554 case Token::TK_TagDirective:
555 OS << "Tag-Directive: ";
556 break;
557 case Token::TK_DocumentStart:
558 OS << "Document-Start: ";
559 break;
560 case Token::TK_DocumentEnd:
561 OS << "Document-End: ";
562 break;
563 case Token::TK_BlockEntry:
564 OS << "Block-Entry: ";
565 break;
566 case Token::TK_BlockEnd:
567 OS << "Block-End: ";
568 break;
569 case Token::TK_BlockSequenceStart:
570 OS << "Block-Sequence-Start: ";
571 break;
572 case Token::TK_BlockMappingStart:
573 OS << "Block-Mapping-Start: ";
574 break;
575 case Token::TK_FlowEntry:
576 OS << "Flow-Entry: ";
577 break;
578 case Token::TK_FlowSequenceStart:
579 OS << "Flow-Sequence-Start: ";
580 break;
581 case Token::TK_FlowSequenceEnd:
582 OS << "Flow-Sequence-End: ";
583 break;
584 case Token::TK_FlowMappingStart:
585 OS << "Flow-Mapping-Start: ";
586 break;
587 case Token::TK_FlowMappingEnd:
588 OS << "Flow-Mapping-End: ";
589 break;
590 case Token::TK_Key:
591 OS << "Key: ";
592 break;
593 case Token::TK_Value:
594 OS << "Value: ";
595 break;
596 case Token::TK_Scalar:
597 OS << "Scalar: ";
598 break;
599 case Token::TK_Alias:
600 OS << "Alias: ";
601 break;
602 case Token::TK_Anchor:
603 OS << "Anchor: ";
604 break;
605 case Token::TK_Tag:
606 OS << "Tag: ";
607 break;
608 case Token::TK_Error:
609 break;
610 }
611 OS << T.Range << "\n";
612 if (T.Kind == Token::TK_StreamEnd)
613 break;
614 else if (T.Kind == Token::TK_Error)
615 return false;
616 }
617 return true;
618 }
619
scanTokens(StringRef Input)620 bool yaml::scanTokens(StringRef Input) {
621 llvm::SourceMgr SM;
622 llvm::yaml::Scanner scanner(Input, SM);
623 for (;;) {
624 llvm::yaml::Token T = scanner.getNext();
625 if (T.Kind == Token::TK_StreamEnd)
626 break;
627 else if (T.Kind == Token::TK_Error)
628 return false;
629 }
630 return true;
631 }
632
escape(StringRef Input)633 std::string yaml::escape(StringRef Input) {
634 std::string EscapedInput;
635 for (StringRef::iterator i = Input.begin(), e = Input.end(); i != e; ++i) {
636 if (*i == '\\')
637 EscapedInput += "\\\\";
638 else if (*i == '"')
639 EscapedInput += "\\\"";
640 else if (*i == 0)
641 EscapedInput += "\\0";
642 else if (*i == 0x07)
643 EscapedInput += "\\a";
644 else if (*i == 0x08)
645 EscapedInput += "\\b";
646 else if (*i == 0x09)
647 EscapedInput += "\\t";
648 else if (*i == 0x0A)
649 EscapedInput += "\\n";
650 else if (*i == 0x0B)
651 EscapedInput += "\\v";
652 else if (*i == 0x0C)
653 EscapedInput += "\\f";
654 else if (*i == 0x0D)
655 EscapedInput += "\\r";
656 else if (*i == 0x1B)
657 EscapedInput += "\\e";
658 else if ((unsigned char)*i < 0x20) { // Control characters not handled above.
659 std::string HexStr = utohexstr(*i);
660 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr;
661 } else if (*i & 0x80) { // UTF-8 multiple code unit subsequence.
662 UTF8Decoded UnicodeScalarValue
663 = decodeUTF8(StringRef(i, Input.end() - i));
664 if (UnicodeScalarValue.second == 0) {
665 // Found invalid char.
666 SmallString<4> Val;
667 encodeUTF8(0xFFFD, Val);
668 EscapedInput.insert(EscapedInput.end(), Val.begin(), Val.end());
669 // FIXME: Error reporting.
670 return EscapedInput;
671 }
672 if (UnicodeScalarValue.first == 0x85)
673 EscapedInput += "\\N";
674 else if (UnicodeScalarValue.first == 0xA0)
675 EscapedInput += "\\_";
676 else if (UnicodeScalarValue.first == 0x2028)
677 EscapedInput += "\\L";
678 else if (UnicodeScalarValue.first == 0x2029)
679 EscapedInput += "\\P";
680 else {
681 std::string HexStr = utohexstr(UnicodeScalarValue.first);
682 if (HexStr.size() <= 2)
683 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr;
684 else if (HexStr.size() <= 4)
685 EscapedInput += "\\u" + std::string(4 - HexStr.size(), '0') + HexStr;
686 else if (HexStr.size() <= 8)
687 EscapedInput += "\\U" + std::string(8 - HexStr.size(), '0') + HexStr;
688 }
689 i += UnicodeScalarValue.second - 1;
690 } else
691 EscapedInput.push_back(*i);
692 }
693 return EscapedInput;
694 }
695
Scanner(StringRef Input,SourceMgr & sm)696 Scanner::Scanner(StringRef Input, SourceMgr &sm)
697 : SM(sm)
698 , Indent(-1)
699 , Column(0)
700 , Line(0)
701 , FlowLevel(0)
702 , IsStartOfStream(true)
703 , IsSimpleKeyAllowed(true)
704 , Failed(false) {
705 InputBuffer = MemoryBuffer::getMemBuffer(Input, "YAML");
706 SM.AddNewSourceBuffer(InputBuffer, SMLoc());
707 Current = InputBuffer->getBufferStart();
708 End = InputBuffer->getBufferEnd();
709 }
710
peekNext()711 Token &Scanner::peekNext() {
712 // If the current token is a possible simple key, keep parsing until we
713 // can confirm.
714 bool NeedMore = false;
715 while (true) {
716 if (TokenQueue.empty() || NeedMore) {
717 if (!fetchMoreTokens()) {
718 TokenQueue.clear();
719 TokenQueue.push_back(Token());
720 return TokenQueue.front();
721 }
722 }
723 assert(!TokenQueue.empty() &&
724 "fetchMoreTokens lied about getting tokens!");
725
726 removeStaleSimpleKeyCandidates();
727 SimpleKey SK;
728 SK.Tok = TokenQueue.front();
729 if (std::find(SimpleKeys.begin(), SimpleKeys.end(), SK)
730 == SimpleKeys.end())
731 break;
732 else
733 NeedMore = true;
734 }
735 return TokenQueue.front();
736 }
737
getNext()738 Token Scanner::getNext() {
739 Token Ret = peekNext();
740 // TokenQueue can be empty if there was an error getting the next token.
741 if (!TokenQueue.empty())
742 TokenQueue.pop_front();
743
744 // There cannot be any referenced Token's if the TokenQueue is empty. So do a
745 // quick deallocation of them all.
746 if (TokenQueue.empty()) {
747 TokenQueue.Alloc.Reset();
748 }
749
750 return Ret;
751 }
752
skip_nb_char(StringRef::iterator Position)753 StringRef::iterator Scanner::skip_nb_char(StringRef::iterator Position) {
754 if (Position == End)
755 return Position;
756 // Check 7 bit c-printable - b-char.
757 if ( *Position == 0x09
758 || (*Position >= 0x20 && *Position <= 0x7E))
759 return Position + 1;
760
761 // Check for valid UTF-8.
762 if (uint8_t(*Position) & 0x80) {
763 UTF8Decoded u8d = decodeUTF8(Position);
764 if ( u8d.second != 0
765 && u8d.first != 0xFEFF
766 && ( u8d.first == 0x85
767 || ( u8d.first >= 0xA0
768 && u8d.first <= 0xD7FF)
769 || ( u8d.first >= 0xE000
770 && u8d.first <= 0xFFFD)
771 || ( u8d.first >= 0x10000
772 && u8d.first <= 0x10FFFF)))
773 return Position + u8d.second;
774 }
775 return Position;
776 }
777
skip_b_break(StringRef::iterator Position)778 StringRef::iterator Scanner::skip_b_break(StringRef::iterator Position) {
779 if (Position == End)
780 return Position;
781 if (*Position == 0x0D) {
782 if (Position + 1 != End && *(Position + 1) == 0x0A)
783 return Position + 2;
784 return Position + 1;
785 }
786
787 if (*Position == 0x0A)
788 return Position + 1;
789 return Position;
790 }
791
792
skip_s_white(StringRef::iterator Position)793 StringRef::iterator Scanner::skip_s_white(StringRef::iterator Position) {
794 if (Position == End)
795 return Position;
796 if (*Position == ' ' || *Position == '\t')
797 return Position + 1;
798 return Position;
799 }
800
skip_ns_char(StringRef::iterator Position)801 StringRef::iterator Scanner::skip_ns_char(StringRef::iterator Position) {
802 if (Position == End)
803 return Position;
804 if (*Position == ' ' || *Position == '\t')
805 return Position;
806 return skip_nb_char(Position);
807 }
808
skip_while(SkipWhileFunc Func,StringRef::iterator Position)809 StringRef::iterator Scanner::skip_while( SkipWhileFunc Func
810 , StringRef::iterator Position) {
811 while (true) {
812 StringRef::iterator i = (this->*Func)(Position);
813 if (i == Position)
814 break;
815 Position = i;
816 }
817 return Position;
818 }
819
is_ns_hex_digit(const char C)820 static bool is_ns_hex_digit(const char C) {
821 return (C >= '0' && C <= '9')
822 || (C >= 'a' && C <= 'z')
823 || (C >= 'A' && C <= 'Z');
824 }
825
is_ns_word_char(const char C)826 static bool is_ns_word_char(const char C) {
827 return C == '-'
828 || (C >= 'a' && C <= 'z')
829 || (C >= 'A' && C <= 'Z');
830 }
831
scan_ns_uri_char()832 StringRef Scanner::scan_ns_uri_char() {
833 StringRef::iterator Start = Current;
834 while (true) {
835 if (Current == End)
836 break;
837 if (( *Current == '%'
838 && Current + 2 < End
839 && is_ns_hex_digit(*(Current + 1))
840 && is_ns_hex_digit(*(Current + 2)))
841 || is_ns_word_char(*Current)
842 || StringRef(Current, 1).find_first_of("#;/?:@&=+$,_.!~*'()[]")
843 != StringRef::npos) {
844 ++Current;
845 ++Column;
846 } else
847 break;
848 }
849 return StringRef(Start, Current - Start);
850 }
851
scan_ns_plain_one_line()852 StringRef Scanner::scan_ns_plain_one_line() {
853 StringRef::iterator start = Current;
854 // The first character must already be verified.
855 ++Current;
856 while (true) {
857 if (Current == End) {
858 break;
859 } else if (*Current == ':') {
860 // Check if the next character is a ns-char.
861 if (Current + 1 == End)
862 break;
863 StringRef::iterator i = skip_ns_char(Current + 1);
864 if (Current + 1 != i) {
865 Current = i;
866 Column += 2; // Consume both the ':' and ns-char.
867 } else
868 break;
869 } else if (*Current == '#') {
870 // Check if the previous character was a ns-char.
871 // The & 0x80 check is to check for the trailing byte of a utf-8
872 if (*(Current - 1) & 0x80 || skip_ns_char(Current - 1) == Current) {
873 ++Current;
874 ++Column;
875 } else
876 break;
877 } else {
878 StringRef::iterator i = skip_nb_char(Current);
879 if (i == Current)
880 break;
881 Current = i;
882 ++Column;
883 }
884 }
885 return StringRef(start, Current - start);
886 }
887
consume(uint32_t Expected)888 bool Scanner::consume(uint32_t Expected) {
889 if (Expected >= 0x80)
890 report_fatal_error("Not dealing with this yet");
891 if (Current == End)
892 return false;
893 if (uint8_t(*Current) >= 0x80)
894 report_fatal_error("Not dealing with this yet");
895 if (uint8_t(*Current) == Expected) {
896 ++Current;
897 ++Column;
898 return true;
899 }
900 return false;
901 }
902
skip(uint32_t Distance)903 void Scanner::skip(uint32_t Distance) {
904 Current += Distance;
905 Column += Distance;
906 }
907
isBlankOrBreak(StringRef::iterator Position)908 bool Scanner::isBlankOrBreak(StringRef::iterator Position) {
909 if (Position == End)
910 return false;
911 if ( *Position == ' ' || *Position == '\t'
912 || *Position == '\r' || *Position == '\n')
913 return true;
914 return false;
915 }
916
saveSimpleKeyCandidate(TokenQueueT::iterator Tok,unsigned AtColumn,bool IsRequired)917 void Scanner::saveSimpleKeyCandidate( TokenQueueT::iterator Tok
918 , unsigned AtColumn
919 , bool IsRequired) {
920 if (IsSimpleKeyAllowed) {
921 SimpleKey SK;
922 SK.Tok = Tok;
923 SK.Line = Line;
924 SK.Column = AtColumn;
925 SK.IsRequired = IsRequired;
926 SK.FlowLevel = FlowLevel;
927 SimpleKeys.push_back(SK);
928 }
929 }
930
removeStaleSimpleKeyCandidates()931 void Scanner::removeStaleSimpleKeyCandidates() {
932 for (SmallVectorImpl<SimpleKey>::iterator i = SimpleKeys.begin();
933 i != SimpleKeys.end();) {
934 if (i->Line != Line || i->Column + 1024 < Column) {
935 if (i->IsRequired)
936 setError( "Could not find expected : for simple key"
937 , i->Tok->Range.begin());
938 i = SimpleKeys.erase(i);
939 } else
940 ++i;
941 }
942 }
943
removeSimpleKeyCandidatesOnFlowLevel(unsigned Level)944 void Scanner::removeSimpleKeyCandidatesOnFlowLevel(unsigned Level) {
945 if (!SimpleKeys.empty() && (SimpleKeys.end() - 1)->FlowLevel == Level)
946 SimpleKeys.pop_back();
947 }
948
unrollIndent(int ToColumn)949 bool Scanner::unrollIndent(int ToColumn) {
950 Token T;
951 // Indentation is ignored in flow.
952 if (FlowLevel != 0)
953 return true;
954
955 while (Indent > ToColumn) {
956 T.Kind = Token::TK_BlockEnd;
957 T.Range = StringRef(Current, 1);
958 TokenQueue.push_back(T);
959 Indent = Indents.pop_back_val();
960 }
961
962 return true;
963 }
964
rollIndent(int ToColumn,Token::TokenKind Kind,TokenQueueT::iterator InsertPoint)965 bool Scanner::rollIndent( int ToColumn
966 , Token::TokenKind Kind
967 , TokenQueueT::iterator InsertPoint) {
968 if (FlowLevel)
969 return true;
970 if (Indent < ToColumn) {
971 Indents.push_back(Indent);
972 Indent = ToColumn;
973
974 Token T;
975 T.Kind = Kind;
976 T.Range = StringRef(Current, 0);
977 TokenQueue.insert(InsertPoint, T);
978 }
979 return true;
980 }
981
scanToNextToken()982 void Scanner::scanToNextToken() {
983 while (true) {
984 while (*Current == ' ' || *Current == '\t') {
985 skip(1);
986 }
987
988 // Skip comment.
989 if (*Current == '#') {
990 while (true) {
991 // This may skip more than one byte, thus Column is only incremented
992 // for code points.
993 StringRef::iterator i = skip_nb_char(Current);
994 if (i == Current)
995 break;
996 Current = i;
997 ++Column;
998 }
999 }
1000
1001 // Skip EOL.
1002 StringRef::iterator i = skip_b_break(Current);
1003 if (i == Current)
1004 break;
1005 Current = i;
1006 ++Line;
1007 Column = 0;
1008 // New lines may start a simple key.
1009 if (!FlowLevel)
1010 IsSimpleKeyAllowed = true;
1011 }
1012 }
1013
scanStreamStart()1014 bool Scanner::scanStreamStart() {
1015 IsStartOfStream = false;
1016
1017 EncodingInfo EI = getUnicodeEncoding(currentInput());
1018
1019 Token T;
1020 T.Kind = Token::TK_StreamStart;
1021 T.Range = StringRef(Current, EI.second);
1022 TokenQueue.push_back(T);
1023 Current += EI.second;
1024 return true;
1025 }
1026
scanStreamEnd()1027 bool Scanner::scanStreamEnd() {
1028 // Force an ending new line if one isn't present.
1029 if (Column != 0) {
1030 Column = 0;
1031 ++Line;
1032 }
1033
1034 unrollIndent(-1);
1035 SimpleKeys.clear();
1036 IsSimpleKeyAllowed = false;
1037
1038 Token T;
1039 T.Kind = Token::TK_StreamEnd;
1040 T.Range = StringRef(Current, 0);
1041 TokenQueue.push_back(T);
1042 return true;
1043 }
1044
scanDirective()1045 bool Scanner::scanDirective() {
1046 // Reset the indentation level.
1047 unrollIndent(-1);
1048 SimpleKeys.clear();
1049 IsSimpleKeyAllowed = false;
1050
1051 StringRef::iterator Start = Current;
1052 consume('%');
1053 StringRef::iterator NameStart = Current;
1054 Current = skip_while(&Scanner::skip_ns_char, Current);
1055 StringRef Name(NameStart, Current - NameStart);
1056 Current = skip_while(&Scanner::skip_s_white, Current);
1057
1058 if (Name == "YAML") {
1059 Current = skip_while(&Scanner::skip_ns_char, Current);
1060 Token T;
1061 T.Kind = Token::TK_VersionDirective;
1062 T.Range = StringRef(Start, Current - Start);
1063 TokenQueue.push_back(T);
1064 return true;
1065 }
1066 return false;
1067 }
1068
scanDocumentIndicator(bool IsStart)1069 bool Scanner::scanDocumentIndicator(bool IsStart) {
1070 unrollIndent(-1);
1071 SimpleKeys.clear();
1072 IsSimpleKeyAllowed = false;
1073
1074 Token T;
1075 T.Kind = IsStart ? Token::TK_DocumentStart : Token::TK_DocumentEnd;
1076 T.Range = StringRef(Current, 3);
1077 skip(3);
1078 TokenQueue.push_back(T);
1079 return true;
1080 }
1081
scanFlowCollectionStart(bool IsSequence)1082 bool Scanner::scanFlowCollectionStart(bool IsSequence) {
1083 Token T;
1084 T.Kind = IsSequence ? Token::TK_FlowSequenceStart
1085 : Token::TK_FlowMappingStart;
1086 T.Range = StringRef(Current, 1);
1087 skip(1);
1088 TokenQueue.push_back(T);
1089
1090 // [ and { may begin a simple key.
1091 saveSimpleKeyCandidate(TokenQueue.back(), Column - 1, false);
1092
1093 // And may also be followed by a simple key.
1094 IsSimpleKeyAllowed = true;
1095 ++FlowLevel;
1096 return true;
1097 }
1098
scanFlowCollectionEnd(bool IsSequence)1099 bool Scanner::scanFlowCollectionEnd(bool IsSequence) {
1100 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1101 IsSimpleKeyAllowed = false;
1102 Token T;
1103 T.Kind = IsSequence ? Token::TK_FlowSequenceEnd
1104 : Token::TK_FlowMappingEnd;
1105 T.Range = StringRef(Current, 1);
1106 skip(1);
1107 TokenQueue.push_back(T);
1108 if (FlowLevel)
1109 --FlowLevel;
1110 return true;
1111 }
1112
scanFlowEntry()1113 bool Scanner::scanFlowEntry() {
1114 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1115 IsSimpleKeyAllowed = true;
1116 Token T;
1117 T.Kind = Token::TK_FlowEntry;
1118 T.Range = StringRef(Current, 1);
1119 skip(1);
1120 TokenQueue.push_back(T);
1121 return true;
1122 }
1123
scanBlockEntry()1124 bool Scanner::scanBlockEntry() {
1125 rollIndent(Column, Token::TK_BlockSequenceStart, TokenQueue.end());
1126 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1127 IsSimpleKeyAllowed = true;
1128 Token T;
1129 T.Kind = Token::TK_BlockEntry;
1130 T.Range = StringRef(Current, 1);
1131 skip(1);
1132 TokenQueue.push_back(T);
1133 return true;
1134 }
1135
scanKey()1136 bool Scanner::scanKey() {
1137 if (!FlowLevel)
1138 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end());
1139
1140 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
1141 IsSimpleKeyAllowed = !FlowLevel;
1142
1143 Token T;
1144 T.Kind = Token::TK_Key;
1145 T.Range = StringRef(Current, 1);
1146 skip(1);
1147 TokenQueue.push_back(T);
1148 return true;
1149 }
1150
scanValue()1151 bool Scanner::scanValue() {
1152 // If the previous token could have been a simple key, insert the key token
1153 // into the token queue.
1154 if (!SimpleKeys.empty()) {
1155 SimpleKey SK = SimpleKeys.pop_back_val();
1156 Token T;
1157 T.Kind = Token::TK_Key;
1158 T.Range = SK.Tok->Range;
1159 TokenQueueT::iterator i, e;
1160 for (i = TokenQueue.begin(), e = TokenQueue.end(); i != e; ++i) {
1161 if (i == SK.Tok)
1162 break;
1163 }
1164 assert(i != e && "SimpleKey not in token queue!");
1165 i = TokenQueue.insert(i, T);
1166
1167 // We may also need to add a Block-Mapping-Start token.
1168 rollIndent(SK.Column, Token::TK_BlockMappingStart, i);
1169
1170 IsSimpleKeyAllowed = false;
1171 } else {
1172 if (!FlowLevel)
1173 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end());
1174 IsSimpleKeyAllowed = !FlowLevel;
1175 }
1176
1177 Token T;
1178 T.Kind = Token::TK_Value;
1179 T.Range = StringRef(Current, 1);
1180 skip(1);
1181 TokenQueue.push_back(T);
1182 return true;
1183 }
1184
1185 // Forbidding inlining improves performance by roughly 20%.
1186 // FIXME: Remove once llvm optimizes this to the faster version without hints.
1187 LLVM_ATTRIBUTE_NOINLINE static bool
1188 wasEscaped(StringRef::iterator First, StringRef::iterator Position);
1189
1190 // Returns whether a character at 'Position' was escaped with a leading '\'.
1191 // 'First' specifies the position of the first character in the string.
wasEscaped(StringRef::iterator First,StringRef::iterator Position)1192 static bool wasEscaped(StringRef::iterator First,
1193 StringRef::iterator Position) {
1194 assert(Position - 1 >= First);
1195 StringRef::iterator I = Position - 1;
1196 // We calculate the number of consecutive '\'s before the current position
1197 // by iterating backwards through our string.
1198 while (I >= First && *I == '\\') --I;
1199 // (Position - 1 - I) now contains the number of '\'s before the current
1200 // position. If it is odd, the character at 'Position' was escaped.
1201 return (Position - 1 - I) % 2 == 1;
1202 }
1203
scanFlowScalar(bool IsDoubleQuoted)1204 bool Scanner::scanFlowScalar(bool IsDoubleQuoted) {
1205 StringRef::iterator Start = Current;
1206 unsigned ColStart = Column;
1207 if (IsDoubleQuoted) {
1208 do {
1209 ++Current;
1210 while (Current != End && *Current != '"')
1211 ++Current;
1212 // Repeat until the previous character was not a '\' or was an escaped
1213 // backslash.
1214 } while ( Current != End
1215 && *(Current - 1) == '\\'
1216 && wasEscaped(Start + 1, Current));
1217 } else {
1218 skip(1);
1219 while (true) {
1220 // Skip a ' followed by another '.
1221 if (Current + 1 < End && *Current == '\'' && *(Current + 1) == '\'') {
1222 skip(2);
1223 continue;
1224 } else if (*Current == '\'')
1225 break;
1226 StringRef::iterator i = skip_nb_char(Current);
1227 if (i == Current) {
1228 i = skip_b_break(Current);
1229 if (i == Current)
1230 break;
1231 Current = i;
1232 Column = 0;
1233 ++Line;
1234 } else {
1235 if (i == End)
1236 break;
1237 Current = i;
1238 ++Column;
1239 }
1240 }
1241 }
1242 skip(1); // Skip ending quote.
1243 Token T;
1244 T.Kind = Token::TK_Scalar;
1245 T.Range = StringRef(Start, Current - Start);
1246 TokenQueue.push_back(T);
1247
1248 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1249
1250 IsSimpleKeyAllowed = false;
1251
1252 return true;
1253 }
1254
scanPlainScalar()1255 bool Scanner::scanPlainScalar() {
1256 StringRef::iterator Start = Current;
1257 unsigned ColStart = Column;
1258 unsigned LeadingBlanks = 0;
1259 assert(Indent >= -1 && "Indent must be >= -1 !");
1260 unsigned indent = static_cast<unsigned>(Indent + 1);
1261 while (true) {
1262 if (*Current == '#')
1263 break;
1264
1265 while (!isBlankOrBreak(Current)) {
1266 if ( FlowLevel && *Current == ':'
1267 && !(isBlankOrBreak(Current + 1) || *(Current + 1) == ',')) {
1268 setError("Found unexpected ':' while scanning a plain scalar", Current);
1269 return false;
1270 }
1271
1272 // Check for the end of the plain scalar.
1273 if ( (*Current == ':' && isBlankOrBreak(Current + 1))
1274 || ( FlowLevel
1275 && (StringRef(Current, 1).find_first_of(",:?[]{}")
1276 != StringRef::npos)))
1277 break;
1278
1279 StringRef::iterator i = skip_nb_char(Current);
1280 if (i == Current)
1281 break;
1282 Current = i;
1283 ++Column;
1284 }
1285
1286 // Are we at the end?
1287 if (!isBlankOrBreak(Current))
1288 break;
1289
1290 // Eat blanks.
1291 StringRef::iterator Tmp = Current;
1292 while (isBlankOrBreak(Tmp)) {
1293 StringRef::iterator i = skip_s_white(Tmp);
1294 if (i != Tmp) {
1295 if (LeadingBlanks && (Column < indent) && *Tmp == '\t') {
1296 setError("Found invalid tab character in indentation", Tmp);
1297 return false;
1298 }
1299 Tmp = i;
1300 ++Column;
1301 } else {
1302 i = skip_b_break(Tmp);
1303 if (!LeadingBlanks)
1304 LeadingBlanks = 1;
1305 Tmp = i;
1306 Column = 0;
1307 ++Line;
1308 }
1309 }
1310
1311 if (!FlowLevel && Column < indent)
1312 break;
1313
1314 Current = Tmp;
1315 }
1316 if (Start == Current) {
1317 setError("Got empty plain scalar", Start);
1318 return false;
1319 }
1320 Token T;
1321 T.Kind = Token::TK_Scalar;
1322 T.Range = StringRef(Start, Current - Start);
1323 TokenQueue.push_back(T);
1324
1325 // Plain scalars can be simple keys.
1326 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1327
1328 IsSimpleKeyAllowed = false;
1329
1330 return true;
1331 }
1332
scanAliasOrAnchor(bool IsAlias)1333 bool Scanner::scanAliasOrAnchor(bool IsAlias) {
1334 StringRef::iterator Start = Current;
1335 unsigned ColStart = Column;
1336 skip(1);
1337 while(true) {
1338 if ( *Current == '[' || *Current == ']'
1339 || *Current == '{' || *Current == '}'
1340 || *Current == ','
1341 || *Current == ':')
1342 break;
1343 StringRef::iterator i = skip_ns_char(Current);
1344 if (i == Current)
1345 break;
1346 Current = i;
1347 ++Column;
1348 }
1349
1350 if (Start == Current) {
1351 setError("Got empty alias or anchor", Start);
1352 return false;
1353 }
1354
1355 Token T;
1356 T.Kind = IsAlias ? Token::TK_Alias : Token::TK_Anchor;
1357 T.Range = StringRef(Start, Current - Start);
1358 TokenQueue.push_back(T);
1359
1360 // Alias and anchors can be simple keys.
1361 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1362
1363 IsSimpleKeyAllowed = false;
1364
1365 return true;
1366 }
1367
scanBlockScalar(bool IsLiteral)1368 bool Scanner::scanBlockScalar(bool IsLiteral) {
1369 StringRef::iterator Start = Current;
1370 skip(1); // Eat | or >
1371 while(true) {
1372 StringRef::iterator i = skip_nb_char(Current);
1373 if (i == Current) {
1374 if (Column == 0)
1375 break;
1376 i = skip_b_break(Current);
1377 if (i != Current) {
1378 // We got a line break.
1379 Column = 0;
1380 ++Line;
1381 Current = i;
1382 continue;
1383 } else {
1384 // There was an error, which should already have been printed out.
1385 return false;
1386 }
1387 }
1388 Current = i;
1389 ++Column;
1390 }
1391
1392 if (Start == Current) {
1393 setError("Got empty block scalar", Start);
1394 return false;
1395 }
1396
1397 Token T;
1398 T.Kind = Token::TK_Scalar;
1399 T.Range = StringRef(Start, Current - Start);
1400 TokenQueue.push_back(T);
1401 return true;
1402 }
1403
scanTag()1404 bool Scanner::scanTag() {
1405 StringRef::iterator Start = Current;
1406 unsigned ColStart = Column;
1407 skip(1); // Eat !.
1408 if (Current == End || isBlankOrBreak(Current)); // An empty tag.
1409 else if (*Current == '<') {
1410 skip(1);
1411 scan_ns_uri_char();
1412 if (!consume('>'))
1413 return false;
1414 } else {
1415 // FIXME: Actually parse the c-ns-shorthand-tag rule.
1416 Current = skip_while(&Scanner::skip_ns_char, Current);
1417 }
1418
1419 Token T;
1420 T.Kind = Token::TK_Tag;
1421 T.Range = StringRef(Start, Current - Start);
1422 TokenQueue.push_back(T);
1423
1424 // Tags can be simple keys.
1425 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false);
1426
1427 IsSimpleKeyAllowed = false;
1428
1429 return true;
1430 }
1431
fetchMoreTokens()1432 bool Scanner::fetchMoreTokens() {
1433 if (IsStartOfStream)
1434 return scanStreamStart();
1435
1436 scanToNextToken();
1437
1438 if (Current == End)
1439 return scanStreamEnd();
1440
1441 removeStaleSimpleKeyCandidates();
1442
1443 unrollIndent(Column);
1444
1445 if (Column == 0 && *Current == '%')
1446 return scanDirective();
1447
1448 if (Column == 0 && Current + 4 <= End
1449 && *Current == '-'
1450 && *(Current + 1) == '-'
1451 && *(Current + 2) == '-'
1452 && (Current + 3 == End || isBlankOrBreak(Current + 3)))
1453 return scanDocumentIndicator(true);
1454
1455 if (Column == 0 && Current + 4 <= End
1456 && *Current == '.'
1457 && *(Current + 1) == '.'
1458 && *(Current + 2) == '.'
1459 && (Current + 3 == End || isBlankOrBreak(Current + 3)))
1460 return scanDocumentIndicator(false);
1461
1462 if (*Current == '[')
1463 return scanFlowCollectionStart(true);
1464
1465 if (*Current == '{')
1466 return scanFlowCollectionStart(false);
1467
1468 if (*Current == ']')
1469 return scanFlowCollectionEnd(true);
1470
1471 if (*Current == '}')
1472 return scanFlowCollectionEnd(false);
1473
1474 if (*Current == ',')
1475 return scanFlowEntry();
1476
1477 if (*Current == '-' && isBlankOrBreak(Current + 1))
1478 return scanBlockEntry();
1479
1480 if (*Current == '?' && (FlowLevel || isBlankOrBreak(Current + 1)))
1481 return scanKey();
1482
1483 if (*Current == ':' && (FlowLevel || isBlankOrBreak(Current + 1)))
1484 return scanValue();
1485
1486 if (*Current == '*')
1487 return scanAliasOrAnchor(true);
1488
1489 if (*Current == '&')
1490 return scanAliasOrAnchor(false);
1491
1492 if (*Current == '!')
1493 return scanTag();
1494
1495 if (*Current == '|' && !FlowLevel)
1496 return scanBlockScalar(true);
1497
1498 if (*Current == '>' && !FlowLevel)
1499 return scanBlockScalar(false);
1500
1501 if (*Current == '\'')
1502 return scanFlowScalar(false);
1503
1504 if (*Current == '"')
1505 return scanFlowScalar(true);
1506
1507 // Get a plain scalar.
1508 StringRef FirstChar(Current, 1);
1509 if (!(isBlankOrBreak(Current)
1510 || FirstChar.find_first_of("-?:,[]{}#&*!|>'\"%@`") != StringRef::npos)
1511 || (*Current == '-' && !isBlankOrBreak(Current + 1))
1512 || (!FlowLevel && (*Current == '?' || *Current == ':')
1513 && isBlankOrBreak(Current + 1))
1514 || (!FlowLevel && *Current == ':'
1515 && Current + 2 < End
1516 && *(Current + 1) == ':'
1517 && !isBlankOrBreak(Current + 2)))
1518 return scanPlainScalar();
1519
1520 setError("Unrecognized character while tokenizing.");
1521 return false;
1522 }
1523
Stream(StringRef Input,SourceMgr & SM)1524 Stream::Stream(StringRef Input, SourceMgr &SM)
1525 : scanner(new Scanner(Input, SM))
1526 , CurrentDoc(0) {}
1527
~Stream()1528 Stream::~Stream() {}
1529
failed()1530 bool Stream::failed() { return scanner->failed(); }
1531
printError(Node * N,const Twine & Msg)1532 void Stream::printError(Node *N, const Twine &Msg) {
1533 SmallVector<SMRange, 1> Ranges;
1534 Ranges.push_back(N->getSourceRange());
1535 scanner->printError( N->getSourceRange().Start
1536 , SourceMgr::DK_Error
1537 , Msg
1538 , Ranges);
1539 }
1540
handleYAMLDirective(const Token & t)1541 void Stream::handleYAMLDirective(const Token &t) {
1542 // TODO: Ensure version is 1.x.
1543 }
1544
begin()1545 document_iterator Stream::begin() {
1546 if (CurrentDoc)
1547 report_fatal_error("Can only iterate over the stream once");
1548
1549 // Skip Stream-Start.
1550 scanner->getNext();
1551
1552 CurrentDoc.reset(new Document(*this));
1553 return document_iterator(CurrentDoc);
1554 }
1555
end()1556 document_iterator Stream::end() {
1557 return document_iterator();
1558 }
1559
skip()1560 void Stream::skip() {
1561 for (document_iterator i = begin(), e = end(); i != e; ++i)
1562 i->skip();
1563 }
1564
Node(unsigned int Type,OwningPtr<Document> & D,StringRef A)1565 Node::Node(unsigned int Type, OwningPtr<Document> &D, StringRef A)
1566 : Doc(D)
1567 , TypeID(Type)
1568 , Anchor(A) {
1569 SMLoc Start = SMLoc::getFromPointer(peekNext().Range.begin());
1570 SourceRange = SMRange(Start, Start);
1571 }
1572
peekNext()1573 Token &Node::peekNext() {
1574 return Doc->peekNext();
1575 }
1576
getNext()1577 Token Node::getNext() {
1578 return Doc->getNext();
1579 }
1580
parseBlockNode()1581 Node *Node::parseBlockNode() {
1582 return Doc->parseBlockNode();
1583 }
1584
getAllocator()1585 BumpPtrAllocator &Node::getAllocator() {
1586 return Doc->NodeAllocator;
1587 }
1588
setError(const Twine & Msg,Token & Tok) const1589 void Node::setError(const Twine &Msg, Token &Tok) const {
1590 Doc->setError(Msg, Tok);
1591 }
1592
failed() const1593 bool Node::failed() const {
1594 return Doc->failed();
1595 }
1596
1597
1598
getValue(SmallVectorImpl<char> & Storage) const1599 StringRef ScalarNode::getValue(SmallVectorImpl<char> &Storage) const {
1600 // TODO: Handle newlines properly. We need to remove leading whitespace.
1601 if (Value[0] == '"') { // Double quoted.
1602 // Pull off the leading and trailing "s.
1603 StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
1604 // Search for characters that would require unescaping the value.
1605 StringRef::size_type i = UnquotedValue.find_first_of("\\\r\n");
1606 if (i != StringRef::npos)
1607 return unescapeDoubleQuoted(UnquotedValue, i, Storage);
1608 return UnquotedValue;
1609 } else if (Value[0] == '\'') { // Single quoted.
1610 // Pull off the leading and trailing 's.
1611 StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
1612 StringRef::size_type i = UnquotedValue.find('\'');
1613 if (i != StringRef::npos) {
1614 // We're going to need Storage.
1615 Storage.clear();
1616 Storage.reserve(UnquotedValue.size());
1617 for (; i != StringRef::npos; i = UnquotedValue.find('\'')) {
1618 StringRef Valid(UnquotedValue.begin(), i);
1619 Storage.insert(Storage.end(), Valid.begin(), Valid.end());
1620 Storage.push_back('\'');
1621 UnquotedValue = UnquotedValue.substr(i + 2);
1622 }
1623 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end());
1624 return StringRef(Storage.begin(), Storage.size());
1625 }
1626 return UnquotedValue;
1627 }
1628 // Plain or block.
1629 return Value.rtrim(" ");
1630 }
1631
unescapeDoubleQuoted(StringRef UnquotedValue,StringRef::size_type i,SmallVectorImpl<char> & Storage) const1632 StringRef ScalarNode::unescapeDoubleQuoted( StringRef UnquotedValue
1633 , StringRef::size_type i
1634 , SmallVectorImpl<char> &Storage)
1635 const {
1636 // Use Storage to build proper value.
1637 Storage.clear();
1638 Storage.reserve(UnquotedValue.size());
1639 for (; i != StringRef::npos; i = UnquotedValue.find_first_of("\\\r\n")) {
1640 // Insert all previous chars into Storage.
1641 StringRef Valid(UnquotedValue.begin(), i);
1642 Storage.insert(Storage.end(), Valid.begin(), Valid.end());
1643 // Chop off inserted chars.
1644 UnquotedValue = UnquotedValue.substr(i);
1645
1646 assert(!UnquotedValue.empty() && "Can't be empty!");
1647
1648 // Parse escape or line break.
1649 switch (UnquotedValue[0]) {
1650 case '\r':
1651 case '\n':
1652 Storage.push_back('\n');
1653 if ( UnquotedValue.size() > 1
1654 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
1655 UnquotedValue = UnquotedValue.substr(1);
1656 UnquotedValue = UnquotedValue.substr(1);
1657 break;
1658 default:
1659 if (UnquotedValue.size() == 1)
1660 // TODO: Report error.
1661 break;
1662 UnquotedValue = UnquotedValue.substr(1);
1663 switch (UnquotedValue[0]) {
1664 default: {
1665 Token T;
1666 T.Range = StringRef(UnquotedValue.begin(), 1);
1667 setError("Unrecognized escape code!", T);
1668 return "";
1669 }
1670 case '\r':
1671 case '\n':
1672 // Remove the new line.
1673 if ( UnquotedValue.size() > 1
1674 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
1675 UnquotedValue = UnquotedValue.substr(1);
1676 // If this was just a single byte newline, it will get skipped
1677 // below.
1678 break;
1679 case '0':
1680 Storage.push_back(0x00);
1681 break;
1682 case 'a':
1683 Storage.push_back(0x07);
1684 break;
1685 case 'b':
1686 Storage.push_back(0x08);
1687 break;
1688 case 't':
1689 case 0x09:
1690 Storage.push_back(0x09);
1691 break;
1692 case 'n':
1693 Storage.push_back(0x0A);
1694 break;
1695 case 'v':
1696 Storage.push_back(0x0B);
1697 break;
1698 case 'f':
1699 Storage.push_back(0x0C);
1700 break;
1701 case 'r':
1702 Storage.push_back(0x0D);
1703 break;
1704 case 'e':
1705 Storage.push_back(0x1B);
1706 break;
1707 case ' ':
1708 Storage.push_back(0x20);
1709 break;
1710 case '"':
1711 Storage.push_back(0x22);
1712 break;
1713 case '/':
1714 Storage.push_back(0x2F);
1715 break;
1716 case '\\':
1717 Storage.push_back(0x5C);
1718 break;
1719 case 'N':
1720 encodeUTF8(0x85, Storage);
1721 break;
1722 case '_':
1723 encodeUTF8(0xA0, Storage);
1724 break;
1725 case 'L':
1726 encodeUTF8(0x2028, Storage);
1727 break;
1728 case 'P':
1729 encodeUTF8(0x2029, Storage);
1730 break;
1731 case 'x': {
1732 if (UnquotedValue.size() < 3)
1733 // TODO: Report error.
1734 break;
1735 unsigned int UnicodeScalarValue;
1736 if (UnquotedValue.substr(1, 2).getAsInteger(16, UnicodeScalarValue))
1737 // TODO: Report error.
1738 UnicodeScalarValue = 0xFFFD;
1739 encodeUTF8(UnicodeScalarValue, Storage);
1740 UnquotedValue = UnquotedValue.substr(2);
1741 break;
1742 }
1743 case 'u': {
1744 if (UnquotedValue.size() < 5)
1745 // TODO: Report error.
1746 break;
1747 unsigned int UnicodeScalarValue;
1748 if (UnquotedValue.substr(1, 4).getAsInteger(16, UnicodeScalarValue))
1749 // TODO: Report error.
1750 UnicodeScalarValue = 0xFFFD;
1751 encodeUTF8(UnicodeScalarValue, Storage);
1752 UnquotedValue = UnquotedValue.substr(4);
1753 break;
1754 }
1755 case 'U': {
1756 if (UnquotedValue.size() < 9)
1757 // TODO: Report error.
1758 break;
1759 unsigned int UnicodeScalarValue;
1760 if (UnquotedValue.substr(1, 8).getAsInteger(16, UnicodeScalarValue))
1761 // TODO: Report error.
1762 UnicodeScalarValue = 0xFFFD;
1763 encodeUTF8(UnicodeScalarValue, Storage);
1764 UnquotedValue = UnquotedValue.substr(8);
1765 break;
1766 }
1767 }
1768 UnquotedValue = UnquotedValue.substr(1);
1769 }
1770 }
1771 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end());
1772 return StringRef(Storage.begin(), Storage.size());
1773 }
1774
getKey()1775 Node *KeyValueNode::getKey() {
1776 if (Key)
1777 return Key;
1778 // Handle implicit null keys.
1779 {
1780 Token &t = peekNext();
1781 if ( t.Kind == Token::TK_BlockEnd
1782 || t.Kind == Token::TK_Value
1783 || t.Kind == Token::TK_Error) {
1784 return Key = new (getAllocator()) NullNode(Doc);
1785 }
1786 if (t.Kind == Token::TK_Key)
1787 getNext(); // skip TK_Key.
1788 }
1789
1790 // Handle explicit null keys.
1791 Token &t = peekNext();
1792 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Value) {
1793 return Key = new (getAllocator()) NullNode(Doc);
1794 }
1795
1796 // We've got a normal key.
1797 return Key = parseBlockNode();
1798 }
1799
getValue()1800 Node *KeyValueNode::getValue() {
1801 if (Value)
1802 return Value;
1803 getKey()->skip();
1804 if (failed())
1805 return Value = new (getAllocator()) NullNode(Doc);
1806
1807 // Handle implicit null values.
1808 {
1809 Token &t = peekNext();
1810 if ( t.Kind == Token::TK_BlockEnd
1811 || t.Kind == Token::TK_FlowMappingEnd
1812 || t.Kind == Token::TK_Key
1813 || t.Kind == Token::TK_FlowEntry
1814 || t.Kind == Token::TK_Error) {
1815 return Value = new (getAllocator()) NullNode(Doc);
1816 }
1817
1818 if (t.Kind != Token::TK_Value) {
1819 setError("Unexpected token in Key Value.", t);
1820 return Value = new (getAllocator()) NullNode(Doc);
1821 }
1822 getNext(); // skip TK_Value.
1823 }
1824
1825 // Handle explicit null values.
1826 Token &t = peekNext();
1827 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Key) {
1828 return Value = new (getAllocator()) NullNode(Doc);
1829 }
1830
1831 // We got a normal value.
1832 return Value = parseBlockNode();
1833 }
1834
increment()1835 void MappingNode::increment() {
1836 if (failed()) {
1837 IsAtEnd = true;
1838 CurrentEntry = 0;
1839 return;
1840 }
1841 if (CurrentEntry) {
1842 CurrentEntry->skip();
1843 if (Type == MT_Inline) {
1844 IsAtEnd = true;
1845 CurrentEntry = 0;
1846 return;
1847 }
1848 }
1849 Token T = peekNext();
1850 if (T.Kind == Token::TK_Key || T.Kind == Token::TK_Scalar) {
1851 // KeyValueNode eats the TK_Key. That way it can detect null keys.
1852 CurrentEntry = new (getAllocator()) KeyValueNode(Doc);
1853 } else if (Type == MT_Block) {
1854 switch (T.Kind) {
1855 case Token::TK_BlockEnd:
1856 getNext();
1857 IsAtEnd = true;
1858 CurrentEntry = 0;
1859 break;
1860 default:
1861 setError("Unexpected token. Expected Key or Block End", T);
1862 case Token::TK_Error:
1863 IsAtEnd = true;
1864 CurrentEntry = 0;
1865 }
1866 } else {
1867 switch (T.Kind) {
1868 case Token::TK_FlowEntry:
1869 // Eat the flow entry and recurse.
1870 getNext();
1871 return increment();
1872 case Token::TK_FlowMappingEnd:
1873 getNext();
1874 case Token::TK_Error:
1875 // Set this to end iterator.
1876 IsAtEnd = true;
1877 CurrentEntry = 0;
1878 break;
1879 default:
1880 setError( "Unexpected token. Expected Key, Flow Entry, or Flow "
1881 "Mapping End."
1882 , T);
1883 IsAtEnd = true;
1884 CurrentEntry = 0;
1885 }
1886 }
1887 }
1888
increment()1889 void SequenceNode::increment() {
1890 if (failed()) {
1891 IsAtEnd = true;
1892 CurrentEntry = 0;
1893 return;
1894 }
1895 if (CurrentEntry)
1896 CurrentEntry->skip();
1897 Token T = peekNext();
1898 if (SeqType == ST_Block) {
1899 switch (T.Kind) {
1900 case Token::TK_BlockEntry:
1901 getNext();
1902 CurrentEntry = parseBlockNode();
1903 if (CurrentEntry == 0) { // An error occurred.
1904 IsAtEnd = true;
1905 CurrentEntry = 0;
1906 }
1907 break;
1908 case Token::TK_BlockEnd:
1909 getNext();
1910 IsAtEnd = true;
1911 CurrentEntry = 0;
1912 break;
1913 default:
1914 setError( "Unexpected token. Expected Block Entry or Block End."
1915 , T);
1916 case Token::TK_Error:
1917 IsAtEnd = true;
1918 CurrentEntry = 0;
1919 }
1920 } else if (SeqType == ST_Indentless) {
1921 switch (T.Kind) {
1922 case Token::TK_BlockEntry:
1923 getNext();
1924 CurrentEntry = parseBlockNode();
1925 if (CurrentEntry == 0) { // An error occurred.
1926 IsAtEnd = true;
1927 CurrentEntry = 0;
1928 }
1929 break;
1930 default:
1931 case Token::TK_Error:
1932 IsAtEnd = true;
1933 CurrentEntry = 0;
1934 }
1935 } else if (SeqType == ST_Flow) {
1936 switch (T.Kind) {
1937 case Token::TK_FlowEntry:
1938 // Eat the flow entry and recurse.
1939 getNext();
1940 WasPreviousTokenFlowEntry = true;
1941 return increment();
1942 case Token::TK_FlowSequenceEnd:
1943 getNext();
1944 case Token::TK_Error:
1945 // Set this to end iterator.
1946 IsAtEnd = true;
1947 CurrentEntry = 0;
1948 break;
1949 case Token::TK_StreamEnd:
1950 case Token::TK_DocumentEnd:
1951 case Token::TK_DocumentStart:
1952 setError("Could not find closing ]!", T);
1953 // Set this to end iterator.
1954 IsAtEnd = true;
1955 CurrentEntry = 0;
1956 break;
1957 default:
1958 if (!WasPreviousTokenFlowEntry) {
1959 setError("Expected , between entries!", T);
1960 IsAtEnd = true;
1961 CurrentEntry = 0;
1962 break;
1963 }
1964 // Otherwise it must be a flow entry.
1965 CurrentEntry = parseBlockNode();
1966 if (!CurrentEntry) {
1967 IsAtEnd = true;
1968 }
1969 WasPreviousTokenFlowEntry = false;
1970 break;
1971 }
1972 }
1973 }
1974
Document(Stream & S)1975 Document::Document(Stream &S) : stream(S), Root(0) {
1976 if (parseDirectives())
1977 expectToken(Token::TK_DocumentStart);
1978 Token &T = peekNext();
1979 if (T.Kind == Token::TK_DocumentStart)
1980 getNext();
1981 }
1982
skip()1983 bool Document::skip() {
1984 if (stream.scanner->failed())
1985 return false;
1986 if (!Root)
1987 getRoot();
1988 Root->skip();
1989 Token &T = peekNext();
1990 if (T.Kind == Token::TK_StreamEnd)
1991 return false;
1992 if (T.Kind == Token::TK_DocumentEnd) {
1993 getNext();
1994 return skip();
1995 }
1996 return true;
1997 }
1998
peekNext()1999 Token &Document::peekNext() {
2000 return stream.scanner->peekNext();
2001 }
2002
getNext()2003 Token Document::getNext() {
2004 return stream.scanner->getNext();
2005 }
2006
setError(const Twine & Message,Token & Location) const2007 void Document::setError(const Twine &Message, Token &Location) const {
2008 stream.scanner->setError(Message, Location.Range.begin());
2009 }
2010
failed() const2011 bool Document::failed() const {
2012 return stream.scanner->failed();
2013 }
2014
parseBlockNode()2015 Node *Document::parseBlockNode() {
2016 Token T = peekNext();
2017 // Handle properties.
2018 Token AnchorInfo;
2019 parse_property:
2020 switch (T.Kind) {
2021 case Token::TK_Alias:
2022 getNext();
2023 return new (NodeAllocator) AliasNode(stream.CurrentDoc, T.Range.substr(1));
2024 case Token::TK_Anchor:
2025 if (AnchorInfo.Kind == Token::TK_Anchor) {
2026 setError("Already encountered an anchor for this node!", T);
2027 return 0;
2028 }
2029 AnchorInfo = getNext(); // Consume TK_Anchor.
2030 T = peekNext();
2031 goto parse_property;
2032 case Token::TK_Tag:
2033 getNext(); // Skip TK_Tag.
2034 T = peekNext();
2035 goto parse_property;
2036 default:
2037 break;
2038 }
2039
2040 switch (T.Kind) {
2041 case Token::TK_BlockEntry:
2042 // We got an unindented BlockEntry sequence. This is not terminated with
2043 // a BlockEnd.
2044 // Don't eat the TK_BlockEntry, SequenceNode needs it.
2045 return new (NodeAllocator) SequenceNode( stream.CurrentDoc
2046 , AnchorInfo.Range.substr(1)
2047 , SequenceNode::ST_Indentless);
2048 case Token::TK_BlockSequenceStart:
2049 getNext();
2050 return new (NodeAllocator)
2051 SequenceNode( stream.CurrentDoc
2052 , AnchorInfo.Range.substr(1)
2053 , SequenceNode::ST_Block);
2054 case Token::TK_BlockMappingStart:
2055 getNext();
2056 return new (NodeAllocator)
2057 MappingNode( stream.CurrentDoc
2058 , AnchorInfo.Range.substr(1)
2059 , MappingNode::MT_Block);
2060 case Token::TK_FlowSequenceStart:
2061 getNext();
2062 return new (NodeAllocator)
2063 SequenceNode( stream.CurrentDoc
2064 , AnchorInfo.Range.substr(1)
2065 , SequenceNode::ST_Flow);
2066 case Token::TK_FlowMappingStart:
2067 getNext();
2068 return new (NodeAllocator)
2069 MappingNode( stream.CurrentDoc
2070 , AnchorInfo.Range.substr(1)
2071 , MappingNode::MT_Flow);
2072 case Token::TK_Scalar:
2073 getNext();
2074 return new (NodeAllocator)
2075 ScalarNode( stream.CurrentDoc
2076 , AnchorInfo.Range.substr(1)
2077 , T.Range);
2078 case Token::TK_Key:
2079 // Don't eat the TK_Key, KeyValueNode expects it.
2080 return new (NodeAllocator)
2081 MappingNode( stream.CurrentDoc
2082 , AnchorInfo.Range.substr(1)
2083 , MappingNode::MT_Inline);
2084 case Token::TK_DocumentStart:
2085 case Token::TK_DocumentEnd:
2086 case Token::TK_StreamEnd:
2087 default:
2088 // TODO: Properly handle tags. "[!!str ]" should resolve to !!str "", not
2089 // !!null null.
2090 return new (NodeAllocator) NullNode(stream.CurrentDoc);
2091 case Token::TK_Error:
2092 return 0;
2093 }
2094 llvm_unreachable("Control flow shouldn't reach here.");
2095 return 0;
2096 }
2097
parseDirectives()2098 bool Document::parseDirectives() {
2099 bool isDirective = false;
2100 while (true) {
2101 Token T = peekNext();
2102 if (T.Kind == Token::TK_TagDirective) {
2103 handleTagDirective(getNext());
2104 isDirective = true;
2105 } else if (T.Kind == Token::TK_VersionDirective) {
2106 stream.handleYAMLDirective(getNext());
2107 isDirective = true;
2108 } else
2109 break;
2110 }
2111 return isDirective;
2112 }
2113
expectToken(int TK)2114 bool Document::expectToken(int TK) {
2115 Token T = getNext();
2116 if (T.Kind != TK) {
2117 setError("Unexpected token", T);
2118 return false;
2119 }
2120 return true;
2121 }
2122