Lines Matching full:token
15 /// `Token` represents a single lexeme of an XML document. These lexemes
18 pub enum Token { enum
59 impl fmt::Display for Token { implementation
62 Token::Chunk(s) => write!(f, "{}", s), in fmt()
63 Token::Character(c) | Token::Whitespace(c) => write!(f, "{}", c), in fmt()
65 Token::OpeningTagStart => "<", in fmt()
66 Token::ProcessingInstructionStart => "<?", in fmt()
67 Token::DoctypeStart => "<!DOCTYPE", in fmt()
68 Token::ClosingTagStart => "</", in fmt()
69 Token::CommentStart => "<!--", in fmt()
70 Token::CDataStart => "<![CDATA[", in fmt()
71 Token::TagEnd => ">", in fmt()
72 Token::EmptyTagEnd => "/>", in fmt()
73 Token::ProcessingInstructionEnd => "?>", in fmt()
74 Token::CommentEnd => "-->", in fmt()
75 Token::CDataEnd => "]]>", in fmt()
76 Token::ReferenceStart => "&", in fmt()
77 Token::ReferenceEnd => ";", in fmt()
78 Token::EqualsSign => "=", in fmt()
79 Token::SingleQuote => "'", in fmt()
80 Token::DoubleQuote => "\"", in fmt()
87 impl Token { implementation
90 Token::OpeningTagStart => Some("<"), in as_static_str()
91 Token::ProcessingInstructionStart => Some("<?"), in as_static_str()
92 Token::DoctypeStart => Some("<!DOCTYPE"), in as_static_str()
93 Token::ClosingTagStart => Some("</"), in as_static_str()
94 Token::CommentStart => Some("<!--"), in as_static_str()
95 Token::CDataStart => Some("<![CDATA["), in as_static_str()
96 Token::TagEnd => Some(">"), in as_static_str()
97 Token::EmptyTagEnd => Some("/>"), in as_static_str()
98 Token::ProcessingInstructionEnd => Some("?>"), in as_static_str()
99 Token::CommentEnd => Some("-->"), in as_static_str()
100 Token::CDataEnd => Some("]]>"), in as_static_str()
101 Token::ReferenceStart => Some("&"), in as_static_str()
102 Token::ReferenceEnd => Some(";"), in as_static_str()
103 Token::EqualsSign => Some("="), in as_static_str()
104 Token::SingleQuote => Some("'"), in as_static_str()
105 Token::DoubleQuote => Some("\""), in as_static_str()
106 Token::Chunk(s) => Some(s), in as_static_str()
111 // using String.push_str(token.to_string()) is simply way too slow
117 Token::Character(c) | Token::Whitespace(c) => target.push(c), in push_to_string()
124 /// Returns `true` if this token contains data that can be interpreted
129 Token::Whitespace(_) | Token::Chunk(_) | Token::Character(_) | Token::CommentEnd | in contains_char_data()
130 … Token::TagEnd | Token::EqualsSign | Token::DoubleQuote | Token::SingleQuote | Token::CDataEnd | in contains_char_data()
131 Token::ProcessingInstructionEnd | Token::EmptyTagEnd => true, in contains_char_data()
136 /// Returns `true` if this token corresponds to a white space character.
140 Token::Whitespace(_) => true, in is_whitespace()
186 /// `Result` represents lexing result. It is either a token or an error message.
187 pub type Result = result::Result<Option<Token>, Error>;
232 /// Returns the position of the last token produced by the lexer
274 /// Tries to read the next token from the buffer.
282 …/// * `Ok(Some(token)) where token: Token` - in case a complete-token has been read from the strea…
333 Ok(Some(Token::Character('?'))), in next_token()
335 Ok(Some(Token::Character('/'))), in next_token()
337 Ok(Some(Token::Character('-'))), in next_token()
339 Ok(Some(Token::Character(']'))), in next_token()
341 Ok(Some(Token::Chunk("]]"))), in next_token()
388 fn move_to_with(&mut self, st: State, token: Token) -> Result { in move_to_with()
390 Ok(Some(token)) in move_to_with()
394 fn move_to_with_unread(&mut self, st: State, cs: &[char], token: Token) -> Result { in move_to_with_unread()
396 self.move_to_with(st, token) in move_to_with_unread()
402 self.move_to_with(State::Normal, Token::Chunk(chunk)) in handle_error()
404 Err(self.error(format!("Unexpected token '{}' before '{}'", chunk, c))) in handle_error()
412 '>' => Ok(Some(Token::TagEnd)), in normal()
414 '=' => Ok(Some(Token::EqualsSign)), in normal()
415 '"' => Ok(Some(Token::DoubleQuote)), in normal()
416 '\'' => Ok(Some(Token::SingleQuote)), in normal()
420 '&' => Ok(Some(Token::ReferenceStart)), in normal()
421 ';' => Ok(Some(Token::ReferenceEnd)), in normal()
422 _ if is_whitespace_char(c) => Ok(Some(Token::Whitespace(c))), in normal()
423 _ => Ok(Some(Token::Character(c))) in normal()
430 … '?' => self.move_to_with(State::Normal, Token::ProcessingInstructionStart), in tag_opened()
431 '/' => self.move_to_with(State::Normal, Token::ClosingTagStart), in tag_opened()
433 …_ if is_whitespace_char(c) => self.move_to_with_unread(State::Normal, &[c], Token::OpeningTagStart… in tag_opened()
434 …_ if is_name_char(c) => self.move_to_with_unread(State::Normal, &[c], Token::OpeningTagStart… in tag_opened()
452 '-' => self.move_to_with(State::Normal, Token::CommentStart), in comment_started()
466 CDATA ; '[' ; "<![CDATA" ; self.move_to_with(State::Normal, Token::CDataStart) in cdata_started()
479 … DOCTYP ; 'E' ; "<!DOCTYP" ; self.move_to_with(State::DoctypeFinishing(1), Token::DoctypeStart) in doctype_started()
487 '>' if d == 1 => self.move_to_with(State::Normal, Token::TagEnd), in doctype_finishing()
496 '>' => self.move_to_with(State::Normal, Token::ProcessingInstructionEnd), in processing_instruction_closing()
497 _ => self.move_to_with_unread(State::Normal, &[c], Token::Character('?')), in processing_instruction_closing()
504 '>' => self.move_to_with(State::Normal, Token::EmptyTagEnd), in empty_element_closing()
505 _ => self.move_to_with_unread(State::Normal, &[c], Token::Character('/')), in empty_element_closing()
514 _ => self.move_to_with_unread(State::Normal, &[c], Token::Character('-')) in comment_closing()
517 '>' => self.move_to_with(State::Normal, Token::CommentEnd), in comment_closing()
524 … _ => self.move_to_with_unread(State::Normal, &[c], Token::Chunk("--")) in comment_closing()
534 _ => self.move_to_with_unread(State::Normal, &[c], Token::Character(']')) in cdata_closing()
537 '>' => self.move_to_with(State::Normal, Token::CDataEnd), in cdata_closing()
538 _ => self.move_to_with_unread(State::Normal, &[']', c], Token::Character(']')) in cdata_closing()
549 use super::{Lexer, Token};
587 Token::OpeningTagStart in simple_lexer_test()
588 Token::Character('a') in simple_lexer_test()
589 Token::Whitespace(' ') in simple_lexer_test()
590 Token::Character('p') in simple_lexer_test()
591 Token::EqualsSign in simple_lexer_test()
592 Token::SingleQuote in simple_lexer_test()
593 Token::Character('q') in simple_lexer_test()
594 Token::SingleQuote in simple_lexer_test()
595 Token::TagEnd in simple_lexer_test()
596 Token::Whitespace(' ') in simple_lexer_test()
597 Token::Character('x') in simple_lexer_test()
598 Token::OpeningTagStart in simple_lexer_test()
599 Token::Character('b') in simple_lexer_test()
600 Token::Whitespace(' ') in simple_lexer_test()
601 Token::Character('z') in simple_lexer_test()
602 Token::EqualsSign in simple_lexer_test()
603 Token::DoubleQuote in simple_lexer_test()
604 Token::Character('y') in simple_lexer_test()
605 Token::DoubleQuote in simple_lexer_test()
606 Token::TagEnd in simple_lexer_test()
607 Token::Character('d') in simple_lexer_test()
608 Token::Whitespace('\t') in simple_lexer_test()
609 Token::ClosingTagStart in simple_lexer_test()
610 Token::Character('b') in simple_lexer_test()
611 Token::TagEnd in simple_lexer_test()
612 Token::ClosingTagStart in simple_lexer_test()
613 Token::Character('a') in simple_lexer_test()
614 Token::TagEnd in simple_lexer_test()
615 Token::OpeningTagStart in simple_lexer_test()
616 Token::Character('p') in simple_lexer_test()
617 Token::EmptyTagEnd in simple_lexer_test()
618 Token::Whitespace(' ') in simple_lexer_test()
619 Token::ProcessingInstructionStart in simple_lexer_test()
620 Token::Character('n') in simple_lexer_test()
621 Token::Character('m') in simple_lexer_test()
622 Token::Whitespace(' ') in simple_lexer_test()
623 Token::ProcessingInstructionEnd in simple_lexer_test()
624 Token::Whitespace(' ') in simple_lexer_test()
625 Token::CommentStart in simple_lexer_test()
626 Token::Whitespace(' ') in simple_lexer_test()
627 Token::Character('a') in simple_lexer_test()
628 Token::Whitespace(' ') in simple_lexer_test()
629 Token::Character('c') in simple_lexer_test()
630 Token::Whitespace(' ') in simple_lexer_test()
631 Token::CommentEnd in simple_lexer_test()
632 Token::Whitespace(' ') in simple_lexer_test()
633 Token::ReferenceStart in simple_lexer_test()
634 Token::Character('n') in simple_lexer_test()
635 Token::Character('b') in simple_lexer_test()
636 Token::Character('s') in simple_lexer_test()
637 Token::Character('p') in simple_lexer_test()
638 Token::ReferenceEnd in simple_lexer_test()
650 Token::Character('?') in special_chars_test()
651 Token::Character('x') in special_chars_test()
652 Token::Character('!') in special_chars_test()
653 Token::Character('+') in special_chars_test()
654 Token::Whitespace(' ') in special_chars_test()
655 Token::Character('/') in special_chars_test()
656 Token::Character('/') in special_chars_test()
657 Token::Whitespace(' ') in special_chars_test()
658 Token::Character('-') in special_chars_test()
659 Token::Character('|') in special_chars_test()
660 Token::Whitespace(' ') in special_chars_test()
661 Token::Character(']') in special_chars_test()
662 Token::Character('z') in special_chars_test()
663 Token::Chunk("]]") in special_chars_test()
675 Token::OpeningTagStart in cdata_test()
676 Token::Character('a') in cdata_test()
677 Token::TagEnd in cdata_test()
678 Token::CDataStart in cdata_test()
679 Token::Character('x') in cdata_test()
680 Token::Whitespace(' ') in cdata_test()
681 Token::Character('y') in cdata_test()
682 Token::Whitespace(' ') in cdata_test()
683 Token::Character('?') in cdata_test()
684 Token::CDataEnd in cdata_test()
685 Token::Whitespace(' ') in cdata_test()
686 Token::ClosingTagStart in cdata_test()
687 Token::Character('a') in cdata_test()
688 Token::TagEnd in cdata_test()
699 Token::OpeningTagStart in doctype_test()
700 Token::Character('a') in doctype_test()
701 Token::TagEnd in doctype_test()
702 Token::DoctypeStart in doctype_test()
703 Token::TagEnd in doctype_test()
704 Token::Whitespace(' ') in doctype_test()
715 Token::OpeningTagStart in doctype_with_internal_subset_test()
716 Token::Character('a') in doctype_with_internal_subset_test()
717 Token::TagEnd in doctype_with_internal_subset_test()
718 Token::DoctypeStart in doctype_with_internal_subset_test()
719 Token::TagEnd in doctype_with_internal_subset_test()
720 Token::Whitespace(' ') in doctype_with_internal_subset_test()
728 ($data:expr ; $token:expr) => ({ in end_of_stream_handling_ok()
730 assert_oks!(for lex and buf ; $token); in end_of_stream_handling_ok()
734 eof_check!("?" ; Token::Character('?')); in end_of_stream_handling_ok()
735 eof_check!("/" ; Token::Character('/')); in end_of_stream_handling_ok()
736 eof_check!("-" ; Token::Character('-')); in end_of_stream_handling_ok()
737 eof_check!("]" ; Token::Character(']')); in end_of_stream_handling_ok()
738 eof_check!("]]" ; Token::Chunk("]]")); in end_of_stream_handling_ok()
766 "Unexpected token '<!' before 'x'" in error_in_comment_or_cdata_prefix()
772 Token::Chunk("<!") in error_in_comment_or_cdata_prefix()
773 Token::Character('x') in error_in_comment_or_cdata_prefix()
782 "Unexpected token '<!-' before '\t'" in error_in_comment_started()
788 Token::Chunk("<!-") in error_in_comment_started()
789 Token::Whitespace('\t') in error_in_comment_started()
799 "Unexpected token '--' before 'x'" in error_in_comment_two_dashes_not_at_end()
804 Token::Chunk("--") in error_in_comment_two_dashes_not_at_end()
805 Token::Character('x') in error_in_comment_two_dashes_not_at_end()
817 Token::Chunk($chunk)
818 Token::Character($app)
826 check_case!("<![", '['; "<![[" ; 0, 0, "Unexpected token '<![' before '['"); in error_in_cdata_started()
827 check_case!("<![C", '['; "<![C[" ; 0, 0, "Unexpected token '<![C' before '['"); in error_in_cdata_started()
828 check_case!("<![CD", '['; "<![CD[" ; 0, 0, "Unexpected token '<![CD' before '['"); in error_in_cdata_started()
829 check_case!("<![CDA", '['; "<![CDA[" ; 0, 0, "Unexpected token '<![CDA' before '['"); in error_in_cdata_started()
830 check_case!("<![CDAT", '['; "<![CDAT[" ; 0, 0, "Unexpected token '<![CDAT' before '['"); in error_in_cdata_started()
831 check_case!("<![CDATA", '|'; "<![CDATA|" ; 0, 0, "Unexpected token '<![CDATA' before '|'"); in error_in_cdata_started()
836 check_case!("<!D", 'a'; "<!Da" ; 0, 0, "Unexpected token '<!D' before 'a'"); in error_in_doctype_started()
837 check_case!("<!DO", 'b'; "<!DOb" ; 0, 0, "Unexpected token '<!DO' before 'b'"); in error_in_doctype_started()
838 check_case!("<!DOC", 'c'; "<!DOCc" ; 0, 0, "Unexpected token '<!DOC' before 'c'"); in error_in_doctype_started()
839 check_case!("<!DOCT", 'd'; "<!DOCTd" ; 0, 0, "Unexpected token '<!DOCT' before 'd'"); in error_in_doctype_started()
840 check_case!("<!DOCTY", 'e'; "<!DOCTYe" ; 0, 0, "Unexpected token '<!DOCTY' before 'e'"); in error_in_doctype_started()
841 check_case!("<!DOCTYP", 'f'; "<!DOCTYPf" ; 0, 0, "Unexpected token '<!DOCTYP' before 'f'"); in error_in_doctype_started()
853 Token::CDataStart in issue_98_cdata_ending_with_right_bracket()
854 Token::Character('F') in issue_98_cdata_ending_with_right_bracket()
855 Token::Character('o') in issue_98_cdata_ending_with_right_bracket()
856 Token::Character('o') in issue_98_cdata_ending_with_right_bracket()
857 Token::Whitespace(' ') in issue_98_cdata_ending_with_right_bracket()
858 Token::Character('[') in issue_98_cdata_ending_with_right_bracket()
859 Token::Character('B') in issue_98_cdata_ending_with_right_bracket()
860 Token::Character('a') in issue_98_cdata_ending_with_right_bracket()
861 Token::Character('r') in issue_98_cdata_ending_with_right_bracket()
862 Token::Character(']') in issue_98_cdata_ending_with_right_bracket()
863 Token::CDataEnd in issue_98_cdata_ending_with_right_bracket()