1 use super::*;
2
3 use expect_test::{expect, Expect};
4
check_raw_str(s: &str, expected: Result<u8, RawStrError>)5 fn check_raw_str(s: &str, expected: Result<u8, RawStrError>) {
6 let s = &format!("r{}", s);
7 let mut cursor = Cursor::new(s);
8 cursor.bump();
9 let res = cursor.raw_double_quoted_string(0);
10 assert_eq!(res, expected);
11 }
12
13 #[test]
test_naked_raw_str()14 fn test_naked_raw_str() {
15 check_raw_str(r#""abc""#, Ok(0));
16 }
17
18 #[test]
test_raw_no_start()19 fn test_raw_no_start() {
20 check_raw_str(r##""abc"#"##, Ok(0));
21 }
22
23 #[test]
test_too_many_terminators()24 fn test_too_many_terminators() {
25 // this error is handled in the parser later
26 check_raw_str(r###"#"abc"##"###, Ok(1));
27 }
28
29 #[test]
test_unterminated()30 fn test_unterminated() {
31 check_raw_str(
32 r#"#"abc"#,
33 Err(RawStrError::NoTerminator { expected: 1, found: 0, possible_terminator_offset: None }),
34 );
35 check_raw_str(
36 r###"##"abc"#"###,
37 Err(RawStrError::NoTerminator {
38 expected: 2,
39 found: 1,
40 possible_terminator_offset: Some(7),
41 }),
42 );
43 // We're looking for "# not just any #
44 check_raw_str(
45 r###"##"abc#"###,
46 Err(RawStrError::NoTerminator { expected: 2, found: 0, possible_terminator_offset: None }),
47 )
48 }
49
50 #[test]
test_invalid_start()51 fn test_invalid_start() {
52 check_raw_str(r##"#~"abc"#"##, Err(RawStrError::InvalidStarter { bad_char: '~' }));
53 }
54
55 #[test]
test_unterminated_no_pound()56 fn test_unterminated_no_pound() {
57 // https://github.com/rust-lang/rust/issues/70677
58 check_raw_str(
59 r#"""#,
60 Err(RawStrError::NoTerminator { expected: 0, found: 0, possible_terminator_offset: None }),
61 );
62 }
63
64 #[test]
test_too_many_hashes()65 fn test_too_many_hashes() {
66 let max_count = u8::MAX;
67 let hashes1 = "#".repeat(max_count as usize);
68 let hashes2 = "#".repeat(max_count as usize + 1);
69 let middle = "\"abc\"";
70 let s1 = [&hashes1, middle, &hashes1].join("");
71 let s2 = [&hashes2, middle, &hashes2].join("");
72
73 // Valid number of hashes (255 = 2^8 - 1 = u8::MAX).
74 check_raw_str(&s1, Ok(255));
75
76 // One more hash sign (256 = 2^8) becomes too many.
77 check_raw_str(&s2, Err(RawStrError::TooManyDelimiters { found: u32::from(max_count) + 1 }));
78 }
79
80 #[test]
test_valid_shebang()81 fn test_valid_shebang() {
82 // https://github.com/rust-lang/rust/issues/70528
83 let input = "#!/usr/bin/rustrun\nlet x = 5;";
84 assert_eq!(strip_shebang(input), Some(18));
85 }
86
87 #[test]
test_invalid_shebang_valid_rust_syntax()88 fn test_invalid_shebang_valid_rust_syntax() {
89 // https://github.com/rust-lang/rust/issues/70528
90 let input = "#! [bad_attribute]";
91 assert_eq!(strip_shebang(input), None);
92 }
93
94 #[test]
test_shebang_second_line()95 fn test_shebang_second_line() {
96 // Because shebangs are interpreted by the kernel, they must be on the first line
97 let input = "\n#!/bin/bash";
98 assert_eq!(strip_shebang(input), None);
99 }
100
101 #[test]
test_shebang_space()102 fn test_shebang_space() {
103 let input = "#! /bin/bash";
104 assert_eq!(strip_shebang(input), Some(input.len()));
105 }
106
107 #[test]
test_shebang_empty_shebang()108 fn test_shebang_empty_shebang() {
109 let input = "#! \n[attribute(foo)]";
110 assert_eq!(strip_shebang(input), None);
111 }
112
113 #[test]
test_invalid_shebang_comment()114 fn test_invalid_shebang_comment() {
115 let input = "#!//bin/ami/a/comment\n[";
116 assert_eq!(strip_shebang(input), None)
117 }
118
119 #[test]
test_invalid_shebang_another_comment()120 fn test_invalid_shebang_another_comment() {
121 let input = "#!/*bin/ami/a/comment*/\n[attribute";
122 assert_eq!(strip_shebang(input), None)
123 }
124
125 #[test]
test_shebang_valid_rust_after()126 fn test_shebang_valid_rust_after() {
127 let input = "#!/*bin/ami/a/comment*/\npub fn main() {}";
128 assert_eq!(strip_shebang(input), Some(23))
129 }
130
131 #[test]
test_shebang_followed_by_attrib()132 fn test_shebang_followed_by_attrib() {
133 let input = "#!/bin/rust-scripts\n#![allow_unused(true)]";
134 assert_eq!(strip_shebang(input), Some(19));
135 }
136
check_lexing(src: &str, expect: Expect)137 fn check_lexing(src: &str, expect: Expect) {
138 let actual: String = tokenize(src).map(|token| format!("{:?}\n", token)).collect();
139 expect.assert_eq(&actual)
140 }
141
142 #[test]
smoke_test()143 fn smoke_test() {
144 check_lexing(
145 "/* my source file */ fn main() { println!(\"zebra\"); }\n",
146 expect![[r#"
147 Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 }
148 Token { kind: Whitespace, len: 1 }
149 Token { kind: Ident, len: 2 }
150 Token { kind: Whitespace, len: 1 }
151 Token { kind: Ident, len: 4 }
152 Token { kind: OpenParen, len: 1 }
153 Token { kind: CloseParen, len: 1 }
154 Token { kind: Whitespace, len: 1 }
155 Token { kind: OpenBrace, len: 1 }
156 Token { kind: Whitespace, len: 1 }
157 Token { kind: Ident, len: 7 }
158 Token { kind: Bang, len: 1 }
159 Token { kind: OpenParen, len: 1 }
160 Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 7 }, len: 7 }
161 Token { kind: CloseParen, len: 1 }
162 Token { kind: Semi, len: 1 }
163 Token { kind: Whitespace, len: 1 }
164 Token { kind: CloseBrace, len: 1 }
165 Token { kind: Whitespace, len: 1 }
166 "#]],
167 )
168 }
169
170 #[test]
comment_flavors()171 fn comment_flavors() {
172 check_lexing(
173 r"
174 // line
175 //// line as well
176 /// outer doc line
177 //! inner doc line
178 /* block */
179 /**/
180 /*** also block */
181 /** outer doc block */
182 /*! inner doc block */
183 ",
184 expect![[r#"
185 Token { kind: Whitespace, len: 1 }
186 Token { kind: LineComment { doc_style: None }, len: 7 }
187 Token { kind: Whitespace, len: 1 }
188 Token { kind: LineComment { doc_style: None }, len: 17 }
189 Token { kind: Whitespace, len: 1 }
190 Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 }
191 Token { kind: Whitespace, len: 1 }
192 Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 }
193 Token { kind: Whitespace, len: 1 }
194 Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
195 Token { kind: Whitespace, len: 1 }
196 Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 }
197 Token { kind: Whitespace, len: 1 }
198 Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 }
199 Token { kind: Whitespace, len: 1 }
200 Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 }
201 Token { kind: Whitespace, len: 1 }
202 Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 }
203 Token { kind: Whitespace, len: 1 }
204 "#]],
205 )
206 }
207
208 #[test]
nested_block_comments()209 fn nested_block_comments() {
210 check_lexing(
211 "/* /* */ */'a'",
212 expect![[r#"
213 Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
214 Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
215 "#]],
216 )
217 }
218
219 #[test]
characters()220 fn characters() {
221 check_lexing(
222 "'a' ' ' '\\n'",
223 expect![[r#"
224 Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
225 Token { kind: Whitespace, len: 1 }
226 Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
227 Token { kind: Whitespace, len: 1 }
228 Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 4 }, len: 4 }
229 "#]],
230 );
231 }
232
233 #[test]
lifetime()234 fn lifetime() {
235 check_lexing(
236 "'abc",
237 expect![[r#"
238 Token { kind: Lifetime { starts_with_number: false }, len: 4 }
239 "#]],
240 );
241 }
242
243 #[test]
raw_string()244 fn raw_string() {
245 check_lexing(
246 "r###\"\"#a\\b\x00c\"\"###",
247 expect![[r#"
248 Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 17 }, len: 17 }
249 "#]],
250 )
251 }
252
253 #[test]
literal_suffixes()254 fn literal_suffixes() {
255 check_lexing(
256 r####"
257 'a'
258 b'a'
259 "a"
260 b"a"
261 1234
262 0b101
263 0xABC
264 1.0
265 1.0e10
266 2us
267 r###"raw"###suffix
268 br###"raw"###suffix
269 "####,
270 expect![[r#"
271 Token { kind: Whitespace, len: 1 }
272 Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
273 Token { kind: Whitespace, len: 1 }
274 Token { kind: Literal { kind: Byte { terminated: true }, suffix_start: 4 }, len: 4 }
275 Token { kind: Whitespace, len: 1 }
276 Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 3 }, len: 3 }
277 Token { kind: Whitespace, len: 1 }
278 Token { kind: Literal { kind: ByteStr { terminated: true }, suffix_start: 4 }, len: 4 }
279 Token { kind: Whitespace, len: 1 }
280 Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 4 }, len: 4 }
281 Token { kind: Whitespace, len: 1 }
282 Token { kind: Literal { kind: Int { base: Binary, empty_int: false }, suffix_start: 5 }, len: 5 }
283 Token { kind: Whitespace, len: 1 }
284 Token { kind: Literal { kind: Int { base: Hexadecimal, empty_int: false }, suffix_start: 5 }, len: 5 }
285 Token { kind: Whitespace, len: 1 }
286 Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 3 }, len: 3 }
287 Token { kind: Whitespace, len: 1 }
288 Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 6 }, len: 6 }
289 Token { kind: Whitespace, len: 1 }
290 Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 1 }, len: 3 }
291 Token { kind: Whitespace, len: 1 }
292 Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 12 }, len: 18 }
293 Token { kind: Whitespace, len: 1 }
294 Token { kind: Literal { kind: RawByteStr { n_hashes: Some(3) }, suffix_start: 13 }, len: 19 }
295 Token { kind: Whitespace, len: 1 }
296 "#]],
297 )
298 }
299