1 #![warn(rust_2018_idioms)]
2
3 use tokio_util::codec::{AnyDelimiterCodec, BytesCodec, Decoder, Encoder, LinesCodec};
4
5 use bytes::{BufMut, Bytes, BytesMut};
6
7 #[test]
bytes_decoder()8 fn bytes_decoder() {
9 let mut codec = BytesCodec::new();
10 let buf = &mut BytesMut::new();
11 buf.put_slice(b"abc");
12 assert_eq!("abc", codec.decode(buf).unwrap().unwrap());
13 assert_eq!(None, codec.decode(buf).unwrap());
14 assert_eq!(None, codec.decode(buf).unwrap());
15 buf.put_slice(b"a");
16 assert_eq!("a", codec.decode(buf).unwrap().unwrap());
17 }
18
19 #[test]
bytes_encoder()20 fn bytes_encoder() {
21 let mut codec = BytesCodec::new();
22
23 // Default capacity of BytesMut
24 #[cfg(target_pointer_width = "64")]
25 const INLINE_CAP: usize = 4 * 8 - 1;
26 #[cfg(target_pointer_width = "32")]
27 const INLINE_CAP: usize = 4 * 4 - 1;
28
29 let mut buf = BytesMut::new();
30 codec
31 .encode(Bytes::from_static(&[0; INLINE_CAP + 1]), &mut buf)
32 .unwrap();
33
34 // Default capacity of Framed Read
35 const INITIAL_CAPACITY: usize = 8 * 1024;
36
37 let mut buf = BytesMut::with_capacity(INITIAL_CAPACITY);
38 codec
39 .encode(Bytes::from_static(&[0; INITIAL_CAPACITY + 1]), &mut buf)
40 .unwrap();
41 codec
42 .encode(BytesMut::from(&b"hello"[..]), &mut buf)
43 .unwrap();
44 }
45
46 #[test]
lines_decoder()47 fn lines_decoder() {
48 let mut codec = LinesCodec::new();
49 let buf = &mut BytesMut::new();
50 buf.reserve(200);
51 buf.put_slice(b"line 1\nline 2\r\nline 3\n\r\n\r");
52 assert_eq!("line 1", codec.decode(buf).unwrap().unwrap());
53 assert_eq!("line 2", codec.decode(buf).unwrap().unwrap());
54 assert_eq!("line 3", codec.decode(buf).unwrap().unwrap());
55 assert_eq!("", codec.decode(buf).unwrap().unwrap());
56 assert_eq!(None, codec.decode(buf).unwrap());
57 assert_eq!(None, codec.decode_eof(buf).unwrap());
58 buf.put_slice(b"k");
59 assert_eq!(None, codec.decode(buf).unwrap());
60 assert_eq!("\rk", codec.decode_eof(buf).unwrap().unwrap());
61 assert_eq!(None, codec.decode(buf).unwrap());
62 assert_eq!(None, codec.decode_eof(buf).unwrap());
63 }
64
65 #[test]
lines_decoder_max_length()66 fn lines_decoder_max_length() {
67 const MAX_LENGTH: usize = 6;
68
69 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
70 let buf = &mut BytesMut::new();
71
72 buf.reserve(200);
73 buf.put_slice(b"line 1 is too long\nline 2\nline 3\r\nline 4\n\r\n\r");
74
75 assert!(codec.decode(buf).is_err());
76
77 let line = codec.decode(buf).unwrap().unwrap();
78 assert!(
79 line.len() <= MAX_LENGTH,
80 "{:?}.len() <= {:?}",
81 line,
82 MAX_LENGTH
83 );
84 assert_eq!("line 2", line);
85
86 assert!(codec.decode(buf).is_err());
87
88 let line = codec.decode(buf).unwrap().unwrap();
89 assert!(
90 line.len() <= MAX_LENGTH,
91 "{:?}.len() <= {:?}",
92 line,
93 MAX_LENGTH
94 );
95 assert_eq!("line 4", line);
96
97 let line = codec.decode(buf).unwrap().unwrap();
98 assert!(
99 line.len() <= MAX_LENGTH,
100 "{:?}.len() <= {:?}",
101 line,
102 MAX_LENGTH
103 );
104 assert_eq!("", line);
105
106 assert_eq!(None, codec.decode(buf).unwrap());
107 assert_eq!(None, codec.decode_eof(buf).unwrap());
108 buf.put_slice(b"k");
109 assert_eq!(None, codec.decode(buf).unwrap());
110
111 let line = codec.decode_eof(buf).unwrap().unwrap();
112 assert!(
113 line.len() <= MAX_LENGTH,
114 "{:?}.len() <= {:?}",
115 line,
116 MAX_LENGTH
117 );
118 assert_eq!("\rk", line);
119
120 assert_eq!(None, codec.decode(buf).unwrap());
121 assert_eq!(None, codec.decode_eof(buf).unwrap());
122
123 // Line that's one character too long. This could cause an out of bounds
124 // error if we peek at the next characters using slice indexing.
125 buf.put_slice(b"aaabbbc");
126 assert!(codec.decode(buf).is_err());
127 }
128
129 #[test]
lines_decoder_max_length_underrun()130 fn lines_decoder_max_length_underrun() {
131 const MAX_LENGTH: usize = 6;
132
133 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
134 let buf = &mut BytesMut::new();
135
136 buf.reserve(200);
137 buf.put_slice(b"line ");
138 assert_eq!(None, codec.decode(buf).unwrap());
139 buf.put_slice(b"too l");
140 assert!(codec.decode(buf).is_err());
141 buf.put_slice(b"ong\n");
142 assert_eq!(None, codec.decode(buf).unwrap());
143
144 buf.put_slice(b"line 2");
145 assert_eq!(None, codec.decode(buf).unwrap());
146 buf.put_slice(b"\n");
147 assert_eq!("line 2", codec.decode(buf).unwrap().unwrap());
148 }
149
150 #[test]
lines_decoder_max_length_bursts()151 fn lines_decoder_max_length_bursts() {
152 const MAX_LENGTH: usize = 10;
153
154 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
155 let buf = &mut BytesMut::new();
156
157 buf.reserve(200);
158 buf.put_slice(b"line ");
159 assert_eq!(None, codec.decode(buf).unwrap());
160 buf.put_slice(b"too l");
161 assert_eq!(None, codec.decode(buf).unwrap());
162 buf.put_slice(b"ong\n");
163 assert!(codec.decode(buf).is_err());
164 }
165
166 #[test]
lines_decoder_max_length_big_burst()167 fn lines_decoder_max_length_big_burst() {
168 const MAX_LENGTH: usize = 10;
169
170 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
171 let buf = &mut BytesMut::new();
172
173 buf.reserve(200);
174 buf.put_slice(b"line ");
175 assert_eq!(None, codec.decode(buf).unwrap());
176 buf.put_slice(b"too long!\n");
177 assert!(codec.decode(buf).is_err());
178 }
179
180 #[test]
lines_decoder_max_length_newline_between_decodes()181 fn lines_decoder_max_length_newline_between_decodes() {
182 const MAX_LENGTH: usize = 5;
183
184 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
185 let buf = &mut BytesMut::new();
186
187 buf.reserve(200);
188 buf.put_slice(b"hello");
189 assert_eq!(None, codec.decode(buf).unwrap());
190
191 buf.put_slice(b"\nworld");
192 assert_eq!("hello", codec.decode(buf).unwrap().unwrap());
193 }
194
195 // Regression test for [infinite loop bug](https://github.com/tokio-rs/tokio/issues/1483)
196 #[test]
lines_decoder_discard_repeat()197 fn lines_decoder_discard_repeat() {
198 const MAX_LENGTH: usize = 1;
199
200 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
201 let buf = &mut BytesMut::new();
202
203 buf.reserve(200);
204 buf.put_slice(b"aa");
205 assert!(codec.decode(buf).is_err());
206 buf.put_slice(b"a");
207 assert_eq!(None, codec.decode(buf).unwrap());
208 }
209
210 // Regression test for [subsequent calls to LinesCodec decode does not return the desired results bug](https://github.com/tokio-rs/tokio/issues/3555)
211 #[test]
lines_decoder_max_length_underrun_twice()212 fn lines_decoder_max_length_underrun_twice() {
213 const MAX_LENGTH: usize = 11;
214
215 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
216 let buf = &mut BytesMut::new();
217
218 buf.reserve(200);
219 buf.put_slice(b"line ");
220 assert_eq!(None, codec.decode(buf).unwrap());
221 buf.put_slice(b"too very l");
222 assert!(codec.decode(buf).is_err());
223 buf.put_slice(b"aaaaaaaaaaaaaaaaaaaaaaa");
224 assert_eq!(None, codec.decode(buf).unwrap());
225 buf.put_slice(b"ong\nshort\n");
226 assert_eq!("short", codec.decode(buf).unwrap().unwrap());
227 }
228
229 #[test]
lines_encoder()230 fn lines_encoder() {
231 let mut codec = LinesCodec::new();
232 let mut buf = BytesMut::new();
233
234 codec.encode("line 1", &mut buf).unwrap();
235 assert_eq!("line 1\n", buf);
236
237 codec.encode("line 2", &mut buf).unwrap();
238 assert_eq!("line 1\nline 2\n", buf);
239 }
240
241 #[test]
any_delimiters_decoder_any_character()242 fn any_delimiters_decoder_any_character() {
243 let mut codec = AnyDelimiterCodec::new(b",;\n\r".to_vec(), b",".to_vec());
244 let buf = &mut BytesMut::new();
245 buf.reserve(200);
246 buf.put_slice(b"chunk 1,chunk 2;chunk 3\n\r");
247 assert_eq!("chunk 1", codec.decode(buf).unwrap().unwrap());
248 assert_eq!("chunk 2", codec.decode(buf).unwrap().unwrap());
249 assert_eq!("chunk 3", codec.decode(buf).unwrap().unwrap());
250 assert_eq!("", codec.decode(buf).unwrap().unwrap());
251 assert_eq!(None, codec.decode(buf).unwrap());
252 assert_eq!(None, codec.decode_eof(buf).unwrap());
253 buf.put_slice(b"k");
254 assert_eq!(None, codec.decode(buf).unwrap());
255 assert_eq!("k", codec.decode_eof(buf).unwrap().unwrap());
256 assert_eq!(None, codec.decode(buf).unwrap());
257 assert_eq!(None, codec.decode_eof(buf).unwrap());
258 }
259
260 #[test]
any_delimiters_decoder_max_length()261 fn any_delimiters_decoder_max_length() {
262 const MAX_LENGTH: usize = 7;
263
264 let mut codec =
265 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
266 let buf = &mut BytesMut::new();
267
268 buf.reserve(200);
269 buf.put_slice(b"chunk 1 is too long\nchunk 2\nchunk 3\r\nchunk 4\n\r\n");
270
271 assert!(codec.decode(buf).is_err());
272
273 let chunk = codec.decode(buf).unwrap().unwrap();
274 assert!(
275 chunk.len() <= MAX_LENGTH,
276 "{:?}.len() <= {:?}",
277 chunk,
278 MAX_LENGTH
279 );
280 assert_eq!("chunk 2", chunk);
281
282 let chunk = codec.decode(buf).unwrap().unwrap();
283 assert!(
284 chunk.len() <= MAX_LENGTH,
285 "{:?}.len() <= {:?}",
286 chunk,
287 MAX_LENGTH
288 );
289 assert_eq!("chunk 3", chunk);
290
291 // \r\n cause empty chunk
292 let chunk = codec.decode(buf).unwrap().unwrap();
293 assert!(
294 chunk.len() <= MAX_LENGTH,
295 "{:?}.len() <= {:?}",
296 chunk,
297 MAX_LENGTH
298 );
299 assert_eq!("", chunk);
300
301 let chunk = codec.decode(buf).unwrap().unwrap();
302 assert!(
303 chunk.len() <= MAX_LENGTH,
304 "{:?}.len() <= {:?}",
305 chunk,
306 MAX_LENGTH
307 );
308 assert_eq!("chunk 4", chunk);
309
310 let chunk = codec.decode(buf).unwrap().unwrap();
311 assert!(
312 chunk.len() <= MAX_LENGTH,
313 "{:?}.len() <= {:?}",
314 chunk,
315 MAX_LENGTH
316 );
317 assert_eq!("", chunk);
318
319 let chunk = codec.decode(buf).unwrap().unwrap();
320 assert!(
321 chunk.len() <= MAX_LENGTH,
322 "{:?}.len() <= {:?}",
323 chunk,
324 MAX_LENGTH
325 );
326 assert_eq!("", chunk);
327
328 assert_eq!(None, codec.decode(buf).unwrap());
329 assert_eq!(None, codec.decode_eof(buf).unwrap());
330 buf.put_slice(b"k");
331 assert_eq!(None, codec.decode(buf).unwrap());
332
333 let chunk = codec.decode_eof(buf).unwrap().unwrap();
334 assert!(
335 chunk.len() <= MAX_LENGTH,
336 "{:?}.len() <= {:?}",
337 chunk,
338 MAX_LENGTH
339 );
340 assert_eq!("k", chunk);
341
342 assert_eq!(None, codec.decode(buf).unwrap());
343 assert_eq!(None, codec.decode_eof(buf).unwrap());
344
345 // Delimiter that's one character too long. This could cause an out of bounds
346 // error if we peek at the next characters using slice indexing.
347 buf.put_slice(b"aaabbbcc");
348 assert!(codec.decode(buf).is_err());
349 }
350
351 #[test]
any_delimiter_decoder_max_length_underrun()352 fn any_delimiter_decoder_max_length_underrun() {
353 const MAX_LENGTH: usize = 7;
354
355 let mut codec =
356 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
357 let buf = &mut BytesMut::new();
358
359 buf.reserve(200);
360 buf.put_slice(b"chunk ");
361 assert_eq!(None, codec.decode(buf).unwrap());
362 buf.put_slice(b"too l");
363 assert!(codec.decode(buf).is_err());
364 buf.put_slice(b"ong\n");
365 assert_eq!(None, codec.decode(buf).unwrap());
366
367 buf.put_slice(b"chunk 2");
368 assert_eq!(None, codec.decode(buf).unwrap());
369 buf.put_slice(b",");
370 assert_eq!("chunk 2", codec.decode(buf).unwrap().unwrap());
371 }
372
373 #[test]
any_delimiter_decoder_max_length_underrun_twice()374 fn any_delimiter_decoder_max_length_underrun_twice() {
375 const MAX_LENGTH: usize = 11;
376
377 let mut codec =
378 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
379 let buf = &mut BytesMut::new();
380
381 buf.reserve(200);
382 buf.put_slice(b"chunk ");
383 assert_eq!(None, codec.decode(buf).unwrap());
384 buf.put_slice(b"too very l");
385 assert!(codec.decode(buf).is_err());
386 buf.put_slice(b"aaaaaaaaaaaaaaaaaaaaaaa");
387 assert_eq!(None, codec.decode(buf).unwrap());
388 buf.put_slice(b"ong\nshort\n");
389 assert_eq!("short", codec.decode(buf).unwrap().unwrap());
390 }
391 #[test]
any_delimiter_decoder_max_length_bursts()392 fn any_delimiter_decoder_max_length_bursts() {
393 const MAX_LENGTH: usize = 11;
394
395 let mut codec =
396 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
397 let buf = &mut BytesMut::new();
398
399 buf.reserve(200);
400 buf.put_slice(b"chunk ");
401 assert_eq!(None, codec.decode(buf).unwrap());
402 buf.put_slice(b"too l");
403 assert_eq!(None, codec.decode(buf).unwrap());
404 buf.put_slice(b"ong\n");
405 assert!(codec.decode(buf).is_err());
406 }
407
408 #[test]
any_delimiter_decoder_max_length_big_burst()409 fn any_delimiter_decoder_max_length_big_burst() {
410 const MAX_LENGTH: usize = 11;
411
412 let mut codec =
413 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
414 let buf = &mut BytesMut::new();
415
416 buf.reserve(200);
417 buf.put_slice(b"chunk ");
418 assert_eq!(None, codec.decode(buf).unwrap());
419 buf.put_slice(b"too long!\n");
420 assert!(codec.decode(buf).is_err());
421 }
422
423 #[test]
any_delimiter_decoder_max_length_delimiter_between_decodes()424 fn any_delimiter_decoder_max_length_delimiter_between_decodes() {
425 const MAX_LENGTH: usize = 5;
426
427 let mut codec =
428 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
429 let buf = &mut BytesMut::new();
430
431 buf.reserve(200);
432 buf.put_slice(b"hello");
433 assert_eq!(None, codec.decode(buf).unwrap());
434
435 buf.put_slice(b",world");
436 assert_eq!("hello", codec.decode(buf).unwrap().unwrap());
437 }
438
439 #[test]
any_delimiter_decoder_discard_repeat()440 fn any_delimiter_decoder_discard_repeat() {
441 const MAX_LENGTH: usize = 1;
442
443 let mut codec =
444 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
445 let buf = &mut BytesMut::new();
446
447 buf.reserve(200);
448 buf.put_slice(b"aa");
449 assert!(codec.decode(buf).is_err());
450 buf.put_slice(b"a");
451 assert_eq!(None, codec.decode(buf).unwrap());
452 }
453
454 #[test]
any_delimiter_encoder()455 fn any_delimiter_encoder() {
456 let mut codec = AnyDelimiterCodec::new(b",".to_vec(), b";--;".to_vec());
457 let mut buf = BytesMut::new();
458
459 codec.encode("chunk 1", &mut buf).unwrap();
460 assert_eq!("chunk 1;--;", buf);
461
462 codec.encode("chunk 2", &mut buf).unwrap();
463 assert_eq!("chunk 1;--;chunk 2;--;", buf);
464 }
465