1 /*
2 __ _____ _____ _____
3 __| | __| | | | JSON for Modern C++ (test suite)
4 | | |__ | | | | | | version 3.10.0
5 |_____|_____|_____|_|___| https://github.com/nlohmann/json
6
7 Licensed under the MIT License <http://opensource.org/licenses/MIT>.
8 SPDX-License-Identifier: MIT
9 Copyright (c) 2013-2019 Niels Lohmann <http://nlohmann.me>.
10
11 Permission is hereby granted, free of charge, to any person obtaining a copy
12 of this software and associated documentation files (the "Software"), to deal
13 in the Software without restriction, including without limitation the rights
14 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15 copies of the Software, and to permit persons to whom the Software is
16 furnished to do so, subject to the following conditions:
17
18 The above copyright notice and this permission notice shall be included in all
19 copies or substantial portions of the Software.
20
21 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 SOFTWARE.
28 */
29
30 #include "doctest_compatibility.h"
31
32 // for some reason including this after the json header leads to linker errors with VS 2017...
33 #include <locale>
34 #include <nlohmann/json.hpp>
35 using nlohmann::json;
36
37 #include <fstream>
38 #include <sstream>
39 #include <iomanip>
40 #include <test_data.hpp>
41
skip()42 TEST_CASE("Unicode (1/5)" * doctest::skip())
43 {
44 SECTION("\\uxxxx sequences")
45 {
46 // create an escaped string from a code point
47 const auto codepoint_to_unicode = [](std::size_t cp)
48 {
49 // code points are represented as a six-character sequence: a
50 // reverse solidus, followed by the lowercase letter u, followed
51 // by four hexadecimal digits that encode the character's code
52 // point
53 std::stringstream ss;
54 ss << "\\u" << std::setw(4) << std::setfill('0') << std::hex << cp;
55 return ss.str();
56 };
57
58 SECTION("correct sequences")
59 {
60 // generate all UTF-8 code points; in total, 1112064 code points are
61 // generated: 0x1FFFFF code points - 2048 invalid values between
62 // 0xD800 and 0xDFFF.
63 for (std::size_t cp = 0; cp <= 0x10FFFFu; ++cp)
64 {
65 // string to store the code point as in \uxxxx format
66 std::string json_text = "\"";
67
68 // decide whether to use one or two \uxxxx sequences
69 if (cp < 0x10000u)
70 {
71 // The Unicode standard permanently reserves these code point
72 // values for UTF-16 encoding of the high and low surrogates, and
73 // they will never be assigned a character, so there should be no
74 // reason to encode them. The official Unicode standard says that
75 // no UTF forms, including UTF-16, can encode these code points.
76 if (cp >= 0xD800u && cp <= 0xDFFFu)
77 {
78 // if we would not skip these code points, we would get a
79 // "missing low surrogate" exception
80 continue;
81 }
82
83 // code points in the Basic Multilingual Plane can be
84 // represented with one \uxxxx sequence
85 json_text += codepoint_to_unicode(cp);
86 }
87 else
88 {
89 // To escape an extended character that is not in the Basic
90 // Multilingual Plane, the character is represented as a
91 // 12-character sequence, encoding the UTF-16 surrogate pair
92 const auto codepoint1 = 0xd800u + (((cp - 0x10000u) >> 10) & 0x3ffu);
93 const auto codepoint2 = 0xdc00u + ((cp - 0x10000u) & 0x3ffu);
94 json_text += codepoint_to_unicode(codepoint1) + codepoint_to_unicode(codepoint2);
95 }
96
97 json_text += "\"";
98 CAPTURE(json_text)
99 json _;
100 CHECK_NOTHROW(_ = json::parse(json_text));
101 }
102 }
103
104 SECTION("incorrect sequences")
105 {
106 SECTION("incorrect surrogate values")
107 {
108 json _;
109
110 CHECK_THROWS_AS(_ = json::parse("\"\\uDC00\\uDC00\""), json::parse_error&);
111 CHECK_THROWS_WITH(_ = json::parse("\"\\uDC00\\uDC00\""),
112 "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: '\"\\uDC00'");
113
114 CHECK_THROWS_AS(_ = json::parse("\"\\uD7FF\\uDC00\""), json::parse_error&);
115 CHECK_THROWS_WITH(_ = json::parse("\"\\uD7FF\\uDC00\""),
116 "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: '\"\\uD7FF\\uDC00'");
117
118 CHECK_THROWS_AS(_ = json::parse("\"\\uD800]\""), json::parse_error&);
119 CHECK_THROWS_WITH(_ = json::parse("\"\\uD800]\""),
120 "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800]'");
121
122 CHECK_THROWS_AS(_ = json::parse("\"\\uD800\\v\""), json::parse_error&);
123 CHECK_THROWS_WITH(_ = json::parse("\"\\uD800\\v\""),
124 "[json.exception.parse_error.101] parse error at line 1, column 9: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\v'");
125
126 CHECK_THROWS_AS(_ = json::parse("\"\\uD800\\u123\""), json::parse_error&);
127 CHECK_THROWS_WITH(_ = json::parse("\"\\uD800\\u123\""),
128 "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\uD800\\u123\"'");
129
130 CHECK_THROWS_AS(_ = json::parse("\"\\uD800\\uDBFF\""), json::parse_error&);
131 CHECK_THROWS_WITH(_ = json::parse("\"\\uD800\\uDBFF\""),
132 "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\uDBFF'");
133
134 CHECK_THROWS_AS(_ = json::parse("\"\\uD800\\uE000\""), json::parse_error&);
135 CHECK_THROWS_WITH(_ = json::parse("\"\\uD800\\uE000\""),
136 "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\uE000'");
137 }
138 }
139
140 #if 0
141 SECTION("incorrect sequences")
142 {
143 SECTION("high surrogate without low surrogate")
144 {
145 // D800..DBFF are high surrogates and must be followed by low
146 // surrogates DC00..DFFF; here, nothing follows
147 for (std::size_t cp = 0xD800u; cp <= 0xDBFFu; ++cp)
148 {
149 std::string json_text = "\"" + codepoint_to_unicode(cp) + "\"";
150 CAPTURE(json_text)
151 CHECK_THROWS_AS(json::parse(json_text), json::parse_error&);
152 }
153 }
154
155 SECTION("high surrogate with wrong low surrogate")
156 {
157 // D800..DBFF are high surrogates and must be followed by low
158 // surrogates DC00..DFFF; here a different sequence follows
159 for (std::size_t cp1 = 0xD800u; cp1 <= 0xDBFFu; ++cp1)
160 {
161 for (std::size_t cp2 = 0x0000u; cp2 <= 0xFFFFu; ++cp2)
162 {
163 if (0xDC00u <= cp2 && cp2 <= 0xDFFFu)
164 {
165 continue;
166 }
167
168 std::string json_text = "\"" + codepoint_to_unicode(cp1) + codepoint_to_unicode(cp2) + "\"";
169 CAPTURE(json_text)
170 CHECK_THROWS_AS(json::parse(json_text), json::parse_error&);
171 }
172 }
173 }
174
175 SECTION("low surrogate without high surrogate")
176 {
177 // low surrogates DC00..DFFF must follow high surrogates; here,
178 // they occur alone
179 for (std::size_t cp = 0xDC00u; cp <= 0xDFFFu; ++cp)
180 {
181 std::string json_text = "\"" + codepoint_to_unicode(cp) + "\"";
182 CAPTURE(json_text)
183 CHECK_THROWS_AS(json::parse(json_text), json::parse_error&);
184 }
185 }
186
187 }
188 #endif
189 }
190
191 SECTION("read all unicode characters")
192 {
193 // read a file with all unicode characters stored as single-character
194 // strings in a JSON array
195 std::ifstream f(TEST_DATA_DIRECTORY "/json_nlohmann_tests/all_unicode.json");
196 json j;
197 CHECK_NOTHROW(f >> j);
198
199 // the array has 1112064 + 1 elements (a terminating "null" value)
200 // Note: 1112064 = 0x1FFFFF code points - 2048 invalid values between
201 // 0xD800 and 0xDFFF.
202 CHECK(j.size() == 1112065);
203
204 SECTION("check JSON Pointers")
205 {
206 for (const auto& s : j)
207 {
208 // skip non-string JSON values
209 if (!s.is_string())
210 {
211 continue;
212 }
213
214 auto ptr = s.get<std::string>();
215
216 // tilde must be followed by 0 or 1
217 if (ptr == "~")
218 {
219 ptr += "0";
220 }
221
222 // JSON Pointers must begin with "/"
223 ptr.insert(0, "/");
224
225 CHECK_NOTHROW(json::json_pointer("/" + ptr));
226
227 // check escape/unescape roundtrip
228 auto escaped = nlohmann::detail::escape(ptr);
229 nlohmann::detail::unescape(escaped);
230 CHECK(escaped == ptr);
231 }
232 }
233 }
234
235 SECTION("ignore byte-order-mark")
236 {
237 SECTION("in a stream")
238 {
239 // read a file with a UTF-8 BOM
240 std::ifstream f(TEST_DATA_DIRECTORY "/json_nlohmann_tests/bom.json");
241 json j;
242 CHECK_NOTHROW(f >> j);
243 }
244
245 SECTION("with an iterator")
246 {
247 std::string i = "\xef\xbb\xbf{\n \"foo\": true\n}";
248 json _;
249 CHECK_NOTHROW(_ = json::parse(i.begin(), i.end()));
250 }
251 }
252
253 SECTION("error for incomplete/wrong BOM")
254 {
255 json _;
256 CHECK_THROWS_AS(_ = json::parse("\xef\xbb"), json::parse_error&);
257 CHECK_THROWS_AS(_ = json::parse("\xef\xbb\xbb"), json::parse_error&);
258 }
259 }
260
261 namespace
262 {
263 void roundtrip(bool success_expected, const std::string& s);
264
roundtrip(bool success_expected,const std::string & s)265 void roundtrip(bool success_expected, const std::string& s)
266 {
267 CAPTURE(s)
268 json _;
269
270 // create JSON string value
271 json j = s;
272 // create JSON text
273 std::string ps = std::string("\"") + s + "\"";
274
275 if (success_expected)
276 {
277 // serialization succeeds
278 CHECK_NOTHROW(j.dump());
279
280 // exclude parse test for U+0000
281 if (s[0] != '\0')
282 {
283 // parsing JSON text succeeds
284 CHECK_NOTHROW(_ = json::parse(ps));
285 }
286
287 // roundtrip succeeds
288 CHECK_NOTHROW(_ = json::parse(j.dump()));
289
290 // after roundtrip, the same string is stored
291 json jr = json::parse(j.dump());
292 CHECK(jr.get<std::string>() == s);
293 }
294 else
295 {
296 // serialization fails
297 CHECK_THROWS_AS(j.dump(), json::type_error&);
298
299 // parsing JSON text fails
300 CHECK_THROWS_AS(_ = json::parse(ps), json::parse_error&);
301 }
302 }
303 } // namespace
304
305 TEST_CASE("Markus Kuhn's UTF-8 decoder capability and stress test")
306 {
307 // Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> - 2015-08-28 - CC BY 4.0
308 // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
309
310 SECTION("1 Some correct UTF-8 text")
311 {
312 roundtrip(true, "κόσμε");
313 }
314
315 SECTION("2 Boundary condition test cases")
316 {
317 SECTION("2.1 First possible sequence of a certain length")
318 {
319 // 2.1.1 1 byte (U-00000000)
320 roundtrip(true, std::string("\0", 1));
321 // 2.1.2 2 bytes (U-00000080)
322 roundtrip(true, "\xc2\x80");
323 // 2.1.3 3 bytes (U-00000800)
324 roundtrip(true, "\xe0\xa0\x80");
325 // 2.1.4 4 bytes (U-00010000)
326 roundtrip(true, "\xf0\x90\x80\x80");
327
328 // 2.1.5 5 bytes (U-00200000)
329 roundtrip(false, "\xF8\x88\x80\x80\x80");
330 // 2.1.6 6 bytes (U-04000000)
331 roundtrip(false, "\xFC\x84\x80\x80\x80\x80");
332 }
333
334 SECTION("2.2 Last possible sequence of a certain length")
335 {
336 // 2.2.1 1 byte (U-0000007F)
337 roundtrip(true, "\x7f");
338 // 2.2.2 2 bytes (U-000007FF)
339 roundtrip(true, "\xdf\xbf");
340 // 2.2.3 3 bytes (U-0000FFFF)
341 roundtrip(true, "\xef\xbf\xbf");
342
343 // 2.2.4 4 bytes (U-001FFFFF)
344 roundtrip(false, "\xF7\xBF\xBF\xBF");
345 // 2.2.5 5 bytes (U-03FFFFFF)
346 roundtrip(false, "\xFB\xBF\xBF\xBF\xBF");
347 // 2.2.6 6 bytes (U-7FFFFFFF)
348 roundtrip(false, "\xFD\xBF\xBF\xBF\xBF\xBF");
349 }
350
351 SECTION("2.3 Other boundary conditions")
352 {
353 // 2.3.1 U-0000D7FF = ed 9f bf
354 roundtrip(true, "\xed\x9f\xbf");
355 // 2.3.2 U-0000E000 = ee 80 80
356 roundtrip(true, "\xee\x80\x80");
357 // 2.3.3 U-0000FFFD = ef bf bd
358 roundtrip(true, "\xef\xbf\xbd");
359 // 2.3.4 U-0010FFFF = f4 8f bf bf
360 roundtrip(true, "\xf4\x8f\xbf\xbf");
361
362 // 2.3.5 U-00110000 = f4 90 80 80
363 roundtrip(false, "\xf4\x90\x80\x80");
364 }
365 }
366
367 SECTION("3 Malformed sequences")
368 {
369 SECTION("3.1 Unexpected continuation bytes")
370 {
371 // Each unexpected continuation byte should be separately signalled as a
372 // malformed sequence of its own.
373
374 // 3.1.1 First continuation byte 0x80
375 roundtrip(false, "\x80");
376 // 3.1.2 Last continuation byte 0xbf
377 roundtrip(false, "\xbf");
378
379 // 3.1.3 2 continuation bytes
380 roundtrip(false, "\x80\xbf");
381 // 3.1.4 3 continuation bytes
382 roundtrip(false, "\x80\xbf\x80");
383 // 3.1.5 4 continuation bytes
384 roundtrip(false, "\x80\xbf\x80\xbf");
385 // 3.1.6 5 continuation bytes
386 roundtrip(false, "\x80\xbf\x80\xbf\x80");
387 // 3.1.7 6 continuation bytes
388 roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf");
389 // 3.1.8 7 continuation bytes
390 roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf\x80");
391
392 // 3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf)
393 roundtrip(false, "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf");
394 }
395
396 SECTION("3.2 Lonely start characters")
397 {
398 // 3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf)
399 roundtrip(false, "\xc0 \xc1 \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf");
400 // 3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef)
401 roundtrip(false, "\xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb \xec \xed \xee \xef");
402 // 3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7)
403 roundtrip(false, "\xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7");
404 // 3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb)
405 roundtrip(false, "\xf8 \xf9 \xfa \xfb");
406 // 3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd)
407 roundtrip(false, "\xfc \xfd");
408 }
409
410 SECTION("3.3 Sequences with last continuation byte missing")
411 {
412 // All bytes of an incomplete sequence should be signalled as a single
413 // malformed sequence, i.e., you should see only a single replacement
414 // character in each of the next 10 tests. (Characters as in section 2)
415
416 // 3.3.1 2-byte sequence with last byte missing (U+0000)
417 roundtrip(false, "\xc0");
418 // 3.3.2 3-byte sequence with last byte missing (U+0000)
419 roundtrip(false, "\xe0\x80");
420 // 3.3.3 4-byte sequence with last byte missing (U+0000)
421 roundtrip(false, "\xf0\x80\x80");
422 // 3.3.4 5-byte sequence with last byte missing (U+0000)
423 roundtrip(false, "\xf8\x80\x80\x80");
424 // 3.3.5 6-byte sequence with last byte missing (U+0000)
425 roundtrip(false, "\xfc\x80\x80\x80\x80");
426 // 3.3.6 2-byte sequence with last byte missing (U-000007FF)
427 roundtrip(false, "\xdf");
428 // 3.3.7 3-byte sequence with last byte missing (U-0000FFFF)
429 roundtrip(false, "\xef\xbf");
430 // 3.3.8 4-byte sequence with last byte missing (U-001FFFFF)
431 roundtrip(false, "\xf7\xbf\xbf");
432 // 3.3.9 5-byte sequence with last byte missing (U-03FFFFFF)
433 roundtrip(false, "\xfb\xbf\xbf\xbf");
434 // 3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF)
435 roundtrip(false, "\xfd\xbf\xbf\xbf\xbf");
436 }
437
438 SECTION("3.4 Concatenation of incomplete sequences")
439 {
440 // All the 10 sequences of 3.3 concatenated, you should see 10 malformed
441 // sequences being signalled:
442 roundtrip(false, "\xc0\xe0\x80\xf0\x80\x80\xf8\x80\x80\x80\xfc\x80\x80\x80\x80\xdf\xef\xbf\xf7\xbf\xbf\xfb\xbf\xbf\xbf\xfd\xbf\xbf\xbf\xbf");
443 }
444
445 SECTION("3.5 Impossible bytes")
446 {
447 // The following two bytes cannot appear in a correct UTF-8 string
448
449 // 3.5.1 fe
450 roundtrip(false, "\xfe");
451 // 3.5.2 ff
452 roundtrip(false, "\xff");
453 // 3.5.3 fe fe ff ff
454 roundtrip(false, "\xfe\xfe\xff\xff");
455 }
456 }
457
458 SECTION("4 Overlong sequences")
459 {
460 // The following sequences are not malformed according to the letter of
461 // the Unicode 2.0 standard. However, they are longer then necessary and
462 // a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8
463 // decoder" should reject them just like malformed sequences for two
464 // reasons: (1) It helps to debug applications if overlong sequences are
465 // not treated as valid representations of characters, because this helps
466 // to spot problems more quickly. (2) Overlong sequences provide
467 // alternative representations of characters, that could maliciously be
468 // used to bypass filters that check only for ASCII characters. For
469 // instance, a 2-byte encoded line feed (LF) would not be caught by a
470 // line counter that counts only 0x0a bytes, but it would still be
471 // processed as a line feed by an unsafe UTF-8 decoder later in the
472 // pipeline. From a security point of view, ASCII compatibility of UTF-8
473 // sequences means also, that ASCII characters are *only* allowed to be
474 // represented by ASCII bytes in the range 0x00-0x7f. To ensure this
475 // aspect of ASCII compatibility, use only "safe UTF-8 decoders" that
476 // reject overlong UTF-8 sequences for which a shorter encoding exists.
477
478 SECTION("4.1 Examples of an overlong ASCII character")
479 {
480 // With a safe UTF-8 decoder, all of the following five overlong
481 // representations of the ASCII character slash ("/") should be rejected
482 // like a malformed UTF-8 sequence, for instance by substituting it with
483 // a replacement character. If you see a slash below, you do not have a
484 // safe UTF-8 decoder!
485
486 // 4.1.1 U+002F = c0 af
487 roundtrip(false, "\xc0\xaf");
488 // 4.1.2 U+002F = e0 80 af
489 roundtrip(false, "\xe0\x80\xaf");
490 // 4.1.3 U+002F = f0 80 80 af
491 roundtrip(false, "\xf0\x80\x80\xaf");
492 // 4.1.4 U+002F = f8 80 80 80 af
493 roundtrip(false, "\xf8\x80\x80\x80\xaf");
494 // 4.1.5 U+002F = fc 80 80 80 80 af
495 roundtrip(false, "\xfc\x80\x80\x80\x80\xaf");
496 }
497
498 SECTION("4.2 Maximum overlong sequences")
499 {
500 // Below you see the highest Unicode value that is still resulting in an
501 // overlong sequence if represented with the given number of bytes. This
502 // is a boundary test for safe UTF-8 decoders. All five characters should
503 // be rejected like malformed UTF-8 sequences.
504
505 // 4.2.1 U-0000007F = c1 bf
506 roundtrip(false, "\xc1\xbf");
507 // 4.2.2 U-000007FF = e0 9f bf
508 roundtrip(false, "\xe0\x9f\xbf");
509 // 4.2.3 U-0000FFFF = f0 8f bf bf
510 roundtrip(false, "\xf0\x8f\xbf\xbf");
511 // 4.2.4 U-001FFFFF = f8 87 bf bf bf
512 roundtrip(false, "\xf8\x87\xbf\xbf\xbf");
513 // 4.2.5 U-03FFFFFF = fc 83 bf bf bf bf
514 roundtrip(false, "\xfc\x83\xbf\xbf\xbf\xbf");
515 }
516
517 SECTION("4.3 Overlong representation of the NUL character")
518 {
519 // The following five sequences should also be rejected like malformed
520 // UTF-8 sequences and should not be treated like the ASCII NUL
521 // character.
522
523 // 4.3.1 U+0000 = c0 80
524 roundtrip(false, "\xc0\x80");
525 // 4.3.2 U+0000 = e0 80 80
526 roundtrip(false, "\xe0\x80\x80");
527 // 4.3.3 U+0000 = f0 80 80 80
528 roundtrip(false, "\xf0\x80\x80\x80");
529 // 4.3.4 U+0000 = f8 80 80 80 80
530 roundtrip(false, "\xf8\x80\x80\x80\x80");
531 // 4.3.5 U+0000 = fc 80 80 80 80 80
532 roundtrip(false, "\xfc\x80\x80\x80\x80\x80");
533 }
534 }
535
536 SECTION("5 Illegal code positions")
537 {
538 // The following UTF-8 sequences should be rejected like malformed
539 // sequences, because they never represent valid ISO 10646 characters and
540 // a UTF-8 decoder that accepts them might introduce security problems
541 // comparable to overlong UTF-8 sequences.
542
543 SECTION("5.1 Single UTF-16 surrogates")
544 {
545 // 5.1.1 U+D800 = ed a0 80
546 roundtrip(false, "\xed\xa0\x80");
547 // 5.1.2 U+DB7F = ed ad bf
548 roundtrip(false, "\xed\xad\xbf");
549 // 5.1.3 U+DB80 = ed ae 80
550 roundtrip(false, "\xed\xae\x80");
551 // 5.1.4 U+DBFF = ed af bf
552 roundtrip(false, "\xed\xaf\xbf");
553 // 5.1.5 U+DC00 = ed b0 80
554 roundtrip(false, "\xed\xb0\x80");
555 // 5.1.6 U+DF80 = ed be 80
556 roundtrip(false, "\xed\xbe\x80");
557 // 5.1.7 U+DFFF = ed bf bf
558 roundtrip(false, "\xed\xbf\xbf");
559 }
560
561 SECTION("5.2 Paired UTF-16 surrogates")
562 {
563 // 5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80
564 roundtrip(false, "\xed\xa0\x80\xed\xb0\x80");
565 // 5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf
566 roundtrip(false, "\xed\xa0\x80\xed\xbf\xbf");
567 // 5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80
568 roundtrip(false, "\xed\xad\xbf\xed\xb0\x80");
569 // 5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf
570 roundtrip(false, "\xed\xad\xbf\xed\xbf\xbf");
571 // 5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80
572 roundtrip(false, "\xed\xae\x80\xed\xb0\x80");
573 // 5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf
574 roundtrip(false, "\xed\xae\x80\xed\xbf\xbf");
575 // 5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80
576 roundtrip(false, "\xed\xaf\xbf\xed\xb0\x80");
577 // 5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf
578 roundtrip(false, "\xed\xaf\xbf\xed\xbf\xbf");
579 }
580
581 SECTION("5.3 Noncharacter code positions")
582 {
583 // The following "noncharacters" are "reserved for internal use" by
584 // applications, and according to older versions of the Unicode Standard
585 // "should never be interchanged". Unicode Corrigendum #9 dropped the
586 // latter restriction. Nevertheless, their presence in incoming UTF-8 data
587 // can remain a potential security risk, depending on what use is made of
588 // these codes subsequently. Examples of such internal use:
589 //
590 // - Some file APIs with 16-bit characters may use the integer value -1
591 // = U+FFFF to signal an end-of-file (EOF) or error condition.
592 //
593 // - In some UTF-16 receivers, code point U+FFFE might trigger a
594 // byte-swap operation (to convert between UTF-16LE and UTF-16BE).
595 //
596 // With such internal use of noncharacters, it may be desirable and safer
597 // to block those code points in UTF-8 decoders, as they should never
598 // occur legitimately in incoming UTF-8 data, and could trigger unsafe
599 // behaviour in subsequent processing.
600
601 // Particularly problematic noncharacters in 16-bit applications:
602
603 // 5.3.1 U+FFFE = ef bf be
604 roundtrip(true, "\xef\xbf\xbe");
605 // 5.3.2 U+FFFF = ef bf bf
606 roundtrip(true, "\xef\xbf\xbf");
607
608 // 5.3.3 U+FDD0 .. U+FDEF
609 roundtrip(true, "\xEF\xB7\x90");
610 roundtrip(true, "\xEF\xB7\x91");
611 roundtrip(true, "\xEF\xB7\x92");
612 roundtrip(true, "\xEF\xB7\x93");
613 roundtrip(true, "\xEF\xB7\x94");
614 roundtrip(true, "\xEF\xB7\x95");
615 roundtrip(true, "\xEF\xB7\x96");
616 roundtrip(true, "\xEF\xB7\x97");
617 roundtrip(true, "\xEF\xB7\x98");
618 roundtrip(true, "\xEF\xB7\x99");
619 roundtrip(true, "\xEF\xB7\x9A");
620 roundtrip(true, "\xEF\xB7\x9B");
621 roundtrip(true, "\xEF\xB7\x9C");
622 roundtrip(true, "\xEF\xB7\x9D");
623 roundtrip(true, "\xEF\xB7\x9E");
624 roundtrip(true, "\xEF\xB7\x9F");
625 roundtrip(true, "\xEF\xB7\xA0");
626 roundtrip(true, "\xEF\xB7\xA1");
627 roundtrip(true, "\xEF\xB7\xA2");
628 roundtrip(true, "\xEF\xB7\xA3");
629 roundtrip(true, "\xEF\xB7\xA4");
630 roundtrip(true, "\xEF\xB7\xA5");
631 roundtrip(true, "\xEF\xB7\xA6");
632 roundtrip(true, "\xEF\xB7\xA7");
633 roundtrip(true, "\xEF\xB7\xA8");
634 roundtrip(true, "\xEF\xB7\xA9");
635 roundtrip(true, "\xEF\xB7\xAA");
636 roundtrip(true, "\xEF\xB7\xAB");
637 roundtrip(true, "\xEF\xB7\xAC");
638 roundtrip(true, "\xEF\xB7\xAD");
639 roundtrip(true, "\xEF\xB7\xAE");
640 roundtrip(true, "\xEF\xB7\xAF");
641
642 // 5.3.4 U+nFFFE U+nFFFF (for n = 1..10)
643 roundtrip(true, "\xF0\x9F\xBF\xBF");
644 roundtrip(true, "\xF0\xAF\xBF\xBF");
645 roundtrip(true, "\xF0\xBF\xBF\xBF");
646 roundtrip(true, "\xF1\x8F\xBF\xBF");
647 roundtrip(true, "\xF1\x9F\xBF\xBF");
648 roundtrip(true, "\xF1\xAF\xBF\xBF");
649 roundtrip(true, "\xF1\xBF\xBF\xBF");
650 roundtrip(true, "\xF2\x8F\xBF\xBF");
651 roundtrip(true, "\xF2\x9F\xBF\xBF");
652 roundtrip(true, "\xF2\xAF\xBF\xBF");
653 }
654 }
655 }
656