1 use std::error;
2 use std::fmt;
3 use std::result;
4
5 use hir;
6
7 /// A type alias for errors specific to Unicode handling of classes.
8 pub type Result<T> = result::Result<T, Error>;
9
10 /// An inclusive range of codepoints from a generated file (hence the static
11 /// lifetime).
12 type Range = &'static [(char, char)];
13
14 /// An error that occurs when dealing with Unicode.
15 ///
16 /// We don't impl the Error trait here because these always get converted
17 /// into other public errors. (This error type isn't exported.)
18 #[derive(Debug)]
19 pub enum Error {
20 PropertyNotFound,
21 PropertyValueNotFound,
22 // Not used when unicode-perl is enabled.
23 #[allow(dead_code)]
24 PerlClassNotFound,
25 }
26
27 /// A type alias for errors specific to Unicode case folding.
28 pub type FoldResult<T> = result::Result<T, CaseFoldError>;
29
30 /// An error that occurs when Unicode-aware simple case folding fails.
31 ///
32 /// This error can occur when the case mapping tables necessary for Unicode
33 /// aware case folding are unavailable. This only occurs when the
34 /// `unicode-case` feature is disabled. (The feature is enabled by default.)
35 #[derive(Debug)]
36 pub struct CaseFoldError(());
37
38 impl error::Error for CaseFoldError {}
39
40 impl fmt::Display for CaseFoldError {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result41 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
42 write!(
43 f,
44 "Unicode-aware case folding is not available \
45 (probably because the unicode-case feature is not enabled)"
46 )
47 }
48 }
49
50 /// An error that occurs when the Unicode-aware `\w` class is unavailable.
51 ///
52 /// This error can occur when the data tables necessary for the Unicode aware
53 /// Perl character class `\w` are unavailable. This only occurs when the
54 /// `unicode-perl` feature is disabled. (The feature is enabled by default.)
55 #[derive(Debug)]
56 pub struct UnicodeWordError(());
57
58 impl error::Error for UnicodeWordError {}
59
60 impl fmt::Display for UnicodeWordError {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result61 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
62 write!(
63 f,
64 "Unicode-aware \\w class is not available \
65 (probably because the unicode-perl feature is not enabled)"
66 )
67 }
68 }
69
70 /// Return an iterator over the equivalence class of simple case mappings
71 /// for the given codepoint. The equivalence class does not include the
72 /// given codepoint.
73 ///
74 /// If the equivalence class is empty, then this returns the next scalar
75 /// value that has a non-empty equivalence class, if it exists. If no such
76 /// scalar value exists, then `None` is returned. The point of this behavior
77 /// is to permit callers to avoid calling `simple_fold` more than they need
78 /// to, since there is some cost to fetching the equivalence class.
79 ///
80 /// This returns an error if the Unicode case folding tables are not available.
simple_fold( c: char, ) -> FoldResult<result::Result<impl Iterator<Item = char>, Option<char>>>81 pub fn simple_fold(
82 c: char,
83 ) -> FoldResult<result::Result<impl Iterator<Item = char>, Option<char>>> {
84 #[cfg(not(feature = "unicode-case"))]
85 fn imp(
86 _: char,
87 ) -> FoldResult<result::Result<impl Iterator<Item = char>, Option<char>>>
88 {
89 use std::option::IntoIter;
90 Err::<result::Result<IntoIter<char>, _>, _>(CaseFoldError(()))
91 }
92
93 #[cfg(feature = "unicode-case")]
94 fn imp(
95 c: char,
96 ) -> FoldResult<result::Result<impl Iterator<Item = char>, Option<char>>>
97 {
98 use unicode_tables::case_folding_simple::CASE_FOLDING_SIMPLE;
99
100 Ok(CASE_FOLDING_SIMPLE
101 .binary_search_by_key(&c, |&(c1, _)| c1)
102 .map(|i| CASE_FOLDING_SIMPLE[i].1.iter().map(|&c| c))
103 .map_err(|i| {
104 if i >= CASE_FOLDING_SIMPLE.len() {
105 None
106 } else {
107 Some(CASE_FOLDING_SIMPLE[i].0)
108 }
109 }))
110 }
111
112 imp(c)
113 }
114
115 /// Returns true if and only if the given (inclusive) range contains at least
116 /// one Unicode scalar value that has a non-empty non-trivial simple case
117 /// mapping.
118 ///
119 /// This function panics if `end < start`.
120 ///
121 /// This returns an error if the Unicode case folding tables are not available.
contains_simple_case_mapping( start: char, end: char, ) -> FoldResult<bool>122 pub fn contains_simple_case_mapping(
123 start: char,
124 end: char,
125 ) -> FoldResult<bool> {
126 #[cfg(not(feature = "unicode-case"))]
127 fn imp(_: char, _: char) -> FoldResult<bool> {
128 Err(CaseFoldError(()))
129 }
130
131 #[cfg(feature = "unicode-case")]
132 fn imp(start: char, end: char) -> FoldResult<bool> {
133 use std::cmp::Ordering;
134 use unicode_tables::case_folding_simple::CASE_FOLDING_SIMPLE;
135
136 assert!(start <= end);
137 Ok(CASE_FOLDING_SIMPLE
138 .binary_search_by(|&(c, _)| {
139 if start <= c && c <= end {
140 Ordering::Equal
141 } else if c > end {
142 Ordering::Greater
143 } else {
144 Ordering::Less
145 }
146 })
147 .is_ok())
148 }
149
150 imp(start, end)
151 }
152
153 /// A query for finding a character class defined by Unicode. This supports
154 /// either use of a property name directly, or lookup by property value. The
155 /// former generally refers to Binary properties (see UTS#44, Table 8), but
156 /// as a special exception (see UTS#18, Section 1.2) both general categories
157 /// (an enumeration) and scripts (a catalog) are supported as if each of their
158 /// possible values were a binary property.
159 ///
160 /// In all circumstances, property names and values are normalized and
161 /// canonicalized. That is, `GC == gc == GeneralCategory == general_category`.
162 ///
163 /// The lifetime `'a` refers to the shorter of the lifetimes of property name
164 /// and property value.
165 #[derive(Debug)]
166 pub enum ClassQuery<'a> {
167 /// Return a class corresponding to a Unicode binary property, named by
168 /// a single letter.
169 OneLetter(char),
170 /// Return a class corresponding to a Unicode binary property.
171 ///
172 /// Note that, by special exception (see UTS#18, Section 1.2), both
173 /// general category values and script values are permitted here as if
174 /// they were a binary property.
175 Binary(&'a str),
176 /// Return a class corresponding to all codepoints whose property
177 /// (identified by `property_name`) corresponds to the given value
178 /// (identified by `property_value`).
179 ByValue {
180 /// A property name.
181 property_name: &'a str,
182 /// A property value.
183 property_value: &'a str,
184 },
185 }
186
187 impl<'a> ClassQuery<'a> {
canonicalize(&self) -> Result<CanonicalClassQuery>188 fn canonicalize(&self) -> Result<CanonicalClassQuery> {
189 match *self {
190 ClassQuery::OneLetter(c) => self.canonical_binary(&c.to_string()),
191 ClassQuery::Binary(name) => self.canonical_binary(name),
192 ClassQuery::ByValue { property_name, property_value } => {
193 let property_name = symbolic_name_normalize(property_name);
194 let property_value = symbolic_name_normalize(property_value);
195
196 let canon_name = match canonical_prop(&property_name)? {
197 None => return Err(Error::PropertyNotFound),
198 Some(canon_name) => canon_name,
199 };
200 Ok(match canon_name {
201 "General_Category" => {
202 let canon = match canonical_gencat(&property_value)? {
203 None => return Err(Error::PropertyValueNotFound),
204 Some(canon) => canon,
205 };
206 CanonicalClassQuery::GeneralCategory(canon)
207 }
208 "Script" => {
209 let canon = match canonical_script(&property_value)? {
210 None => return Err(Error::PropertyValueNotFound),
211 Some(canon) => canon,
212 };
213 CanonicalClassQuery::Script(canon)
214 }
215 _ => {
216 let vals = match property_values(canon_name)? {
217 None => return Err(Error::PropertyValueNotFound),
218 Some(vals) => vals,
219 };
220 let canon_val =
221 match canonical_value(vals, &property_value) {
222 None => {
223 return Err(Error::PropertyValueNotFound)
224 }
225 Some(canon_val) => canon_val,
226 };
227 CanonicalClassQuery::ByValue {
228 property_name: canon_name,
229 property_value: canon_val,
230 }
231 }
232 })
233 }
234 }
235 }
236
canonical_binary(&self, name: &str) -> Result<CanonicalClassQuery>237 fn canonical_binary(&self, name: &str) -> Result<CanonicalClassQuery> {
238 let norm = symbolic_name_normalize(name);
239
240 // This is a special case where 'cf' refers to the 'Format' general
241 // category, but where the 'cf' abbreviation is also an abbreviation
242 // for the 'Case_Folding' property. But we want to treat it as
243 // a general category. (Currently, we don't even support the
244 // 'Case_Folding' property. But if we do in the future, users will be
245 // required to spell it out.)
246 if norm != "cf" {
247 if let Some(canon) = canonical_prop(&norm)? {
248 return Ok(CanonicalClassQuery::Binary(canon));
249 }
250 }
251 if let Some(canon) = canonical_gencat(&norm)? {
252 return Ok(CanonicalClassQuery::GeneralCategory(canon));
253 }
254 if let Some(canon) = canonical_script(&norm)? {
255 return Ok(CanonicalClassQuery::Script(canon));
256 }
257 Err(Error::PropertyNotFound)
258 }
259 }
260
261 /// Like ClassQuery, but its parameters have been canonicalized. This also
262 /// differentiates binary properties from flattened general categories and
263 /// scripts.
264 #[derive(Debug, Eq, PartialEq)]
265 enum CanonicalClassQuery {
266 /// The canonical binary property name.
267 Binary(&'static str),
268 /// The canonical general category name.
269 GeneralCategory(&'static str),
270 /// The canonical script name.
271 Script(&'static str),
272 /// An arbitrary association between property and value, both of which
273 /// have been canonicalized.
274 ///
275 /// Note that by construction, the property name of ByValue will never
276 /// be General_Category or Script. Those two cases are subsumed by the
277 /// eponymous variants.
278 ByValue {
279 /// The canonical property name.
280 property_name: &'static str,
281 /// The canonical property value.
282 property_value: &'static str,
283 },
284 }
285
286 /// Looks up a Unicode class given a query. If one doesn't exist, then
287 /// `None` is returned.
class(query: ClassQuery) -> Result<hir::ClassUnicode>288 pub fn class(query: ClassQuery) -> Result<hir::ClassUnicode> {
289 use self::CanonicalClassQuery::*;
290
291 match query.canonicalize()? {
292 Binary(name) => bool_property(name),
293 GeneralCategory(name) => gencat(name),
294 Script(name) => script(name),
295 ByValue { property_name: "Age", property_value } => {
296 let mut class = hir::ClassUnicode::empty();
297 for set in ages(property_value)? {
298 class.union(&hir_class(set));
299 }
300 Ok(class)
301 }
302 ByValue { property_name: "Script_Extensions", property_value } => {
303 script_extension(property_value)
304 }
305 ByValue {
306 property_name: "Grapheme_Cluster_Break",
307 property_value,
308 } => gcb(property_value),
309 ByValue { property_name: "Sentence_Break", property_value } => {
310 sb(property_value)
311 }
312 ByValue { property_name: "Word_Break", property_value } => {
313 wb(property_value)
314 }
315 _ => {
316 // What else should we support?
317 Err(Error::PropertyNotFound)
318 }
319 }
320 }
321
322 /// Returns a Unicode aware class for \w.
323 ///
324 /// This returns an error if the data is not available for \w.
perl_word() -> Result<hir::ClassUnicode>325 pub fn perl_word() -> Result<hir::ClassUnicode> {
326 #[cfg(not(feature = "unicode-perl"))]
327 fn imp() -> Result<hir::ClassUnicode> {
328 Err(Error::PerlClassNotFound)
329 }
330
331 #[cfg(feature = "unicode-perl")]
332 fn imp() -> Result<hir::ClassUnicode> {
333 use unicode_tables::perl_word::PERL_WORD;
334 Ok(hir_class(PERL_WORD))
335 }
336
337 imp()
338 }
339
340 /// Returns a Unicode aware class for \s.
341 ///
342 /// This returns an error if the data is not available for \s.
perl_space() -> Result<hir::ClassUnicode>343 pub fn perl_space() -> Result<hir::ClassUnicode> {
344 #[cfg(not(any(feature = "unicode-perl", feature = "unicode-bool")))]
345 fn imp() -> Result<hir::ClassUnicode> {
346 Err(Error::PerlClassNotFound)
347 }
348
349 #[cfg(all(feature = "unicode-perl", not(feature = "unicode-bool")))]
350 fn imp() -> Result<hir::ClassUnicode> {
351 use unicode_tables::perl_space::WHITE_SPACE;
352 Ok(hir_class(WHITE_SPACE))
353 }
354
355 #[cfg(feature = "unicode-bool")]
356 fn imp() -> Result<hir::ClassUnicode> {
357 use unicode_tables::property_bool::WHITE_SPACE;
358 Ok(hir_class(WHITE_SPACE))
359 }
360
361 imp()
362 }
363
364 /// Returns a Unicode aware class for \d.
365 ///
366 /// This returns an error if the data is not available for \d.
perl_digit() -> Result<hir::ClassUnicode>367 pub fn perl_digit() -> Result<hir::ClassUnicode> {
368 #[cfg(not(any(feature = "unicode-perl", feature = "unicode-gencat")))]
369 fn imp() -> Result<hir::ClassUnicode> {
370 Err(Error::PerlClassNotFound)
371 }
372
373 #[cfg(all(feature = "unicode-perl", not(feature = "unicode-gencat")))]
374 fn imp() -> Result<hir::ClassUnicode> {
375 use unicode_tables::perl_decimal::DECIMAL_NUMBER;
376 Ok(hir_class(DECIMAL_NUMBER))
377 }
378
379 #[cfg(feature = "unicode-gencat")]
380 fn imp() -> Result<hir::ClassUnicode> {
381 use unicode_tables::general_category::DECIMAL_NUMBER;
382 Ok(hir_class(DECIMAL_NUMBER))
383 }
384
385 imp()
386 }
387
388 /// Build a Unicode HIR class from a sequence of Unicode scalar value ranges.
hir_class(ranges: &[(char, char)]) -> hir::ClassUnicode389 pub fn hir_class(ranges: &[(char, char)]) -> hir::ClassUnicode {
390 let hir_ranges: Vec<hir::ClassUnicodeRange> = ranges
391 .iter()
392 .map(|&(s, e)| hir::ClassUnicodeRange::new(s, e))
393 .collect();
394 hir::ClassUnicode::new(hir_ranges)
395 }
396
397 /// Returns true only if the given codepoint is in the `\w` character class.
398 ///
399 /// If the `unicode-perl` feature is not enabled, then this returns an error.
is_word_character(c: char) -> result::Result<bool, UnicodeWordError>400 pub fn is_word_character(c: char) -> result::Result<bool, UnicodeWordError> {
401 #[cfg(not(feature = "unicode-perl"))]
402 fn imp(_: char) -> result::Result<bool, UnicodeWordError> {
403 Err(UnicodeWordError(()))
404 }
405
406 #[cfg(feature = "unicode-perl")]
407 fn imp(c: char) -> result::Result<bool, UnicodeWordError> {
408 use is_word_byte;
409 use std::cmp::Ordering;
410 use unicode_tables::perl_word::PERL_WORD;
411
412 if c <= 0x7F as char && is_word_byte(c as u8) {
413 return Ok(true);
414 }
415 Ok(PERL_WORD
416 .binary_search_by(|&(start, end)| {
417 if start <= c && c <= end {
418 Ordering::Equal
419 } else if start > c {
420 Ordering::Greater
421 } else {
422 Ordering::Less
423 }
424 })
425 .is_ok())
426 }
427
428 imp(c)
429 }
430
431 /// A mapping of property values for a specific property.
432 ///
433 /// The first element of each tuple is a normalized property value while the
434 /// second element of each tuple is the corresponding canonical property
435 /// value.
436 type PropertyValues = &'static [(&'static str, &'static str)];
437
canonical_gencat(normalized_value: &str) -> Result<Option<&'static str>>438 fn canonical_gencat(normalized_value: &str) -> Result<Option<&'static str>> {
439 Ok(match normalized_value {
440 "any" => Some("Any"),
441 "assigned" => Some("Assigned"),
442 "ascii" => Some("ASCII"),
443 _ => {
444 let gencats = property_values("General_Category")?.unwrap();
445 canonical_value(gencats, normalized_value)
446 }
447 })
448 }
449
canonical_script(normalized_value: &str) -> Result<Option<&'static str>>450 fn canonical_script(normalized_value: &str) -> Result<Option<&'static str>> {
451 let scripts = property_values("Script")?.unwrap();
452 Ok(canonical_value(scripts, normalized_value))
453 }
454
455 /// Find the canonical property name for the given normalized property name.
456 ///
457 /// If no such property exists, then `None` is returned.
458 ///
459 /// The normalized property name must have been normalized according to
460 /// UAX44 LM3, which can be done using `symbolic_name_normalize`.
461 ///
462 /// If the property names data is not available, then an error is returned.
canonical_prop(normalized_name: &str) -> Result<Option<&'static str>>463 fn canonical_prop(normalized_name: &str) -> Result<Option<&'static str>> {
464 #[cfg(not(any(
465 feature = "unicode-age",
466 feature = "unicode-bool",
467 feature = "unicode-gencat",
468 feature = "unicode-perl",
469 feature = "unicode-script",
470 feature = "unicode-segment",
471 )))]
472 fn imp(_: &str) -> Result<Option<&'static str>> {
473 Err(Error::PropertyNotFound)
474 }
475
476 #[cfg(any(
477 feature = "unicode-age",
478 feature = "unicode-bool",
479 feature = "unicode-gencat",
480 feature = "unicode-perl",
481 feature = "unicode-script",
482 feature = "unicode-segment",
483 ))]
484 fn imp(name: &str) -> Result<Option<&'static str>> {
485 use unicode_tables::property_names::PROPERTY_NAMES;
486
487 Ok(PROPERTY_NAMES
488 .binary_search_by_key(&name, |&(n, _)| n)
489 .ok()
490 .map(|i| PROPERTY_NAMES[i].1))
491 }
492
493 imp(normalized_name)
494 }
495
496 /// Find the canonical property value for the given normalized property
497 /// value.
498 ///
499 /// The given property values should correspond to the values for the property
500 /// under question, which can be found using `property_values`.
501 ///
502 /// If no such property value exists, then `None` is returned.
503 ///
504 /// The normalized property value must have been normalized according to
505 /// UAX44 LM3, which can be done using `symbolic_name_normalize`.
canonical_value( vals: PropertyValues, normalized_value: &str, ) -> Option<&'static str>506 fn canonical_value(
507 vals: PropertyValues,
508 normalized_value: &str,
509 ) -> Option<&'static str> {
510 vals.binary_search_by_key(&normalized_value, |&(n, _)| n)
511 .ok()
512 .map(|i| vals[i].1)
513 }
514
515 /// Return the table of property values for the given property name.
516 ///
517 /// If the property values data is not available, then an error is returned.
property_values( canonical_property_name: &'static str, ) -> Result<Option<PropertyValues>>518 fn property_values(
519 canonical_property_name: &'static str,
520 ) -> Result<Option<PropertyValues>> {
521 #[cfg(not(any(
522 feature = "unicode-age",
523 feature = "unicode-bool",
524 feature = "unicode-gencat",
525 feature = "unicode-perl",
526 feature = "unicode-script",
527 feature = "unicode-segment",
528 )))]
529 fn imp(_: &'static str) -> Result<Option<PropertyValues>> {
530 Err(Error::PropertyValueNotFound)
531 }
532
533 #[cfg(any(
534 feature = "unicode-age",
535 feature = "unicode-bool",
536 feature = "unicode-gencat",
537 feature = "unicode-perl",
538 feature = "unicode-script",
539 feature = "unicode-segment",
540 ))]
541 fn imp(name: &'static str) -> Result<Option<PropertyValues>> {
542 use unicode_tables::property_values::PROPERTY_VALUES;
543
544 Ok(PROPERTY_VALUES
545 .binary_search_by_key(&name, |&(n, _)| n)
546 .ok()
547 .map(|i| PROPERTY_VALUES[i].1))
548 }
549
550 imp(canonical_property_name)
551 }
552
553 // This is only used in some cases, but small enough to just let it be dead
554 // instead of figuring out (and maintaining) the right set of features.
555 #[allow(dead_code)]
property_set( name_map: &'static [(&'static str, Range)], canonical: &'static str, ) -> Option<Range>556 fn property_set(
557 name_map: &'static [(&'static str, Range)],
558 canonical: &'static str,
559 ) -> Option<Range> {
560 name_map
561 .binary_search_by_key(&canonical, |x| x.0)
562 .ok()
563 .map(|i| name_map[i].1)
564 }
565
566 /// Returns an iterator over Unicode Age sets. Each item corresponds to a set
567 /// of codepoints that were added in a particular revision of Unicode. The
568 /// iterator yields items in chronological order.
569 ///
570 /// If the given age value isn't valid or if the data isn't available, then an
571 /// error is returned instead.
ages(canonical_age: &str) -> Result<impl Iterator<Item = Range>>572 fn ages(canonical_age: &str) -> Result<impl Iterator<Item = Range>> {
573 #[cfg(not(feature = "unicode-age"))]
574 fn imp(_: &str) -> Result<impl Iterator<Item = Range>> {
575 use std::option::IntoIter;
576 Err::<IntoIter<Range>, _>(Error::PropertyNotFound)
577 }
578
579 #[cfg(feature = "unicode-age")]
580 fn imp(canonical_age: &str) -> Result<impl Iterator<Item = Range>> {
581 use unicode_tables::age;
582
583 const AGES: &'static [(&'static str, Range)] = &[
584 ("V1_1", age::V1_1),
585 ("V2_0", age::V2_0),
586 ("V2_1", age::V2_1),
587 ("V3_0", age::V3_0),
588 ("V3_1", age::V3_1),
589 ("V3_2", age::V3_2),
590 ("V4_0", age::V4_0),
591 ("V4_1", age::V4_1),
592 ("V5_0", age::V5_0),
593 ("V5_1", age::V5_1),
594 ("V5_2", age::V5_2),
595 ("V6_0", age::V6_0),
596 ("V6_1", age::V6_1),
597 ("V6_2", age::V6_2),
598 ("V6_3", age::V6_3),
599 ("V7_0", age::V7_0),
600 ("V8_0", age::V8_0),
601 ("V9_0", age::V9_0),
602 ("V10_0", age::V10_0),
603 ("V11_0", age::V11_0),
604 ("V12_0", age::V12_0),
605 ("V12_1", age::V12_1),
606 ("V13_0", age::V13_0),
607 ];
608 assert_eq!(AGES.len(), age::BY_NAME.len(), "ages are out of sync");
609
610 let pos = AGES.iter().position(|&(age, _)| canonical_age == age);
611 match pos {
612 None => Err(Error::PropertyValueNotFound),
613 Some(i) => Ok(AGES[..i + 1].iter().map(|&(_, classes)| classes)),
614 }
615 }
616
617 imp(canonical_age)
618 }
619
620 /// Returns the Unicode HIR class corresponding to the given general category.
621 ///
622 /// Name canonicalization is assumed to be performed by the caller.
623 ///
624 /// If the given general category could not be found, or if the general
625 /// category data is not available, then an error is returned.
gencat(canonical_name: &'static str) -> Result<hir::ClassUnicode>626 fn gencat(canonical_name: &'static str) -> Result<hir::ClassUnicode> {
627 #[cfg(not(feature = "unicode-gencat"))]
628 fn imp(_: &'static str) -> Result<hir::ClassUnicode> {
629 Err(Error::PropertyNotFound)
630 }
631
632 #[cfg(feature = "unicode-gencat")]
633 fn imp(name: &'static str) -> Result<hir::ClassUnicode> {
634 use unicode_tables::general_category::BY_NAME;
635 match name {
636 "ASCII" => Ok(hir_class(&[('\0', '\x7F')])),
637 "Any" => Ok(hir_class(&[('\0', '\u{10FFFF}')])),
638 "Assigned" => {
639 let mut cls = gencat("Unassigned")?;
640 cls.negate();
641 Ok(cls)
642 }
643 name => property_set(BY_NAME, name)
644 .map(hir_class)
645 .ok_or(Error::PropertyValueNotFound),
646 }
647 }
648
649 match canonical_name {
650 "Decimal_Number" => perl_digit(),
651 name => imp(name),
652 }
653 }
654
655 /// Returns the Unicode HIR class corresponding to the given script.
656 ///
657 /// Name canonicalization is assumed to be performed by the caller.
658 ///
659 /// If the given script could not be found, or if the script data is not
660 /// available, then an error is returned.
script(canonical_name: &'static str) -> Result<hir::ClassUnicode>661 fn script(canonical_name: &'static str) -> Result<hir::ClassUnicode> {
662 #[cfg(not(feature = "unicode-script"))]
663 fn imp(_: &'static str) -> Result<hir::ClassUnicode> {
664 Err(Error::PropertyNotFound)
665 }
666
667 #[cfg(feature = "unicode-script")]
668 fn imp(name: &'static str) -> Result<hir::ClassUnicode> {
669 use unicode_tables::script::BY_NAME;
670 property_set(BY_NAME, name)
671 .map(hir_class)
672 .ok_or(Error::PropertyValueNotFound)
673 }
674
675 imp(canonical_name)
676 }
677
678 /// Returns the Unicode HIR class corresponding to the given script extension.
679 ///
680 /// Name canonicalization is assumed to be performed by the caller.
681 ///
682 /// If the given script extension could not be found, or if the script data is
683 /// not available, then an error is returned.
script_extension( canonical_name: &'static str, ) -> Result<hir::ClassUnicode>684 fn script_extension(
685 canonical_name: &'static str,
686 ) -> Result<hir::ClassUnicode> {
687 #[cfg(not(feature = "unicode-script"))]
688 fn imp(_: &'static str) -> Result<hir::ClassUnicode> {
689 Err(Error::PropertyNotFound)
690 }
691
692 #[cfg(feature = "unicode-script")]
693 fn imp(name: &'static str) -> Result<hir::ClassUnicode> {
694 use unicode_tables::script_extension::BY_NAME;
695 property_set(BY_NAME, name)
696 .map(hir_class)
697 .ok_or(Error::PropertyValueNotFound)
698 }
699
700 imp(canonical_name)
701 }
702
703 /// Returns the Unicode HIR class corresponding to the given Unicode boolean
704 /// property.
705 ///
706 /// Name canonicalization is assumed to be performed by the caller.
707 ///
708 /// If the given boolean property could not be found, or if the boolean
709 /// property data is not available, then an error is returned.
bool_property(canonical_name: &'static str) -> Result<hir::ClassUnicode>710 fn bool_property(canonical_name: &'static str) -> Result<hir::ClassUnicode> {
711 #[cfg(not(feature = "unicode-bool"))]
712 fn imp(_: &'static str) -> Result<hir::ClassUnicode> {
713 Err(Error::PropertyNotFound)
714 }
715
716 #[cfg(feature = "unicode-bool")]
717 fn imp(name: &'static str) -> Result<hir::ClassUnicode> {
718 use unicode_tables::property_bool::BY_NAME;
719 property_set(BY_NAME, name)
720 .map(hir_class)
721 .ok_or(Error::PropertyNotFound)
722 }
723
724 match canonical_name {
725 "Decimal_Number" => perl_digit(),
726 "White_Space" => perl_space(),
727 name => imp(name),
728 }
729 }
730
731 /// Returns the Unicode HIR class corresponding to the given grapheme cluster
732 /// break property.
733 ///
734 /// Name canonicalization is assumed to be performed by the caller.
735 ///
736 /// If the given property could not be found, or if the corresponding data is
737 /// not available, then an error is returned.
gcb(canonical_name: &'static str) -> Result<hir::ClassUnicode>738 fn gcb(canonical_name: &'static str) -> Result<hir::ClassUnicode> {
739 #[cfg(not(feature = "unicode-segment"))]
740 fn imp(_: &'static str) -> Result<hir::ClassUnicode> {
741 Err(Error::PropertyNotFound)
742 }
743
744 #[cfg(feature = "unicode-segment")]
745 fn imp(name: &'static str) -> Result<hir::ClassUnicode> {
746 use unicode_tables::grapheme_cluster_break::BY_NAME;
747 property_set(BY_NAME, name)
748 .map(hir_class)
749 .ok_or(Error::PropertyValueNotFound)
750 }
751
752 imp(canonical_name)
753 }
754
755 /// Returns the Unicode HIR class corresponding to the given word break
756 /// property.
757 ///
758 /// Name canonicalization is assumed to be performed by the caller.
759 ///
760 /// If the given property could not be found, or if the corresponding data is
761 /// not available, then an error is returned.
wb(canonical_name: &'static str) -> Result<hir::ClassUnicode>762 fn wb(canonical_name: &'static str) -> Result<hir::ClassUnicode> {
763 #[cfg(not(feature = "unicode-segment"))]
764 fn imp(_: &'static str) -> Result<hir::ClassUnicode> {
765 Err(Error::PropertyNotFound)
766 }
767
768 #[cfg(feature = "unicode-segment")]
769 fn imp(name: &'static str) -> Result<hir::ClassUnicode> {
770 use unicode_tables::word_break::BY_NAME;
771 property_set(BY_NAME, name)
772 .map(hir_class)
773 .ok_or(Error::PropertyValueNotFound)
774 }
775
776 imp(canonical_name)
777 }
778
779 /// Returns the Unicode HIR class corresponding to the given sentence
780 /// break property.
781 ///
782 /// Name canonicalization is assumed to be performed by the caller.
783 ///
784 /// If the given property could not be found, or if the corresponding data is
785 /// not available, then an error is returned.
sb(canonical_name: &'static str) -> Result<hir::ClassUnicode>786 fn sb(canonical_name: &'static str) -> Result<hir::ClassUnicode> {
787 #[cfg(not(feature = "unicode-segment"))]
788 fn imp(_: &'static str) -> Result<hir::ClassUnicode> {
789 Err(Error::PropertyNotFound)
790 }
791
792 #[cfg(feature = "unicode-segment")]
793 fn imp(name: &'static str) -> Result<hir::ClassUnicode> {
794 use unicode_tables::sentence_break::BY_NAME;
795 property_set(BY_NAME, name)
796 .map(hir_class)
797 .ok_or(Error::PropertyValueNotFound)
798 }
799
800 imp(canonical_name)
801 }
802
803 /// Like symbolic_name_normalize_bytes, but operates on a string.
symbolic_name_normalize(x: &str) -> String804 fn symbolic_name_normalize(x: &str) -> String {
805 let mut tmp = x.as_bytes().to_vec();
806 let len = symbolic_name_normalize_bytes(&mut tmp).len();
807 tmp.truncate(len);
808 // This should always succeed because `symbolic_name_normalize_bytes`
809 // guarantees that `&tmp[..len]` is always valid UTF-8.
810 //
811 // N.B. We could avoid the additional UTF-8 check here, but it's unlikely
812 // to be worth skipping the additional safety check. A benchmark must
813 // justify it first.
814 String::from_utf8(tmp).unwrap()
815 }
816
817 /// Normalize the given symbolic name in place according to UAX44-LM3.
818 ///
819 /// A "symbolic name" typically corresponds to property names and property
820 /// value aliases. Note, though, that it should not be applied to property
821 /// string values.
822 ///
823 /// The slice returned is guaranteed to be valid UTF-8 for all possible values
824 /// of `slice`.
825 ///
826 /// See: https://unicode.org/reports/tr44/#UAX44-LM3
symbolic_name_normalize_bytes(slice: &mut [u8]) -> &mut [u8]827 fn symbolic_name_normalize_bytes(slice: &mut [u8]) -> &mut [u8] {
828 // I couldn't find a place in the standard that specified that property
829 // names/aliases had a particular structure (unlike character names), but
830 // we assume that it's ASCII only and drop anything that isn't ASCII.
831 let mut start = 0;
832 let mut starts_with_is = false;
833 if slice.len() >= 2 {
834 // Ignore any "is" prefix.
835 starts_with_is = slice[0..2] == b"is"[..]
836 || slice[0..2] == b"IS"[..]
837 || slice[0..2] == b"iS"[..]
838 || slice[0..2] == b"Is"[..];
839 if starts_with_is {
840 start = 2;
841 }
842 }
843 let mut next_write = 0;
844 for i in start..slice.len() {
845 // VALIDITY ARGUMENT: To guarantee that the resulting slice is valid
846 // UTF-8, we ensure that the slice contains only ASCII bytes. In
847 // particular, we drop every non-ASCII byte from the normalized string.
848 let b = slice[i];
849 if b == b' ' || b == b'_' || b == b'-' {
850 continue;
851 } else if b'A' <= b && b <= b'Z' {
852 slice[next_write] = b + (b'a' - b'A');
853 next_write += 1;
854 } else if b <= 0x7F {
855 slice[next_write] = b;
856 next_write += 1;
857 }
858 }
859 // Special case: ISO_Comment has a 'isc' abbreviation. Since we generally
860 // ignore 'is' prefixes, the 'isc' abbreviation gets caught in the cross
861 // fire and ends up creating an alias for 'c' to 'ISO_Comment', but it
862 // is actually an alias for the 'Other' general category.
863 if starts_with_is && next_write == 1 && slice[0] == b'c' {
864 slice[0] = b'i';
865 slice[1] = b's';
866 slice[2] = b'c';
867 next_write = 3;
868 }
869 &mut slice[..next_write]
870 }
871
872 #[cfg(test)]
873 mod tests {
874 use super::{
875 contains_simple_case_mapping, simple_fold, symbolic_name_normalize,
876 symbolic_name_normalize_bytes,
877 };
878
879 #[cfg(feature = "unicode-case")]
simple_fold_ok(c: char) -> impl Iterator<Item = char>880 fn simple_fold_ok(c: char) -> impl Iterator<Item = char> {
881 simple_fold(c).unwrap().unwrap()
882 }
883
884 #[cfg(feature = "unicode-case")]
simple_fold_err(c: char) -> Option<char>885 fn simple_fold_err(c: char) -> Option<char> {
886 match simple_fold(c).unwrap() {
887 Ok(_) => unreachable!("simple_fold returned Ok iterator"),
888 Err(next) => next,
889 }
890 }
891
892 #[cfg(feature = "unicode-case")]
contains_case_map(start: char, end: char) -> bool893 fn contains_case_map(start: char, end: char) -> bool {
894 contains_simple_case_mapping(start, end).unwrap()
895 }
896
897 #[test]
898 #[cfg(feature = "unicode-case")]
simple_fold_k()899 fn simple_fold_k() {
900 let xs: Vec<char> = simple_fold_ok('k').collect();
901 assert_eq!(xs, vec!['K', 'K']);
902
903 let xs: Vec<char> = simple_fold_ok('K').collect();
904 assert_eq!(xs, vec!['k', 'K']);
905
906 let xs: Vec<char> = simple_fold_ok('K').collect();
907 assert_eq!(xs, vec!['K', 'k']);
908 }
909
910 #[test]
911 #[cfg(feature = "unicode-case")]
simple_fold_a()912 fn simple_fold_a() {
913 let xs: Vec<char> = simple_fold_ok('a').collect();
914 assert_eq!(xs, vec!['A']);
915
916 let xs: Vec<char> = simple_fold_ok('A').collect();
917 assert_eq!(xs, vec!['a']);
918 }
919
920 #[test]
921 #[cfg(feature = "unicode-case")]
simple_fold_empty()922 fn simple_fold_empty() {
923 assert_eq!(Some('A'), simple_fold_err('?'));
924 assert_eq!(Some('A'), simple_fold_err('@'));
925 assert_eq!(Some('a'), simple_fold_err('['));
926 assert_eq!(Some('Ⰰ'), simple_fold_err('☃'));
927 }
928
929 #[test]
930 #[cfg(feature = "unicode-case")]
simple_fold_max()931 fn simple_fold_max() {
932 assert_eq!(None, simple_fold_err('\u{10FFFE}'));
933 assert_eq!(None, simple_fold_err('\u{10FFFF}'));
934 }
935
936 #[test]
937 #[cfg(not(feature = "unicode-case"))]
simple_fold_disabled()938 fn simple_fold_disabled() {
939 assert!(simple_fold('a').is_err());
940 }
941
942 #[test]
943 #[cfg(feature = "unicode-case")]
range_contains()944 fn range_contains() {
945 assert!(contains_case_map('A', 'A'));
946 assert!(contains_case_map('Z', 'Z'));
947 assert!(contains_case_map('A', 'Z'));
948 assert!(contains_case_map('@', 'A'));
949 assert!(contains_case_map('Z', '['));
950 assert!(contains_case_map('☃', 'Ⰰ'));
951
952 assert!(!contains_case_map('[', '['));
953 assert!(!contains_case_map('[', '`'));
954
955 assert!(!contains_case_map('☃', '☃'));
956 }
957
958 #[test]
959 #[cfg(not(feature = "unicode-case"))]
range_contains_disabled()960 fn range_contains_disabled() {
961 assert!(contains_simple_case_mapping('a', 'a').is_err());
962 }
963
964 #[test]
965 #[cfg(feature = "unicode-gencat")]
regression_466()966 fn regression_466() {
967 use super::{CanonicalClassQuery, ClassQuery};
968
969 let q = ClassQuery::OneLetter('C');
970 assert_eq!(
971 q.canonicalize().unwrap(),
972 CanonicalClassQuery::GeneralCategory("Other")
973 );
974 }
975
976 #[test]
sym_normalize()977 fn sym_normalize() {
978 let sym_norm = symbolic_name_normalize;
979
980 assert_eq!(sym_norm("Line_Break"), "linebreak");
981 assert_eq!(sym_norm("Line-break"), "linebreak");
982 assert_eq!(sym_norm("linebreak"), "linebreak");
983 assert_eq!(sym_norm("BA"), "ba");
984 assert_eq!(sym_norm("ba"), "ba");
985 assert_eq!(sym_norm("Greek"), "greek");
986 assert_eq!(sym_norm("isGreek"), "greek");
987 assert_eq!(sym_norm("IS_Greek"), "greek");
988 assert_eq!(sym_norm("isc"), "isc");
989 assert_eq!(sym_norm("is c"), "isc");
990 assert_eq!(sym_norm("is_c"), "isc");
991 }
992
993 #[test]
valid_utf8_symbolic()994 fn valid_utf8_symbolic() {
995 let mut x = b"abc\xFFxyz".to_vec();
996 let y = symbolic_name_normalize_bytes(&mut x);
997 assert_eq!(y, b"abcxyz");
998 }
999 }
1000