1# Copyright (C) 2016 and later: Unicode, Inc. and others. 2# License & terms of use: http://www.unicode.org/copyright.html 3# 4# Copyright (C) 2002-2015, International Business Machines Corporation and others. 5# All Rights Reserved. 6# 7# file: sent.txt 8# 9# ICU Sentence Break Rules 10# See Unicode Standard Annex #29. 11# These rules are based on UAX #29 Revision 26 for Unicode Version 8.0 12# 13 14!!quoted_literals_only; 15 16# 17# Character categories as defined in TR 29 18# 19$CR = [\p{Sentence_Break = CR}]; 20$LF = [\p{Sentence_Break = LF}]; 21$Extend = [\p{Sentence_Break = Extend}]; 22$Sep = [\p{Sentence_Break = Sep}]; 23$Format = [\p{Sentence_Break = Format}]; 24$Sp = [\p{Sentence_Break = Sp}]; 25$Lower = [\p{Sentence_Break = Lower}]; 26$Upper = [\p{Sentence_Break = Upper}]; 27$OLetter = [\p{Sentence_Break = OLetter}]; 28$Numeric = [\p{Sentence_Break = Numeric}]; 29$ATerm = [\p{Sentence_Break = ATerm}]; 30$SContinue = [\p{Sentence_Break = SContinue}]; 31$STerm = [\p{Sentence_Break = STerm}]; 32$Close = [\p{Sentence_Break = Close}]; 33 34# 35# Define extended forms of the character classes, 36# incorporate trailing Extend or Format chars. 37# Rules 4 and 5. 38 39$SpEx = $Sp ($Extend | $Format)*; 40$LowerEx = $Lower ($Extend | $Format)*; 41$UpperEx = $Upper ($Extend | $Format)*; 42$OLetterEx = $OLetter ($Extend | $Format)*; 43$NumericEx = $Numeric ($Extend | $Format)*; 44$ATermEx = $ATerm ($Extend | $Format)*; 45$SContinueEx= $SContinue ($Extend | $Format)*; 46$STermEx = $STerm ($Extend | $Format)*; 47$CloseEx = $Close ($Extend | $Format)*; 48 49 50## ------------------------------------------------- 51 52!!chain; 53!!forward; 54 55# Rule 3 - break after separators. Keep CR/LF together. 56# 57$CR $LF; 58 59 60# Rule 4 - Break after $Sep. 61# Rule 5 - Ignore $Format and $Extend 62# 63[^$Sep $CR $LF]? ($Extend | $Format)*; 64 65 66# Rule 6 67$ATermEx $NumericEx; 68 69# Rule 7 70($UpperEx | $LowerEx) $ATermEx $UpperEx; 71 72#Rule 8 73$NotLettersEx = [^$OLetter $Upper $Lower $Sep $CR $LF $ATerm $STerm] ($Extend | $Format)*; 74$ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower; 75 76# Rule 8a 77($STermEx | $ATermEx) $CloseEx* $SpEx* ($SContinueEx | $STermEx | $ATermEx); 78 79#Rule 9, 10, 11 80($STermEx | $ATermEx) $CloseEx* $SpEx* ($Sep | $CR | $LF)?; 81 82#Rule 12 83[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* .; 84[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* ([$Sep $LF $CR {eof}] | $CR $LF){100}; 85 86## ------------------------------------------------- 87 88!!safe_reverse; 89 90$SpEx_R = ($Extend | $Format)* $Sp; 91$ATermEx_R = ($Extend | $Format)* $ATerm; 92$STermEx_R = ($Extend | $Format)* $STerm; 93$CloseEx_R = ($Extend | $Format)* $Close; 94 95[{bof}] (.? | $LF $CR) [^$Sep $CR $LF]* [$Sep $CR $LF {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*; 96#.*; 97 98# Explanation for this rule: 99# 100# It needs to back over 101# The $Sep at which we probably begin 102# All of the non $Sep chars leading to the preceding $Sep 103# The preceding $Sep, which will be the second one that the rule matches. 104# Any immediately preceding STerm or ATerm sequences. We need to see these 105# to get the correct rule status when moving forwards again. 106# 107# [{bof}] inhibit rule chaining. Without this, rule would loop on itself and match 108# the entire string. TODO: can bof be replaced with ^ 109# 110# (.? | $LF $CR) Match one $Sep instance. Use .? rather than $Sep because position might be 111# at the beginning of the string at this point, and we don't want to fail. 112# Can only use {eof} once, and it is used later. 113# 114