• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1--- source/data/brkitr/word.txt	2010-09-17 18:22:35.000000000 -0700
2+++ source/data/brkitr/word.txt	2011-01-25 18:14:09.445378000 -0800
3@@ -29,29 +29,49 @@
4 $Newline      = [\p{Word_Break = Newline}];
5 $Extend       = [\p{Word_Break = Extend}];
6 $Format       = [\p{Word_Break = Format}];
7+$Hiragana     = [:Hiragana:];
8 $Katakana     = [\p{Word_Break = Katakana}];
9+$Han          = [:Han:];
10 $ALetter      = [\p{Word_Break = ALetter}];
11-$MidNumLet    = [\p{Word_Break = MidNumLet}];
12+# Remove two full stop characters from $MidNumLet and add them to $MidNum
13+# to break a hostname into its components at the cost of breaking
14+# 'e.g.' and 'i.e.' as well.
15+# $MidNumLet is used in rules 6/7 (rules of our interest) and rules 11/12.
16+# Because it's OR'd with $MidNum in rules 11/12, rules 11/12 are not affected
17+# while rules 6/7 are reverted to the old behavior we want.
18+$MidNumLet    = [[\p{Word_Break = MidNumLet}] - [\u002E \uFF0E]];
19 $MidLetter    = [\p{Word_Break = MidLetter}];
20-$MidNum       = [\p{Word_Break = MidNum}];
21-$Numeric      = [\p{Word_Break = Numeric}];
22+$MidNum       = [\p{Word_Break = MidNum}[\u002E \uFF0E]];
23+$Numeric      = [\p{Word_Break = Numeric}[\uff10-\uff19]]; #includes fullwidth digits
24 $ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
25
26+# Extra sets not to break 'HebrewLetter U+0022 HebrewLetter'.
27+$HebrewLet    = [\p{Word_Break = ALetter} & \p{Script = Hebrew} - [\u05F3]];
28+# U+05F3 is ALetter and U+05F4 is MidLetter so that they're covered by
29+# the current rule 6/7.
30+$HebrewMidLet = [\u0022];
31
32 #   Dictionary character set, for triggering language-based break engines. Currently
33-#   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
34-#   5.0 or later as the definition of Complex_Context was corrected to include all
35+#   limited to LineBreak=Complex_Context and CJK. Note that this set only works
36+#   in Unicode 5.0 or later as the definition of Complex_Context was corrected to include all
37 #   characters requiring dictionary break.
38
39-$dictionary   = [:LineBreak = Complex_Context:];
40 $Control        = [\p{Grapheme_Cluster_Break = Control}];
41-$ALetterPlus  = [$ALetter [$dictionary-$Extend-$Control]];   # Note:  default ALetter does not
42-                                                             #  include the dictionary characters.
43+$HangulSyllable = [\uac00-\ud7a3];
44+$ComplexContext = [:LineBreak = Complex_Context:];
45+$KanaKanji      = [$Han $Hiragana $Katakana];
46+$dictionaryCJK  = [$KanaKanji $HangulSyllable];
47+$dictionary     = [$ComplexContext $dictionaryCJK];
48+
49+# leave CJK scripts out of ALetterPlus
50+$ALetterPlus  = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
51+
52
53 #
54 #  Rules 4    Ignore Format and Extend characters,
55 #             except when they appear at the beginning of a region of text.
56 #
57+# TODO: check if handling of katakana in dictionary makes rules incorrect/void.
58 $KatakanaEx     = $Katakana     ($Extend |  $Format)*;
59 $ALetterEx      = $ALetterPlus  ($Extend |  $Format)*;
60 $MidNumLetEx    = $MidNumLet    ($Extend |  $Format)*;
61@@ -59,8 +79,8 @@
62 $MidNumEx       = $MidNum       ($Extend |  $Format)*;
63 $NumericEx      = $Numeric      ($Extend |  $Format)*;
64 $ExtendNumLetEx = $ExtendNumLet ($Extend |  $Format)*;
65+$HebrewLetEx    = $HebrewLet    ($Extend |  $Format)*;
66
67-$Hiragana       = [\p{script=Hiragana}];
68 $Ideographic    = [\p{Ideographic}];
69 $HiraganaEx     = $Hiragana     ($Extend |  $Format)*;
70 $IdeographicEx  = $Ideographic  ($Extend |  $Format)*;
71@@ -79,12 +99,14 @@
72 #          begins with a group of Format chars, or with a "word" consisting of a single
73 #          char that is not in any of the listed word break categories followed by
74 #          format char(s).
75-[^$CR $LF $Newline]? ($Extend |  $Format)+;
76+ #          format char(s), or is not a CJK dictionary character.
77+[^$CR $LF $Newline $dictionaryCJK]? ($Extend |  $Format)+;
78
79 $NumericEx {100};
80 $ALetterEx {200};
81-$KatakanaEx {300};       # note:  these status values override those from rule 5
82-$HiraganaEx {300};       #        by virtual of being numerically larger.
83+$HangulSyllable {200};
84+$KatakanaEx {400}; #originally 300
85+$HiraganaEx {400}; #originally 300
86 $IdeographicEx {400};    #
87
88 #
89@@ -96,6 +118,9 @@
90 # rule 6 and 7
91 $ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200};
92
93+# Chrome addition
94+$HebrewLetEx $HebrewMidLet $HebrewLetEx {200};
95+
96 # rule 8
97
98 $NumericEx $NumericEx {100};
99@@ -114,19 +139,25 @@
100
101 # rule 13
102
103-$KatakanaEx  $KatakanaEx {300};
104+# To be consistent with '$KanaKanji $KanaKanji', changed
105+# from 300 to 400.
106+# See also TestRuleStatus in intltest/rbbiapts.cpp
107+$KatakanaEx  $KatakanaEx {400};
108
109 # rule 13a/b
110
111 $ALetterEx      $ExtendNumLetEx {200};    #  (13a)
112 $NumericEx      $ExtendNumLetEx {100};    #  (13a)
113-$KatakanaEx     $ExtendNumLetEx {300};    #  (13a)
114+$KatakanaEx     $ExtendNumLetEx {400};    #  (13a)
115 $ExtendNumLetEx $ExtendNumLetEx {200};    #  (13a)
116
117 $ExtendNumLetEx $ALetterEx  {200};    #  (13b)
118 $ExtendNumLetEx $NumericEx  {100};    #  (13b)
119-$ExtendNumLetEx $KatakanaEx {300};    #  (13b)
120-
121+$ExtendNumLetEx $KatakanaEx {400};    #  (13b)
122+
123+# special handling for CJK characters: chain for later dictionary segmentation
124+$HangulSyllable $HangulSyllable {200};
125+$KanaKanji $KanaKanji {400}; #different rule status if both kanji and kana found
126
127
128 ## -------------------------------------------------
129@@ -139,13 +170,16 @@
130 $BackMidNumEx      = ($Format | $Extend)* $MidNum;
131 $BackMidLetterEx   = ($Format | $Extend)* $MidLetter;
132 $BackKatakanaEx    = ($Format | $Extend)* $Katakana;
133+$BackHiraganaEx    = ($Extend | $Format)* $Hiragana;
134 $BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet;
135+$BackHebrewLetEx   = ($Format | $Extend)* $HebrewLet;
136+
137
138 # rule 3
139 $LF $CR;
140
141 # rule 4
142-($Format | $Extend)*  [^$CR $LF $Newline]?;
143+($Format | $Extend)*  [^$CR $LF $Newline $dictionaryCJK]?;
144
145 # rule 5
146
147@@ -155,6 +189,8 @@
148
149 $BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx;
150
151+# Chrome addition
152+$BackHebrewLetEx $HebrewMidLet $BackHebrewLetEx;
153
154 # rule 8
155
156@@ -181,6 +217,10 @@
157 $BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
158 ($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
159
160+# special handling for CJK characters: chain for later dictionary segmentation
161+$HangulSyllable $HangulSyllable;
162+$KanaKanji $KanaKanji; #different rule status if both kanji and kana found
163+
164 ## -------------------------------------------------
165
166 !!safe_reverse;
167--- source/data/brkitr/line.txt	2010-09-17 18:22:35.000000000 -0700
168+++ source/data/brkitr/line.txt	2011-07-22 13:46:19.923562000 -0700
169@@ -11,6 +11,9 @@
170 #         TODO:  Rule LB 8 remains as it was in Unicode 5.2
171 #         This is only because of a limitation of ICU break engine implementation,
172 #         not because the older behavior is desirable.
173+#
174+#         CHROME: Hebrew tailoring was incorporatd as well as some
175+#         other minor changes (CL, OP, ID).
176
177 #
178 #  Character Classes defined by TR 14.
179@@ -55,15 +58,22 @@
180 #
181 #           See rule LB 19 for an example.
182 #
183+$SmallHira = [\u3041 \u3043 \u3045 \u3047 \u3049 \u3063 \u3083 \u3085 \u3087 \u308E \u3095 \u3096];
184+$SmallKata = [\u30A1 \u30A3 \u30A5 \u30A7 \u30A9 \u30C3 \u30E3 \u30E5 \u30E7 \u30EE \u30F5 \u30F6];
185+$SmallKataExt = [\u31F0 \u31F1 \u31F2 \u31F3 \u31F4 \u31F5 \u31F6 \u31F7 \u31F8 \u31F9 \u31FA \u31FB \u31FC \u31FD \u31FE \u31FF];
186+$SmallKanaAndProlongedMark = [[$SmallHira] [$SmallKata] [$SmallKataExt] [\u30FC]];
187+
188
189 $AI = [:LineBreak =  Ambiguous:];
190-$AL = [:LineBreak =  Alphabetic:];
191-$BA = [:LineBreak =  Break_After:];
192+$AL = [[:LineBreak =  Alphabetic:] - [[:Hebrew:] & [:Letter:]] - [\u23B4\u23B5]];
193+$HL = [[:Hebrew:] & [:Letter:]];
194+$BA = [[:LineBreak =  Break_After:] - [\u2010]];
195+$HH = [\u2010];
196 $BB = [:LineBreak =  Break_Before:];
197 $BK = [:LineBreak =  Mandatory_Break:];
198 $B2 = [:LineBreak =  Break_Both:];
199 $CB = [:LineBreak =  Contingent_Break:];
200-$CL = [:LineBreak =  Close_Punctuation:];
201+$CL = [[:LineBreak =  Close_Punctuation:] [\uFE51\uFE10\u23B5]];
202 $CM = [:LineBreak =  Combining_Mark:];
203 $CP = [:LineBreak =  Close_Parenthesis:];
204 $CR = [:LineBreak =  Carriage_Return:];
205@@ -72,17 +82,17 @@
206 $HY = [:LineBreak =  Hyphen:];
207 $H2 = [:LineBreak =  H2:];
208 $H3 = [:LineBreak =  H3:];
209-$ID = [:LineBreak =  Ideographic:];
210+$ID = [[[:LineBreak =  Ideographic:] - [\uFE51]] [$SmallKanaAndProlongedMark]];
211 $IN = [:LineBreak =  Inseperable:];
212-$IS = [:LineBreak =  Infix_Numeric:];
213+$IS = [[:LineBreak =  Infix_Numeric:] - [\uFE10]];
214 $JL = [:LineBreak =  JL:];
215 $JV = [:LineBreak =  JV:];
216 $JT = [:LineBreak =  JT:];
217 $LF = [:LineBreak =  Line_Feed:];
218 $NL = [:LineBreak =  Next_Line:];
219-$NS = [:LineBreak =  Nonstarter:];
220+$NS = [[:LineBreak =  Nonstarter:] - [$SmallKanaAndProlongedMark]];
221 $NU = [:LineBreak =  Numeric:];
222-$OP = [:LineBreak =  Open_Punctuation:];
223+$OP = [[:LineBreak =  Open_Punctuation:] \u23B4];
224 $PO = [:LineBreak =  Postfix_Numeric:];
225 $PR = [:LineBreak =  Prefix_Numeric:];
226 $QU = [:LineBreak =  Quotation:];
227@@ -108,13 +118,15 @@
228 #                               XX  (Unknown, unassigned)
229 #                         as $AL  (Alphabetic)
230 #
231-$ALPlus = [$AL $AI $SA $SG $XX];
232+$ALPlus = [$AL $HL $AI $SA $SG $XX];
233
234 #
235 #  Combining Marks.   X $CM*  behaves as if it were X.  Rule LB6.
236 #
237 $ALcm = $ALPlus $CM*;
238+$HLcm = $HL $CM*;
239 $BAcm = $BA $CM*;
240+$HHcm = $HH $CM*;
241 $BBcm = $BB $CM*;
242 $B2cm = $B2 $CM*;
243 $CLcm = $CL $CM*;
244@@ -148,6 +160,7 @@
245 #
246 $ALPlus $CM+;
247 $BA $CM+;
248+$HH $CM+;
249 $BB $CM+;
250 $B2 $CM+;
251 $CL $CM+;
252@@ -190,7 +203,7 @@
253 #            so for this one case we need to manually list out longer sequences.
254 #
255 $AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
256-$AL_FOLLOW_CM   = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $ALPlus];
257+$AL_FOLLOW_CM   = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HH $HY $NS $IN $NU $ALPlus];
258 $AL_FOLLOW      = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
259
260
261@@ -252,7 +265,7 @@
262 # LB 12a  Do not break before NBSP and related characters ...
263 #            [^SP BA HY] x GL
264 #
265-[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GLcm;
266+[[$LB8NonBreaks] - [$SP $BA $HH $HY]] $CM* $GLcm;
267 $CM+ GLcm;
268
269
270@@ -325,7 +338,7 @@
271 # LB 21        x   (BA | HY | NS)
272 #           BB x
273 #
274-$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm);
275+$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm);
276
277 $BBcm [^$CB];                                  #  $BB  x
278 $BBcm $LB20NonBreaks $CM*;
279@@ -381,6 +394,8 @@
280 $CM+ $OPcm;         # The $CM+ is from rule 10, an unattached CM is treated as AL.
281 $CPcm ($ALcm | $NUcm);
282
283+# (LB 31) Add new rule to prevent the break we do not want, this is the behavior change
284+$HLcm ($HY | $HH) $ALcm;
285
286 #
287 #  Reverse Rules.
288@@ -391,6 +406,7 @@
289
290 $CM+ $ALPlus;
291 $CM+ $BA;
292+$CM+ $HH;
293 $CM+ $BB;
294 $CM+ $B2;
295 $CM+ $CL;
296@@ -479,7 +495,7 @@
297 # LB 12a
298 #      [^SP BA HY] x GL
299 #
300-$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
301+$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]];
302
303 # LB 12
304 #     GL  x
305@@ -549,7 +565,7 @@
306 #
307
308 # LB 21
309-$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM];     #  . x (BA | HY | NS)
310+$CM* ($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM];     #  . x (BA | HY | NS)
311
312 $CM* [$LB20NonBreaks-$CM] $CM* $BB;                   #  BB x .
313 [^$CB] $CM* $BB;                                      #
314@@ -597,6 +613,8 @@
315 $CM* $OP $CM* ($ALPlus | $NU);
316 $CM* ($ALPlus | $NU) $CM* $CP;
317
318+# (LB 31) Add new rule to prevent the break we do not want, this is the behavior change
319+$CM* $ALPlus ($HY | $HH) $CM* $HL;
320
321 ## -------------------------------------------------
322
323--- source/data/brkitr/brklocal.mk	1969-12-31 16:00:00.000000000 -0800
324+++ source/data/brkitr/brklocal.mk	2011-01-25 17:54:01.149770000 -0800
325@@ -0,0 +1,48 @@
326+# *   Copyright (C) 1998-2010, International Business Machines
327+# *   Corporation and others.  All Rights Reserved.
328+BRK_RES_CLDR_VERSION = 1.9
329+# A list of txt's to build
330+# Note:
331+#
332+#   If you are thinking of modifying this file, READ THIS.
333+#
334+# Instead of changing this file [unless you want to check it back in],
335+# you should consider creating a 'brklocal.mk' file in this same directory.
336+# Then, you can have your local changes remain even if you upgrade or
337+# reconfigure ICU.
338+#
339+# Example 'brklocal.mk' files:
340+#
341+#  * To add an additional locale to the list:
342+#    _____________________________________________________
343+#    |  BRK_RES_SOURCE_LOCAL =   myLocale.txt ...
344+#
345+#  * To REPLACE the default list and only build with a few
346+#    locales:
347+#    _____________________________________________________
348+#    |  BRK_RES_SOURCE = ar.txt ar_AE.txt en.txt de.txt zh.txt
349+#
350+#
351+# Generated by LDML2ICUConverter, from LDML source files.
352+
353+# Aliases without a corresponding xx.xml file (see icu-config.xml & build.xml)
354+BRK_RES_SYNTHETIC_ALIAS =
355+
356+
357+# All aliases (to not be included under 'installed'), but not including root.
358+BRK_RES_ALIAS_SOURCE = $(BRK_RES_SYNTHETIC_ALIAS)
359+
360+
361+# List of compact trie dictionary files (ctd).
362+BRK_CTD_SOURCE = thaidict.txt cjdict.txt
363+
364+
365+# List of break iterator files (brk).
366+# Chrome change: remove word_ja.txt and line_he.txt
367+BRK_SOURCE =  sent_el.txt word_POSIX.txt line_fi.txt char.txt word.txt line.txt sent.txt title.txt char_th.txt
368+
369+
370+# Ordinary resources
371+# Chrome change: remove ja.txt and he.txt
372+BRK_RES_SOURCE = el.txt en.txt en_US.txt en_US_POSIX.txt\
373+ fi.txt   th.txt
374