1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 *******************************************************************************
5 *
6 * Copyright (C) 2009-2016, International Business Machines
7 * Corporation and others. All Rights Reserved.
8 *
9 *******************************************************************************
10 * file name: n2builder.cpp
11 * encoding: UTF-8
12 * tab size: 8 (not used)
13 * indentation:4
14 *
15 * created on: 2009nov25
16 * created by: Markus W. Scherer
17 *
18 * Builds Normalizer2 data and writes a binary .nrm file.
19 * For the file format see source/common/normalizer2impl.h.
20 */
21
22 #include "unicode/utypes.h"
23 #include "n2builder.h"
24
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <vector>
29 #include "unicode/errorcode.h"
30 #include "unicode/localpointer.h"
31 #include "unicode/putil.h"
32 #include "unicode/ucptrie.h"
33 #include "unicode/udata.h"
34 #include "unicode/umutablecptrie.h"
35 #include "unicode/uniset.h"
36 #include "unicode/unistr.h"
37 #include "unicode/usetiter.h"
38 #include "unicode/ustring.h"
39 #include "charstr.h"
40 #include "extradata.h"
41 #include "hash.h"
42 #include "normalizer2impl.h"
43 #include "norms.h"
44 #include "toolutil.h"
45 #include "unewdata.h"
46 #include "uvectr32.h"
47 #include "writesrc.h"
48
49 #if !UCONFIG_NO_NORMALIZATION
50
51 /* UDataInfo cf. udata.h */
52 static UDataInfo dataInfo={
53 sizeof(UDataInfo),
54 0,
55
56 U_IS_BIG_ENDIAN,
57 U_CHARSET_FAMILY,
58 U_SIZEOF_UCHAR,
59 0,
60
61 { 0x4e, 0x72, 0x6d, 0x32 }, /* dataFormat="Nrm2" */
62 { 4, 0, 0, 0 }, /* formatVersion */
63 { 11, 0, 0, 0 } /* dataVersion (Unicode version) */
64 };
65
66 U_NAMESPACE_BEGIN
67
68 class HangulIterator {
69 public:
70 struct Range {
71 UChar32 start, end;
72 };
73
HangulIterator()74 HangulIterator() : rangeIndex(0) {}
nextRange()75 const Range *nextRange() {
76 if(rangeIndex<UPRV_LENGTHOF(ranges)) {
77 return ranges+rangeIndex++;
78 } else {
79 return NULL;
80 }
81 }
82 private:
83 static const Range ranges[4];
84 int32_t rangeIndex;
85 };
86
87 const HangulIterator::Range HangulIterator::ranges[4]={
88 { Hangul::JAMO_L_BASE, Hangul::JAMO_L_END },
89 { Hangul::JAMO_V_BASE, Hangul::JAMO_V_END },
90 // JAMO_T_BASE+1: not U+11A7
91 { Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_END },
92 { Hangul::HANGUL_BASE, Hangul::HANGUL_END },
93 };
94
Normalizer2DataBuilder(UErrorCode & errorCode)95 Normalizer2DataBuilder::Normalizer2DataBuilder(UErrorCode &errorCode) :
96 norms(errorCode),
97 phase(0), overrideHandling(OVERRIDE_PREVIOUS), optimization(OPTIMIZE_NORMAL),
98 norm16TrieBytes(nullptr), norm16TrieLength(0) {
99 memset(unicodeVersion, 0, sizeof(unicodeVersion));
100 memset(indexes, 0, sizeof(indexes));
101 memset(smallFCD, 0, sizeof(smallFCD));
102 }
103
~Normalizer2DataBuilder()104 Normalizer2DataBuilder::~Normalizer2DataBuilder() {
105 delete[] norm16TrieBytes;
106 }
107
108 void
setUnicodeVersion(const char * v)109 Normalizer2DataBuilder::setUnicodeVersion(const char *v) {
110 UVersionInfo nullVersion={ 0, 0, 0, 0 };
111 UVersionInfo version;
112 u_versionFromString(version, v);
113 if( 0!=memcmp(version, unicodeVersion, U_MAX_VERSION_LENGTH) &&
114 0!=memcmp(nullVersion, unicodeVersion, U_MAX_VERSION_LENGTH)
115 ) {
116 char buffer[U_MAX_VERSION_STRING_LENGTH];
117 u_versionToString(unicodeVersion, buffer);
118 fprintf(stderr, "gennorm2 error: multiple inconsistent Unicode version numbers %s vs. %s\n",
119 buffer, v);
120 exit(U_ILLEGAL_ARGUMENT_ERROR);
121 }
122 memcpy(unicodeVersion, version, U_MAX_VERSION_LENGTH);
123 }
124
checkNormForMapping(Norm * p,UChar32 c)125 Norm *Normalizer2DataBuilder::checkNormForMapping(Norm *p, UChar32 c) {
126 if(p!=NULL) {
127 if(p->mappingType!=Norm::NONE) {
128 if( overrideHandling==OVERRIDE_NONE ||
129 (overrideHandling==OVERRIDE_PREVIOUS && p->mappingPhase==phase)
130 ) {
131 fprintf(stderr,
132 "error in gennorm2 phase %d: "
133 "not permitted to override mapping for U+%04lX from phase %d\n",
134 (int)phase, (long)c, (int)p->mappingPhase);
135 exit(U_INVALID_FORMAT_ERROR);
136 }
137 delete p->mapping;
138 p->mapping=NULL;
139 }
140 p->mappingPhase=phase;
141 }
142 return p;
143 }
144
setOverrideHandling(OverrideHandling oh)145 void Normalizer2DataBuilder::setOverrideHandling(OverrideHandling oh) {
146 overrideHandling=oh;
147 ++phase;
148 }
149
setCC(UChar32 c,uint8_t cc)150 void Normalizer2DataBuilder::setCC(UChar32 c, uint8_t cc) {
151 norms.createNorm(c)->cc=cc;
152 norms.ccSet.add(c);
153 }
154
isWellFormed(const UnicodeString & s)155 static UBool isWellFormed(const UnicodeString &s) {
156 UErrorCode errorCode=U_ZERO_ERROR;
157 u_strToUTF8(NULL, 0, NULL, toUCharPtr(s.getBuffer()), s.length(), &errorCode);
158 return U_SUCCESS(errorCode) || errorCode==U_BUFFER_OVERFLOW_ERROR;
159 }
160
setOneWayMapping(UChar32 c,const UnicodeString & m)161 void Normalizer2DataBuilder::setOneWayMapping(UChar32 c, const UnicodeString &m) {
162 if(!isWellFormed(m)) {
163 fprintf(stderr,
164 "error in gennorm2 phase %d: "
165 "illegal one-way mapping from U+%04lX to malformed string\n",
166 (int)phase, (long)c);
167 exit(U_INVALID_FORMAT_ERROR);
168 }
169 Norm *p=checkNormForMapping(norms.createNorm(c), c);
170 p->mapping=new UnicodeString(m);
171 p->mappingType=Norm::ONE_WAY;
172 p->setMappingCP();
173 norms.mappingSet.add(c);
174 }
175
setRoundTripMapping(UChar32 c,const UnicodeString & m)176 void Normalizer2DataBuilder::setRoundTripMapping(UChar32 c, const UnicodeString &m) {
177 if(U_IS_SURROGATE(c)) {
178 fprintf(stderr,
179 "error in gennorm2 phase %d: "
180 "illegal round-trip mapping from surrogate code point U+%04lX\n",
181 (int)phase, (long)c);
182 exit(U_INVALID_FORMAT_ERROR);
183 }
184 if(!isWellFormed(m)) {
185 fprintf(stderr,
186 "error in gennorm2 phase %d: "
187 "illegal round-trip mapping from U+%04lX to malformed string\n",
188 (int)phase, (long)c);
189 exit(U_INVALID_FORMAT_ERROR);
190 }
191 int32_t numCP=u_countChar32(toUCharPtr(m.getBuffer()), m.length());
192 if(numCP!=2) {
193 fprintf(stderr,
194 "error in gennorm2 phase %d: "
195 "illegal round-trip mapping from U+%04lX to %d!=2 code points\n",
196 (int)phase, (long)c, (int)numCP);
197 exit(U_INVALID_FORMAT_ERROR);
198 }
199 Norm *p=checkNormForMapping(norms.createNorm(c), c);
200 p->mapping=new UnicodeString(m);
201 p->mappingType=Norm::ROUND_TRIP;
202 p->mappingCP=U_SENTINEL;
203 norms.mappingSet.add(c);
204 }
205
removeMapping(UChar32 c)206 void Normalizer2DataBuilder::removeMapping(UChar32 c) {
207 // createNorm(c), not getNorm(c), to record a non-mapping and detect conflicting data.
208 Norm *p=checkNormForMapping(norms.createNorm(c), c);
209 p->mappingType=Norm::REMOVED;
210 norms.mappingSet.add(c);
211 }
212
mappingHasCompBoundaryAfter(const BuilderReorderingBuffer & buffer,Norm::MappingType mappingType) const213 UBool Normalizer2DataBuilder::mappingHasCompBoundaryAfter(const BuilderReorderingBuffer &buffer,
214 Norm::MappingType mappingType) const {
215 if(buffer.isEmpty()) {
216 return FALSE; // Maps-to-empty-string is no boundary of any kind.
217 }
218 int32_t lastStarterIndex=buffer.lastStarterIndex();
219 if(lastStarterIndex<0) {
220 return FALSE; // no starter
221 }
222 const int32_t lastIndex=buffer.length()-1;
223 if(mappingType==Norm::ONE_WAY && lastStarterIndex<lastIndex && buffer.ccAt(lastIndex)>1) {
224 // One-way mapping where after the last starter is at least one combining mark
225 // with a combining class greater than 1,
226 // which means that another combining mark can reorder before it.
227 // By contrast, in a round-trip mapping this does not prevent a boundary as long as
228 // the starter or composite does not combine-forward with a following combining mark.
229 return FALSE;
230 }
231 UChar32 starter=buffer.charAt(lastStarterIndex);
232 if(lastStarterIndex==0 && norms.combinesBack(starter)) {
233 // The last starter is at the beginning of the mapping and combines backward.
234 return FALSE;
235 }
236 if(Hangul::isJamoL(starter) ||
237 (Hangul::isJamoV(starter) &&
238 0<lastStarterIndex && Hangul::isJamoL(buffer.charAt(lastStarterIndex-1)))) {
239 // A Jamo leading consonant or an LV pair combines-forward if it is at the end,
240 // otherwise it is blocked.
241 return lastStarterIndex!=lastIndex;
242 }
243 // Note: There can be no Hangul syllable in the fully decomposed mapping.
244
245 // Multiple starters can combine into one.
246 // Look for the first of the last sequence of starters, excluding Jamos.
247 int32_t i=lastStarterIndex;
248 UChar32 c;
249 while(0<i && buffer.ccAt(i-1)==0 && !Hangul::isJamo(c=buffer.charAt(i-1))) {
250 starter=c;
251 --i;
252 }
253 // Compose as far as possible, and see if further compositions with
254 // characters following this mapping are possible.
255 const Norm *starterNorm=norms.getNorm(starter);
256 if(i==lastStarterIndex &&
257 (starterNorm==nullptr || starterNorm->compositions==nullptr)) {
258 return TRUE; // The last starter does not combine forward.
259 }
260 uint8_t prevCC=0;
261 while(++i<buffer.length()) {
262 uint8_t cc=buffer.ccAt(i); // !=0 if after last starter
263 if(i>lastStarterIndex && norms.combinesWithCCBetween(*starterNorm, prevCC, cc)) {
264 // The starter combines with a mark that reorders before the current one.
265 return FALSE;
266 }
267 UChar32 c=buffer.charAt(i);
268 if(starterNorm!=nullptr && (prevCC<cc || prevCC==0) &&
269 norms.getNormRef(c).combinesBack && (starter=starterNorm->combine(c))>=0) {
270 // The starter combines with c into a composite replacement starter.
271 starterNorm=norms.getNorm(starter);
272 if(i>=lastStarterIndex &&
273 (starterNorm==nullptr || starterNorm->compositions==nullptr)) {
274 return TRUE; // The composite does not combine further.
275 }
276 // Keep prevCC because we "removed" the combining mark.
277 } else if(cc==0) {
278 starterNorm=norms.getNorm(c);
279 if(i==lastStarterIndex &&
280 (starterNorm==nullptr || starterNorm->compositions==nullptr)) {
281 return TRUE; // The new starter does not combine forward.
282 }
283 prevCC=0;
284 } else {
285 prevCC=cc;
286 }
287 }
288 if(prevCC==0) {
289 return FALSE; // forward-combining starter at the very end
290 }
291 if(norms.combinesWithCCBetween(*starterNorm, prevCC, 256)) {
292 // The starter combines with another mark.
293 return FALSE;
294 }
295 return TRUE;
296 }
297
mappingRecomposes(const BuilderReorderingBuffer & buffer) const298 UBool Normalizer2DataBuilder::mappingRecomposes(const BuilderReorderingBuffer &buffer) const {
299 if(buffer.lastStarterIndex()<0) {
300 return FALSE; // no starter
301 }
302 const Norm *starterNorm=nullptr;
303 uint8_t prevCC=0;
304 for(int32_t i=0; i<buffer.length(); ++i) {
305 UChar32 c=buffer.charAt(i);
306 uint8_t cc=buffer.ccAt(i);
307 if(starterNorm!=nullptr && (prevCC<cc || prevCC==0) &&
308 norms.getNormRef(c).combinesBack && starterNorm->combine(c)>=0) {
309 return TRUE; // normal composite
310 } else if(cc==0) {
311 if(Hangul::isJamoL(c)) {
312 if((i+1)<buffer.length() && Hangul::isJamoV(buffer.charAt(i+1))) {
313 return TRUE; // Hangul syllable
314 }
315 starterNorm=nullptr;
316 } else {
317 starterNorm=norms.getNorm(c);
318 }
319 }
320 prevCC=cc;
321 }
322 return FALSE;
323 }
324
postProcess(Norm & norm)325 void Normalizer2DataBuilder::postProcess(Norm &norm) {
326 // Prerequisites: Compositions are built, mappings are recursively decomposed.
327 // Mappings are not yet in canonical order.
328 //
329 // This function works on a Norm struct. We do not know which code point(s) map(s) to it.
330 // Therefore, we cannot compute algorithmic mapping deltas here.
331 // Error conditions are checked, but printed later when we do know the offending code point.
332 if(norm.hasMapping()) {
333 if(norm.mapping->length()>Normalizer2Impl::MAPPING_LENGTH_MASK) {
334 norm.error="mapping longer than maximum of 31";
335 return;
336 }
337 // Ensure canonical order.
338 BuilderReorderingBuffer buffer;
339 if(norm.rawMapping!=nullptr) {
340 norms.reorder(*norm.rawMapping, buffer);
341 buffer.reset();
342 }
343 norms.reorder(*norm.mapping, buffer);
344 if(buffer.isEmpty()) {
345 // A character that is deleted (maps to an empty string) must
346 // get the worst-case lccc and tccc values because arbitrary
347 // characters on both sides will become adjacent.
348 norm.leadCC=1;
349 norm.trailCC=0xff;
350 } else {
351 norm.leadCC=buffer.ccAt(0);
352 norm.trailCC=buffer.ccAt(buffer.length()-1);
353 }
354
355 norm.hasCompBoundaryBefore=
356 !buffer.isEmpty() && norm.leadCC==0 && !norms.combinesBack(buffer.charAt(0));
357 norm.hasCompBoundaryAfter=
358 norm.compositions==nullptr && mappingHasCompBoundaryAfter(buffer, norm.mappingType);
359
360 if(norm.combinesBack) {
361 norm.error="combines-back and decomposes, not possible in Unicode normalization";
362 } else if(norm.mappingType==Norm::ROUND_TRIP) {
363 if(norm.compositions!=NULL) {
364 norm.type=Norm::YES_NO_COMBINES_FWD;
365 } else {
366 norm.type=Norm::YES_NO_MAPPING_ONLY;
367 }
368 } else { // one-way mapping
369 if(norm.compositions!=NULL) {
370 norm.error="combines-forward and has a one-way mapping, "
371 "not possible in Unicode normalization";
372 } else if(buffer.isEmpty()) {
373 norm.type=Norm::NO_NO_EMPTY;
374 } else if(!norm.hasCompBoundaryBefore) {
375 norm.type=Norm::NO_NO_COMP_NO_MAYBE_CC;
376 } else if(mappingRecomposes(buffer)) {
377 norm.type=Norm::NO_NO_COMP_BOUNDARY_BEFORE;
378 } else {
379 // The mapping is comp-normalized.
380 norm.type=Norm::NO_NO_COMP_YES;
381 }
382 }
383 } else { // no mapping
384 norm.leadCC=norm.trailCC=norm.cc;
385
386 norm.hasCompBoundaryBefore=
387 norm.cc==0 && !norm.combinesBack;
388 norm.hasCompBoundaryAfter=
389 norm.cc==0 && !norm.combinesBack && norm.compositions==nullptr;
390
391 if(norm.combinesBack) {
392 if(norm.compositions!=nullptr) {
393 // Earlier code checked ccc=0.
394 norm.type=Norm::MAYBE_YES_COMBINES_FWD;
395 } else {
396 norm.type=Norm::MAYBE_YES_SIMPLE; // any ccc
397 }
398 } else if(norm.compositions!=nullptr) {
399 // Earlier code checked ccc=0.
400 norm.type=Norm::YES_YES_COMBINES_FWD;
401 } else if(norm.cc!=0) {
402 norm.type=Norm::YES_YES_WITH_CC;
403 } else {
404 norm.type=Norm::INERT;
405 }
406 }
407 }
408
409 class Norm16Writer : public Norms::Enumerator {
410 public:
Norm16Writer(UMutableCPTrie * trie,Norms & n,Normalizer2DataBuilder & b)411 Norm16Writer(UMutableCPTrie *trie, Norms &n, Normalizer2DataBuilder &b) :
412 Norms::Enumerator(n), builder(b), norm16Trie(trie) {}
rangeHandler(UChar32 start,UChar32 end,Norm & norm)413 void rangeHandler(UChar32 start, UChar32 end, Norm &norm) U_OVERRIDE {
414 builder.writeNorm16(norm16Trie, start, end, norm);
415 }
416 Normalizer2DataBuilder &builder;
417 UMutableCPTrie *norm16Trie;
418 };
419
setSmallFCD(UChar32 c)420 void Normalizer2DataBuilder::setSmallFCD(UChar32 c) {
421 UChar32 lead= c<=0xffff ? c : U16_LEAD(c);
422 smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7);
423 }
424
writeNorm16(UMutableCPTrie * norm16Trie,UChar32 start,UChar32 end,Norm & norm)425 void Normalizer2DataBuilder::writeNorm16(UMutableCPTrie *norm16Trie, UChar32 start, UChar32 end, Norm &norm) {
426 if((norm.leadCC|norm.trailCC)!=0) {
427 for(UChar32 c=start; c<=end; ++c) {
428 setSmallFCD(c);
429 }
430 }
431
432 int32_t norm16;
433 switch(norm.type) {
434 case Norm::INERT:
435 norm16=Normalizer2Impl::INERT;
436 break;
437 case Norm::YES_YES_COMBINES_FWD:
438 norm16=norm.offset*2;
439 break;
440 case Norm::YES_NO_COMBINES_FWD:
441 norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO]+norm.offset*2;
442 break;
443 case Norm::YES_NO_MAPPING_ONLY:
444 norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]+norm.offset*2;
445 break;
446 case Norm::NO_NO_COMP_YES:
447 norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO]+norm.offset*2;
448 break;
449 case Norm::NO_NO_COMP_BOUNDARY_BEFORE:
450 norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE]+norm.offset*2;
451 break;
452 case Norm::NO_NO_COMP_NO_MAYBE_CC:
453 norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_NO_MAYBE_CC]+norm.offset*2;
454 break;
455 case Norm::NO_NO_EMPTY:
456 norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO_EMPTY]+norm.offset*2;
457 break;
458 case Norm::NO_NO_DELTA:
459 {
460 // Positive offset from minNoNoDelta, shifted left for additional bits.
461 int32_t offset=(norm.offset+Normalizer2Impl::MAX_DELTA)<<Normalizer2Impl::DELTA_SHIFT;
462 if(norm.trailCC==0) {
463 // DELTA_TCCC_0==0
464 } else if(norm.trailCC==1) {
465 offset|=Normalizer2Impl::DELTA_TCCC_1;
466 } else {
467 offset|=Normalizer2Impl::DELTA_TCCC_GT_1;
468 }
469 norm16=getMinNoNoDelta()+offset;
470 break;
471 }
472 case Norm::MAYBE_YES_COMBINES_FWD:
473 norm16=indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]+norm.offset*2;
474 break;
475 case Norm::MAYBE_YES_SIMPLE:
476 norm16=Normalizer2Impl::MIN_NORMAL_MAYBE_YES+norm.cc*2; // ccc=0..255
477 break;
478 case Norm::YES_YES_WITH_CC:
479 U_ASSERT(norm.cc!=0);
480 norm16=Normalizer2Impl::MIN_YES_YES_WITH_CC-2+norm.cc*2; // ccc=1..255
481 break;
482 default: // Should not occur.
483 exit(U_INTERNAL_PROGRAM_ERROR);
484 }
485 U_ASSERT((norm16&1)==0);
486 if(norm.hasCompBoundaryAfter) {
487 norm16|=Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER;
488 }
489 IcuToolErrorCode errorCode("gennorm2/writeNorm16()");
490 umutablecptrie_setRange(norm16Trie, start, end, (uint32_t)norm16, errorCode);
491
492 // Set the minimum code points for real data lookups in the quick check loops.
493 UBool isDecompNo=
494 (Norm::YES_NO_COMBINES_FWD<=norm.type && norm.type<=Norm::NO_NO_DELTA) ||
495 norm.cc!=0;
496 if(isDecompNo && start<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) {
497 indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=start;
498 }
499 UBool isCompNoMaybe= norm.type>=Norm::NO_NO_COMP_YES;
500 if(isCompNoMaybe && start<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) {
501 indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=start;
502 }
503 if(norm.leadCC!=0 && start<indexes[Normalizer2Impl::IX_MIN_LCCC_CP]) {
504 indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=start;
505 }
506 }
507
setHangulData(UMutableCPTrie * norm16Trie)508 void Normalizer2DataBuilder::setHangulData(UMutableCPTrie *norm16Trie) {
509 HangulIterator hi;
510 const HangulIterator::Range *range;
511 // Check that none of the Hangul/Jamo code points have data.
512 while((range=hi.nextRange())!=NULL) {
513 for(UChar32 c=range->start; c<=range->end; ++c) {
514 if(umutablecptrie_get(norm16Trie, c)>Normalizer2Impl::INERT) {
515 fprintf(stderr,
516 "gennorm2 error: "
517 "illegal mapping/composition/ccc data for Hangul or Jamo U+%04lX\n",
518 (long)c);
519 exit(U_INVALID_FORMAT_ERROR);
520 }
521 }
522 }
523 // Set data for algorithmic runtime handling.
524 IcuToolErrorCode errorCode("gennorm2/setHangulData()");
525
526 // Jamo V/T are maybeYes
527 if(Hangul::JAMO_V_BASE<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) {
528 indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=Hangul::JAMO_V_BASE;
529 }
530 umutablecptrie_setRange(norm16Trie, Hangul::JAMO_L_BASE, Hangul::JAMO_L_END,
531 Normalizer2Impl::JAMO_L, errorCode);
532 umutablecptrie_setRange(norm16Trie, Hangul::JAMO_V_BASE, Hangul::JAMO_V_END,
533 Normalizer2Impl::JAMO_VT, errorCode);
534 // JAMO_T_BASE+1: not U+11A7
535 umutablecptrie_setRange(norm16Trie, Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_END,
536 Normalizer2Impl::JAMO_VT, errorCode);
537
538 // Hangul LV encoded as minYesNo
539 uint32_t lv=indexes[Normalizer2Impl::IX_MIN_YES_NO];
540 // Hangul LVT encoded as minYesNoMappingsOnly|HAS_COMP_BOUNDARY_AFTER
541 uint32_t lvt=indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]|
542 Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER;
543 if(Hangul::HANGUL_BASE<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) {
544 indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=Hangul::HANGUL_BASE;
545 }
546 // Set the first LV, then write all other Hangul syllables as LVT,
547 // then overwrite the remaining LV.
548 umutablecptrie_set(norm16Trie, Hangul::HANGUL_BASE, lv, errorCode);
549 umutablecptrie_setRange(norm16Trie, Hangul::HANGUL_BASE+1, Hangul::HANGUL_END, lvt, errorCode);
550 UChar32 c=Hangul::HANGUL_BASE;
551 while((c+=Hangul::JAMO_T_COUNT)<=Hangul::HANGUL_END) {
552 umutablecptrie_set(norm16Trie, c, lv, errorCode);
553 }
554 errorCode.assertSuccess();
555 }
556
processData()557 LocalUCPTriePointer Normalizer2DataBuilder::processData() {
558 // Build composition lists before recursive decomposition,
559 // so that we still have the raw, pair-wise mappings.
560 CompositionBuilder compBuilder(norms);
561 norms.enumRanges(compBuilder);
562
563 // Recursively decompose all mappings.
564 Decomposer decomposer(norms);
565 do {
566 decomposer.didDecompose=FALSE;
567 norms.enumRanges(decomposer);
568 } while(decomposer.didDecompose);
569
570 // Set the Norm::Type and other properties.
571 int32_t normsLength=norms.length();
572 for(int32_t i=1; i<normsLength; ++i) {
573 postProcess(norms.getNormRefByIndex(i));
574 }
575
576 // Write the properties, mappings and composition lists to
577 // appropriate parts of the "extra data" array.
578 ExtraData extra(norms, optimization==OPTIMIZE_FAST);
579 norms.enumRanges(extra);
580
581 extraData=extra.yesYesCompositions;
582 indexes[Normalizer2Impl::IX_MIN_YES_NO]=extraData.length()*2;
583 extraData.append(extra.yesNoMappingsAndCompositions);
584 indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]=extraData.length()*2;
585 extraData.append(extra.yesNoMappingsOnly);
586 indexes[Normalizer2Impl::IX_MIN_NO_NO]=extraData.length()*2;
587 extraData.append(extra.noNoMappingsCompYes);
588 indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE]=extraData.length()*2;
589 extraData.append(extra.noNoMappingsCompBoundaryBefore);
590 indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_NO_MAYBE_CC]=extraData.length()*2;
591 extraData.append(extra.noNoMappingsCompNoMaybeCC);
592 indexes[Normalizer2Impl::IX_MIN_NO_NO_EMPTY]=extraData.length()*2;
593 extraData.append(extra.noNoMappingsEmpty);
594 indexes[Normalizer2Impl::IX_LIMIT_NO_NO]=extraData.length()*2;
595
596 // Pad the maybeYesCompositions length to a multiple of 4,
597 // so that NO_NO_DELTA bits 2..1 can be used without subtracting the center.
598 while(extra.maybeYesCompositions.length()&3) {
599 extra.maybeYesCompositions.append((UChar)0);
600 }
601 extraData.insert(0, extra.maybeYesCompositions);
602 indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]=
603 Normalizer2Impl::MIN_NORMAL_MAYBE_YES-
604 extra.maybeYesCompositions.length()*2;
605
606 // Pad to even length for 4-byte alignment of following data.
607 if(extraData.length()&1) {
608 extraData.append((UChar)0);
609 }
610
611 int32_t minNoNoDelta=getMinNoNoDelta();
612 U_ASSERT((minNoNoDelta&7)==0);
613 if(indexes[Normalizer2Impl::IX_LIMIT_NO_NO]>minNoNoDelta) {
614 fprintf(stderr,
615 "gennorm2 error: "
616 "data structure overflow, too much mapping composition data\n");
617 exit(U_BUFFER_OVERFLOW_ERROR);
618 }
619
620 // writeNorm16() and setHangulData() reduce these as needed.
621 indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=0x110000;
622 indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=0x110000;
623 indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=0x110000;
624
625 IcuToolErrorCode errorCode("gennorm2/processData()");
626 UMutableCPTrie *norm16Trie = umutablecptrie_open(
627 Normalizer2Impl::INERT, Normalizer2Impl::INERT, errorCode);
628 errorCode.assertSuccess();
629
630 // Map each code point to its norm16 value,
631 // including the properties that fit directly,
632 // and the offset to the "extra data" if necessary.
633 Norm16Writer norm16Writer(norm16Trie, norms, *this);
634 norms.enumRanges(norm16Writer);
635 // TODO: iterate via getRange() instead of callback?
636
637 setHangulData(norm16Trie);
638
639 // Look for the "worst" norm16 value of any supplementary code point
640 // corresponding to a lead surrogate, and set it as that surrogate's value.
641 // Enables UTF-16 quick check inner loops to look at only code units.
642 //
643 // We could be more sophisticated:
644 // We could collect a bit set for whether there are values in the different
645 // norm16 ranges (yesNo, maybeYes, yesYesWithCC etc.)
646 // and select the best value that only breaks the composition and/or decomposition
647 // inner loops if necessary.
648 // However, that seems like overkill for an optimization for supplementary characters.
649 //
650 // First check that surrogate code *points* are inert.
651 // The parser should have rejected values/mappings for them.
652 uint32_t value;
653 UChar32 end = umutablecptrie_getRange(norm16Trie, 0xd800, UCPMAP_RANGE_NORMAL, 0,
654 nullptr, nullptr, &value);
655 if (value != Normalizer2Impl::INERT || end < 0xdfff) {
656 fprintf(stderr,
657 "gennorm2 error: not all surrogate code points are inert: U+d800..U+%04x=%lx\n",
658 (int)end, (long)value);
659 exit(U_INTERNAL_PROGRAM_ERROR);
660 }
661 uint32_t maxNorm16 = 0;
662 // ANDing values yields 0 bits where any value has a 0.
663 // Used for worst-case HAS_COMP_BOUNDARY_AFTER.
664 uint32_t andedNorm16 = 0;
665 end = 0;
666 for (UChar32 start = 0x10000;;) {
667 if (start > end) {
668 end = umutablecptrie_getRange(norm16Trie, start, UCPMAP_RANGE_NORMAL, 0,
669 nullptr, nullptr, &value);
670 if (end < 0) { break; }
671 }
672 if ((start & 0x3ff) == 0) {
673 // Data for a new lead surrogate.
674 maxNorm16 = andedNorm16 = value;
675 } else {
676 if (value > maxNorm16) {
677 maxNorm16 = value;
678 }
679 andedNorm16 &= value;
680 }
681 // Intersect each range with the code points for one lead surrogate.
682 UChar32 leadEnd = start | 0x3ff;
683 if (leadEnd <= end) {
684 // End of the supplementary block for a lead surrogate.
685 if (maxNorm16 >= (uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]) {
686 // Set noNo ("worst" value) if it got into "less-bad" maybeYes or ccc!=0.
687 // Otherwise it might end up at something like JAMO_VT which stays in
688 // the inner decomposition quick check loop.
689 maxNorm16 = (uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO];
690 }
691 maxNorm16 =
692 (maxNorm16 & ~Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER)|
693 (andedNorm16 & Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER);
694 if (maxNorm16 != Normalizer2Impl::INERT) {
695 umutablecptrie_set(norm16Trie, U16_LEAD(start), maxNorm16, errorCode);
696 }
697 if (value == Normalizer2Impl::INERT) {
698 // Potentially skip inert supplementary blocks for several lead surrogates.
699 start = (end + 1) & ~0x3ff;
700 } else {
701 start = leadEnd + 1;
702 }
703 } else {
704 start = end + 1;
705 }
706 }
707
708 // Adjust supplementary minimum code points to break quick check loops at their lead surrogates.
709 // For an empty data file, minCP=0x110000 turns into 0xdc00 (first trail surrogate)
710 // which is harmless.
711 // As a result, the minimum code points are always BMP code points.
712 int32_t minCP=indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP];
713 if(minCP>=0x10000) {
714 indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=U16_LEAD(minCP);
715 }
716 minCP=indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP];
717 if(minCP>=0x10000) {
718 indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=U16_LEAD(minCP);
719 }
720 minCP=indexes[Normalizer2Impl::IX_MIN_LCCC_CP];
721 if(minCP>=0x10000) {
722 indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=U16_LEAD(minCP);
723 }
724
725 LocalUCPTriePointer builtTrie(
726 umutablecptrie_buildImmutable(norm16Trie, UCPTRIE_TYPE_FAST, UCPTRIE_VALUE_BITS_16, errorCode));
727 norm16TrieLength=ucptrie_toBinary(builtTrie.getAlias(), nullptr, 0, errorCode);
728 if(errorCode.get()!=U_BUFFER_OVERFLOW_ERROR) {
729 fprintf(stderr, "gennorm2 error: unable to build/serialize the normalization trie - %s\n",
730 errorCode.errorName());
731 exit(errorCode.reset());
732 }
733 umutablecptrie_close(norm16Trie);
734 errorCode.reset();
735 norm16TrieBytes=new uint8_t[norm16TrieLength];
736 ucptrie_toBinary(builtTrie.getAlias(), norm16TrieBytes, norm16TrieLength, errorCode);
737 errorCode.assertSuccess();
738
739 int32_t offset=(int32_t)sizeof(indexes);
740 indexes[Normalizer2Impl::IX_NORM_TRIE_OFFSET]=offset;
741 offset+=norm16TrieLength;
742 indexes[Normalizer2Impl::IX_EXTRA_DATA_OFFSET]=offset;
743 offset+=extraData.length()*2;
744 indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET]=offset;
745 offset+=sizeof(smallFCD);
746 int32_t totalSize=offset;
747 for(int32_t i=Normalizer2Impl::IX_RESERVED3_OFFSET; i<=Normalizer2Impl::IX_TOTAL_SIZE; ++i) {
748 indexes[i]=totalSize;
749 }
750
751 if(beVerbose) {
752 printf("size of normalization trie: %5ld bytes\n", (long)norm16TrieLength);
753 printf("size of 16-bit extra data: %5ld uint16_t\n", (long)extraData.length());
754 printf("size of small-FCD data: %5ld bytes\n", (long)sizeof(smallFCD));
755 printf("size of binary data file contents: %5ld bytes\n", (long)totalSize);
756 printf("minDecompNoCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]);
757 printf("minCompNoMaybeCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]);
758 printf("minLcccCodePoint: U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_LCCC_CP]);
759 printf("minYesNo: (with compositions) 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO]);
760 printf("minYesNoMappingsOnly: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]);
761 printf("minNoNo: (comp-normalized) 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO]);
762 printf("minNoNoCompBoundaryBefore: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE]);
763 printf("minNoNoCompNoMaybeCC: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO_COMP_NO_MAYBE_CC]);
764 printf("minNoNoEmpty: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO_EMPTY]);
765 printf("limitNoNo: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]);
766 printf("minNoNoDelta: 0x%04x\n", (int)minNoNoDelta);
767 printf("minMaybeYes: 0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]);
768 }
769
770 UVersionInfo nullVersion={ 0, 0, 0, 0 };
771 if(0==memcmp(nullVersion, unicodeVersion, 4)) {
772 u_versionFromString(unicodeVersion, U_UNICODE_VERSION);
773 }
774 memcpy(dataInfo.dataVersion, unicodeVersion, 4);
775 return builtTrie;
776 }
777
writeBinaryFile(const char * filename)778 void Normalizer2DataBuilder::writeBinaryFile(const char *filename) {
779 processData();
780
781 IcuToolErrorCode errorCode("gennorm2/writeBinaryFile()");
782 UNewDataMemory *pData=
783 udata_create(NULL, NULL, filename, &dataInfo,
784 haveCopyright ? U_COPYRIGHT_STRING : NULL, errorCode);
785 if(errorCode.isFailure()) {
786 fprintf(stderr, "gennorm2 error: unable to create the output file %s - %s\n",
787 filename, errorCode.errorName());
788 exit(errorCode.reset());
789 }
790 udata_writeBlock(pData, indexes, sizeof(indexes));
791 udata_writeBlock(pData, norm16TrieBytes, norm16TrieLength);
792 udata_writeUString(pData, toUCharPtr(extraData.getBuffer()), extraData.length());
793 udata_writeBlock(pData, smallFCD, sizeof(smallFCD));
794 int32_t writtenSize=udata_finish(pData, errorCode);
795 if(errorCode.isFailure()) {
796 fprintf(stderr, "gennorm2: error %s writing the output file\n", errorCode.errorName());
797 exit(errorCode.reset());
798 }
799 int32_t totalSize=indexes[Normalizer2Impl::IX_TOTAL_SIZE];
800 if(writtenSize!=totalSize) {
801 fprintf(stderr, "gennorm2 error: written size %ld != calculated size %ld\n",
802 (long)writtenSize, (long)totalSize);
803 exit(U_INTERNAL_PROGRAM_ERROR);
804 }
805 }
806
807 void
writeCSourceFile(const char * filename)808 Normalizer2DataBuilder::writeCSourceFile(const char *filename) {
809 LocalUCPTriePointer norm16Trie = processData();
810
811 IcuToolErrorCode errorCode("gennorm2/writeCSourceFile()");
812 const char *basename=findBasename(filename);
813 CharString path(filename, (int32_t)(basename-filename), errorCode);
814 CharString dataName(basename, errorCode);
815 const char *extension=strrchr(basename, '.');
816 if(extension!=NULL) {
817 dataName.truncate((int32_t)(extension-basename));
818 }
819 const char *name=dataName.data();
820 errorCode.assertSuccess();
821
822 FILE *f=usrc_create(path.data(), basename, 2016, "icu/source/tools/gennorm2/n2builder.cpp");
823 if(f==NULL) {
824 fprintf(stderr, "gennorm2/writeCSourceFile() error: unable to create the output file %s\n",
825 filename);
826 exit(U_FILE_ACCESS_ERROR);
827 }
828 fputs("#ifdef INCLUDED_FROM_NORMALIZER2_CPP\n\n", f);
829
830 char line[100];
831 sprintf(line, "static const UVersionInfo %s_formatVersion={", name);
832 usrc_writeArray(f, line, dataInfo.formatVersion, 8, 4, "};\n");
833 sprintf(line, "static const UVersionInfo %s_dataVersion={", name);
834 usrc_writeArray(f, line, dataInfo.dataVersion, 8, 4, "};\n\n");
835 sprintf(line, "static const int32_t %s_indexes[Normalizer2Impl::IX_COUNT]={\n", name);
836 usrc_writeArray(f, line, indexes, 32, Normalizer2Impl::IX_COUNT, "\n};\n\n");
837
838 usrc_writeUCPTrie(f, name, norm16Trie.getAlias());
839
840 sprintf(line, "static const uint16_t %s_extraData[%%ld]={\n", name);
841 usrc_writeArray(f, line, extraData.getBuffer(), 16, extraData.length(), "\n};\n\n");
842 sprintf(line, "static const uint8_t %s_smallFCD[%%ld]={\n", name);
843 usrc_writeArray(f, line, smallFCD, 8, sizeof(smallFCD), "\n};\n\n");
844
845 fputs("#endif // INCLUDED_FROM_NORMALIZER2_CPP\n", f);
846 fclose(f);
847 }
848
849 namespace {
850
equalStrings(const UnicodeString * s1,const UnicodeString * s2)851 bool equalStrings(const UnicodeString *s1, const UnicodeString *s2) {
852 if(s1 == nullptr) {
853 return s2 == nullptr;
854 } else if(s2 == nullptr) {
855 return false;
856 } else {
857 return *s1 == *s2;
858 }
859 }
860
861 const char *typeChars = "?-=>";
862
writeMapping(FILE * f,const UnicodeString * m)863 void writeMapping(FILE *f, const UnicodeString *m) {
864 if(m != nullptr && !m->isEmpty()) {
865 int32_t i = 0;
866 UChar32 c = m->char32At(i);
867 fprintf(f, "%04lX", (long)c);
868 while((i += U16_LENGTH(c)) < m->length()) {
869 c = m->char32At(i);
870 fprintf(f, " %04lX", (long)c);
871 }
872 }
873 fputs("\n", f);
874 }
875
876 } // namespace
877
878 void
writeDataFile(const char * filename,bool writeRemoved) const879 Normalizer2DataBuilder::writeDataFile(const char *filename, bool writeRemoved) const {
880 // Do not processData() before writing the input-syntax data file.
881 FILE *f = fopen(filename, "w");
882 if(f == nullptr) {
883 fprintf(stderr, "gennorm2/writeDataFile() error: unable to create the output file %s\n",
884 filename);
885 exit(U_FILE_ACCESS_ERROR);
886 return;
887 }
888
889 if(unicodeVersion[0] != 0 || unicodeVersion[1] != 0 ||
890 unicodeVersion[2] != 0 || unicodeVersion[3] != 0) {
891 char uv[U_MAX_VERSION_STRING_LENGTH];
892 u_versionToString(unicodeVersion, uv);
893 fprintf(f, "* Unicode %s\n\n", uv);
894 }
895
896 UnicodeSetIterator ccIter(norms.ccSet);
897 UChar32 start = U_SENTINEL;
898 UChar32 end = U_SENTINEL;
899 uint8_t prevCC = 0;
900 bool done = false;
901 bool didWrite = false;
902 do {
903 UChar32 c;
904 uint8_t cc;
905 if(ccIter.next() && !ccIter.isString()) {
906 c = ccIter.getCodepoint();
907 cc = norms.getCC(c);
908 } else {
909 c = 0x110000;
910 cc = 0;
911 done = true;
912 }
913 if(cc == prevCC && c == (end + 1)) {
914 end = c;
915 } else {
916 if(prevCC != 0) {
917 if(start == end) {
918 fprintf(f, "%04lX:%d\n", (long)start, (int)prevCC);
919 } else {
920 fprintf(f, "%04lX..%04lX:%d\n", (long)start, (long)end, (int)prevCC);
921 }
922 didWrite = true;
923 }
924 start = end = c;
925 prevCC = cc;
926 }
927 } while(!done);
928 if(didWrite) {
929 fputs("\n", f);
930 }
931
932 UnicodeSetIterator mIter(norms.mappingSet);
933 start = U_SENTINEL;
934 end = U_SENTINEL;
935 const UnicodeString *prevMapping = nullptr;
936 Norm::MappingType prevType = Norm::NONE;
937 done = false;
938 do {
939 UChar32 c;
940 const Norm *norm;
941 if(mIter.next() && !mIter.isString()) {
942 c = mIter.getCodepoint();
943 norm = norms.getNorm(c);
944 } else {
945 c = 0x110000;
946 norm = nullptr;
947 done = true;
948 }
949 const UnicodeString *mapping;
950 Norm::MappingType type;
951 if(norm == nullptr) {
952 mapping = nullptr;
953 type = Norm::NONE;
954 } else {
955 type = norm->mappingType;
956 if(type == Norm::NONE) {
957 mapping = nullptr;
958 } else {
959 mapping = norm->mapping;
960 }
961 }
962 if(type == prevType && equalStrings(mapping, prevMapping) && c == (end + 1)) {
963 end = c;
964 } else {
965 if(writeRemoved ? prevType != Norm::NONE : prevType > Norm::REMOVED) {
966 if(start == end) {
967 fprintf(f, "%04lX%c", (long)start, typeChars[prevType]);
968 } else {
969 fprintf(f, "%04lX..%04lX%c", (long)start, (long)end, typeChars[prevType]);
970 }
971 writeMapping(f, prevMapping);
972 }
973 start = end = c;
974 prevMapping = mapping;
975 prevType = type;
976 }
977 } while(!done);
978
979 fclose(f);
980 }
981
982 void
computeDiff(const Normalizer2DataBuilder & b1,const Normalizer2DataBuilder & b2,Normalizer2DataBuilder & diff)983 Normalizer2DataBuilder::computeDiff(const Normalizer2DataBuilder &b1,
984 const Normalizer2DataBuilder &b2,
985 Normalizer2DataBuilder &diff) {
986 // Compute diff = b1 - b2
987 // so that we should be able to get b1 = b2 + diff.
988 if(0 != memcmp(b1.unicodeVersion, b2.unicodeVersion, U_MAX_VERSION_LENGTH)) {
989 memcpy(diff.unicodeVersion, b1.unicodeVersion, U_MAX_VERSION_LENGTH);
990 }
991
992 UnicodeSet ccSet(b1.norms.ccSet);
993 ccSet.addAll(b2.norms.ccSet);
994 UnicodeSetIterator ccIter(ccSet);
995 while(ccIter.next() && !ccIter.isString()) {
996 UChar32 c = ccIter.getCodepoint();
997 uint8_t cc1 = b1.norms.getCC(c);
998 uint8_t cc2 = b2.norms.getCC(c);
999 if(cc1 != cc2) {
1000 diff.setCC(c, cc1);
1001 }
1002 }
1003
1004 UnicodeSet mSet(b1.norms.mappingSet);
1005 mSet.addAll(b2.norms.mappingSet);
1006 UnicodeSetIterator mIter(mSet);
1007 while(mIter.next() && !mIter.isString()) {
1008 UChar32 c = mIter.getCodepoint();
1009 const Norm *norm1 = b1.norms.getNorm(c);
1010 const Norm *norm2 = b2.norms.getNorm(c);
1011 const UnicodeString *mapping1;
1012 Norm::MappingType type1;
1013 if(norm1 == nullptr || !norm1->hasMapping()) {
1014 mapping1 = nullptr;
1015 type1 = Norm::NONE;
1016 } else {
1017 mapping1 = norm1->mapping;
1018 type1 = norm1->mappingType;
1019 }
1020 const UnicodeString *mapping2;
1021 Norm::MappingType type2;
1022 if(norm2 == nullptr || !norm2->hasMapping()) {
1023 mapping2 = nullptr;
1024 type2 = Norm::NONE;
1025 } else {
1026 mapping2 = norm2->mapping;
1027 type2 = norm2->mappingType;
1028 }
1029 if(type1 == type2 && equalStrings(mapping1, mapping2)) {
1030 // Nothing to do.
1031 } else if(type1 == Norm::NONE) {
1032 diff.removeMapping(c);
1033 } else if(type1 == Norm::ROUND_TRIP) {
1034 diff.setRoundTripMapping(c, *mapping1);
1035 } else if(type1 == Norm::ONE_WAY) {
1036 diff.setOneWayMapping(c, *mapping1);
1037 }
1038 }
1039 }
1040
1041 U_NAMESPACE_END
1042
1043 #endif /* #if !UCONFIG_NO_NORMALIZATION */
1044
1045 /*
1046 * Hey, Emacs, please set the following:
1047 *
1048 * Local Variables:
1049 * indent-tabs-mode: nil
1050 * End:
1051 */
1052