// © 2017 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html /* ******************************************************************************* * * Copyright (C) 2004-2016, International Business Machines * Corporation and others. All Rights Reserved. * ******************************************************************************* * file name: bidipropsbuilder.cpp (was genbidi/store.c) * encoding: US-ASCII * tab size: 8 (not used) * indentation:4 * * created on: 2004dec30 * created by: Markus W. Scherer * * Store Unicode bidi/shaping properties efficiently for * random access. */ #include #include #include "unicode/utypes.h" #include "unicode/uchar.h" #include "unicode/uniset.h" #include "cmemory.h" #include "cstring.h" #include "ppucd.h" #include "uarrsort.h" #include "unicode/udata.h" #include "unewdata.h" #include "utrie2.h" #include "writesrc.h" #include "ubidi_props.h" #include "genprops.h" /* Unicode bidi/shaping properties file format --------------------------------- The file format prepared and written here contains several data structures that store indexes or data. Before the data contents described below, there are the headers required by the udata API for loading ICU data. Especially, a UDataInfo structure precedes the actual data. It contains platform properties values and the file format version. The following is a description of format version 2.2 . The file contains the following structures: const int32_t indexes[i0] with values i0, i1, ...: (see UBIDI_IX_... constants for names of indexes) i0 indexLength; -- length of indexes[] (UBIDI_IX_TOP) i1 dataLength; -- length in bytes of the post-header data (incl. indexes[]) i2 trieSize; -- size in bytes of the bidi/shaping properties trie i3 mirrorLength; -- length in uint32_t of the bidi mirroring array i4 jgStart; -- first code point with Joining_Group data i5 jgLimit; -- limit code point for Joining_Group data -- i6, i7 new in format version 2.2: i6 jgStart2; -- first code point with Joining_Group data, second range i7 jgLimit2; -- limit code point for Joining_Group data, second range i8..i14 reservedIndexes; -- reserved values; 0 for now i15 maxValues; -- maximum code values for enumerated properties bits 23..16 contain the max value for Joining_Group, otherwise the bits are used like enum fields in the trie word Serialized trie, see utrie2.h; const uint32_t mirrors[mirrorLength]; const uint8_t jgArray[i5-i4]; -- (i5-i4)+(i7-i6) is always a multiple of 4 const uint8_t jgArray2[i7-i6]; -- new in format version 2.2 Trie data word: Bits 15..13 signed delta to bidi mirroring code point (add delta to input code point) 0 no such code point (source maps to itself) -3..-1, 1..3 delta -4 look in mirrors table 12 is mirrored 11 Bidi_Control 10 Join_Control 9.. 8 Bidi_Paired_Bracket_Type(bpt) -- new in format version 2.1 7.. 5 Joining_Type 4.. 0 BiDi category Mirrors: Stores some of the bidi mirroring data, where each code point maps to at most one other. Most code points do not have a mirroring code point; most that do have a signed delta stored in the trie data value. Only those where the delta does not fit into the trie data are stored in this table. Logically, this is a two-column table with source and mirror code points. Physically, the table is compressed by taking advantage of the fact that each mirror code point is also a source code point (each of them is a mirror of the other). Therefore, both logical columns contain the same set of code points, which needs to be stored only once. The table stores source code points, and also for each the index of its mirror code point in the same table, in a simple array of uint32_t. Bits 31..21 index to mirror code point (unsigned) 20.. 0 source code point The table is sorted by source code points. Joining_Group array: The Joining_Group values do not fit into the 16-bit trie, but the data is also limited to a small range of code points (Arabic and Syriac) and not well compressible. The start and limit code points for the range are stored in the indexes[] array, and the jgArray[] stores a byte for each of these code points, containing the Joining_Group value. All code points outside of this range have No_Joining_Group (0). ICU 54 adds jgArray2[] for a second range. --- Changes in format version 2.2 --- Addition of second range for Joining_Group values (i6, i7), for 10800..10FFF, including Unicode 7.0 10AC0..10AFF Manichaean. --- Changes in format version 2.1 --- Addition of Bidi_Paired_Bracket_Type(bpt) values. (Trie data bits 9..8 were reserved.) --- Changes in format version 2 --- Change from UTrie to UTrie2. ----------------------------------------------------------------------------- */ U_NAMESPACE_USE /* UDataInfo cf. udata.h */ static UDataInfo dataInfo={ sizeof(UDataInfo), 0, U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, U_SIZEOF_UCHAR, 0, /* dataFormat="BiDi" */ { UBIDI_FMT_0, UBIDI_FMT_1, UBIDI_FMT_2, UBIDI_FMT_3 }, { 2, 2, 0, 0 }, /* formatVersion */ { 6, 0, 0, 0 } /* dataVersion */ }; /* -------------------------------------------------------------------------- */ class BiDiPropsBuilder : public PropsBuilder { public: BiDiPropsBuilder(UErrorCode &errorCode); virtual ~BiDiPropsBuilder(); virtual void setUnicodeVersion(const UVersionInfo version); virtual void setProps(const UniProps &, const UnicodeSet &newValues, UErrorCode &errorCode); virtual void build(UErrorCode &errorCode); virtual void writeCSourceFile(const char *path, UErrorCode &errorCode); virtual void writeBinaryData(const char *path, UBool withCopyright, UErrorCode &errorCode); private: int32_t encodeBidiMirroringGlyph(UChar32 src, UChar32 end, UChar32 mirror, UErrorCode &errorCode); void makeMirror(UErrorCode &errorCode); static const UChar32 MIN_JG_START=0x600; static const UChar32 MAX_JG_LIMIT=0x8ff+1; static const UChar32 MIN_JG_START2=0x10800; static const UChar32 MAX_JG_LIMIT2=0x10fff+1; UnicodeSet relevantProps; UTrie2 *pTrie; uint8_t jgArray[MAX_JG_LIMIT-MIN_JG_START]; uint8_t jgArray2[MAX_JG_LIMIT2-MIN_JG_START2]; uint32_t mirrors[UBIDI_MAX_MIRROR_INDEX+1][2]; int32_t mirrorTop; }; BiDiPropsBuilder::BiDiPropsBuilder(UErrorCode &errorCode) : pTrie(NULL), mirrorTop(0) { // This builder encodes the following properties. relevantProps. add(UCHAR_BIDI_CONTROL). add(UCHAR_BIDI_MIRRORED). add(UCHAR_BIDI_CLASS). add(UCHAR_BIDI_MIRRORING_GLYPH). add(UCHAR_JOIN_CONTROL). add(UCHAR_JOINING_GROUP). add(UCHAR_JOINING_TYPE); pTrie=utrie2_open(0, 0, &errorCode); if(U_FAILURE(errorCode)) { fprintf(stderr, "genprops error: bidipropsbuilder utrie2_open() failed - %s\n", u_errorName(errorCode)); } uprv_memset(jgArray, U_JG_NO_JOINING_GROUP, sizeof(jgArray)); uprv_memset(jgArray2, U_JG_NO_JOINING_GROUP, sizeof(jgArray2)); } BiDiPropsBuilder::~BiDiPropsBuilder() { utrie2_close(pTrie); } void BiDiPropsBuilder::setUnicodeVersion(const UVersionInfo version) { uprv_memcpy(dataInfo.dataVersion, version, 4); } /* bidi mirroring table ----------------------------------------------------- */ int32_t BiDiPropsBuilder::encodeBidiMirroringGlyph(UChar32 src, UChar32 end, UChar32 mirror, UErrorCode &errorCode) { if(U_FAILURE(errorCode) || mirror<0) { return 0; } if(src!=end) { fprintf(stderr, "genprops error: range U+%04lX..U+%04lX all with the same " "Bidi_Mirroring_Glyph U+%04lX\n", (long)src, (long)end, (long)mirror); errorCode=U_ILLEGAL_ARGUMENT_ERROR; return 0; } int32_t delta=mirror-src; if(delta==0) { return 0; /* mapping to self=no mapping */ } if(delta else bpb=bmg\n"); return; } int32_t delta=encodeBidiMirroringGlyph(start, end, props.bmg, errorCode); uint32_t value=(uint32_t)delta<0x1fffff) { continue; /* this entry already has an index */ } /* search for the mirror code point in the source column */ int32_t start, limit, step; if(c%04lx->?\n", (long)mirrors[i][0], (long)mirrors[i][1]); errorCode=U_ILLEGAL_ARGUMENT_ERROR; } if(c==mirrors[j][0]) { /* * found the mirror code point c in the source column, * set both entries' indexes to each other */ if(UBIDI_GET_MIRROR_CODE_POINT(mirrors[i][0])!=UBIDI_GET_MIRROR_CODE_POINT(mirrors[j][1])) { /* roundtrip check fails */ fprintf(stderr, "genprops error: bidi mirrors do not roundtrip - %04lx->%04lx->%04lx\n", (long)mirrors[i][0], (long)mirrors[i][1], (long)mirrors[j][1]); errorCode=U_ILLEGAL_ARGUMENT_ERROR; } else { mirrors[i][1]|=(uint32_t)j<