1<!-- © 2019 and later: Unicode, Inc. and others. 2 License & terms of use: http://www.unicode.org/copyright.html --> 3 4<!--================================================================================ 5 Setup: 6 Follow the installation instructions in README.txt in this directory. 7 8 To build ICU data files: 9 1: Determine the CLDR base directory and set the CLDR_DIR environment variable. 10 2: Determine the flags required (see the list of properties below). 11 3: Run: ant -f build-icu-data.xml -D<flag-name>=<flag-value>... 12 ================================================================================--> 13<!-- TODO: Add things like copying of a template directory and deleting previous files 14 (perhaps always generate into a temporary directory and copy back to avoid having 15 inconsistent state when the conversion is cancelled). --> 16<project name="Convert" default="all" basedir="."> 17 18 <target name="all" depends="init-args, prepare-jar, clean, convert"/> 19 20 <!-- Initialize the properties which were not already set on the command line. --> 21 <target name="init-args"> 22 <property environment="env"/> 23 <!-- Inherit properties from environment variable unless specified. As usual 24 with Ant, this is messier than it should be. All we are saying here is: 25 "Use the property if explicitly set, otherwise use the environment variable." 26 We cannot just set the property to the environment variable, since expansion 27 fails for non existant properties, and you are left with a literal value of 28 "${env.CLDR_DATA_DIR}". --> 29 <condition property="cldrDataDir" value="${env.CLDR_DATA_DIR}"> 30 <isset property="env.CLDR_DATA_DIR"/> 31 </condition> 32 <fail unless="cldrDataDir" 33 message="Set the CLDR_DATA_DIR environment variable (or cldrDataDir property) to the CLDR data directory (typically ending in '/production')"/> 34 35 <!-- Ant does not inherit this from the user's environment (and it can matter). 36 This is only needed because we have to "exec" a new Ant task below. --> 37 <condition property="javaHome" value="${env.JAVA_HOME}"> 38 <isset property="env.JAVA_HOME"/> 39 </condition> 40 41 <!-- The output directory into which to write the converted ICU data. By default 42 this will overwrite (without deletion) the ICU data files in this ICU release, 43 so it is recommended that for testing, it be set to another value. --> 44 <property name="outDir" value="${basedir}/../../../icu4c/source/data/"/> 45 46 <!-- The directory in which the additional ICU XML data is stored. --> 47 <property name="specialsDir" value="${basedir}/../../../icu4c/source/data/xml"/> 48 49 <!-- Default value for ICU version (icuver.txt). Update this for each release. --> 50 <property name="icuVersion" value="68.2.0.0"/> 51 52 <!-- Default value for ICU data version (icuver.txt). Update this for each release. --> 53 <property name="icuDataVersion" value="68.2.0.0"/> 54 55 <!-- An override for the CLDR version string (icuver.txt and others). This will be 56 extracted from the CLDR library used for building the data if not set here. --> 57 <property name="cldrVersion" value=""/> 58 59 <!-- The minimum draft status for CLDR data to be used in the conversion. See 60 CldrDraftStatus for more details. --> 61 <property name="minDraftStatus" value="contributed"/> 62 63 <!-- A regular expression to match the locale IDs to be generated (useful for 64 debugging specific regions). This is applied after locale ID specifications 65 have been expanded into full locale IDs, so the value "en" will NOT match 66 "en_GB" or "en_001" etc. --> 67 <property name="localeIdFilter" value=""/> 68 69 <!-- Whether to synthetically generate "pseudo locale" data ("en_XA" and "ar_XB"). --> 70 <property name="includePseudoLocales" value="false"/> 71 72 <!-- Whether to emit a debug report containing some possibly useful information after 73 the conversion has finished. --> 74 <!-- TODO: Currently this isn't hugely useful, so find out what people want. --> 75 <property name="emitReport" value="false"/> 76 77 <!-- List of output "types" to be generated (e.g. "rbnf,plurals,locales"); an empty 78 list means "build everything". 79 80 Note that the grouping of types is based on the legacy converter behaviour and 81 is not always directly associated with an output directory (e.g. "locales" 82 produces locale data for curr/, lang/, main/, region/, unit/, zone/ but NOT 83 coll/, brkitr/ or rbnf/). 84 85 Pass in the value "HELP" (or any invalid value) to see the full list of types. --> 86 <!-- TODO: Find out what common use cases are and use them. --> 87 <property name="outputTypes" value=""/> 88 89 <!-- Override to force the 'clean' task to delete files it cannot determine to be 90 auto-generated by this tool. This is useful if the file header changes since 91 the heading is what's used to recognize auto-generated files. --> 92 <property name="forceDelete" value="false"/> 93 </target> 94 95 <!-- Build a standalone JAR which is called by Ant (and which avoids needing to mess 96 about making Ant know the Maven class-path). --> 97 <target name="prepare-jar" depends="init-args"> 98 <exec executable="mvn" searchpath="true" failonerror="true"> 99 <arg value="compile"/> 100 </exec> 101 </target> 102 103 <!-- Somewhat hacky wrapper target which invokes the real conversion task. 104 This is done so we can set the environment variable of the new process and 105 effectively overwrite the CLDR_DIR value. If ever the CLDR library doesn't 106 need to use CLDR_DIR at runtime to find the production data, this can all be 107 removed. --> 108 <target name="convert" depends="init-args, prepare-jar"> 109 <exec executable="ant" searchpath="true" failonerror="true"> 110 <!-- The CLDR library wants CLDR_DIR set, to the data directory. --> 111 <env key="CLDR_DIR" value="${cldrDataDir}" /> 112 <!-- Force inherit JAVA_HOME (this can be important). --> 113 <env key="JAVA_HOME" value="${javaHome}" /> 114 <!-- Initial Ant command line with all the "interesting" bit in. --> 115 <arg line="-f build-icu-data.xml convert-impl -DcldrDir=${cldrDataDir}"/> 116 <!-- List all properties in the "convert-impl" task (except cldrDir). --> 117 <arg value="-DoutDir=${outDir}"/> 118 <arg value="-DspecialsDir=${specialsDir}"/> 119 <arg value="-DoutputTypes=${outputTypes}"/> 120 <arg value="-DicuVersion=${icuVersion}"/> 121 <arg value="-DicuDataVersion=${icuDataVersion}"/> 122 <arg value="-DcldrVersion=${cldrVersion}"/> 123 <arg value="-DminDraftStatus=${minDraftStatus}"/> 124 <arg value="-DlocaleIdFilter=${localeIdFilter}"/> 125 <arg value="-DincludePseudoLocales=${includePseudoLocales}"/> 126 <arg value="-DemitReport=${emitReport}"/> 127 </exec> 128 </target> 129 130 <!-- Do the actual CLDR data conversion, based on the command line arguments, built in 131 default properties and the configuration in the "<convert>" element below. --> 132 <target name="convert-impl"> 133 <taskdef name="convert" classname="org.unicode.icu.tool.cldrtoicu.ant.ConvertIcuDataTask"> 134 <classpath> 135 <pathelement path="target/cldr-to-icu-1.0-SNAPSHOT-jar-with-dependencies.jar"/> 136 </classpath> 137 </taskdef> 138 <convert cldrDir="${cldrDir}" outputDir="${outDir}" specialsDir="${specialsDir}" 139 outputTypes="${outputTypes}" cldrVersion="${cldrVersion}" 140 icuVersion="${icuVersion}" icuDataVersion="${icuDataVersion}" 141 minimalDraftStatus="${minDraftStatus}" localeIdFilter="${localeIdFilter}" 142 includePseudoLocales="${includePseudoLocales}" emitReport="${emitReport}"> 143 144 <!-- The primary set of locale IDs to be generated by default. The IDs in this list are 145 automatically expanded to include default scripts and all available regions. The 146 rules are: 147 148 1) Base languages are expanded to include default scripts (e.g. "en" -> "en_Latn"). 149 2) All region and variant subtags are added for any base language or language+script 150 (e.g. "en" -> "en_GB" or "shi_Latn" -> "shi_Latn_MA"). 151 152 If a non-default script is desired it should be listed explicitly (e.g. "sr_Latn"). 153 154 Locale IDs with deprecated subtags (which become aliases) must still be listed in 155 full (e.g. "en_RH" or "sr_Latn_YU"). 156 --> 157 <localeIds> 158 // A 159 af, agq, ak, am, ar, ars, as, asa, ast, az, az_AZ, az_Cyrl 160 161 // B 162 bas, be, bem, bez, bg, bm, bn, bo, br, brx, bs, bs_BA, bs_Cyrl 163 164 // C 165 ca, ccp, ce, ceb, cgg, chr, ckb, cs, cy 166 167 // D 168 da, dav, de, dje, doi, dsb, dua, dyo, dz 169 170 // E 171 ebu, ee, el, en, en_NH, en_RH, eo, es, et, eu, ewo 172 173 // F 174 fa, ff, ff_Adlm, ff_CM, ff_GN, ff_MR, ff_SN, fi, fil, fo, fr, fur, fy 175 176 // G 177 ga, gd, gl, gsw, gu, guz, gv 178 179 // H 180 ha, haw, he, hi, hr, hsb, hu, hy 181 182 // I 183 ia, id, ig, ii, in, in_ID, is, it, iw, iw_IL 184 185 // J 186 ja, jgo, jmc, jv 187 188 // K 189 ka, kab, kam, kde, kea, khq, ki, kk, kkj, kl, kln, km, kn, ko, kok, ks 190 ks_IN, ksb, ksf, ksh, ku, kw, ky 191 192 // L 193 lag, lb, lg, lkt, ln, lo, lrc, lt, lu, luo, luy, lv 194 195 // M 196 mai, mas, mer, mfe, mg, mgh, mgo, mi, mk, ml, mn, mni, mni_IN, mo, mr, ms 197 mt, mua, my, mzn 198 199 // N 200 naq, nb, nd, ne, nl, nmg, nn, nnh, no, no_NO, no_NO_NY, nus, nyn 201 202 // O 203 om, or, os 204 205 // P 206 pa, pa_Arab, pa_IN, pa_PK, pcm, pl, ps, pt 207 208 // Q 209 qu 210 211 // R 212 rm, rn, ro, rof, ru, rw, rwk 213 214 // S 215 sa, sah, saq, sat, sat_IN, sbp, sd, sd_Deva, sd_PK, se, seh, ses, sg, sh, sh_BA, sh_CS, sh_YU 216 shi, shi_Latn, shi_MA, si, sk, sl, smn, sn, so, sq, sr, sr_BA, sr_CS, sr_Cyrl_CS, sr_Cyrl_YU, sr_Latn 217 sr_Latn_CS, sr_Latn_YU, sr_ME, sr_RS, sr_XK, sr_YU, su, su_ID, sv, sw 218 219 // T 220 ta, te, teo, tg, th, ti, tk, tl, tl_PH, to, tr, tt, twq, tzm 221 222 // U 223 ug, uk, ur, uz, uz_AF, uz_Arab, uz_Cyrl, uz_UZ 224 225 // V 226 vai, vai_LR, vai_Latn, vi, vun 227 228 // W 229 wae, wo 230 231 // X 232 xh, xog 233 234 // Y 235 yav, yi, yo, yue, yue_CN, yue_HK, yue_Hans 236 237 // Z 238 zgh, zh, zh_CN, zh_HK, zh_Hant, zh_MO, zh_SG, zh_TW, zu 239 </localeIds> 240 241 <!-- The following elements configure directories in which a subset of the available 242 locales IDs should be generated. Unlike the main <localeId> element, these 243 filters must specify all locale IDs in full (but since they mostly select base 244 languages, this isn't a big deal). 245 246 As well as allowing some data directories to have a subset of available data (via 247 the <localeIds> element) there are also mechanisms for controlling aliasing and 248 the locale parent relation which allows the sharing of some ICU data in cases 249 where it would otherwise need to be copied. The two mechanisms are: 250 251 1: inheritLanguageSubtag: Used to rewrite the parent of a locale ID from "root" to 252 its language subtag (e.g. "zh_Hant" has a natural parent of "root", but to allow 253 some base language data to be shared it can be made to have a parent of "zh"). 254 255 2: forcedAlias: Used to add aliases for specific directories in order to affect the 256 ICU behaviour in special cases. 257 258 Between them these mechanisms are known as "tailorings" of the affected locales. --> 259 <!-- TODO: Explain why these special cases are needed/different. --> 260 261 <!-- Collation data is large, but also more sharable than other data, which is why there 262 are a number of aliases and parent remappings for this directory. --> 263 <directory dir="coll" inheritLanguageSubtag="bs_Cyrl, sr_Latn, zh_Hant"> 264 <!-- These aliases are to avoid needing to copy and maintain the same collation data 265 for "zh" and "yue". The maximized versions of "yue_Hans" is "yue_Hans_CN" (vs 266 "zh_Hans_CN"), and for "yue" it's "yue_Hant_HK" (vs "zh_Hant_HK"), so the 267 aliases are effectively just rewriting the base language. --> 268 <forcedAlias source="yue" target="zh_Hant"/> 269 <forcedAlias source="yue_Hant" target="zh_Hant"/> 270 <forcedAlias source="yue_CN" target="zh_Hans"/> 271 <forcedAlias source="yue_Hans" target="zh_Hans"/> 272 <forcedAlias source="yue_Hans_CN" target="zh_Hans"/> 273 274 <!-- TODO: Find out and document this properly. --> 275 <forcedAlias source="sr_ME" target="sr_Cyrl_ME"/> 276 277 <localeIds> 278 root, 279 280 // A-B 281 af, am, ars, ar, as, az, be, bg, bn, bo, br, bs_Cyrl, bs, 282 283 // C-F 284 ca, ceb, chr, cs, cy, da, de_AT, de, dsb, dz, ee, el, en, 285 en_US_POSIX, en_US, eo, es, et, fa_AF, fa, ff_Adlm, ff, fil, fi, fo, fr_CA, fr, 286 287 // G-J 288 ga, gl, gu, ha, haw, he, hi, hr, hsb, hu, hy, 289 id_ID, id, ig, in, in_ID, is, it, iw_IL, iw, ja, 290 291 // K-P 292 ka, kk, kl, km, kn, kok, ko, ku, ky, lb, lkt, ln, lo, lt, lv, 293 mk, ml, mn, mo, mr, ms, mt, my, nb, ne, nl, nn, no_NO, no, 294 om, or, pa_IN, pa, pa_Guru, pl, ps, pt, 295 296 // R-T 297 ro, ru, sa, se, sh_BA, sh_CS, sh, sh_YU, si, sk, sl, smn, sq, 298 sr_BA, sr_Cyrl_ME, sr_Latn, sr_ME, sr_RS, sr, sv, sw, 299 ta, te, th, tk, to, tr, 300 301 // U-Z 302 ug, uk, ur, uz, vi, wae, wo, xh, yi, yo, yue_CN, yue_Hans_CN, yue_Hans 303 yue_Hant, yue, zh_CN, zh_Hans, zh_Hant, zh_HK, zh_MO, zh_SG, zh_TW, zh, zu 304 </localeIds> 305 </directory> 306 307 <directory dir="rbnf"> 308 <!-- It is not at all clear why this is being done. It's certainly not exactly the 309 same as above, since (a) the alias is reversed (b) "zh_Hant" does exist, with 310 different data than "yue", so this alias is not just rewriting the base 311 language. --> 312 <!-- TODO: Find out and document this properly. --> 313 <forcedAlias source="zh_Hant_HK" target="yue"/> 314 315 <localeIds> 316 root, 317 318 // A-E 319 af, ak, am, ars, ar, az, be, bg, bs, ca, ccp, chr, cs, cy, 320 da, de_CH, de, ee, el, en_001, en_IN, en, eo, es_419, es_DO, 321 es_GT, es_HN, es_MX, es_NI, es_PA, es_PR, es_SV, es, es_US, et, 322 323 // F-P 324 fa_AF, fa, ff, fil, fi, fo, fr_BE, fr_CH, fr, ga, he, hi, hr, 325 hu, hy, id, in, is, it, iw, ja, ka, kl, km, ko, ky, lb, 326 lo, lrc, lt, lv, mk, ms, mt, my, nb, nl, nn, no, pl, pt_PT, pt, 327 328 // Q-Z 329 qu, ro, ru, se, sh, sk, sl, sq, sr_Latn, sr, su, sv, sw, ta, th, tr, 330 uk, vi, yue_Hans, yue, zh_Hant_HK, zh_Hant, zh_HK, zh_MO, zh_TW, zh 331 </localeIds> 332 </directory> 333 334 <directory dir="brkitr" inheritLanguageSubtag="zh_Hant"> 335 <localeIds> 336 root, 337 de, el, en, en_US_POSIX, en_US, es, fr, it, ja, pt, ru, zh_Hant, zh 338 </localeIds> 339 </directory> 340 341 <!-- GLOBAL ALIASES --> 342 343 <!-- Some spoken languages (e.g. "ars") inherit all their data from a written language 344 (e.g. "ar_SA"). However CLDR doesn't currently support a way to represent that 345 relationship. Unlike deprecated languages for which an alias can be inferred from 346 the "languageAlias" CLDR data, there's no way in CLDR to represent the fact that 347 we want "ars" (a non-deprecated language) to inherit the data of "ar_SA". 348 349 This alias is the first example of potentially many cases where ICU needs to 350 generate an alias in order to affect "sideways inheritance" for spoken languages, 351 and at some stage it should probably be supported properly in the CLDR data. --> 352 <forcedAlias source="ars" target="ar_SA"/> 353 354 <!-- A legacy global alias (note that "no_NO_NY" is not even structurally valid). --> 355 <forcedAlias source="no_NO_NY" target="nn_NO"/> 356 357 <!-- ALTERNATE VALUES --> 358 359 <!-- The following elements configure alternate values for some special case paths. 360 The target path will only be replaced if both it, and the source path, exist in 361 the CLDR data (paths will not be modified if only the source path exists). 362 363 Since the paths must represent the same semantic type of data, they must be in the 364 same "namespace" (same element names) and must not contain value attributes. Thus 365 they can only differ by distinguishing attributes (either added or modified). 366 367 This feature is typically used to select alternate translations (e.g. short forms) 368 for certain paths. --> 369 <!-- <altPath target="//path/to/value[@attr='foo']" 370 source="//path/to/value[@attr='bar']" 371 locales="xx,yy_ZZ"/> --> 372 <!-- Android patch (b/36123938) begin. --> 373 <altPath target="//ldml/localeDisplayNames/territories/territory[@type='FK']" 374 source="//ldml/localeDisplayNames/territories/territory[@type='FK'][@alt='variant']"/> 375 <altPath target="//ldml/localeDisplayNames/territories/territory[@type='HK']" 376 source="//ldml/localeDisplayNames/territories/territory[@type='HK'][@alt='short']"/> 377 <altPath target="//ldml/localeDisplayNames/territories/territory[@type='MK']" 378 source="//ldml/localeDisplayNames/territories/territory[@type='MK'][@alt='variant']"/> 379 <altPath target="//ldml/localeDisplayNames/territories/territory[@type='MO']" 380 source="//ldml/localeDisplayNames/territories/territory[@type='MO'][@alt='short']"/> 381 <!-- Android patch (b/36123938) end. --> 382 <!-- Android patch (b/8264703) begin. --> 383 <altPath target="//ldml/localeDisplayNames/territories/territory[@type='PS']" 384 source="//ldml/localeDisplayNames/territories/territory[@type='PS'][@alt='short']"/> 385 <!-- Android patch (b/8264703) end. --> 386 <!-- Android patch (b/21295835) begin. --> 387 <altPath target="//ldml/numbers/currencies/currency[@type='UAH']/symbol" 388 source="//ldml/numbers/currencies/currency[@type='UAH']/symbol[@alt='variant']"/> 389 <!-- Android patch (b/21295835) end. --> 390 </convert> 391 </target> 392 393 <target name="clean" depends="init-args, prepare-jar"> 394 <taskdef name="outputDirectories" classname="org.unicode.icu.tool.cldrtoicu.ant.CleanOutputDirectoryTask"> 395 <classpath> 396 <pathelement path="target/cldr-to-icu-1.0-SNAPSHOT-jar-with-dependencies.jar"/> 397 </classpath> 398 </taskdef> 399 400 <!-- If a directory is listed here, then every file in it is assumed to be automatically 401 generated by the conversion tool, unless it is explicitly listed in a <retain> element. 402 The tool then checks every file to determine if it has the expected header present, 403 indiciating that it was automatically generated, before deleting it. 404 405 If unexpected files are found, the "clean" task will fail without deleting anything 406 (unless'forceDelete' is set to override this). Note that even if 'forceDelete' is set, 407 the files listed explicitly below will never be deleted by this process. 408 409 This two-step approach minimizes the risk that the conversion process will ever 410 accidentally delete a manually maintained file. 411 --> 412 <outputDirectories root="${outDir}" forceDelete="${forceDelete}"> 413 <dir name="brkitr"> 414 <retain path="dictionaries"/> 415 <retain path="rules"/> 416 </dir> 417 <dir name="coll"> 418 <!-- Legacy files whose file names aren't supported for automatic generation. 419 Simple to maintain manually and unlikely to ever change again. --> 420 <retain path="de__PHONEBOOK.txt"/> 421 <retain path="de_.txt"/> 422 <retain path="es__TRADITIONAL.txt"/> 423 <retain path="es_.txt"/> 424 </dir> 425 <dir name="curr"/> 426 <dir name="lang"/> 427 <dir name="locales"/> 428 <dir name="misc"> 429 <!-- Machine generated files produced by different tools. 430 Possibly worth moving into the new LDML conversion tool one day. --> 431 <retain path="currencyNumericCodes.txt"/> 432 <retain path="zoneinfo64.txt"/> 433 <!-- Project file (not ICU data), unlikely to ever be auto-generated. --> 434 <retain path="icudata.rc"/> 435 <!-- Small high-level metadata file, stable and easy to maintain manually. --> 436 <retain path="icustd.txt"/> 437 </dir> 438 <dir name="rbnf"/> 439 <dir name="region"/> 440 <dir name="translit"> 441 <!-- Small, easy to maintain, special case top-level files. --> 442 <retain path="en.txt"/> 443 <retain path="el.txt"/> 444 </dir> 445 <dir name="unit"/> 446 <dir name="zone"> 447 <!-- Manually edited to support TZ database name compatibility. --> 448 <retain path="tzdbNames.txt"/> 449 </dir> 450 </outputDirectories> 451 </target> 452</project> 453 454