1 package org.unicode.cldr.ant; 2 3 import java.util.ArrayList; 4 import java.util.Arrays; 5 import java.util.Collections; 6 import java.util.List; 7 import java.util.Map; 8 import java.util.Set; 9 10 import org.apache.tools.ant.Task; 11 import org.unicode.cldr.ant.CLDRBuild.Paths; 12 import org.unicode.cldr.icu.LDMLConstants; 13 import org.unicode.cldr.icu.ResourceSplitter.SplitInfo; 14 import org.unicode.cldr.util.CLDRConfig; 15 import org.unicode.cldr.util.CoverageInfo; 16 import org.unicode.cldr.util.Level; 17 import org.unicode.cldr.util.StandardCodes; 18 import org.unicode.cldr.util.XPathParts; 19 import org.w3c.dom.Node; 20 21 /** 22 * All tools that would like to make use of CLDR Build process 23 * through the ant plug-in should extend this class. For implementing 24 * the processArgs method basically move the implementation of main into 25 * this method and add code to deal with situation where localesMap field 26 * is set, see {@link org.unicode.cldr.icu.LDML2ICUConverter#processArgs(String[])}. 27 * The subclasses are also expected to invoke computeConvertibleXPaths method 28 * for all the xpaths in the file that they are currently processing and at 29 * every leaf node should verify if an XPath is convertible or not. Please see 30 * {@link org.unicode.cldr.icu.LDML2ICUConverter#isNodeNotConvertible(Node, StringBuilder)}. 31 * 32 * @author ram 33 * 34 */ 35 public abstract class CLDRConverterTool { 36 /** 37 * Information from the deprecates build rules. 38 */ 39 protected AliasDeprecates aliasDeprecates; 40 41 /** 42 * Map of locales that need to processed. 43 * Key : locale name 44 * Value: draft attribute 45 */ 46 private Map<String, String> localesMap; 47 48 private Set<String> includedLocales; 49 50 /** 51 * List of xpaths to include or exclude 52 */ 53 protected List<Task> pathList; 54 55 /** 56 * Override fallbacks list 57 */ 58 protected List<CLDRBuild.Paths> overrideFallbackList; 59 60 /** 61 * Information used by ResourceSplitter, if not null. 62 */ 63 protected List<SplitInfo> splitInfos; 64 65 /** 66 * Object that holds information about aliases on the 67 * <alias from="in" to="id" /> elements. 68 * 69 * @author ram 70 * 71 */ 72 public static class Alias { 73 public final String from; 74 public final String to; 75 public final String xpath; 76 public final String rbPath; 77 public final String value; 78 Alias(String from, String to, String xpath, String rbPath, String value)79 public Alias(String from, String to, String xpath, String rbPath, String value) { 80 this.from = from; 81 this.to = to; 82 this.xpath = xpath; 83 this.rbPath = rbPath; 84 this.value = value; 85 } 86 } 87 88 public static class AliasDeprecates { 89 public final List<Alias> aliasList; 90 public final List<String> aliasLocaleList; 91 public final List<String> emptyLocaleList; 92 AliasDeprecates(List<Alias> aliasList, List<String> aliasLocaleList, List<String> emptyLocaleList)93 public AliasDeprecates(List<Alias> aliasList, List<String> aliasLocaleList, 94 List<String> emptyLocaleList) { 95 this.aliasList = aliasList; 96 this.aliasLocaleList = aliasLocaleList; 97 this.emptyLocaleList = emptyLocaleList; 98 } 99 } 100 101 /** 102 * Process the arguments 103 * 104 * @param args 105 */ processArgs(String[] args)106 public abstract void processArgs(String[] args); 107 108 /** 109 * For support and interpretation of 110 * <deprecates> 111 * <alias from="no_NO_NY" to="nn_NO" /> 112 * <alias from="en_RH" to="en_ZW" /> 113 * <aliasLocale locale="zh_SG" /> 114 * <aliasLocale locale="zh_TW" /> 115 * <emptyLocale locale="hi_" /> 116 * <emptyLocale locale="zh_" /> 117 * </deprecates> 118 */ setAliasDeprecates(AliasDeprecates aliasDeprecates)119 public void setAliasDeprecates(AliasDeprecates aliasDeprecates) { 120 this.aliasDeprecates = aliasDeprecates; 121 } 122 123 /** 124 * 125 * @param map 126 */ setLocalesMap(Map<String, String> map)127 public void setLocalesMap(Map<String, String> map) { 128 localesMap = map; 129 } 130 setIncludedLocales(Set<String> set)131 public void setIncludedLocales(Set<String> set) { 132 includedLocales = set; 133 } 134 135 /** 136 * Sets the list of objects that contain information in 137 * include and exclude elements 138 * 139 * <include xpath="//ldml/.* /dateTimeElements/.*" draft=".*"/> 140 * <exclude xpath="//ldml/.* /language.*" preferAlt="proposed" draft=".*"/> 141 * 142 * @param list 143 */ setPathList(List<Task> list)144 public void setPathList(List<Task> list) { 145 pathList = list; 146 } 147 148 /** 149 * Set the fallback override list 150 */ setOverrideFallbackList(List<Paths> list)151 public void setOverrideFallbackList(List<Paths> list) { 152 // overrideFallbackList = list; 153 } 154 setSplitInfos(List<SplitInfo> infos)155 public void setSplitInfos(List<SplitInfo> infos) { 156 this.splitInfos = Collections.unmodifiableList(infos); 157 } 158 mergeOverrideFallbackNodes(Node main, String locale)159 protected Node mergeOverrideFallbackNodes(Node main, String locale) { 160 // for (int i = 0; i < overrideFallbackList.size(); i++) { 161 // CLDRBuild.Paths path = overrideFallbackList.get(i); 162 // if (CLDRBuild.matchesLocale(path.locales, locale)){ 163 // //TODO write the merging algorithm 164 // } 165 // } 166 return main; 167 } 168 169 /** 170 * Computes the convertible xpaths by walking through the xpathList given and applying the rules 171 * in children of <path> elements. 172 * 173 * @param xpathList 174 * A sorted list of all xpaths for the current run 175 * @param localeName 176 * The name of locale being processed 177 * @return an ArrayList of the computed convertible xpaths 178 */ computeConvertibleXPaths( List<String> xpathList, boolean exemplarsContainA_Z, String localeName, String supplementalDir)179 protected List<String> computeConvertibleXPaths( 180 List<String> xpathList, boolean exemplarsContainA_Z, String localeName, 181 String supplementalDir) { 182 /* 183 * Assumptions: 184 * 1. Vetted nodes do not have draft attribute 185 * 2. Nodes with draft attribute set and alt atrribute not set do not have a vetted 186 * counterpart 187 * 3. Nodes with alt attribute may or may not have a draft attribute 188 * 4. If no draft field is set in the preferences object assume vetted node is requested 189 */ 190 191 // fast path 192 String draft = getLocalesMap() == null ? null : getLocalesMap().get(localeName + ".xml"); 193 XPathParts parts = new XPathParts(null, null); 194 if (draft != null) { 195 for (int i = 0; i < xpathList.size(); i++) { 196 parts = parts.set(xpathList.get(i)); 197 Map<String, String> attr = parts.getAttributes(parts.size() - 1); 198 String draftVal = attr.get(LDMLConstants.DRAFT); 199 String altVal = attr.get(LDMLConstants.ALT); 200 if (draftVal != null && !draftVal.matches(draft)) { 201 xpathList.remove(i); 202 } 203 // remove xpaths with alt attribute set 204 if (altVal != null) { 205 xpathList.remove(i); 206 } 207 } 208 return xpathList; 209 } 210 211 if (pathList == null) { 212 // include everything! 213 return xpathList; 214 } 215 216 ArrayList<String> myXPathList = new ArrayList<String>(xpathList.size()); 217 StandardCodes sc = StandardCodes.make(); 218 // Instantiate CoverageInfo outside the loop 219 CoverageInfo covInfo = CLDRConfig.getInstance().getCoverageInfo(); 220 // iterator of xpaths of the current CLDR file being processed 221 // this map only contains xpaths of the leaf nodes 222 for (int i = 0; i < xpathList.size(); i++) { 223 String xpath = xpathList.get(i); 224 parts = parts.set(xpath); 225 Map<String, String> attr = parts.getAttributes(parts.size() - 1); 226 227 boolean include = false; 228 229 for (Task obj : pathList) { 230 if (obj instanceof CLDRBuild.CoverageLevel) { 231 CLDRBuild.CoverageLevel level = (CLDRBuild.CoverageLevel) obj; 232 if (level.locales != null) { 233 List<String> localeList = Arrays.asList(level.locales.split("\\s+")); 234 if (CLDRBuild.matchesLocale(localeList, localeName) == false) { 235 continue; 236 } 237 } 238 239 // process further only if the current locale is part of the given group and org 240 if (level.group != null 241 && !sc.isLocaleInGroup(localeName, level.group, level.org)) { 242 continue; 243 } 244 245 Level cv = Level.get(level.level); 246 // only include the xpaths that have the coverage level at least the coverage 247 // level specified by the locale 248 if (covInfo.getCoverageLevel(xpath, localeName).compareTo(cv) <= 0) { 249 String draftVal = attr.get(LDMLConstants.DRAFT); 250 if (level.draft != null) { 251 if (draftVal == null 252 && (level.draft.equals("false") || level.draft.equals(".*"))) { 253 include = true; 254 } else if (draftVal != null && draftVal.matches(level.draft)) { 255 include = true; 256 } else { 257 include = false; 258 } 259 } else { 260 if (draftVal == null) { 261 include = true; 262 } 263 } 264 } 265 } else if (obj instanceof CLDRBuild.Exclude) { 266 CLDRBuild.Exclude exc = (CLDRBuild.Exclude) obj; 267 // fast path if locale attribute is set 268 if (exc.locales != null 269 && CLDRBuild.matchesLocale(exc.locales, localeName) == false) { 270 continue; 271 } 272 if (exc.xpath != null && xpath.matches(exc.xpath)) { 273 /* 274 * Now starts struggle for figuring out which xpaths should be excluded 275 * The following cases need to be handled: 276 * 1. <exclude xpath="//ldml/localeDisplayNames/languages/.*" draft="false"> 277 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] then 278 * include = true 279 * if xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 280 * include = false 281 * 2. <exclude xpath="//ldml/localeDisplayNames/languages/.*" draft="true"> 282 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] then 283 * include = false 284 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='test'] then 285 * include = true 286 * if xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 287 * include = true 288 * 3. <exclude xpath="//ldml/localeDisplayNames/languages/.*" draft=".*"> 289 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] 290 * or xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='test'] 291 * or xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 292 * include = false 293 * 4. <exclude xpath="//ldml/localeDisplayNames/languages/.*" draft="false" preferAlt='true'> 294 * if xp of //ldml/localeDisplayNames/languages/language[@type='en' alt='.*'] exists then 295 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] 296 * or xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='test'] 297 * or xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 298 * include = true 299 * else 300 * apply rules for processing draft and alt attribute together. 301 * else 302 * if xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 303 * include = false 304 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='test'] then 305 * include = true 306 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] then 307 * include = true 308 */ 309 String draftVal = attr.get(LDMLConstants.DRAFT); 310 String altVal = attr.get(LDMLConstants.ALT); 311 boolean altExc = false, draftExc = false; 312 if (exc.alt == null && altVal == null) { 313 altExc = true; 314 } else if (exc.alt == null && altVal != null) { 315 altExc = true; 316 } else if (exc.alt != null && altVal == null) { 317 altExc = false; 318 } else { 319 if (altVal.matches(exc.alt)) { 320 altExc = true; 321 } 322 } 323 if (exc.draft == null && draftVal == null) { 324 draftExc = true; 325 } else if (exc.draft != null && draftVal == null) { 326 if ((exc.draft.equals("false") || exc.draft.equals(".*"))) { 327 draftExc = true; 328 } 329 } else if (exc.draft == null && draftVal != null) { 330 draftExc = false; 331 } else { 332 if (draftVal.matches(exc.draft)) { 333 draftExc = true; 334 } 335 } 336 if (altExc == true && draftExc == true) { 337 include = false; 338 } 339 } 340 } else if (obj instanceof CLDRBuild.Include) { 341 CLDRBuild.Include inc = (CLDRBuild.Include) obj; 342 // fast path if locale attribute is set 343 if (inc.locales != null 344 && CLDRBuild.matchesLocale(inc.locales, localeName) == false) { 345 continue; 346 } 347 if (inc.xpath != null && xpath.matches(inc.xpath)) { 348 /* 349 * The following cases need to be handled: 350 * 1. <include xpath="//ldml/localeDisplayNames/languages/.*" draft="false"> 351 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] then 352 * include = false 353 * if xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 354 * include = true 355 * 2. <include xpath="//ldml/localeDisplayNames/languages/.*" draft="true"> 356 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] then 357 * include = true 358 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='test'] then 359 * include = false 360 * if xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 361 * include = false 362 * 3. <include xpath="//ldml/localeDisplayNames/languages/.*" draft=".*"> 363 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] 364 * or xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='test'] 365 * or xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 366 * include = true 367 * 4. <include xpath="//ldml/localeDisplayNames/languages/.*" draft="false" preferAlt='true'> 368 * if xp of //ldml/localeDisplayNames/languages/language[@type='en' alt='.*'] exists then 369 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] 370 * or xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='test'] 371 * or xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 372 * include = false 373 * else 374 * apply rules for processing draft and alt attribute together. 375 * else 376 * if xp is //ldml/localeDisplayNames/languages/language[@type='en'] then 377 * include = true 378 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='test'] then 379 * include = false 380 * if xp is //ldml/localeDisplayNames/languages/language[@type='en' and draft='true'] then 381 * include = false 382 */ 383 String draftVal = attr.get(LDMLConstants.DRAFT); 384 String altVal = attr.get(LDMLConstants.ALT); 385 boolean altInc = false; 386 if (inc.alt == null && altVal == null) { 387 altInc = true; 388 } else if (inc.alt == null && altVal != null) { 389 altInc = false; 390 } else if (inc.alt != null && altVal == null) { 391 // the current xpath does not have the alt attribute set 392 // since the list is sorted we can be sure that if the 393 // next xpath matches the current one and there is an alt 394 // attibute available for this path, that next entry is 395 // where we should expect to find it. 396 // now check if next xpath contains alt attribute 397 if (i + 1 < xpathList.size()) { 398 String nxp = xpathList.get(i + 1); 399 XPathParts nparts = (new XPathParts(null, null)).set(nxp); 400 // make sure the type attribute is the same 401 if (parts.isLike(nparts)) { 402 Map<String, String> nattr = nparts.getAttributes(nparts.size() - 1); 403 if (nattr != null) { 404 altVal = nattr.get(LDMLConstants.ALT); 405 if (altVal != null && altVal.matches(inc.alt)) { 406 draftVal = nattr.get(LDMLConstants.DRAFT); 407 xpath = nxp; 408 i++; 409 altInc = true; 410 } 411 } 412 } 413 } 414 } else { 415 if (altVal.matches(inc.alt)) { 416 altInc = true; 417 } 418 } 419 boolean draftInc = false; 420 if (inc.draft == null && draftVal == null) { 421 draftInc = true; 422 } else if (inc.draft != null && draftVal == null) { 423 if ((inc.draft.equals("false") || inc.draft.equals(".*"))) { 424 draftInc = true; 425 } 426 } else if (inc.draft == null && draftVal != null) { 427 draftInc = false; 428 } else { 429 if (draftVal.matches(inc.draft)) { 430 draftInc = true; 431 } 432 } 433 if (altInc == true && draftInc == true) { 434 include = true; 435 } 436 } 437 } else { 438 System.err.println( 439 "ERROR: computeConvertibleXPath method cannot handle object of type: " 440 + obj.getClass().toString()); 441 } 442 } 443 if (include == true) { 444 myXPathList.add(xpath); 445 } 446 } 447 448 return myXPathList; 449 } 450 getLocalesMap()451 protected Map<String, String> getLocalesMap() { 452 return localesMap; 453 } 454 getIncludedLocales()455 protected Set<String> getIncludedLocales() { 456 return includedLocales; 457 } 458 } 459