1 package org.unicode.cldr.ant; 2 3 import java.io.File; 4 import java.io.FileFilter; 5 import java.util.ArrayList; 6 import java.util.Arrays; 7 import java.util.Collections; 8 import java.util.HashMap; 9 import java.util.HashSet; 10 import java.util.List; 11 import java.util.Map; 12 import java.util.Set; 13 import java.util.TreeMap; 14 import java.util.regex.Pattern; 15 16 import org.apache.tools.ant.BuildException; 17 import org.apache.tools.ant.Task; 18 import org.unicode.cldr.ant.CLDRConverterTool.AliasDeprecates; 19 import org.unicode.cldr.icu.ResourceSplitter.SplitInfo; 20 import org.unicode.cldr.util.PatternCache; 21 22 import com.ibm.icu.dev.tool.UOption; 23 24 public class CLDRBuild extends Task { 25 private String toolName; 26 private String srcFile; 27 private String destFile; 28 private boolean noArgs; 29 private List<Run> runs = new ArrayList<Run>(); 30 31 private UOption srcDir = UOption.SOURCEDIR(); 32 private UOption destDir = UOption.DESTDIR(); 33 34 private static class PatternFilter implements FileFilter { 35 private final Pattern filePattern; 36 PatternFilter(String filePattern)37 public PatternFilter(String filePattern) { 38 this.filePattern = filePattern == null ? null : PatternCache.get(filePattern); 39 } 40 accept(File pathname)41 public boolean accept(File pathname) { 42 return filePattern != null && filePattern.matcher(pathname.getName()).matches(); 43 } 44 } 45 matchesLocale(List<String> locales, String locale)46 public static boolean matchesLocale(List<String> locales, String locale) { 47 for (String localePattern : locales) { 48 if (localePattern.equals(locale) || locale.matches(localePattern)) { 49 return true; 50 } 51 } 52 return false; 53 } 54 getLocalesList(Config config, String src, String dest)55 public Map<String, String> getLocalesList(Config config, String src, String dest) { 56 File srcdir = new File(src); 57 File[] srcFiles = srcdir.listFiles(new PatternFilter(srcFile)); 58 File destdir = new File(dest); 59 File[] destFiles = destdir.listFiles(new PatternFilter(destFile)); 60 61 Map<String, String> ret = new TreeMap<String, String>(); 62 63 if (config != null) { 64 List<InExclude> localesList = config.locales.localesList; 65 for (InExclude inex : localesList) { 66 for (File file : srcFiles) { 67 String fileName = file.getName(); 68 if (inex.matchesFileName(fileName)) { 69 if (inex.include) { 70 ret.put(fileName, inex.draft); 71 } else { 72 ret.remove(fileName); 73 } 74 } 75 } 76 } 77 } else { 78 for (File file : srcFiles) { 79 ret.put(file.getName(), ".*"); 80 } 81 } 82 83 // Only build the files that need to be built 84 if (srcFile == null) { 85 // Don't rebuild dstFiles that already exist 86 for (File file : destFiles) { 87 if (file.exists()) { 88 ret.remove(file.getName()); 89 } 90 } 91 } else if (srcFiles.length > 0) { 92 // Don't rebuild files that are newer than the corresponding source file 93 94 // In the grand scheme of things, the number of files is relatively 95 // small and n * m operations isn't noticeable, so don't optimize. 96 // The previous code tried to optimize but the optimization was broken 97 // so this performs about the same as before. 98 for (File dstFile : destFiles) { 99 String destName = stripExtension(dstFile.getName()); 100 for (File srcFile : srcFiles) { 101 String srcName = stripExtension(srcFile.getName()); 102 if (srcName.equals(destName) && dstFile.lastModified() > srcFile.lastModified()) { 103 ret.remove(srcFile.getName()); 104 } 105 } 106 } 107 } 108 109 if (ret.size() == 0 && destFiles.length == 1) { 110 return null; 111 } 112 113 return ret; 114 } 115 getIncludedLocales(Config config)116 public Set<String> getIncludedLocales(Config config) { 117 118 Set<String> ret = new HashSet<String>(); 119 if (config != null) { 120 List<InExclude> localesList = config.locales.localesList; 121 for (InExclude inex : localesList) { 122 if (inex.include) { 123 for (String str : inex.locales) { 124 ret.add(str); 125 } 126 } 127 } 128 } 129 return ret; 130 } 131 stripExtension(String fileName)132 private static String stripExtension(String fileName) { 133 int index = fileName.lastIndexOf('.'); 134 return index == -1 ? fileName : fileName.substring(0, index); 135 } 136 exitWithException(Throwable t)137 static void exitWithException(Throwable t) { 138 errln(t.getMessage()); 139 t.printStackTrace(System.err); 140 System.exit(-1); 141 } 142 exitWithError(String msg)143 static void exitWithError(String msg) { 144 errln(msg); 145 System.exit(-1); 146 } 147 errln(String msg)148 static void errln(String msg) { 149 System.err.println("ERROR: " + msg); 150 } 151 warnln(String msg)152 static void warnln(String msg) { 153 System.out.println("WARNING: " + msg); 154 } 155 infoln(String msg)156 static void infoln(String msg) { 157 System.out.println("INFO: " + msg); 158 } 159 getDirString(Args runArgs, UOption key)160 private String getDirString(Args runArgs, UOption key) { 161 String value = runArgs.map.get("--" + key.longName); 162 if (value == null) { 163 value = runArgs.map.get("-" + key.shortName); 164 } 165 return value; 166 } 167 168 // The method executing the task 169 @Override execute()170 public void execute() throws BuildException { 171 if (toolName == null) { 172 throw new BuildException("Tool name not set"); 173 } 174 175 try { 176 for (Run run : runs) { 177 178 Args runArgs = run.args; 179 180 Set<String> includedLocales = getIncludedLocales(run.config); 181 Map<String, String> localesMap = getLocalesList( 182 run.config, getDirString(runArgs, srcDir), getDirString(runArgs, destDir)); 183 if (localesMap == null || (localesMap.size() == 0 && !noArgs)) { 184 continue; 185 } 186 187 List<String> argList = new ArrayList<String>(); 188 StringBuilder printArgs = new StringBuilder(); 189 for (Map.Entry<String, String> e : runArgs.map.entrySet()) { 190 String key = e.getKey(); 191 String value = e.getValue(); 192 printArgs.append(key).append(' '); 193 argList.add(key); 194 if (value != null && value.length() > 0) { 195 printArgs.append(value).append(' '); 196 argList.add(value); 197 } 198 } 199 200 Object obj = createObject(toolName); 201 if (!(obj instanceof CLDRConverterTool)) { 202 exitWithError(toolName + " not a subclass of CLDRConverterTool!"); 203 } 204 205 CLDRConverterTool tool = (CLDRConverterTool) obj; 206 tool.setLocalesMap(localesMap); 207 tool.setIncludedLocales(includedLocales); 208 209 if (run.deprecates != null) { 210 AliasDeprecates aliasDeprecates = new AliasDeprecates( 211 run.deprecates.aliasList, 212 run.deprecates.aliasLocaleList, 213 run.deprecates.emptyLocaleList); 214 tool.setAliasDeprecates(aliasDeprecates); 215 } 216 217 if (run.config != null) { 218 if (run.config.paths != null) { 219 tool.setPathList(run.config.paths.pathList); 220 } 221 222 if (run.config.ofb != null) { 223 tool.setOverrideFallbackList(run.config.ofb.pathsList); 224 } 225 } 226 227 if (run.remapper != null) { 228 List<SplitInfo> infos = new ArrayList<SplitInfo>(); 229 for (Remap remap : run.remapper.remaps) { 230 infos.add(new SplitInfo(remap.sourcePath, remap.targetDir, remap.targetPath)); 231 } 232 tool.setSplitInfos(infos); 233 } 234 235 tool.processArgs(argList.toArray(new String[argList.size()])); 236 } 237 } catch (Throwable t) { 238 t.printStackTrace(); 239 } 240 } 241 createObject(String className)242 private static Object createObject(String className) { 243 Object object = null; 244 try { 245 Class<?> classDefinition = Class.forName(className); 246 object = classDefinition.newInstance(); 247 } catch (InstantiationException e) { 248 exitWithException(e); 249 } catch (IllegalAccessException e) { 250 exitWithException(e); 251 } catch (ClassNotFoundException e) { 252 exitWithException(e); 253 } catch (Throwable t) { 254 exitWithException(t); 255 } 256 return object; 257 } 258 addConfiguredRun(Run run)259 public void addConfiguredRun(Run run) { 260 runs.add(run); 261 } 262 setToolName(String name)263 public void setToolName(String name) { 264 toolName = name; 265 } 266 setSrcFile(String sf)267 public void setSrcFile(String sf) { 268 srcFile = sf; 269 } 270 setDestFile(String df)271 public void setDestFile(String df) { 272 destFile = df; 273 } 274 setNoArgs(String bool)275 public void setNoArgs(String bool) { 276 noArgs = bool.equals("true"); 277 } 278 279 public static class Run extends Task { 280 String type; 281 Args args; 282 Config config; 283 Deprecates deprecates; 284 Remapper remapper; 285 setType(String type)286 public void setType(String type) { 287 this.type = type; 288 } 289 addConfiguredArgs(Args args)290 public void addConfiguredArgs(Args args) { 291 this.args = args; 292 } 293 addConfiguredConfig(Config config)294 public void addConfiguredConfig(Config config) { 295 this.config = config; 296 } 297 addConfiguredDeprecates(Deprecates deprecates)298 public void addConfiguredDeprecates(Deprecates deprecates) { 299 this.deprecates = deprecates; 300 } 301 addConfiguredRemapper(Remapper remapper)302 public void addConfiguredRemapper(Remapper remapper) { 303 if (remapper.remaps.isEmpty()) { 304 exitWithError("remaps must not be empty"); 305 } 306 this.remapper = remapper; 307 } 308 } 309 310 public static class Args extends Task { 311 Map<String, String> map = new HashMap<String, String>(); 312 addConfiguredArg(Arg arg)313 public void addConfiguredArg(Arg arg) { 314 if (arg.name == null) { 315 throw new IllegalArgumentException("argument missing name"); 316 } 317 map.put(arg.name, arg.value); 318 } 319 } 320 321 public static class Arg extends Task { 322 String name; 323 String value; 324 setName(String name)325 public void setName(String name) { 326 this.name = name; 327 } 328 setValue(String value)329 public void setValue(String value) { 330 this.value = value; 331 } 332 } 333 334 public static class Config extends Task { 335 Locales locales; 336 Paths paths; 337 OverrideFallback ofb; 338 String type; 339 addConfiguredLocales(Locales loc)340 public void addConfiguredLocales(Locales loc) { 341 if (locales != null) { 342 exitWithError("Multiple <locales> elements not supported"); 343 } 344 locales = loc; 345 } 346 addConfiguredPaths(Paths ps)347 public void addConfiguredPaths(Paths ps) { 348 if (paths != null) { 349 exitWithError("Multiple <paths> elements not supported"); 350 } 351 paths = ps; 352 } 353 addConfiguredOverrideFallback(OverrideFallback ofb)354 public void addConfiguredOverrideFallback(OverrideFallback ofb) { 355 if (this.ofb != null) { 356 exitWithError("Multiple <overrideFallback> elements not allowed!"); 357 } 358 this.ofb = ofb; 359 } 360 setType(String type)361 public void setType(String type) { 362 this.type = type; 363 } 364 } 365 366 public static class Locales extends Task { 367 List<InExclude> localesList = new ArrayList<InExclude>(); 368 addConfiguredInclude(Include include)369 public void addConfiguredInclude(Include include) { 370 addInEx(include); 371 } 372 addConfiguredExclude(Exclude exclude)373 public void addConfiguredExclude(Exclude exclude) { 374 addInEx(exclude); 375 } 376 addInEx(InExclude inex)377 private void addInEx(InExclude inex) { 378 inex.validate(); 379 localesList.add(inex); 380 } 381 } 382 383 public static class InExclude extends Task { 384 static final List<String> ANY = Collections.emptyList(); 385 386 final boolean include; 387 List<String> locales; 388 String draft; 389 String xpath; 390 String alt; 391 InExclude(boolean include)392 protected InExclude(boolean include) { 393 this.include = include; 394 } 395 setDraft(String draft)396 public void setDraft(String draft) { 397 this.draft = draft; 398 } 399 setLocales(String locales)400 public void setLocales(String locales) { 401 if (".*".equals(locales)) { 402 this.locales = ANY; 403 } else { 404 this.locales = Arrays.asList(locales.split("\\s+")); 405 } 406 } 407 setXpath(String xpath)408 public void setXpath(String xpath) { 409 this.xpath = xpath; 410 } 411 setAlt(String alt)412 public void setAlt(String alt) { 413 this.alt = alt; 414 } 415 validate()416 void validate() { 417 if (locales == null) { 418 exitWithError("locales attribute not set for include/exclude element!"); 419 } 420 } 421 matchesFileName(String fileName)422 boolean matchesFileName(String fileName) { 423 if (locales == ANY) { 424 return true; 425 } 426 String localePattern = fileName.substring(0, fileName.indexOf(".xml")); 427 return matchesLocale(locales, localePattern); 428 } 429 430 @Override equals(Object o)431 public boolean equals(Object o) { 432 if (!(o instanceof InExclude)) { 433 return false; 434 } 435 436 if (o == this) { 437 return true; 438 } 439 440 InExclude rhs = (InExclude) o; 441 return include == rhs.include && 442 equalLists(locales, rhs.locales) && 443 equalStrings(draft, rhs.draft) && 444 equalStrings(xpath, rhs.xpath) && 445 equalStrings(alt, rhs.alt); 446 } 447 448 @Override hashCode()449 public int hashCode() { 450 return hash(locales, hash(draft, hash(xpath, hash(alt, 0)))); 451 } 452 equalStrings(String lhs, String rhs)453 private boolean equalStrings(String lhs, String rhs) { 454 return lhs == rhs || (lhs != null && lhs.equals(rhs)); 455 } 456 equalLists(List<? extends T> lhs, List<? extends T> rhs)457 private <T> boolean equalLists(List<? extends T> lhs, List<? extends T> rhs) { 458 return lhs == rhs || (lhs != null && lhs.equals(rhs)); 459 } 460 hash(Object rhs, int hash)461 private int hash(Object rhs, int hash) { 462 return rhs == null ? hash : (hash * 31) ^ rhs.hashCode(); 463 } 464 } 465 466 public static class Include extends InExclude { Include()467 public Include() { 468 super(true); 469 } 470 } 471 472 public static class Exclude extends InExclude { Exclude()473 public Exclude() { 474 super(false); 475 } 476 } 477 478 public static class Deprecates extends Task { 479 List<String> aliasLocaleList; 480 List<String> emptyLocaleList; 481 List<CLDRConverterTool.Alias> aliasList; 482 addConfiguredAlias(Alias alias)483 public void addConfiguredAlias(Alias alias) { 484 if (aliasList == null) { 485 aliasList = new ArrayList<CLDRConverterTool.Alias>(); 486 } 487 aliasList.add(new CLDRConverterTool.Alias(alias.from, alias.to, alias.xpath, alias.rbPath, alias.value)); 488 } 489 addConfiguredEmptyLocale(EmptyLocale alias)490 public void addConfiguredEmptyLocale(EmptyLocale alias) { 491 if (emptyLocaleList == null) { 492 emptyLocaleList = new ArrayList<String>(); 493 } 494 emptyLocaleList.add(alias.locale); 495 } 496 addConfiguredAliasLocale(AliasLocale alias)497 public void addConfiguredAliasLocale(AliasLocale alias) { 498 if (aliasLocaleList == null) { 499 aliasLocaleList = new ArrayList<String>(); 500 } 501 aliasLocaleList.add(alias.locale); 502 } 503 } 504 505 public static class Alias extends Task { 506 String from; 507 String to; 508 // TODO(jchye): remove xpath field after old converter is deleted. 509 String xpath; 510 String rbPath; 511 String value; 512 setFrom(String from)513 public void setFrom(String from) { 514 this.from = from; 515 } 516 setTo(String to)517 public void setTo(String to) { 518 this.to = to; 519 } 520 setXpath(String xpath)521 public void setXpath(String xpath) { 522 this.xpath = xpath; 523 } 524 setRbPath(String rbPath)525 public void setRbPath(String rbPath) { 526 this.rbPath = rbPath; 527 } 528 setValue(String value)529 public void setValue(String value) { 530 this.value = value; 531 } 532 } 533 534 public static class AliasLocale extends Task { 535 String locale; 536 setLocale(String locale)537 public void setLocale(String locale) { 538 this.locale = locale; 539 } 540 } 541 542 public static class EmptyLocale extends Task { 543 String locale; 544 String list; 545 setLocale(String locale)546 public void setLocale(String locale) { 547 this.locale = locale; 548 } 549 setList(String list)550 public void setList(String list) { 551 this.list = list; 552 } 553 } 554 555 public static class Paths extends Task { 556 public String fallback; 557 public String locales; 558 public String draft; 559 560 private List<Task> pathList = new ArrayList<Task>(); 561 addConfiguredInclude(Include include)562 public void addConfiguredInclude(Include include) { 563 pathList.add(include); 564 } 565 addConfiguredExclude(Exclude exclude)566 public void addConfiguredExclude(Exclude exclude) { 567 pathList.add(exclude); 568 } 569 setFallback(String fallback)570 public void setFallback(String fallback) { 571 this.fallback = fallback; 572 } 573 setLocales(String locales)574 public void setLocales(String locales) { 575 this.locales = locales; 576 } 577 setDraft(String draft)578 public void setDraft(String draft) { 579 this.draft = draft; 580 } 581 addConfiguredCoverageLevel(CoverageLevel level)582 public void addConfiguredCoverageLevel(CoverageLevel level) { 583 level.validate(); 584 pathList.add(level); 585 } 586 } 587 588 public static class CoverageLevel extends Task { 589 public String group; 590 public String level; 591 public String locales; 592 public String draft; 593 public String org; 594 setDraft(String draft)595 public void setDraft(String draft) { 596 this.draft = draft; 597 } 598 setLevel(String level)599 public void setLevel(String level) { 600 this.level = level; 601 } 602 setLocales(String locales)603 public void setLocales(String locales) { 604 this.locales = locales; 605 } 606 setOrg(String org)607 public void setOrg(String org) { 608 this.org = org; 609 } 610 setGroup(String group)611 public void setGroup(String group) { 612 this.group = group; 613 } 614 validate()615 void validate() { 616 if ((group == null) != (org == null)) { 617 exitWithError("Invalid specification of coverageLevel element; org && group not set!"); 618 } 619 620 if (level == null) { 621 exitWithError("Invalid specification of coverageLevel element; level not set!"); 622 } 623 } 624 } 625 626 public static class OverrideFallback extends Task { 627 List<Paths> pathsList = new ArrayList<Paths>(); 628 addConfiguredPaths(Paths paths)629 public void addConfiguredPaths(Paths paths) { 630 pathsList.add(paths); 631 } 632 } 633 634 public static class Remap extends Task { 635 public String sourcePath; 636 public String targetPath; 637 public String targetDir; 638 setSourcePath(String sourcePath)639 public void setSourcePath(String sourcePath) { 640 this.sourcePath = sourcePath; 641 } 642 setTargetPath(String targetPath)643 public void setTargetPath(String targetPath) { 644 this.targetPath = targetPath; 645 } 646 setTargetDir(String targetDir)647 public void setTargetDir(String targetDir) { 648 this.targetDir = targetDir; 649 } 650 } 651 652 public static class Remapper extends Task { 653 public String baseDir; 654 List<Remap> remaps = new ArrayList<Remap>(); 655 setBaseDir(String baseDir)656 public void setBaseDir(String baseDir) { 657 this.baseDir = baseDir; 658 } 659 addConfiguredRemap(Remap remap)660 public void addConfiguredRemap(Remap remap) { 661 if (remap.sourcePath == null || remap.sourcePath.trim().isEmpty()) { 662 exitWithError("remap source path must not be empty"); 663 } 664 remap.sourcePath = remap.sourcePath.trim(); 665 666 if (remap.targetPath != null && remap.targetPath.trim().isEmpty()) { 667 remap.targetPath = null; 668 } 669 670 if (remap.targetDir != null && remap.targetDir.trim().isEmpty()) { 671 remap.targetDir = null; 672 } 673 674 remaps.add(remap); 675 } 676 } 677 } 678