• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 package org.unicode.cldr.ant;
2 
3 import java.io.File;
4 import java.io.FileFilter;
5 import java.util.ArrayList;
6 import java.util.Arrays;
7 import java.util.Collections;
8 import java.util.HashMap;
9 import java.util.HashSet;
10 import java.util.List;
11 import java.util.Map;
12 import java.util.Set;
13 import java.util.TreeMap;
14 import java.util.regex.Pattern;
15 
16 import org.apache.tools.ant.BuildException;
17 import org.apache.tools.ant.Task;
18 import org.unicode.cldr.ant.CLDRConverterTool.AliasDeprecates;
19 import org.unicode.cldr.icu.ResourceSplitter.SplitInfo;
20 import org.unicode.cldr.util.PatternCache;
21 
22 import com.ibm.icu.dev.tool.UOption;
23 
24 public class CLDRBuild extends Task {
25     private String toolName;
26     private String srcFile;
27     private String destFile;
28     private boolean noArgs;
29     private List<Run> runs = new ArrayList<Run>();
30 
31     private UOption srcDir = UOption.SOURCEDIR();
32     private UOption destDir = UOption.DESTDIR();
33 
34     private static class PatternFilter implements FileFilter {
35         private final Pattern filePattern;
36 
PatternFilter(String filePattern)37         public PatternFilter(String filePattern) {
38             this.filePattern = filePattern == null ? null : PatternCache.get(filePattern);
39         }
40 
accept(File pathname)41         public boolean accept(File pathname) {
42             return filePattern != null && filePattern.matcher(pathname.getName()).matches();
43         }
44     }
45 
matchesLocale(List<String> locales, String locale)46     public static boolean matchesLocale(List<String> locales, String locale) {
47         for (String localePattern : locales) {
48             if (localePattern.equals(locale) || locale.matches(localePattern)) {
49                 return true;
50             }
51         }
52         return false;
53     }
54 
getLocalesList(Config config, String src, String dest)55     public Map<String, String> getLocalesList(Config config, String src, String dest) {
56         File srcdir = new File(src);
57         File[] srcFiles = srcdir.listFiles(new PatternFilter(srcFile));
58         File destdir = new File(dest);
59         File[] destFiles = destdir.listFiles(new PatternFilter(destFile));
60 
61         Map<String, String> ret = new TreeMap<String, String>();
62 
63         if (config != null) {
64             List<InExclude> localesList = config.locales.localesList;
65             for (InExclude inex : localesList) {
66                 for (File file : srcFiles) {
67                     String fileName = file.getName();
68                     if (inex.matchesFileName(fileName)) {
69                         if (inex.include) {
70                             ret.put(fileName, inex.draft);
71                         } else {
72                             ret.remove(fileName);
73                         }
74                     }
75                 }
76             }
77         } else {
78             for (File file : srcFiles) {
79                 ret.put(file.getName(), ".*");
80             }
81         }
82 
83         // Only build the files that need to be built
84         if (srcFile == null) {
85             // Don't rebuild dstFiles that already exist
86             for (File file : destFiles) {
87                 if (file.exists()) {
88                     ret.remove(file.getName());
89                 }
90             }
91         } else if (srcFiles.length > 0) {
92             // Don't rebuild files that are newer than the corresponding source file
93 
94             // In the grand scheme of things, the number of files is relatively
95             // small and n * m operations isn't noticeable, so don't optimize.
96             // The previous code tried to optimize but the optimization was broken
97             // so this performs about the same as before.
98             for (File dstFile : destFiles) {
99                 String destName = stripExtension(dstFile.getName());
100                 for (File srcFile : srcFiles) {
101                     String srcName = stripExtension(srcFile.getName());
102                     if (srcName.equals(destName) && dstFile.lastModified() > srcFile.lastModified()) {
103                         ret.remove(srcFile.getName());
104                     }
105                 }
106             }
107         }
108 
109         if (ret.size() == 0 && destFiles.length == 1) {
110             return null;
111         }
112 
113         return ret;
114     }
115 
getIncludedLocales(Config config)116     public Set<String> getIncludedLocales(Config config) {
117 
118         Set<String> ret = new HashSet<String>();
119         if (config != null) {
120             List<InExclude> localesList = config.locales.localesList;
121             for (InExclude inex : localesList) {
122                 if (inex.include) {
123                     for (String str : inex.locales) {
124                         ret.add(str);
125                     }
126                 }
127             }
128         }
129         return ret;
130     }
131 
stripExtension(String fileName)132     private static String stripExtension(String fileName) {
133         int index = fileName.lastIndexOf('.');
134         return index == -1 ? fileName : fileName.substring(0, index);
135     }
136 
exitWithException(Throwable t)137     static void exitWithException(Throwable t) {
138         errln(t.getMessage());
139         t.printStackTrace(System.err);
140         System.exit(-1);
141     }
142 
exitWithError(String msg)143     static void exitWithError(String msg) {
144         errln(msg);
145         System.exit(-1);
146     }
147 
errln(String msg)148     static void errln(String msg) {
149         System.err.println("ERROR: " + msg);
150     }
151 
warnln(String msg)152     static void warnln(String msg) {
153         System.out.println("WARNING: " + msg);
154     }
155 
infoln(String msg)156     static void infoln(String msg) {
157         System.out.println("INFO: " + msg);
158     }
159 
getDirString(Args runArgs, UOption key)160     private String getDirString(Args runArgs, UOption key) {
161         String value = runArgs.map.get("--" + key.longName);
162         if (value == null) {
163             value = runArgs.map.get("-" + key.shortName);
164         }
165         return value;
166     }
167 
168     // The method executing the task
169     @Override
execute()170     public void execute() throws BuildException {
171         if (toolName == null) {
172             throw new BuildException("Tool name not set");
173         }
174 
175         try {
176             for (Run run : runs) {
177 
178                 Args runArgs = run.args;
179 
180                 Set<String> includedLocales = getIncludedLocales(run.config);
181                 Map<String, String> localesMap = getLocalesList(
182                     run.config, getDirString(runArgs, srcDir), getDirString(runArgs, destDir));
183                 if (localesMap == null || (localesMap.size() == 0 && !noArgs)) {
184                     continue;
185                 }
186 
187                 List<String> argList = new ArrayList<String>();
188                 StringBuilder printArgs = new StringBuilder();
189                 for (Map.Entry<String, String> e : runArgs.map.entrySet()) {
190                     String key = e.getKey();
191                     String value = e.getValue();
192                     printArgs.append(key).append(' ');
193                     argList.add(key);
194                     if (value != null && value.length() > 0) {
195                         printArgs.append(value).append(' ');
196                         argList.add(value);
197                     }
198                 }
199 
200                 Object obj = createObject(toolName);
201                 if (!(obj instanceof CLDRConverterTool)) {
202                     exitWithError(toolName + " not a subclass of CLDRConverterTool!");
203                 }
204 
205                 CLDRConverterTool tool = (CLDRConverterTool) obj;
206                 tool.setLocalesMap(localesMap);
207                 tool.setIncludedLocales(includedLocales);
208 
209                 if (run.deprecates != null) {
210                     AliasDeprecates aliasDeprecates = new AliasDeprecates(
211                         run.deprecates.aliasList,
212                         run.deprecates.aliasLocaleList,
213                         run.deprecates.emptyLocaleList);
214                     tool.setAliasDeprecates(aliasDeprecates);
215                 }
216 
217                 if (run.config != null) {
218                     if (run.config.paths != null) {
219                         tool.setPathList(run.config.paths.pathList);
220                     }
221 
222                     if (run.config.ofb != null) {
223                         tool.setOverrideFallbackList(run.config.ofb.pathsList);
224                     }
225                 }
226 
227                 if (run.remapper != null) {
228                     List<SplitInfo> infos = new ArrayList<SplitInfo>();
229                     for (Remap remap : run.remapper.remaps) {
230                         infos.add(new SplitInfo(remap.sourcePath, remap.targetDir, remap.targetPath));
231                     }
232                     tool.setSplitInfos(infos);
233                 }
234 
235                 tool.processArgs(argList.toArray(new String[argList.size()]));
236             }
237         } catch (Throwable t) {
238             t.printStackTrace();
239         }
240     }
241 
createObject(String className)242     private static Object createObject(String className) {
243         Object object = null;
244         try {
245             Class<?> classDefinition = Class.forName(className);
246             object = classDefinition.newInstance();
247         } catch (InstantiationException e) {
248             exitWithException(e);
249         } catch (IllegalAccessException e) {
250             exitWithException(e);
251         } catch (ClassNotFoundException e) {
252             exitWithException(e);
253         } catch (Throwable t) {
254             exitWithException(t);
255         }
256         return object;
257     }
258 
addConfiguredRun(Run run)259     public void addConfiguredRun(Run run) {
260         runs.add(run);
261     }
262 
setToolName(String name)263     public void setToolName(String name) {
264         toolName = name;
265     }
266 
setSrcFile(String sf)267     public void setSrcFile(String sf) {
268         srcFile = sf;
269     }
270 
setDestFile(String df)271     public void setDestFile(String df) {
272         destFile = df;
273     }
274 
setNoArgs(String bool)275     public void setNoArgs(String bool) {
276         noArgs = bool.equals("true");
277     }
278 
279     public static class Run extends Task {
280         String type;
281         Args args;
282         Config config;
283         Deprecates deprecates;
284         Remapper remapper;
285 
setType(String type)286         public void setType(String type) {
287             this.type = type;
288         }
289 
addConfiguredArgs(Args args)290         public void addConfiguredArgs(Args args) {
291             this.args = args;
292         }
293 
addConfiguredConfig(Config config)294         public void addConfiguredConfig(Config config) {
295             this.config = config;
296         }
297 
addConfiguredDeprecates(Deprecates deprecates)298         public void addConfiguredDeprecates(Deprecates deprecates) {
299             this.deprecates = deprecates;
300         }
301 
addConfiguredRemapper(Remapper remapper)302         public void addConfiguredRemapper(Remapper remapper) {
303             if (remapper.remaps.isEmpty()) {
304                 exitWithError("remaps must not be empty");
305             }
306             this.remapper = remapper;
307         }
308     }
309 
310     public static class Args extends Task {
311         Map<String, String> map = new HashMap<String, String>();
312 
addConfiguredArg(Arg arg)313         public void addConfiguredArg(Arg arg) {
314             if (arg.name == null) {
315                 throw new IllegalArgumentException("argument missing name");
316             }
317             map.put(arg.name, arg.value);
318         }
319     }
320 
321     public static class Arg extends Task {
322         String name;
323         String value;
324 
setName(String name)325         public void setName(String name) {
326             this.name = name;
327         }
328 
setValue(String value)329         public void setValue(String value) {
330             this.value = value;
331         }
332     }
333 
334     public static class Config extends Task {
335         Locales locales;
336         Paths paths;
337         OverrideFallback ofb;
338         String type;
339 
addConfiguredLocales(Locales loc)340         public void addConfiguredLocales(Locales loc) {
341             if (locales != null) {
342                 exitWithError("Multiple <locales> elements not supported");
343             }
344             locales = loc;
345         }
346 
addConfiguredPaths(Paths ps)347         public void addConfiguredPaths(Paths ps) {
348             if (paths != null) {
349                 exitWithError("Multiple <paths> elements not supported");
350             }
351             paths = ps;
352         }
353 
addConfiguredOverrideFallback(OverrideFallback ofb)354         public void addConfiguredOverrideFallback(OverrideFallback ofb) {
355             if (this.ofb != null) {
356                 exitWithError("Multiple <overrideFallback> elements not allowed!");
357             }
358             this.ofb = ofb;
359         }
360 
setType(String type)361         public void setType(String type) {
362             this.type = type;
363         }
364     }
365 
366     public static class Locales extends Task {
367         List<InExclude> localesList = new ArrayList<InExclude>();
368 
addConfiguredInclude(Include include)369         public void addConfiguredInclude(Include include) {
370             addInEx(include);
371         }
372 
addConfiguredExclude(Exclude exclude)373         public void addConfiguredExclude(Exclude exclude) {
374             addInEx(exclude);
375         }
376 
addInEx(InExclude inex)377         private void addInEx(InExclude inex) {
378             inex.validate();
379             localesList.add(inex);
380         }
381     }
382 
383     public static class InExclude extends Task {
384         static final List<String> ANY = Collections.emptyList();
385 
386         final boolean include;
387         List<String> locales;
388         String draft;
389         String xpath;
390         String alt;
391 
InExclude(boolean include)392         protected InExclude(boolean include) {
393             this.include = include;
394         }
395 
setDraft(String draft)396         public void setDraft(String draft) {
397             this.draft = draft;
398         }
399 
setLocales(String locales)400         public void setLocales(String locales) {
401             if (".*".equals(locales)) {
402                 this.locales = ANY;
403             } else {
404                 this.locales = Arrays.asList(locales.split("\\s+"));
405             }
406         }
407 
setXpath(String xpath)408         public void setXpath(String xpath) {
409             this.xpath = xpath;
410         }
411 
setAlt(String alt)412         public void setAlt(String alt) {
413             this.alt = alt;
414         }
415 
validate()416         void validate() {
417             if (locales == null) {
418                 exitWithError("locales attribute not set for include/exclude element!");
419             }
420         }
421 
matchesFileName(String fileName)422         boolean matchesFileName(String fileName) {
423             if (locales == ANY) {
424                 return true;
425             }
426             String localePattern = fileName.substring(0, fileName.indexOf(".xml"));
427             return matchesLocale(locales, localePattern);
428         }
429 
430         @Override
equals(Object o)431         public boolean equals(Object o) {
432             if (!(o instanceof InExclude)) {
433                 return false;
434             }
435 
436             if (o == this) {
437                 return true;
438             }
439 
440             InExclude rhs = (InExclude) o;
441             return include == rhs.include &&
442                 equalLists(locales, rhs.locales) &&
443                 equalStrings(draft, rhs.draft) &&
444                 equalStrings(xpath, rhs.xpath) &&
445                 equalStrings(alt, rhs.alt);
446         }
447 
448         @Override
hashCode()449         public int hashCode() {
450             return hash(locales, hash(draft, hash(xpath, hash(alt, 0))));
451         }
452 
equalStrings(String lhs, String rhs)453         private boolean equalStrings(String lhs, String rhs) {
454             return lhs == rhs || (lhs != null && lhs.equals(rhs));
455         }
456 
equalLists(List<? extends T> lhs, List<? extends T> rhs)457         private <T> boolean equalLists(List<? extends T> lhs, List<? extends T> rhs) {
458             return lhs == rhs || (lhs != null && lhs.equals(rhs));
459         }
460 
hash(Object rhs, int hash)461         private int hash(Object rhs, int hash) {
462             return rhs == null ? hash : (hash * 31) ^ rhs.hashCode();
463         }
464     }
465 
466     public static class Include extends InExclude {
Include()467         public Include() {
468             super(true);
469         }
470     }
471 
472     public static class Exclude extends InExclude {
Exclude()473         public Exclude() {
474             super(false);
475         }
476     }
477 
478     public static class Deprecates extends Task {
479         List<String> aliasLocaleList;
480         List<String> emptyLocaleList;
481         List<CLDRConverterTool.Alias> aliasList;
482 
addConfiguredAlias(Alias alias)483         public void addConfiguredAlias(Alias alias) {
484             if (aliasList == null) {
485                 aliasList = new ArrayList<CLDRConverterTool.Alias>();
486             }
487             aliasList.add(new CLDRConverterTool.Alias(alias.from, alias.to, alias.xpath, alias.rbPath, alias.value));
488         }
489 
addConfiguredEmptyLocale(EmptyLocale alias)490         public void addConfiguredEmptyLocale(EmptyLocale alias) {
491             if (emptyLocaleList == null) {
492                 emptyLocaleList = new ArrayList<String>();
493             }
494             emptyLocaleList.add(alias.locale);
495         }
496 
addConfiguredAliasLocale(AliasLocale alias)497         public void addConfiguredAliasLocale(AliasLocale alias) {
498             if (aliasLocaleList == null) {
499                 aliasLocaleList = new ArrayList<String>();
500             }
501             aliasLocaleList.add(alias.locale);
502         }
503     }
504 
505     public static class Alias extends Task {
506         String from;
507         String to;
508         // TODO(jchye): remove xpath field after old converter is deleted.
509         String xpath;
510         String rbPath;
511         String value;
512 
setFrom(String from)513         public void setFrom(String from) {
514             this.from = from;
515         }
516 
setTo(String to)517         public void setTo(String to) {
518             this.to = to;
519         }
520 
setXpath(String xpath)521         public void setXpath(String xpath) {
522             this.xpath = xpath;
523         }
524 
setRbPath(String rbPath)525         public void setRbPath(String rbPath) {
526             this.rbPath = rbPath;
527         }
528 
setValue(String value)529         public void setValue(String value) {
530             this.value = value;
531         }
532     }
533 
534     public static class AliasLocale extends Task {
535         String locale;
536 
setLocale(String locale)537         public void setLocale(String locale) {
538             this.locale = locale;
539         }
540     }
541 
542     public static class EmptyLocale extends Task {
543         String locale;
544         String list;
545 
setLocale(String locale)546         public void setLocale(String locale) {
547             this.locale = locale;
548         }
549 
setList(String list)550         public void setList(String list) {
551             this.list = list;
552         }
553     }
554 
555     public static class Paths extends Task {
556         public String fallback;
557         public String locales;
558         public String draft;
559 
560         private List<Task> pathList = new ArrayList<Task>();
561 
addConfiguredInclude(Include include)562         public void addConfiguredInclude(Include include) {
563             pathList.add(include);
564         }
565 
addConfiguredExclude(Exclude exclude)566         public void addConfiguredExclude(Exclude exclude) {
567             pathList.add(exclude);
568         }
569 
setFallback(String fallback)570         public void setFallback(String fallback) {
571             this.fallback = fallback;
572         }
573 
setLocales(String locales)574         public void setLocales(String locales) {
575             this.locales = locales;
576         }
577 
setDraft(String draft)578         public void setDraft(String draft) {
579             this.draft = draft;
580         }
581 
addConfiguredCoverageLevel(CoverageLevel level)582         public void addConfiguredCoverageLevel(CoverageLevel level) {
583             level.validate();
584             pathList.add(level);
585         }
586     }
587 
588     public static class CoverageLevel extends Task {
589         public String group;
590         public String level;
591         public String locales;
592         public String draft;
593         public String org;
594 
setDraft(String draft)595         public void setDraft(String draft) {
596             this.draft = draft;
597         }
598 
setLevel(String level)599         public void setLevel(String level) {
600             this.level = level;
601         }
602 
setLocales(String locales)603         public void setLocales(String locales) {
604             this.locales = locales;
605         }
606 
setOrg(String org)607         public void setOrg(String org) {
608             this.org = org;
609         }
610 
setGroup(String group)611         public void setGroup(String group) {
612             this.group = group;
613         }
614 
validate()615         void validate() {
616             if ((group == null) != (org == null)) {
617                 exitWithError("Invalid specification of coverageLevel element; org && group not set!");
618             }
619 
620             if (level == null) {
621                 exitWithError("Invalid specification of coverageLevel element; level not set!");
622             }
623         }
624     }
625 
626     public static class OverrideFallback extends Task {
627         List<Paths> pathsList = new ArrayList<Paths>();
628 
addConfiguredPaths(Paths paths)629         public void addConfiguredPaths(Paths paths) {
630             pathsList.add(paths);
631         }
632     }
633 
634     public static class Remap extends Task {
635         public String sourcePath;
636         public String targetPath;
637         public String targetDir;
638 
setSourcePath(String sourcePath)639         public void setSourcePath(String sourcePath) {
640             this.sourcePath = sourcePath;
641         }
642 
setTargetPath(String targetPath)643         public void setTargetPath(String targetPath) {
644             this.targetPath = targetPath;
645         }
646 
setTargetDir(String targetDir)647         public void setTargetDir(String targetDir) {
648             this.targetDir = targetDir;
649         }
650     }
651 
652     public static class Remapper extends Task {
653         public String baseDir;
654         List<Remap> remaps = new ArrayList<Remap>();
655 
setBaseDir(String baseDir)656         public void setBaseDir(String baseDir) {
657             this.baseDir = baseDir;
658         }
659 
addConfiguredRemap(Remap remap)660         public void addConfiguredRemap(Remap remap) {
661             if (remap.sourcePath == null || remap.sourcePath.trim().isEmpty()) {
662                 exitWithError("remap source path must not be empty");
663             }
664             remap.sourcePath = remap.sourcePath.trim();
665 
666             if (remap.targetPath != null && remap.targetPath.trim().isEmpty()) {
667                 remap.targetPath = null;
668             }
669 
670             if (remap.targetDir != null && remap.targetDir.trim().isEmpty()) {
671                 remap.targetDir = null;
672             }
673 
674             remaps.add(remap);
675         }
676     }
677 }
678