• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  ******************************************************************************
3  * Copyright (C) 2005-2014, International Business Machines Corporation and   *
4  * others. All Rights Reserved.                                               *
5  ******************************************************************************
6  */
7 
8 package org.unicode.cldr.test;
9 
10 import java.io.BufferedReader;
11 import java.io.IOException;
12 import java.text.ParsePosition;
13 import java.util.ArrayList;
14 import java.util.Arrays;
15 import java.util.EnumSet;
16 import java.util.HashMap;
17 import java.util.Iterator;
18 import java.util.List;
19 import java.util.Locale;
20 import java.util.Map;
21 import java.util.Set;
22 import java.util.TreeSet;
23 import java.util.regex.Matcher;
24 import java.util.regex.Pattern;
25 
26 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype;
27 import org.unicode.cldr.util.CLDRFile;
28 import org.unicode.cldr.util.CLDRInfo.CandidateInfo;
29 import org.unicode.cldr.util.CLDRInfo.PathValueInfo;
30 import org.unicode.cldr.util.CLDRInfo.UserInfo;
31 import org.unicode.cldr.util.CLDRLocale;
32 import org.unicode.cldr.util.CldrUtility;
33 import org.unicode.cldr.util.Factory;
34 import org.unicode.cldr.util.InternalCldrException;
35 import org.unicode.cldr.util.Level;
36 import org.unicode.cldr.util.PathHeader;
37 import org.unicode.cldr.util.PathHeader.SurveyToolStatus;
38 import org.unicode.cldr.util.PatternCache;
39 import org.unicode.cldr.util.RegexFileParser;
40 import org.unicode.cldr.util.RegexFileParser.RegexLineParser;
41 import org.unicode.cldr.util.StandardCodes;
42 import org.unicode.cldr.util.TransliteratorUtilities;
43 import org.unicode.cldr.util.VoteResolver;
44 
45 import com.ibm.icu.dev.util.ElapsedTimer;
46 import com.ibm.icu.impl.Row.R3;
47 import com.ibm.icu.text.ListFormatter;
48 import com.ibm.icu.text.MessageFormat;
49 import com.ibm.icu.text.Transliterator;
50 import com.ibm.icu.util.ICUUncheckedIOException;
51 
52 /**
53  * This class provides a foundation for both console-driven CLDR tests, and
54  * Survey Tool Tests.
55  * <p>
56  * To add a test, subclass CLDRFile and override handleCheck and possibly setCldrFileToCheck. Then put the test into
57  * getCheckAll.
58  * <p>
59  * To use the test, take a look at the main in ConsoleCheckCLDR. Note that you need to call setDisplayInformation with
60  * the CLDRFile for the locale that you want the display information (eg names for codes) to be in.<br>
61  * Some options are passed in the Map options. Examples: boolean SHOW_TIMES = options.containsKey("SHOW_TIMES"); // for
62  * printing times for doing setCldrFileToCheck.
63  * <p>
64  * Some errors/warnings will be explicitly filtered out when calling CheckCLDR's check() method.
65  * The full list of filters can be found in org/unicode/cldr/util/data/CheckCLDR-exceptions.txt.
66  *
67  * @author davis
68  */
69 abstract public class CheckCLDR {
70     private static CLDRFile displayInformation;
71 
72     private CLDRFile cldrFileToCheck;
73     private CLDRFile englishFile = null;
74 
75     private boolean skipTest = false;
76     private Phase phase;
77     private Map<Subtype, List<Pattern>> filtersForLocale = new HashMap<Subtype, List<Pattern>>();
78 
79     public enum InputMethod {
80         DIRECT, BULK
81     }
82 
83     public enum StatusAction {
84         /**
85          * Allow voting and add new values (in Change column).
86          */
87         ALLOW,
88         /**
89          * Allow voting and ticket (in Change column).
90          */
91         ALLOW_VOTING_AND_TICKET,
92         /**
93          * Allow voting but no add new values (in Change column).
94          */
95         ALLOW_VOTING_BUT_NO_ADD,
96         /**
97          * Only allow filing a ticket.
98          */
99         ALLOW_TICKET_ONLY,
100         /**
101          * Disallow (for various reasons)
102          */
103         FORBID_ERRORS(true), FORBID_READONLY(true), FORBID_UNLESS_DATA_SUBMISSION(true), FORBID_COVERAGE(true), FORBID_NEEDS_TICKET(true);
104 
105         private final boolean isForbidden;
106 
StatusAction()107         private StatusAction() {
108             isForbidden = false;
109         };
110 
StatusAction(boolean isForbidden)111         private StatusAction(boolean isForbidden) {
112             this.isForbidden = isForbidden;
113         };
114 
isForbidden()115         public boolean isForbidden() {
116             return isForbidden;
117         }
118 
canShow()119         public boolean canShow() {
120             return !isForbidden;
121         }
122 
123         /**
124          * @deprecated
125          */
126         public static final StatusAction FORBID = FORBID_READONLY;
127         /**
128          * @deprecated
129          */
130         public static final StatusAction SHOW_VOTING_AND_ADD = ALLOW;
131         /**
132          * @deprecated
133          */
134         public static final StatusAction SHOW_VOTING_AND_TICKET = ALLOW_VOTING_AND_TICKET;
135         /**
136          * @deprecated
137          */
138         public static final StatusAction SHOW_VOTING_BUT_NO_ADD = ALLOW_VOTING_BUT_NO_ADD;
139         /**
140          * @deprecated
141          */
142         public static final StatusAction FORBID_HAS_ERROR = FORBID_ERRORS;
143     }
144 
145     private static final HashMap<String, Phase> PHASE_NAMES = new HashMap<String, Phase>();
146 
147     public enum Phase {
148         BUILD, SUBMISSION, VETTING, FINAL_TESTING("RESOLUTION");
Phase(String... alternateName)149         Phase(String... alternateName) {
150             for (String name : alternateName) {
151                 PHASE_NAMES.put(name.toUpperCase(Locale.ENGLISH), this);
152             }
153         }
154 
forString(String value)155         public static Phase forString(String value) {
156             if (value == null) {
157                 return org.unicode.cldr.util.CLDRConfig.getInstance().getPhase();
158             }
159             value = value.toUpperCase(Locale.ENGLISH);
160             Phase result = PHASE_NAMES.get(value);
161             return result != null ? result
162                 : Phase.valueOf(value);
163         }
164 
165         /**
166          * Return whether or not to show a row, and if so, how.
167          *
168          * @param pathValueInfo
169          * @param inputMethod
170          * @param status
171          * @param userInfo
172          *            null if there is no userInfo (nobody logged in).
173          * @return
174          */
getShowRowAction( PathValueInfo pathValueInfo, InputMethod inputMethod, PathHeader.SurveyToolStatus status, UserInfo userInfo )175         public StatusAction getShowRowAction(
176             PathValueInfo pathValueInfo,
177             InputMethod inputMethod,
178             PathHeader.SurveyToolStatus status,
179             UserInfo userInfo // can get voterInfo from this.
180         ) {
181 
182             // always forbid deprecated items - don't show.
183             if (status == SurveyToolStatus.DEPRECATED) {
184                 return StatusAction.FORBID_READONLY;
185             }
186 
187             if (status == SurveyToolStatus.READ_ONLY) {
188                 return StatusAction.ALLOW_TICKET_ONLY;
189             }
190 
191             // always forbid bulk import except in data submission.
192             if (inputMethod == InputMethod.BULK && this != Phase.SUBMISSION) {
193                 return StatusAction.FORBID_UNLESS_DATA_SUBMISSION;
194             }
195 
196             // if TC+, allow anything else, even suppressed items and errors
197             if (userInfo != null && userInfo.getVoterInfo().getLevel().compareTo(VoteResolver.Level.tc) >= 0) {
198                 return StatusAction.ALLOW;
199             }
200 
201             // if the coverage level is optional, disallow everything
202             if (pathValueInfo.getCoverageLevel().compareTo(Level.COMPREHENSIVE) > 0) {
203                 return StatusAction.FORBID_COVERAGE;
204             }
205 
206             if (status == SurveyToolStatus.HIDE) {
207                 return StatusAction.FORBID_READONLY;
208             }
209 
210             if (this == Phase.SUBMISSION) {
211                 return (status == SurveyToolStatus.READ_WRITE || status == SurveyToolStatus.LTR_ALWAYS)
212                     ? StatusAction.ALLOW
213                     : StatusAction.ALLOW_VOTING_AND_TICKET;
214             }
215 
216             // We are not in submission.
217             // Only allow ADD if we have an error or warning
218             ValueStatus valueStatus = ValueStatus.NONE;
219             CandidateInfo winner = pathValueInfo.getCurrentItem();
220             // Only check winning value for errors/warnings per ticket #8677
221             // We used to check all candidates.
222 //            for (CandidateInfo value : pathValueInfo.getValues()) {
223             valueStatus = getValueStatus(winner, valueStatus);
224             if (valueStatus != ValueStatus.NONE) {
225                 return (status == SurveyToolStatus.READ_WRITE || status == SurveyToolStatus.LTR_ALWAYS)
226                     ? StatusAction.ALLOW
227                     : StatusAction.ALLOW_VOTING_AND_TICKET;
228             }
229 //            }
230 
231             // No warnings, so allow just voting.
232             return StatusAction.ALLOW_VOTING_BUT_NO_ADD;
233         }
234 
235         /**
236          * getAcceptNewItemAction. MUST only be called if getShowRowAction(...).canShow()
237          * TODO Consider moving Phase, StatusAction, etc into CLDRInfo.
238          *
239          * @param enteredValue
240          *            If null, means an abstention.
241          *            If voting for an existing value, pathValueInfo.getValues().contains(enteredValue) MUST be true
242          * @param pathValueInfo
243          * @param inputMethod
244          * @param status
245          * @param userInfo
246          * @return
247          */
getAcceptNewItemAction( CandidateInfo enteredValue, PathValueInfo pathValueInfo, InputMethod inputMethod, PathHeader.SurveyToolStatus status, UserInfo userInfo )248         public StatusAction getAcceptNewItemAction(
249             CandidateInfo enteredValue,
250             PathValueInfo pathValueInfo,
251             InputMethod inputMethod,
252             PathHeader.SurveyToolStatus status,
253             UserInfo userInfo // can get voterInfo from this.
254         ) {
255             if (status != SurveyToolStatus.READ_WRITE && status != SurveyToolStatus.LTR_ALWAYS) {
256                 return StatusAction.FORBID_READONLY; // not writable.
257             }
258 
259             // only logged in users can add items.
260             if (userInfo == null) {
261                 return StatusAction.FORBID_ERRORS;
262             }
263 
264             // we can always abstain
265             if (enteredValue == null) {
266                 return StatusAction.ALLOW;
267             }
268 
269             // if TC+, allow anything else, even suppressed items and errors
270             if (userInfo.getVoterInfo().getLevel().compareTo(VoteResolver.Level.tc) >= 0) {
271                 return StatusAction.ALLOW;
272             }
273 
274             // Disallow errors.
275             ValueStatus valueStatus = getValueStatus(enteredValue, ValueStatus.NONE);
276             if (valueStatus == ValueStatus.ERROR) {
277                 return StatusAction.FORBID_ERRORS;
278             }
279 
280             // Allow items if submission
281             if (this == Phase.SUBMISSION) {
282                 return StatusAction.ALLOW;
283             }
284 
285             // Voting for an existing value is ok
286             valueStatus = ValueStatus.NONE;
287             for (CandidateInfo value : pathValueInfo.getValues()) {
288                 if (value == enteredValue) {
289                     return StatusAction.ALLOW;
290                 }
291                 valueStatus = getValueStatus(value, valueStatus);
292             }
293 
294             // If there were any errors/warnings on other values, allow
295             if (valueStatus != ValueStatus.NONE) {
296                 return StatusAction.ALLOW;
297             }
298 
299             // Otherwise (we are vetting, but with no errors or warnings)
300             // DISALLOW NEW STUFF
301 
302             return StatusAction.FORBID_UNLESS_DATA_SUBMISSION;
303         }
304 
305         enum ValueStatus {
306             ERROR, WARNING, NONE
307         }
308 
getValueStatus(CandidateInfo value, ValueStatus previous)309         private ValueStatus getValueStatus(CandidateInfo value, ValueStatus previous) {
310             if (previous == ValueStatus.ERROR || value == null) {
311                 return previous;
312             }
313 
314             for (CheckStatus item : value.getCheckStatusList()) {
315                 CheckStatus.Type type = item.getType();
316                 if (type.equals(CheckStatus.Type.Error)) {
317                     if (CheckStatus.crossCheckSubtypes.contains(item.getSubtype())) {
318                         return ValueStatus.WARNING;
319                     } else {
320                         return ValueStatus.ERROR;
321                     }
322                 } else if (type.equals(CheckStatus.Type.Warning)) {
323                     previous = ValueStatus.WARNING;
324                 }
325             }
326             return previous;
327         }
328     }
329 
330     public static final class Options implements Comparable<Options> {
331 
332         public enum Option {
333             locale, CoverageLevel_requiredLevel("CoverageLevel.requiredLevel"), CoverageLevel_localeType(
334                 "CoverageLevel.localeType"), SHOW_TIMES, phase, lgWarningCheck, CheckCoverage_skip("CheckCoverage.skip"), exemplarErrors;
335 
336             private String key;
337 
getKey()338             public String getKey() {
339                 return key;
340             }
341 
Option(String key)342             Option(String key) {
343                 this.key = key;
344             }
345 
Option()346             Option() {
347                 this.key = name();
348             }
349         };
350 
351         private static StandardCodes sc = StandardCodes.make();
352 
353         private final boolean DEBUG_OPTS = false;
354 
355         String options[] = new String[Option.values().length];
356         CLDRLocale locale = null;
357 
358         private final String key; // for fast compare
359 
360         /**
361          * Adopt some other map
362          * @param fromOptions
363          */
Options(Map<String, String> fromOptions)364         public Options(Map<String, String> fromOptions) {
365             clear();
366             setAll(fromOptions);
367             key = null; // no key = slow compare
368         }
369 
setAll(Map<String, String> fromOptions)370         private void setAll(Map<String, String> fromOptions) {
371             for (Map.Entry<String, String> e : fromOptions.entrySet()) {
372                 set(e.getKey(), e.getValue());
373             }
374         }
375 
376         /**
377          * @param key
378          * @param value
379          */
set(String key, String value)380         public void set(String key, String value) {
381             // TODO- cache the map
382             for (Option o : Option.values()) {
383                 if (o.getKey().equals(key)) {
384                     set(o, value);
385                     return;
386                 }
387             }
388             throw new IllegalArgumentException("Unknown CLDR option: '" + key + "' - valid keys are: " + Options.getValidKeys());
389         }
390 
getValidKeys()391         private static String getValidKeys() {
392             Set<String> allkeys = new TreeSet<String>();
393             for (Option o : Option.values()) {
394                 allkeys.add(o.getKey());
395             }
396             return ListFormatter.getInstance().format(allkeys);
397         }
398 
Options()399         public Options() {
400             clear();
401             key = "".intern(); // null Options.
402         }
403 
404         /**
405          * Deep clone
406          * @param options2
407          */
Options(Options options2)408         public Options(Options options2) {
409             this.options = Arrays.copyOf(options2.options, options2.options.length);
410             this.key = options2.key;
411         }
412 
Options(CLDRLocale locale, CheckCLDR.Phase testPhase, String requiredLevel, String localeType)413         public Options(CLDRLocale locale, CheckCLDR.Phase testPhase, String requiredLevel, String localeType) {
414             this.locale = locale;
415             options = new String[Option.values().length];
416             StringBuilder sb = new StringBuilder();
417             set(Option.locale, locale.getBaseName());
418             sb.append(locale.getBaseName()).append('/');
419             set(Option.CoverageLevel_requiredLevel, requiredLevel);
420             sb.append(requiredLevel).append('/');
421             set(Option.CoverageLevel_localeType, localeType);
422             sb.append(localeType).append('/');
423             set(Option.phase, testPhase.name().toLowerCase());
424             sb.append(localeType).append('/');
425             key = sb.toString().intern();
426         }
427 
428         @Override
clone()429         public Options clone() {
430             return new Options(this);
431         }
432 
433         @Override
equals(Object other)434         public boolean equals(Object other) {
435             if (this == other) return true;
436             if (!(other instanceof Options)) return false;
437             if (this.key != null && ((Options) other).key != null) {
438                 return (this.key == ((Options) other).key);
439             } else {
440                 return this.compareTo((Options) other) == 0;
441             }
442         }
443 
clear(Option o)444         private Options clear(Option o) {
445             set(o, null);
446             return this;
447         }
448 
clear()449         private Options clear() {
450             for (int i = 0; i < options.length; i++) {
451                 options[i] = null;
452             }
453             return this;
454         }
455 
set(Option o, String v)456         private Options set(Option o, String v) {
457             options[o.ordinal()] = v;
458             if (DEBUG_OPTS) System.err.println("Setting " + o + " = " + v);
459             return this;
460         }
461 
get(Option o)462         public String get(Option o) {
463             final String v = options[o.ordinal()];
464             if (DEBUG_OPTS) System.err.println("Getting " + o + " = " + v);
465             return v;
466         }
467 
getLocale()468         public CLDRLocale getLocale() {
469             if (locale != null) return locale;
470             return CLDRLocale.getInstance(get(Option.locale));
471         }
472 
getRequiredLevel(String localeID)473         public Level getRequiredLevel(String localeID) {
474             Level result;
475             // see if there is an explicit level
476             String localeType = get(Option.CoverageLevel_requiredLevel);
477             if (localeType != null) {
478                 result = Level.get(localeType);
479                 if (result != Level.UNDETERMINED) {
480                     return result;
481                 }
482             }
483             // otherwise, see if there is an organization level
484             return sc.getLocaleCoverageLevel("Cldr", localeID);
485         }
486 
contains(Option o)487         public boolean contains(Option o) {
488             String s = get(o);
489             return (s != null && !s.isEmpty());
490         }
491 
492         @Override
compareTo(Options other)493         public int compareTo(Options other) {
494             if (other == this) return 0;
495             if (key != null && other.key != null) {
496                 if (key == other.key) return 0;
497                 return key.compareTo(other.key);
498             }
499             for (int i = 0; i < options.length; i++) {
500                 final String s1 = options[i];
501                 final String s2 = other.options[i];
502                 if (s1 == null && s2 == null) {
503                     // no difference
504                 } else if (s1 == null) {
505                     return -1;
506                 } else if (s2 == null) {
507                     return 1;
508                 } else {
509                     int rv = s1.compareTo(s2);
510                     if (rv != 0) {
511                         return rv;
512                     }
513                 }
514             }
515             return 0;
516         }
517 
518         @Override
hashCode()519         public int hashCode() {
520             if (key != null) return key.hashCode();
521 
522             int h = 1;
523             for (int i = 0; i < options.length; i++) {
524                 if (options[i] == null) {
525                     h *= 11;
526                 } else {
527                     h = (h * 11) + options[i].hashCode();
528                 }
529             }
530             return h;
531         }
532 
533         @Override
toString()534         public String toString() {
535             if (key != null) return "Options:" + key;
536             StringBuilder sb = new StringBuilder();
537             for (Option o : Option.values()) {
538                 if (options[o.ordinal()] != null) {
539                     sb.append(o)
540                         .append('=')
541                         .append(options[o.ordinal()])
542                         .append(' ');
543                 }
544             }
545             return sb.toString();
546         }
547     };
548 
isSkipTest()549     public boolean isSkipTest() {
550         return skipTest;
551     }
552 
553     // this should only be set for the test in setCldrFileToCheck
setSkipTest(boolean skipTest)554     public void setSkipTest(boolean skipTest) {
555         this.skipTest = skipTest;
556     }
557 
558     /**
559      * Here is where the list of all checks is found.
560      *
561      * @param nameMatcher
562      *            Regex pattern that determines which checks are run,
563      *            based on their class name (such as .* for all checks, .*Collisions.* for CheckDisplayCollisions, etc.)
564      * @return
565      */
getCheckAll(Factory factory, String nameMatcher)566     public static CompoundCheckCLDR getCheckAll(Factory factory, String nameMatcher) {
567         return new CompoundCheckCLDR()
568             .setFilter(Pattern.compile(nameMatcher, Pattern.CASE_INSENSITIVE).matcher(""))
569             //.add(new CheckAttributeValues(factory))
570             .add(new CheckChildren(factory))
571             .add(new CheckCoverage(factory))
572             .add(new CheckDates(factory))
573             .add(new CheckForCopy(factory))
574             .add(new CheckDisplayCollisions(factory))
575             .add(new CheckExemplars(factory))
576             .add(new CheckForExemplars(factory))
577             .add(new CheckForInheritanceMarkers())
578             .add(new CheckNames())
579             .add(new CheckNumbers(factory))
580             // .add(new CheckZones()) // this doesn't work; many spurious errors that user can't correct
581             .add(new CheckMetazones())
582             .add(new CheckLogicalGroupings(factory))
583             .add(new CheckAlt())
584             .add(new CheckCurrencies())
585             .add(new CheckCasing())
586             .add(new CheckConsistentCasing(factory)) // this doesn't work; many spurious errors that user can't correct
587             .add(new CheckQuotes())
588             .add(new CheckUnits())
589             .add(new CheckWidths())
590             .add(new CheckPlaceHolders())
591             .add(new CheckNew(factory)) // this is at the end; it will check for other certain other errors and warnings and
592         // not add a message if there are any.
593         ;
594     }
595 
596     /**
597      * These determine what language is used to display information. Must be set before use.
598      *
599      * @param locale
600      * @return
601      */
getDisplayInformation()602     public static synchronized CLDRFile getDisplayInformation() {
603         return displayInformation;
604     }
605 
setDisplayInformation(CLDRFile inputDisplayInformation)606     public static synchronized void setDisplayInformation(CLDRFile inputDisplayInformation) {
607         displayInformation = inputDisplayInformation;
608     }
609 
610     /**
611      * [Warnings - please zoom in] dates/timeZoneNames/singleCountries
612      * (empty)
613      * [refs][hide] Ref: [Zoom...]
614      * [Warnings - please zoom in] dates/timeZoneNames/hours {0}/{1} {0}/{1}
615      * [refs][hide] Ref: [Zoom...]
616      * [Warnings - please zoom in] dates/timeZoneNames/hour +HH:mm;-HH:mm
617      * +HH:mm;-HH:mm
618      * [refs][hide] Ref: [Zoom...]
619      * [ok] layout/orientation (empty)
620      * [refs][hide] Ref: [Zoom...]
621      * [ok] dates/localizedPatternChars GyMdkHmsSEDFwWahKzYeugAZvcL
622      * GaMjkHmsSEDFwWxhKzAeugXZvcL
623      * [refs][hide] Ref: [Zoom...]
624      */
625 
626     /**
627      * Get the CLDRFile.
628      *
629      * @param cldrFileToCheck
630      */
getCldrFileToCheck()631     public final CLDRFile getCldrFileToCheck() {
632         return cldrFileToCheck;
633     }
634 
635     /**
636      * Don't override this, use the other setCldrFileToCheck which takes an Options instead of a Map<>
637      * @param cldrFileToCheck
638      * @param options
639      * @param possibleErrors
640      * @return
641      * @see #setCldrFileToCheck(CLDRFile, Options, List)
642      */
setCldrFileToCheck(CLDRFile cldrFileToCheck, Map<String, String> options, List<CheckStatus> possibleErrors)643     final public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Map<String, String> options,
644         List<CheckStatus> possibleErrors) {
645         return setCldrFileToCheck(cldrFileToCheck, new Options(options), possibleErrors);
646     }
647 
648     /**
649      * Set the CLDRFile. Must be done before calling check. If null is called, just skip
650      * Often subclassed for initializing. If so, make the first 2 lines:
651      * if (cldrFileToCheck == null) return this;
652      * super.setCldrFileToCheck(cldrFileToCheck);
653      * do stuff
654      *
655      * @param cldrFileToCheck
656      * @param options (not currently used)
657      * @param possibleErrors
658      *            TODO
659      */
setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)660     public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options,
661         List<CheckStatus> possibleErrors) {
662         this.cldrFileToCheck = cldrFileToCheck;
663 
664         // Shortlist error filters for this locale.
665         loadFilters();
666         String locale = cldrFileToCheck.getLocaleID();
667         filtersForLocale.clear();
668         for (R3<Pattern, Subtype, Pattern> filter : allFilters) {
669             if (filter.get0() == null || !filter.get0().matcher(locale).matches()) continue;
670             Subtype subtype = filter.get1();
671             List<Pattern> xpaths = filtersForLocale.get(subtype);
672             if (xpaths == null) {
673                 filtersForLocale.put(subtype, xpaths = new ArrayList<Pattern>());
674             }
675             xpaths.add(filter.get2());
676         }
677         return this;
678     }
679 
680     /**
681      * Status value returned from check
682      */
683     public static class CheckStatus {
684         public static final Type alertType = Type.Comment,
685             warningType = Type.Warning,
686             errorType = Type.Error,
687             exampleType = Type.Example,
688             demoType = Type.Demo;
689 
690         public enum Type {
691             Comment, Warning, Error, Example, Demo
692         };
693 
694         public enum Subtype {
695             none, noUnproposedVariant, deprecatedAttribute, illegalPlural, invalidLocale, incorrectCasing,
696             valueMustBeOverridden,
697             valueAlwaysOverridden, nullChildFile, internalError, coverageLevel, missingPluralInfo,
698             currencySymbolTooWide, incorrectDatePattern, abbreviatedDateFieldTooWide, displayCollision,
699             illegalExemplarSet, missingAuxiliaryExemplars, extraPlaceholders, missingPlaceholders,
700             shouldntHavePlaceholders, couldNotAccessExemplars, noExemplarCharacters, modifiedEnglishValue,
701             invalidCurrencyMatchSet, multipleMetazoneMappings, noMetazoneMapping, noMetazoneMappingAfter1970,
702             noMetazoneMappingBeforeNow, cannotCreateZoneFormatter, insufficientCoverage, missingLanguageTerritoryInfo,
703             missingEuroCountryInfo, deprecatedAttributeWithReplacement, missingOrExtraDateField, internalUnicodeSetFormattingError,
704             auxiliaryExemplarsOverlap, missingPunctuationCharacters,
705 
706             charactersNotInCurrencyExemplars, asciiCharactersNotInCurrencyExemplars,
707             charactersNotInMainOrAuxiliaryExemplars, asciiCharactersNotInMainOrAuxiliaryExemplars,
708 
709             narrowDateFieldTooWide, illegalCharactersInExemplars, orientationDisagreesWithExemplars,
710             inconsistentDatePattern, inconsistentTimePattern, missingDatePattern, illegalDatePattern,
711             missingMainExemplars, mustNotStartOrEndWithSpace, illegalCharactersInNumberPattern,
712             numberPatternNotCanonical, currencyPatternMissingCurrencySymbol, missingMinusSign,
713             badNumericType, percentPatternMissingPercentSymbol, illegalNumberFormat, unexpectedAttributeValue,
714             metazoneContainsDigit, tooManyGroupingSeparators, inconsistentPluralFormat, sameAsEnglishOrCode,
715             dateSymbolCollision, incompleteLogicalGroup, extraMetazoneString, inconsistentDraftStatus,
716             errorOrWarningInLogicalGroup, valueTooWide, valueTooNarrow, nameContainsYear, patternCannotContainDigits,
717             patternContainsInvalidCharacters, parenthesesNotAllowed, illegalNumberingSystem, unexpectedOrderOfEraYear,
718             invalidPlaceHolder, asciiQuotesNotAllowed, badMinimumGroupingDigits, inconsistentPeriods,
719             inheritanceMarkerNotAllowed, invalidDurationUnitPattern, invalidDelimiter, illegalCharactersInPattern,
720             badParseLenient, tooManyValues;
721 
toString()722             public String toString() {
723                 return TO_STRING.matcher(name()).replaceAll(" $1").toLowerCase();
724             }
725 
726             static Pattern TO_STRING = PatternCache.get("([A-Z])");
727         };
728 
729         public static EnumSet<Subtype> crossCheckSubtypes = EnumSet.of(
730             Subtype.dateSymbolCollision,
731             Subtype.displayCollision,
732             Subtype.inconsistentDraftStatus,
733             Subtype.incompleteLogicalGroup,
734             Subtype.inconsistentPeriods,
735             Subtype.abbreviatedDateFieldTooWide,
736             Subtype.narrowDateFieldTooWide,
737             Subtype.coverageLevel);
738 
739         private Type type;
740         private Subtype subtype = Subtype.none;
741         private String messageFormat;
742         private Object[] parameters;
743         private String htmlMessage;
744         private CheckCLDR cause;
745         private boolean checkOnSubmit = true;
746 
CheckStatus()747         public CheckStatus() {
748 
749         }
750 
isCheckOnSubmit()751         public boolean isCheckOnSubmit() {
752             return checkOnSubmit;
753         }
754 
setCheckOnSubmit(boolean dependent)755         public CheckStatus setCheckOnSubmit(boolean dependent) {
756             this.checkOnSubmit = dependent;
757             return this;
758         }
759 
getType()760         public Type getType() {
761             return type;
762         }
763 
setMainType(CheckStatus.Type type)764         public CheckStatus setMainType(CheckStatus.Type type) {
765             this.type = type;
766             return this;
767         }
768 
getMessage()769         public String getMessage() {
770             String message = messageFormat;
771             if (messageFormat != null && parameters != null) {
772                 try {
773                     String fixedApos = MessageFormat.autoQuoteApostrophe(messageFormat);
774                     MessageFormat format = new MessageFormat(fixedApos);
775                     message = format.format(parameters);
776                 } catch (Exception e) {
777                     message = messageFormat;
778                     System.err.println("MessageFormat Failure: " + subtype + "; " + messageFormat + "; "
779                         + (parameters == null ? null : Arrays.asList(parameters)));
780                     // throw new IllegalArgumentException(subtype + "; " + messageFormat + "; "
781                     // + (parameters == null ? null : Arrays.asList(parameters)), e);
782                 }
783             }
784             Exception[] exceptionParameters = getExceptionParameters();
785             if (exceptionParameters != null) {
786                 for (Exception exception : exceptionParameters) {
787                     message += "; " + exception.getMessage(); // + " \t(" + exception.getClass().getName() + ")";
788                     // for (StackTraceElement item : exception.getStackTrace()) {
789                     // message += "\n\t" + item;
790                     // }
791                 }
792             }
793             return message.replace('\t', ' ');
794         }
795 
796         /**
797          * @deprecated
798          */
getHTMLMessage()799         public String getHTMLMessage() {
800             return htmlMessage;
801         }
802 
803         /**
804          * @deprecated
805          */
setHTMLMessage(String message)806         public CheckStatus setHTMLMessage(String message) {
807             htmlMessage = message;
808             return this;
809         }
810 
setMessage(String message)811         public CheckStatus setMessage(String message) {
812             if (cause == null) {
813                 throw new IllegalArgumentException("Must have cause set.");
814             }
815             this.messageFormat = message;
816             this.parameters = null;
817             return this;
818         }
819 
setMessage(String message, Object... messageArguments)820         public CheckStatus setMessage(String message, Object... messageArguments) {
821             if (cause == null) {
822                 throw new IllegalArgumentException("Must have cause set.");
823             }
824             this.messageFormat = message;
825             this.parameters = messageArguments;
826             return this;
827         }
828 
toString()829         public String toString() {
830             return getType() + ": " + getMessage();
831         }
832 
833         /**
834          * Warning: don't change the contents of the parameters after retrieving.
835          */
getParameters()836         public Object[] getParameters() {
837             return parameters;
838         }
839 
840         /**
841          * Returns any Exception parameters in the status, or null if there are none.
842          *
843          * @return
844          */
getExceptionParameters()845         public Exception[] getExceptionParameters() {
846             if (parameters == null) {
847                 return null;
848             }
849 
850             List<Exception> errors = new ArrayList<Exception>();
851             for (Object o : parameters) {
852                 if (o instanceof Exception) {
853                     errors.add((Exception) o);
854                 }
855             }
856             if (errors.size() == 0) {
857                 return null;
858             }
859             return errors.toArray(new Exception[errors.size()]);
860         }
861 
862         /**
863          * Warning: don't change the contents of the parameters after passing in.
864          */
setParameters(Object[] parameters)865         public CheckStatus setParameters(Object[] parameters) {
866             if (cause == null) {
867                 throw new IllegalArgumentException("Must have cause set.");
868             }
869             this.parameters = parameters;
870             return this;
871         }
872 
getDemo()873         public SimpleDemo getDemo() {
874             return null;
875         }
876 
getCause()877         public CheckCLDR getCause() {
878             return cause;
879         }
880 
setCause(CheckCLDR cause)881         public CheckStatus setCause(CheckCLDR cause) {
882             this.cause = cause;
883             return this;
884         }
885 
getSubtype()886         public Subtype getSubtype() {
887             return subtype;
888         }
889 
setSubtype(Subtype subtype)890         public CheckStatus setSubtype(Subtype subtype) {
891             this.subtype = subtype;
892             return this;
893         }
894 
895         /**
896          * Convenience function: return true if any items in this list are of errorType
897          *
898          * @param result
899          *            the list to check (could be null for empty)
900          * @return true if any items in result are of errorType
901          */
hasError(List<CheckStatus> result)902         public static final boolean hasError(List<CheckStatus> result) {
903             return hasType(result, errorType);
904         }
905 
906         /**
907          * Convenience function: return true if any items in this list are of errorType
908          *
909          * @param result
910          *            the list to check (could be null for empty)
911          * @return true if any items in result are of errorType
912          */
hasType(List<CheckStatus> result, Type type)913         public static boolean hasType(List<CheckStatus> result, Type type) {
914             if (result == null) return false;
915             for (CheckStatus s : result) {
916                 if (s.getType().equals(type)) {
917                     return true;
918                 }
919             }
920             return false;
921         }
922     }
923 
924     public static abstract class SimpleDemo {
925         Map<String, String> internalPostArguments = new HashMap<String, String>();
926 
927         /**
928          * @param postArguments
929          *            A read-write map containing post-style arguments. eg TEXTBOX=abcd, etc. <br>
930          *            The first time this is called, the Map should be empty.
931          * @return true if the map has been changed
932          */
getHTML(Map<String, String> postArguments)933         public abstract String getHTML(Map<String, String> postArguments) throws Exception;
934 
935         /**
936          * Only here for compatibiltiy. Use the other getHTML instead
937          */
getHTML(String path, String fullPath, String value)938         public final String getHTML(String path, String fullPath, String value) throws Exception {
939             return getHTML(internalPostArguments);
940         }
941 
942         /**
943          * THIS IS ONLY FOR COMPATIBILITY: you can call this, then the non-postArguments form of getHTML; or better,
944          * call
945          * getHTML with the postArguments.
946          *
947          * @param postArguments
948          *            A read-write map containing post-style arguments. eg TEXTBOX=abcd, etc.
949          * @return true if the map has been changed
950          */
processPost(Map<String, String> postArguments)951         public final boolean processPost(Map<String, String> postArguments) {
952             internalPostArguments.clear();
953             internalPostArguments.putAll(postArguments);
954             return true;
955         }
956         // /**
957         // * Utility for setting map. Use the paradigm in CheckNumbers.
958         // */
959         // public boolean putIfDifferent(Map inout, String key, String value) {
960         // Object oldValue = inout.put(key, value);
961         // return !value.equals(oldValue);
962         // }
963     }
964 
965     public static abstract class FormatDemo extends SimpleDemo {
966         protected String currentPattern, currentInput, currentFormatted, currentReparsed;
967         protected ParsePosition parsePosition = new ParsePosition(0);
968 
getPattern()969         protected abstract String getPattern();
970 
getSampleInput()971         protected abstract String getSampleInput();
972 
getArguments(Map<String, String> postArguments)973         protected abstract void getArguments(Map<String, String> postArguments);
974 
getHTML(Map<String, String> postArguments)975         public String getHTML(Map<String, String> postArguments) throws Exception {
976             getArguments(postArguments);
977             StringBuffer htmlMessage = new StringBuffer();
978             FormatDemo.appendTitle(htmlMessage);
979             FormatDemo.appendLine(htmlMessage, currentPattern, currentInput, currentFormatted, currentReparsed);
980             htmlMessage.append("</table>");
981             return htmlMessage.toString();
982         }
983 
getPlainText(Map<String, String> postArguments)984         public String getPlainText(Map<String, String> postArguments) {
985             getArguments(postArguments);
986             return MessageFormat.format("<\"\u200E{0}\u200E\", \"{1}\"> \u2192 \"\u200E{2}\u200E\" \u2192 \"{3}\"",
987                 (Object[]) new String[] { currentPattern, currentInput, currentFormatted, currentReparsed });
988         }
989 
990         /**
991          * @param htmlMessage
992          * @param pattern
993          * @param input
994          * @param formatted
995          * @param reparsed
996          */
appendLine(StringBuffer htmlMessage, String pattern, String input, String formatted, String reparsed)997         public static void appendLine(StringBuffer htmlMessage, String pattern, String input, String formatted,
998             String reparsed) {
999             htmlMessage.append("<tr><td><input type='text' name='pattern' value='")
1000                 .append(TransliteratorUtilities.toXML.transliterate(pattern))
1001                 .append("'></td><td><input type='text' name='input' value='")
1002                 .append(TransliteratorUtilities.toXML.transliterate(input))
1003                 .append("'></td><td>")
1004                 .append("<input type='submit' value='Test' name='Test'>")
1005                 .append("</td><td>" + "<input type='text' name='formatted' value='")
1006                 .append(TransliteratorUtilities.toXML.transliterate(formatted))
1007                 .append("'></td><td>" + "<input type='text' name='reparsed' value='")
1008                 .append(TransliteratorUtilities.toXML.transliterate(reparsed))
1009                 .append("'></td></tr>");
1010         }
1011 
1012         /**
1013          * @param htmlMessage
1014          */
appendTitle(StringBuffer htmlMessage)1015         public static void appendTitle(StringBuffer htmlMessage) {
1016             htmlMessage.append("<table border='1' cellspacing='0' cellpadding='2'" +
1017             // " style='border-collapse: collapse' style='width: 100%'" +
1018                 "><tr>" +
1019                 "<th>Pattern</th>" +
1020                 "<th>Unlocalized Input</th>" +
1021                 "<th></th>" +
1022                 "<th>Localized Format</th>" +
1023                 "<th>Re-Parsed</th>" +
1024                 "</tr>");
1025         }
1026     }
1027 
1028     /**
1029      * Wraps the options in an Options and delegates.
1030      * @param path
1031      *            Must be a distinguished path, such as what comes out of CLDRFile.iterator()
1032      * @param fullPath
1033      *            Must be the full path
1034      * @param value
1035      *            the value associated with the path
1036      * @param options
1037      *            A set of test-specific options. Set these with code of the form:<br>
1038      *            options.put("CoverageLevel.localeType", "G0")<br>
1039      *            That is, the key is of the form <testname>.<optiontype>, and the value is of the form <optionvalue>.<br>
1040      *            There is one general option; the following will select only the tests that should be run during this
1041      *            phase.<br>
1042      *            options.put("phase", Phase.<something>);
1043      *            It can be used for new data entry.
1044      * @param result
1045      * @return
1046      * @deprecated use CheckCLDR#check(String, String, String, Options, List)
1047      */
1048     @Deprecated
check(String path, String fullPath, String value, Map<String, String> options, List<CheckStatus> result)1049     public final CheckCLDR check(String path, String fullPath, String value, Map<String, String> options,
1050         List<CheckStatus> result) {
1051         return check(path, fullPath, value, new Options(options), result);
1052     }
1053 
1054     /**
1055      * Checks the path/value in the cldrFileToCheck for correctness, according to some criterion.
1056      * If the path is relevant to the check, there is an alert or warning, then a CheckStatus is added to List.
1057      *
1058      * @param path
1059      *            Must be a distinguished path, such as what comes out of CLDRFile.iterator()
1060      * @param fullPath
1061      *            Must be the full path
1062      * @param value
1063      *            the value associated with the path
1064      * @param result
1065      */
check(String path, String fullPath, String value, Options options, List<CheckStatus> result)1066     public final CheckCLDR check(String path, String fullPath, String value, Options options,
1067         List<CheckStatus> result) {
1068         if (cldrFileToCheck == null) {
1069             throw new InternalCldrException("CheckCLDR problem: cldrFileToCheck must not be null");
1070         }
1071         if (path == null) {
1072             throw new InternalCldrException("CheckCLDR problem: path must not be null");
1073         }
1074         // if (fullPath == null) {
1075         // throw new InternalError("CheckCLDR problem: fullPath must not be null");
1076         // }
1077         // if (value == null) {
1078         // throw new InternalError("CheckCLDR problem: value must not be null");
1079         // }
1080         result.clear();
1081 
1082         /*
1083          * If the item is non-winning, and either inherited or it is code-fallback, then don't run
1084          * any tests on this item.  See http://unicode.org/cldr/trac/ticket/7574
1085          *
1086          * The following conditional formerly used "value == ..." and "value != ...", which in Java doesn't
1087          * mean what it does in some other languages. The condition has been changed to use the equals() method.
1088          * Since value can be null, check for that first.
1089          */
1090         // if (value == cldrFileToCheck.getBaileyValue(path, null, null) && value != cldrFileToCheck.getWinningValue(path)) {
1091         if (value != null
1092                 && value.equals(cldrFileToCheck.getBaileyValue(path, null, null))
1093                 && !value.equals(cldrFileToCheck.getWinningValue(path))) {
1094             return this;
1095         }
1096         // If we're being asked to run tests for an inheritance marker, then we need to change it
1097         // to the "real" value first before running tests. Testing the value CldrUtility.INHERITANCE_MARKER ("↑↑↑") doesn't make sense.
1098         if (CldrUtility.INHERITANCE_MARKER.equals(value)) {
1099             value = cldrFileToCheck.getConstructedBaileyValue(path, null, null);
1100             // If it hasn't changed, then don't run any tests.
1101             if (CldrUtility.INHERITANCE_MARKER.equals(value)) {
1102                 return this;
1103             }
1104         }
1105         CheckCLDR instance = handleCheck(path, fullPath, value, options, result);
1106         Iterator<CheckStatus> iterator = result.iterator();
1107         // Filter out any errors/warnings that match the filter list in CheckCLDR-exceptions.txt.
1108         while (iterator.hasNext()) {
1109             CheckStatus status = iterator.next();
1110             if (shouldExcludeStatus(fullPath, status)) {
1111                 iterator.remove();
1112             }
1113         }
1114         return instance;
1115     }
1116 
1117     /**
1118      * @deprecated use {@link #getExamples(String, String, String, Options, List)}
1119      * @param path
1120      * @param fullPath
1121      * @param value
1122      * @param options
1123      * @param result
1124      * @return
1125      */
1126     @Deprecated
getExamples(String path, String fullPath, String value, Map<String, String> options, List<CheckStatus> result)1127     public final CheckCLDR getExamples(String path, String fullPath, String value, Map<String, String> options,
1128         List<CheckStatus> result) {
1129         return getExamples(path, fullPath, value, new Options(options), result);
1130     }
1131 
1132     /**
1133      * Returns any examples in the result parameter. Both examples and demos can
1134      * be returned. A demo will have getType() == CheckStatus.demoType. In that
1135      * case, there will be no getMessage or getHTMLMessage available; instead,
1136      * call getDemo() to get the demo, then call getHTML() to get the initial
1137      * HTML.
1138      */
getExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result)1139     public final CheckCLDR getExamples(String path, String fullPath, String value, Options options,
1140         List<CheckStatus> result) {
1141         result.clear();
1142         return handleGetExamples(path, fullPath, value, options, result);
1143     }
1144 
handleGetExamples(String path, String fullPath, String value, Options options2, List<CheckStatus> result)1145     protected CheckCLDR handleGetExamples(String path, String fullPath, String value, Options options2,
1146         List<CheckStatus> result) {
1147         return this; // NOOP unless overridden
1148     }
1149 
1150     /**
1151      * This is what the subclasses override.
1152      * If they ever use pathParts or fullPathParts, they need to call initialize() with the respective
1153      * path. Otherwise they must NOT change pathParts or fullPathParts.
1154      * <p>
1155      * If something is found, a CheckStatus is added to result. This can be done multiple times in one call, if multiple
1156      * errors or warnings are found. The CheckStatus may return warnings, errors, examples, or demos. We may expand that
1157      * in the future.
1158      * <p>
1159      * The code to add the CheckStatus will look something like::
1160      *
1161      * <pre>
1162      * result.add(new CheckStatus()
1163      *     .setType(CheckStatus.errorType)
1164      *     .setMessage(&quot;Value should be {0}&quot;, new Object[] { pattern }));
1165      * </pre>
1166      *
1167      * @param options
1168      *            TODO
1169      */
handleCheck(String path, String fullPath, String value, Options options, List<CheckStatus> result)1170     abstract public CheckCLDR handleCheck(String path, String fullPath, String value,
1171         Options options, List<CheckStatus> result);
1172 
1173     /**
1174      * Only for use in ConsoleCheck, for debugging
1175      */
handleFinish()1176     public void handleFinish() {
1177     }
1178 
1179     /**
1180      * Internal class used to bundle up a number of Checks.
1181      *
1182      * @author davis
1183      *
1184      */
1185     static class CompoundCheckCLDR extends CheckCLDR {
1186         private Matcher filter;
1187         private List<CheckCLDR> checkList = new ArrayList<CheckCLDR>();
1188         private List<CheckCLDR> filteredCheckList = new ArrayList<CheckCLDR>();
1189 
add(CheckCLDR item)1190         public CompoundCheckCLDR add(CheckCLDR item) {
1191             checkList.add(item);
1192             if (filter == null) {
1193                 filteredCheckList.add(item);
1194             } else {
1195                 final String className = item.getClass().getName();
1196                 if (filter.reset(className).find()) {
1197                     filteredCheckList.add(item);
1198                 }
1199             }
1200             return this;
1201         }
1202 
handleCheck(String path, String fullPath, String value, Options options, List<CheckStatus> result)1203         public CheckCLDR handleCheck(String path, String fullPath, String value,
1204             Options options, List<CheckStatus> result) {
1205             result.clear();
1206             // If we're being asked to run tests for an inheritance marker, then we need to change it
1207             // to the "real" value first before running tests. Testing the value CldrUtility.INHERITANCE_MARKER ("↑↑↑") doesn't make sense.
1208             if (CldrUtility.INHERITANCE_MARKER.equals(value)) {
1209                 value = getCldrFileToCheck().getConstructedBaileyValue(path, null, null);
1210             }
1211             for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext();) {
1212                 CheckCLDR item = it.next();
1213                 // skip proposed items in final testing.
1214                 if (Phase.FINAL_TESTING == item.getPhase()) {
1215                     if (path.contains("proposed") && path.contains("[@alt=")) {
1216                         continue;
1217                     }
1218                 }
1219                 try {
1220                     if (!item.isSkipTest()) {
1221                         item.handleCheck(path, fullPath, value, options, result);
1222                     }
1223                 } catch (Exception e) {
1224                     addError(result, item, e);
1225                     return this;
1226                 }
1227             }
1228             return this;
1229         }
1230 
1231         @Override
handleFinish()1232         public void handleFinish() {
1233             for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext();) {
1234                 CheckCLDR item = it.next();
1235                 item.handleFinish();
1236             }
1237         }
1238 
handleGetExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result)1239         protected CheckCLDR handleGetExamples(String path, String fullPath, String value, Options options,
1240             List<CheckStatus> result) {
1241             result.clear();
1242             for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext();) {
1243                 CheckCLDR item = it.next();
1244                 try {
1245                     item.handleGetExamples(path, fullPath, value, options, result);
1246                 } catch (Exception e) {
1247                     addError(result, item, e);
1248                     return this;
1249                 }
1250             }
1251             return this;
1252         }
1253 
addError(List<CheckStatus> result, CheckCLDR item, Exception e)1254         private void addError(List<CheckStatus> result, CheckCLDR item, Exception e) {
1255             result.add(new CheckStatus()
1256                 .setCause(this)
1257                 .setMainType(CheckStatus.errorType)
1258                 .setSubtype(Subtype.internalError)
1259                 .setMessage("Internal error in {0}. Exception: {1}, Message: {2}, Trace: {3}",
1260                     new Object[] { item.getClass().getName(), e.getClass().getName(), e,
1261                         Arrays.asList(e.getStackTrace())
1262                     }));
1263         }
1264 
setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)1265         public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options,
1266             List<CheckStatus> possibleErrors) {
1267             ElapsedTimer testTime = null, testOverallTime = null;
1268             if (cldrFileToCheck == null) return this;
1269             boolean SHOW_TIMES = options.contains(Options.Option.SHOW_TIMES);
1270             setPhase(Phase.forString(options.get(Options.Option.phase)));
1271             if (SHOW_TIMES) testOverallTime = new ElapsedTimer("Test setup time for setCldrFileToCheck: {0}");
1272             super.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors);
1273             possibleErrors.clear();
1274 
1275             for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext();) {
1276                 CheckCLDR item = (CheckCLDR) it.next();
1277                 if (SHOW_TIMES)
1278                     testTime = new ElapsedTimer("Test setup time for " + item.getClass().toString() + ": {0}");
1279                 try {
1280                     item.setPhase(getPhase());
1281                     item.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors);
1282                     if (SHOW_TIMES) {
1283                         if (item.isSkipTest()) {
1284                             System.out.println("Disabled : " + testTime);
1285                         } else {
1286                             System.out.println("OK : " + testTime);
1287                         }
1288                     }
1289                 } catch (RuntimeException e) {
1290                     addError(possibleErrors, item, e);
1291                     if (SHOW_TIMES) System.out.println("ERR: " + testTime + " - " + e.toString());
1292                 }
1293             }
1294             if (SHOW_TIMES) System.out.println("Overall: " + testOverallTime + ": {0}");
1295             return this;
1296         }
1297 
getFilter()1298         public Matcher getFilter() {
1299             return filter;
1300         }
1301 
setFilter(Matcher filter)1302         public CompoundCheckCLDR setFilter(Matcher filter) {
1303             this.filter = filter;
1304             filteredCheckList.clear();
1305             for (Iterator<CheckCLDR> it = checkList.iterator(); it.hasNext();) {
1306                 CheckCLDR item = it.next();
1307                 if (filter == null || filter.reset(item.getClass().getName()).matches()) {
1308                     filteredCheckList.add(item);
1309                     item.setCldrFileToCheck(getCldrFileToCheck(), (Options) null, null);
1310                 }
1311             }
1312             return this;
1313         }
1314 
getFilteredTests()1315         public String getFilteredTests() {
1316             return filteredCheckList.toString();
1317         }
1318 
getFilteredTestList()1319         public List<CheckCLDR> getFilteredTestList() {
1320             return filteredCheckList;
1321         }
1322     }
1323 
1324     // static Transliterator prettyPath = getTransliteratorFromFile("ID", "prettyPath.txt");
1325 
getTransliteratorFromFile(String ID, String file)1326     public static Transliterator getTransliteratorFromFile(String ID, String file) {
1327         try {
1328             BufferedReader br = CldrUtility.getUTF8Data(file);
1329             StringBuffer input = new StringBuffer();
1330             while (true) {
1331                 String line = br.readLine();
1332                 if (line == null) break;
1333                 if (line.startsWith("\uFEFF")) line = line.substring(1); // remove BOM
1334                 input.append(line);
1335                 input.append('\n');
1336             }
1337             return Transliterator.createFromRules(ID, input.toString(), Transliterator.FORWARD);
1338         } catch (IOException e) {
1339             throw new ICUUncheckedIOException("Can't open transliterator file " + file, e);
1340         }
1341     }
1342 
getPhase()1343     public Phase getPhase() {
1344         return phase;
1345     }
1346 
setPhase(Phase phase)1347     public void setPhase(Phase phase) {
1348         this.phase = phase;
1349     }
1350 
1351     /**
1352      * A map of error/warning types to their filters.
1353      */
1354     private static List<R3<Pattern, Subtype, Pattern>> allFilters;
1355 
1356     /**
1357      * Loads the set of filters used for CheckCLDR results.
1358      */
loadFilters()1359     private void loadFilters() {
1360         if (allFilters != null) return;
1361         allFilters = new ArrayList<R3<Pattern, Subtype, Pattern>>();
1362         RegexFileParser fileParser = new RegexFileParser();
1363         fileParser.setLineParser(new RegexLineParser() {
1364             @Override
1365             public void parse(String line) {
1366                 String[] fields = line.split("\\s*;\\s*");
1367                 Subtype subtype = Subtype.valueOf(fields[0]);
1368                 Pattern locale = PatternCache.get(fields[1]);
1369                 Pattern xpathRegex = PatternCache.get(fields[2].replaceAll("\\[@", "\\\\[@"));
1370                 allFilters.add(new R3<Pattern, Subtype, Pattern>(locale, subtype, xpathRegex));
1371             }
1372         });
1373         fileParser.parse(CheckCLDR.class, "/org/unicode/cldr/util/data/CheckCLDR-exceptions.txt");
1374     }
1375 
1376     /**
1377      * Checks if a status should be excluded from the list of results returned
1378      * from CheckCLDR.
1379      * @param xpath the xpath that the status belongs to
1380      * @param status the status
1381      * @return true if the status should be included
1382      */
shouldExcludeStatus(String xpath, CheckStatus status)1383     private boolean shouldExcludeStatus(String xpath, CheckStatus status) {
1384         List<Pattern> xpathPatterns = filtersForLocale.get(status.getSubtype());
1385         if (xpathPatterns == null) return false;
1386         for (Pattern xpathPattern : xpathPatterns) {
1387             if (xpathPattern.matcher(xpath).matches()) {
1388                 return true;
1389             }
1390         }
1391         return false;
1392     }
1393 
getEnglishFile()1394     public CLDRFile getEnglishFile() {
1395         return englishFile;
1396     }
1397 
setEnglishFile(CLDRFile englishFile)1398     public void setEnglishFile(CLDRFile englishFile) {
1399         this.englishFile = englishFile;
1400     }
1401 }
1402