• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 package org.unicode.cldr.test;
2 
3 import com.ibm.icu.dev.util.ElapsedTimer;
4 import com.ibm.icu.dev.util.UOption;
5 import com.ibm.icu.impl.Relation;
6 import com.ibm.icu.impl.Row;
7 import com.ibm.icu.lang.UCharacter;
8 import com.ibm.icu.text.Collator;
9 import com.ibm.icu.text.UnicodeSet;
10 import com.ibm.icu.util.ULocale;
11 import java.io.File;
12 import java.io.IOException;
13 import java.io.PrintWriter;
14 import java.util.ArrayList;
15 import java.util.Arrays;
16 import java.util.Collection;
17 import java.util.Comparator;
18 import java.util.EnumSet;
19 import java.util.HashMap;
20 import java.util.Iterator;
21 import java.util.List;
22 import java.util.Map;
23 import java.util.Set;
24 import java.util.TreeMap;
25 import java.util.TreeSet;
26 import java.util.concurrent.ConcurrentHashMap;
27 import java.util.regex.Matcher;
28 import java.util.stream.Stream;
29 import org.unicode.cldr.draft.FileUtilities;
30 import org.unicode.cldr.test.CheckCLDR.CheckStatus;
31 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype;
32 import org.unicode.cldr.test.CheckCLDR.CompoundCheckCLDR;
33 import org.unicode.cldr.test.CheckCLDR.FormatDemo;
34 import org.unicode.cldr.test.CheckCLDR.Options;
35 import org.unicode.cldr.test.CheckCLDR.Phase;
36 import org.unicode.cldr.test.CheckCLDR.SimpleDemo;
37 import org.unicode.cldr.test.TestCache.TestResultBundle;
38 import org.unicode.cldr.tool.Option;
39 import org.unicode.cldr.tool.Option.Params;
40 import org.unicode.cldr.tool.ShowData;
41 import org.unicode.cldr.tool.TablePrinter;
42 import org.unicode.cldr.util.CLDRConfig;
43 import org.unicode.cldr.util.CLDRConfig.Environment;
44 import org.unicode.cldr.util.CLDRFile;
45 import org.unicode.cldr.util.CLDRFile.Status;
46 import org.unicode.cldr.util.CLDRPaths;
47 import org.unicode.cldr.util.CLDRTool;
48 import org.unicode.cldr.util.CldrUtility;
49 import org.unicode.cldr.util.Counter;
50 import org.unicode.cldr.util.CoverageInfo;
51 import org.unicode.cldr.util.Factory;
52 import org.unicode.cldr.util.LanguageTagParser;
53 import org.unicode.cldr.util.Level;
54 import org.unicode.cldr.util.LocaleIDParser;
55 import org.unicode.cldr.util.LogicalGrouping;
56 import org.unicode.cldr.util.Organization;
57 import org.unicode.cldr.util.Pair;
58 import org.unicode.cldr.util.PathDescription;
59 import org.unicode.cldr.util.PathHeader;
60 import org.unicode.cldr.util.PathHeader.SectionId;
61 import org.unicode.cldr.util.PathUtilities;
62 import org.unicode.cldr.util.PatternCache;
63 import org.unicode.cldr.util.SimpleFactory;
64 import org.unicode.cldr.util.StandardCodes;
65 import org.unicode.cldr.util.StringId;
66 import org.unicode.cldr.util.SupplementalDataInfo;
67 import org.unicode.cldr.util.UnicodeSetPrettyPrinter;
68 import org.unicode.cldr.util.VoteResolver;
69 import org.unicode.cldr.util.VoteResolver.CandidateInfo;
70 import org.unicode.cldr.util.VoteResolver.UnknownVoterException;
71 import org.unicode.cldr.util.VoterInfoList;
72 import org.unicode.cldr.util.XMLSource;
73 import org.unicode.cldr.util.XMLSource.SourceLocation;
74 import org.unicode.cldr.util.XPathParts;
75 
76 /**
77  * Console test for CheckCLDR. <br>
78  * Some common source directories:
79  *
80  * <pre>
81  *  -s C:/cvsdata/unicode/cldr/incoming/vetted/main
82  *  -s C:/cvsdata/unicode/cldr/incoming/proposed/main
83  *  -s C:/cvsdata/unicode/cldr/incoming/proposed/main
84  *  -s C:/cvsdata/unicode/cldr/testdata/main
85  * </pre>
86  *
87  * @author markdavis
88  */
89 @CLDRTool(alias = "check", description = "Run CheckCLDR against CLDR data")
90 public class ConsoleCheckCLDR {
91     private static final CLDRConfig CLDR_CONFIG = CLDRConfig.getInstance();
92     public static boolean showStackTrace = false;
93     public static boolean errorsOnly = false;
94     static boolean SHOW_LOCALE = true;
95     static boolean SHOW_EXAMPLES = false;
96     private static boolean CLDR_GITHUB_ANNOTATIONS =
97             (Boolean.parseBoolean(System.getProperty("CLDR_GITHUB_ANNOTATIONS", "false")));
98 
99     // TODO get ride of these in favor of MyOptions
100 
101     private static final int COVERAGE = 2,
102             EXAMPLES = 3,
103             FILE_FILTER = 4,
104             TEST_FILTER = 5,
105             DATE_FORMATS = 6,
106             ORGANIZATION = 7,
107             SHOWALL = 8,
108             PATH_FILTER = 9,
109             ERRORS_ONLY = 10,
110             CHECK_ON_SUBMIT = 11,
111             NO_ALIASES = 12,
112             SOURCE_DIRECTORY = 13,
113             USER = 14,
114             PHASE = 15,
115             GENERATE_HTML = 16,
116             VOTE_RESOLVE = 17,
117             ID_VIEW = 18,
118             SUBTYPE_FILTER = 19,
119             BAILEY = 21,
120             SINGLE_THREAD = 24;
121 
122     static final String SOURCE_DIRS =
123             CLDRPaths.MAIN_DIRECTORY
124                     + ","
125                     + CLDRPaths.ANNOTATIONS_DIRECTORY
126                     + ","
127                     + CLDRPaths.SEED_DIRECTORY;
128 
129     enum MyOptions {
130         coverage(
131                 new Params()
132                         .setHelp("Set the coverage: eg -c comprehensive")
133                         .setMatch("comprehensive|modern|moderate|basic")), // UOption.REQUIRES_ARG
134         examples(
135                 new Params()
136                         .setHelp("Turn on examples (actually a summary of the demo)")
137                         .setFlag('x')), // , 'x', UOption.NO_ARG),
138         file_filter(
139                 new Params()
140                         .setHelp(
141                                 "Pick the locales (files) to check: arg is a regular expression, eg -f fr, or -f fr.*, or -f (fr|en-.*)")
142                         .setDefault(".*")
143                         .setMatch(".*")), // , 'f', UOption.REQUIRES_ARG).setDefault(".*"),
144         test_filter(
145                 new Params()
146                         .setHelp(
147                                 "Filter the Checks: arg is a regular expression, eg -t.*number.*. To check all BUT a given test, use the style -t ((?!.*CheckZones).*)")
148                         .setDefault(".*")
149                         .setMatch(".*")), // , 't', UOption.REQUIRES_ARG).setDefault(".*"),
150         date_formats(
151                 new Params()
152                         .setHelp("Turn on special date format checks")), // , 'd', UOption.NO_ARG),
153         organization(
154                 new Params()
155                         .setHelp(
156                                 "Organization: ibm, google, ....; Uses Locales.txt for to filter locales and set coverage levels")
157                         .setDefault(".*")
158                         .setMatch(".*")), // , 'o', UOption.REQUIRES_ARG),
159         showall(
160                 new Params()
161                         .setHelp("Show all paths, including aliased")
162                         .setFlag('a')), // , 'a', UOption.NO_ARG),
163         path_filter(
164                 new Params()
165                         .setHelp("Pick the paths to check, eg -p.*languages.*")
166                         .setDefault(".*")
167                         .setMatch(".*")), // , 'p', UOption.REQUIRES_ARG).setDefault(".*"),
168         errors_only(
169                 new Params()
170                         .setHelp("Show errors only (with -ef, only final processing errors)")), // ,
171         // 'e', UOption.NO_ARG),
172         check_on_submit(new Params().setHelp("").setFlag('k')), // , 'k', UOption.NO_ARG),
173         noaliases(new Params().setHelp("No aliases")), // , 'n', UOption.NO_ARG),
174         source_directory(
175                 new Params()
176                         .setHelp("Fully qualified source directories. (Conflicts with -S.)")
177                         .setDefault(SOURCE_DIRS)
178                         .setMatch(".*")), // , 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS),
179         user(
180                 new Params()
181                         .setHelp("User, eg -uu148")
182                         .setMatch(".*")), // , 'u', UOption.REQUIRES_ARG),
183         phase(
184                 new Params()
185                         .setHelp("?")
186                         .setMatch(Phase.class)
187                         .setFlag('z')), // , 'z', UOption.REQUIRES_ARG),
188         generate_html(
189                 new Params()
190                         .setHelp("Generate HTML-style chart in directory.")
191                         .setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/")
192                         .setMatch(".*")), // , 'g',
193         // UOption.OPTIONAL_ARG).setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"),
194         vote_resolution(new Params().setHelp("")), // , 'v', UOption.NO_ARG),
195         id_view(new Params().setHelp("")), // , 'i', UOption.NO_ARG),
196         subtype_filter(
197                 new Params()
198                         .setHelp("error/warning subtype filter, eg unexpectedOrderOfEraYear")
199                         .setDefault(".*")
200                         .setMatch(".*")
201                         .setFlag('y')), // , 'y', UOption.REQUIRES_ARG),
202         source_all(
203                 new Params()
204                         .setHelp(
205                                 "Partially qualified directories. Standard subdirectories added if not specified (/main, /annotations, /subdivisions). (Conflicts with -s.)")
206                         .setMatch(".*")
207                         .setFlag('S')
208                         .setDefault("common,seed,exemplars")), // , 'S', <changed>),
209         bailey(
210                 new Params()
211                         .setHelp(
212                                 "check bailey values ("
213                                         + CldrUtility.INHERITANCE_MARKER
214                                         + ")")), // , 'b', UOption.NO_ARG)
215         exemplarError(new Params().setFlag('E').setHelp("include to force strict Exemplar check")),
216         missingPaths(
217                 new Params()
218                         .setHelp(
219                                 "include to show missing and provisional paths, at the specified level")),
220         singleThread(new Params().setHelp("Run in single-thread mode.").setFlag('1'));
221 
222         // BOILERPLATE TO COPY
223         final Option option;
224 
MyOptions(Params params)225         private MyOptions(Params params) {
226             option = new Option(this, params);
227         }
228 
229         private static Option.Options myOptions = new Option.Options();
230 
231         static {
232             for (MyOptions option : MyOptions.values()) {
myOptions.add(option, option.option)233                 myOptions.add(option, option.option);
234             }
235         }
236 
parse(String[] args, boolean showArguments)237         private static Set<String> parse(String[] args, boolean showArguments) {
238             return myOptions.parse(MyOptions.values()[0], args, true);
239         }
240     }
241 
242     // TODO get rid of these in favor of MyOptions
243 
244     private static final UOption[] options = {
245         UOption.HELP_H(),
246         UOption.HELP_QUESTION_MARK(),
247         UOption.create("coverage", 'c', UOption.REQUIRES_ARG),
248         UOption.create("examples", 'x', UOption.NO_ARG),
249         UOption.create("file_filter", 'f', UOption.REQUIRES_ARG).setDefault(".*"),
250         UOption.create("test_filter", 't', UOption.REQUIRES_ARG).setDefault(".*"),
251         UOption.create("date_formats", 'd', UOption.NO_ARG),
252         UOption.create("organization", 'o', UOption.REQUIRES_ARG),
253         UOption.create("showall", 'a', UOption.NO_ARG),
254         UOption.create("path_filter", 'p', UOption.REQUIRES_ARG).setDefault(".*"),
255         UOption.create("errors_only", 'e', UOption.NO_ARG),
256         UOption.create("check-on-submit", 'k', UOption.NO_ARG),
257         UOption.create("noaliases", 'n', UOption.NO_ARG),
258         UOption.create("source_directory", 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS),
259         UOption.create("user", 'u', UOption.REQUIRES_ARG),
260         UOption.create("phase", 'z', UOption.REQUIRES_ARG),
261         UOption.create("generate_html", 'g', UOption.OPTIONAL_ARG)
262                 .setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"),
263         UOption.create("vote resolution", 'v', UOption.NO_ARG),
264         UOption.create("id view", 'i', UOption.NO_ARG),
265         UOption.create("subtype_filter", 'y', UOption.REQUIRES_ARG),
266         UOption.create("source_all", 'S', UOption.OPTIONAL_ARG).setDefault("common,seed,exemplars"),
267         UOption.create("bailey", 'b', UOption.NO_ARG),
268         UOption.create("exemplarError", 'E', UOption.NO_ARG),
269         UOption.create("missingPaths", 'm', UOption.NO_ARG),
270         UOption.create("singleThread", '1', UOption.NO_ARG)
271     };
272 
273     private static final Comparator<String> baseFirstCollator =
274             new Comparator<>() {
275                 LanguageTagParser languageTagParser1 = new LanguageTagParser();
276                 LanguageTagParser languageTagParser2 = new LanguageTagParser();
277 
278                 @Override
279                 public int compare(String o1, String o2) {
280                     String ls1 = languageTagParser1.set(o1).getLanguageScript();
281                     String ls2 = languageTagParser2.set(o2).getLanguageScript();
282                     int result = ls1.compareTo(ls2);
283                     if (result != 0) return result;
284                     return o1.compareTo(o2);
285                 }
286             };
287     private static final boolean PATH_IN_COUNT = false;
288 
289     static Counter<ErrorType> subtotalCount = new Counter<>(true); // new ErrorCount();
290     static Counter<ErrorType> totalCount = new Counter<>(true);
291 
292     private enum RawStatus {
293         missing,
294         provisional,
295         present
296     }
297 
calculateSubtypeFilter(final String subtypeFilterString)298     private static EnumSet<Subtype> calculateSubtypeFilter(final String subtypeFilterString) {
299         EnumSet<Subtype> subtypeFilter = null;
300         if (subtypeFilterString != null) {
301             subtypeFilter = EnumSet.noneOf(Subtype.class);
302             Matcher m = PatternCache.get(subtypeFilterString).matcher("");
303             for (Subtype value : Subtype.values()) {
304                 if (m.reset(value.toString()).find() || m.reset(value.name()).find()) {
305                     subtypeFilter.add(value);
306                 }
307             }
308             if (subtypeFilter.size() == 0) {
309                 throw new IllegalArgumentException("No subtype match for " + subtypeFilterString);
310             }
311         }
312         return subtypeFilter;
313     }
314 
calculatePathFilter(final String pathFilterString)315     static Matcher calculatePathFilter(final String pathFilterString) {
316         if (!pathFilterString.equals(".*")) {
317             return PatternCache.get(pathFilterString).matcher("");
318         } else {
319             return null;
320         }
321     }
322 
calculateCoverageLevel(final String coverageLevelInput, boolean forHtml)323     static Level calculateCoverageLevel(final String coverageLevelInput, boolean forHtml) {
324         Level coverageLevel = null;
325         if (forHtml) {
326             coverageLevel = Level.MODERN; // reset
327         } else if (coverageLevelInput != null) {
328             coverageLevel = Level.get(coverageLevelInput);
329             if (coverageLevel == Level.UNDETERMINED) {
330                 throw new IllegalArgumentException(
331                         "-c"
332                                 + coverageLevelInput
333                                 + "\t is invalid: must be one of: "
334                                 + "basic,moderate,...");
335             }
336         }
337         return coverageLevel;
338     }
339 
340     /**
341      * This will be the test framework way of using these tests.
342      *
343      * @param args
344      * @throws Throwable
345      */
main(String[] args)346     public static void main(String[] args) throws Throwable {
347         // turn off logging to not mess up html and other output.
348         CheckCLDR.setLoggerLevel(java.util.logging.Level.OFF);
349         MyOptions.parse(args, true);
350         ElapsedTimer totalTimer = new ElapsedTimer();
351         UOption.parseArgs(args, options);
352         String factoryFilter = options[FILE_FILTER].value;
353         if (factoryFilter.equals("key")) {
354             factoryFilter =
355                     "(en|ru|nl|fr|de|it|pl|es|tr|th|ja|zh|ko|ar|bg|sr|uk|ca|hr|cs|da|fil|fi|hu|id|lv|lt|no|pt|ro|sk|sl|sv|vi|el|he|fa|hi|am|af|et|is|ms|sw|zu|bn|mr|ta|eu|gl|ur|gu|kn|ml|te|zh_Hant|pt_PT|en_GB)";
356         }
357         String checkFilter = options[TEST_FILTER].value;
358         final String subtypeFilterString = options[SUBTYPE_FILTER].value;
359         final EnumSet<Subtype> subtypeFilter = calculateSubtypeFilter(subtypeFilterString);
360 
361         errorsOnly = options[ERRORS_ONLY].doesOccur;
362         boolean showMissing = MyOptions.missingPaths.option.doesOccur();
363 
364         SHOW_EXAMPLES = options[EXAMPLES].doesOccur;
365         boolean showAll = options[SHOWALL].doesOccur;
366         boolean checkFlexibleDates = options[DATE_FORMATS].doesOccur;
367         final String pathFilterString = options[PATH_FILTER].value;
368         final Matcher pathFilter = calculatePathFilter(pathFilterString);
369         boolean checkOnSubmit = options[CHECK_ON_SUBMIT].doesOccur;
370         boolean noaliases = options[NO_ALIASES].doesOccur;
371 
372         final Level coverageLevel =
373                 calculateCoverageLevel(options[COVERAGE].value, options[GENERATE_HTML].doesOccur);
374 
375         Organization organization =
376                 options[ORGANIZATION].value == null
377                         ? null
378                         : Organization.fromString(options[ORGANIZATION].value);
379         if (organization != null) {
380             Set<Organization> organizations = StandardCodes.make().getLocaleCoverageOrganizations();
381             if (!organizations.contains(organization)) {
382                 throw new IllegalArgumentException(
383                         "-o" + organization + "\t is invalid: must be one of: " + organizations);
384             }
385         }
386         final CLDRConfig cldrConf = CLDR_CONFIG;
387         cldrConf.setEnvironment(Environment.UNITTEST);
388         final Phase phase;
389         if (options[PHASE].doesOccur) {
390             String phaseVal = options[PHASE].value;
391             try {
392                 // no null check for argument; if it is is null, Phase.forString would return the
393                 // one from CLDRConfig
394                 phase = Phase.forString(phaseVal);
395             } catch (IllegalArgumentException e) {
396                 StringBuilder sb = new StringBuilder("Incorrect Phase value");
397                 if (phaseVal != null && !phaseVal.isEmpty()) {
398                     sb.append(" '");
399                     sb.append(phaseVal);
400                     sb.append("'");
401                 }
402                 sb.append(": should be one of ");
403                 for (Phase curPhase : Phase.values()) {
404                     // implicitly does a toString;
405                     sb.append(curPhase);
406                     sb.append(", ");
407                 }
408                 int lastIdx = sb.lastIndexOf(",");
409                 // remove the last comma, if it occurs
410                 if (lastIdx > -1) {
411                     String tmpBuf = sb.substring(0, lastIdx);
412                     sb.setLength(0);
413                     sb.append(tmpBuf);
414                 }
415                 sb.append(".");
416                 // TODO: Reporting should be similar to an error (wrong parameter...), and not
417                 // actually an Exception
418                 throw new IllegalArgumentException(sb.toString(), e);
419             }
420         } else {
421             phase = cldrConf.getPhase();
422         }
423 
424         boolean baileyTest = options[BAILEY].doesOccur;
425 
426         File sourceDirectories[] = null;
427 
428         if (MyOptions.source_all.option.doesOccur()) {
429             if (MyOptions.source_directory.option.doesOccur()) {
430                 throw new IllegalArgumentException("Don't use -s and -S together.");
431             }
432             sourceDirectories =
433                     cldrConf.addStandardSubdirectories(
434                             cldrConf.getCLDRDataDirectories(
435                                     MyOptions.source_all.option.getValue()));
436         } else {
437             String[] sdirs = options[SOURCE_DIRECTORY].value.split(",\\s*");
438             sourceDirectories = new File[sdirs.length];
439             for (int i = 0; i < sdirs.length; ++i) {
440                 sourceDirectories[i] =
441                         new File(
442                                 CldrUtility.checkValidDirectory(
443                                         sdirs[i], "Fix with -s. Use -h for help."));
444             }
445         }
446 
447         if (options[GENERATE_HTML].doesOccur) {
448             ErrorFile.generated_html_directory = options[GENERATE_HTML].value;
449             ErrorFile.generated_html_count =
450                     FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "count.txt");
451         }
452 
453         final boolean sequential =
454                 SHOW_EXAMPLES
455                         || options[GENERATE_HTML].doesOccur
456                         || options[SINGLE_THREAD].doesOccur;
457 
458         idView = options[ID_VIEW].doesOccur;
459 
460         if (options[VOTE_RESOLVE].doesOccur) {
461             resolveVotesDirectory =
462                     CldrUtility.checkValidFile(
463                             CLDRPaths.BASE_DIRECTORY + "incoming/vetted/votes/", true, null);
464             voterInfoList.setVoterToInfo(
465                     CldrUtility.checkValidFile(
466                             CLDRPaths.BASE_DIRECTORY + "incoming/vetted/usersa/usersa.xml",
467                             false,
468                             null));
469             voteResolver = new VoteResolver<>(voterInfoList);
470         }
471 
472         String user = options[USER].value;
473 
474         System.out.println("Source directories:\n");
475         for (File f : sourceDirectories) {
476             System.out.println(
477                     "    " + f.getPath() + "\t(" + PathUtilities.getNormalizedPathString(f) + ")");
478         }
479 
480         // set up the test
481         Factory cldrFactory =
482                 SimpleFactory.make(sourceDirectories, factoryFilter)
483                         .setSupplementalDirectory(new File(CLDRPaths.SUPPLEMENTAL_DIRECTORY));
484         final TestCache testCache = cldrFactory.getTestCache();
485         testCache.setNameMatcher(checkFilter);
486 
487         {
488             // we create an extraneous CompoundCheckCLDR here just to check the filters
489             CompoundCheckCLDR checkCldr = CheckCLDR.getCheckAll(cldrFactory, checkFilter);
490             if (checkCldr.getFilteredTestList().size() == 0) {
491                 throw new IllegalArgumentException("The filter doesn't match any tests.");
492             }
493             System.out.println("filtered tests: " + checkCldr.getFilteredTests());
494         }
495 
496         Factory backCldrFactory = CLDRConfig.getInstance().getMainAndAnnotationsFactory();
497         english = backCldrFactory.make("en", true);
498 
499         CheckCLDR.setDisplayInformation(english);
500         setExampleGenerator(new ExampleGenerator(english, english));
501         PathShower pathShower = new PathShower();
502 
503         // call on the files
504         Set<String> locales = new TreeSet<>(baseFirstCollator);
505         locales.addAll(cldrFactory.getAvailable());
506 
507         Set<String> fatalErrors = new TreeSet<>();
508 
509         showHeaderLine();
510 
511         supplementalDataInfo = SupplementalDataInfo.getInstance(CLDRPaths.SUPPLEMENTAL_DIRECTORY);
512 
513         LocaleIDParser localeIDParser = new LocaleIDParser();
514         PathHeader.Factory pathHeaderFactory = PathHeader.getFactory(english);
515 
516         final Map<String, Level> locale_status =
517                 StandardCodes.make().getLocaleToLevel(organization);
518 
519         final List<String> specialPurposeLocales = new ArrayList<>(Arrays.asList("en_US_POSIX"));
520 
521         // TODO: englishPaths doesn't seem to be used.
522         // final HashSet<String> ep = new HashSet<>();
523         // final CLDRFile displayFile = CheckCLDR.getDisplayInformation();
524         // addPrettyPaths(
525         //         displayFile, pathFilter, pathHeaderFactory, noaliases, true, ep);
526         // addPrettyPaths(
527         //         displayFile,
528         //         displayFile.getExtraPaths(),
529         //         pathFilter,
530         //         pathHeaderFactory,
531         //         noaliases,
532         //         true,
533         //         ep);
534         // final Set<String> englishPaths = Collections.unmodifiableSet(ep); // for robustness
535 
536         // Set up our stream to use. It will be parallel usually, or sequential for HTML.
537         Stream<String> stream;
538 
539         if (sequential) {
540             System.err.println("# Note: running in sequential mode.");
541             stream = locales.stream();
542         } else {
543             stream = locales.parallelStream();
544         }
545 
546         // now, run it
547         stream.forEach(
548                 localeID -> {
549                     if (ErrorFile.writeError != null) {
550                         return; // get out, it's an error.
551                     }
552 
553                     Set<PathHeader> paths = new TreeSet<>(); // CLDRFile.ldmlComparator);
554                     Map<String, String> m = new TreeMap<>();
555                     Map<String, String> options = new HashMap<>();
556                     FlexibleDateFromCLDR fset = new FlexibleDateFromCLDR();
557 
558                     if (CLDRFile.isSupplementalName(localeID)) return;
559                     if (supplementalDataInfo.getDefaultContentLocales().contains(localeID)) {
560                         System.out.println("# Skipping default content locale: " + localeID);
561                         return;
562                     }
563 
564                     // We don't really need to check the POSIX locale, as it is a special purpose
565                     // locale
566                     if (specialPurposeLocales.contains(localeID)) {
567                         System.out.println("# Skipping special purpose locale: " + localeID);
568                         return;
569                     }
570 
571                     boolean isLanguageLocale =
572                             localeID.equals(localeIDParser.set(localeID).getLanguageScript());
573                     options.clear();
574 
575                     if (MyOptions.exemplarError.option.doesOccur()) {
576                         options.put(Options.Option.exemplarErrors.toString(), "true");
577                     }
578 
579                     // if the organization is set, skip any locale that doesn't have a value in
580                     // Locales.txt
581                     Level level = coverageLevel;
582                     if (level == null) {
583                         level = Level.MODERN;
584                     }
585                     if (organization != null) {
586                         if (locale_status == null) return;
587                         level = locale_status.get(localeID);
588                         if (level == null) return;
589                         if (level.compareTo(Level.BASIC) < 0) return;
590                     } else if (!isLanguageLocale) {
591                         // otherwise, skip all language locales
592                         options.put(Options.Option.CheckCoverage_skip.getKey(), "true");
593                     }
594 
595                     // if (organization != null)
596                     // options.put(Options.Option.CoverageLevel_localeType.getKey(),
597                     // organization.toString());
598                     options.put(Options.Option.phase.getKey(), phase.toString());
599 
600                     // also need the locale in the options
601                     options.put(Options.Option.locale.getKey(), localeID);
602 
603                     if (SHOW_LOCALE && sequential) System.out.println();
604 
605                     CLDRFile file;
606                     CLDRFile englishFile = english;
607                     CLDRFile parent = null;
608 
609                     ElapsedTimer timer = new ElapsedTimer();
610                     try {
611                         file = cldrFactory.make(localeID, true);
612                         if (ErrorFile.voteFactory != null) {
613                             ErrorFile.voteFile = ErrorFile.voteFactory.make(localeID, true);
614                         }
615                         final String parentID = LocaleIDParser.getParent(localeID);
616                         if (parentID != null) {
617                             parent = cldrFactory.make(parentID, true);
618                         }
619                     } catch (RuntimeException e) {
620                         fatalErrors.add(localeID);
621                         System.out.println("FATAL ERROR: " + localeID);
622                         e.printStackTrace(System.out);
623                         return;
624                     }
625 
626                     TestResultBundle bundle = testCache.getBundle(new CheckCLDR.Options(options));
627 
628                     // generate HTML if asked for
629                     if (ErrorFile.generated_html_directory != null) {
630                         String baseLanguage = localeIDParser.set(localeID).getLanguageScript();
631 
632                         if (!baseLanguage.equals(ErrorFile.lastBaseLanguage)) {
633                             ErrorFile.lastBaseLanguage = baseLanguage;
634                             try {
635                                 ErrorFile.openErrorFile(localeID, baseLanguage);
636                             } catch (IOException ioe) {
637                                 ErrorFile.writeError = ioe;
638                                 System.err.println(
639                                         "Exception "
640                                                 + ioe
641                                                 + " while trying to open file "
642                                                 + localeID);
643                                 ioe.printStackTrace();
644                                 return;
645                             }
646                         }
647                     }
648 
649                     if (user != null) {
650                         file = new CLDRFile.TestUser(file, user, isLanguageLocale);
651                         if (parent != null) {
652                             parent = new CLDRFile.TestUser(parent, user, isLanguageLocale);
653                         }
654                     }
655 
656                     subtotalCount.clear();
657 
658                     final Set<CheckStatus> possibleProblems = new TreeSet<>();
659                     possibleProblems.addAll(bundle.getPossibleProblems());
660 
661                     paths.clear();
662 
663                     CoverageInfo covInfo = cldrConf.getCoverageInfo();
664                     for (String path : file.fullIterable()) {
665                         if (pathFilter != null && !pathFilter.reset(path).find()) {
666                             continue;
667                         }
668                         if (level != null) {
669                             Level currentLevel = covInfo.getCoverageLevel(path, localeID);
670                             if (currentLevel.compareTo(level) > 0) {
671                                 continue;
672                             }
673                         }
674                         final PathHeader pathHeader = pathHeaderFactory.fromPath(path);
675                         if (pathHeader.getSectionId() != SectionId.Special) {
676                             paths.add(pathHeader);
677                         }
678                     }
679 
680                     UnicodeSet missingExemplars = new UnicodeSet();
681                     UnicodeSet missingCurrencyExemplars = new UnicodeSet();
682                     if (checkFlexibleDates) {
683                         fset.set(file);
684                     }
685                     pathShower.set(localeID);
686 
687                     // only create if we are going to use it
688                     final ExampleGenerator exampleGenerator =
689                             SHOW_EXAMPLES ? new ExampleGenerator(file, englishFile) : null;
690 
691                     int pathCount = 0;
692                     Status otherPath = new Status();
693                     int rawMissingCount = 0;
694                     int rawProvisionalCount = 0;
695                     CLDRFile unresolved = file.getUnresolved();
696 
697                     for (PathHeader pathHeader : paths) {
698                         pathCount++;
699                         String path = pathHeader.getOriginalPath();
700                         String prettyPath =
701                                 pathHeader.toString().replace('\t', '|').replace(' ', '_');
702                         if (!showAll && !file.isWinningPath(path)) {
703                             continue;
704                         }
705                         final String topValue = unresolved.getStringValue(path);
706                         RawStatus rawStatus = RawStatus.present;
707 
708                         if (topValue == null) {
709                             rawStatus = RawStatus.missing;
710                             rawMissingCount++;
711                         }
712 
713                         if (!isLanguageLocale && !baileyTest) {
714                             final String sourceLocaleID = file.getSourceLocaleID(path, otherPath);
715                             if (!localeID.equals(sourceLocaleID)) {
716                                 continue;
717                             }
718                             // also skip aliases
719                             if (!path.equals(otherPath.pathWhereFound)) {
720                                 continue;
721                             }
722                         }
723                         if (path.contains("@alt") && path.contains("proposed")) {
724                             continue;
725                         }
726                         String value = file.getStringValue(path);
727                         if (baileyTest) {
728                             value = CldrUtility.INHERITANCE_MARKER;
729                         }
730 
731                         String fullPath = file.getFullXPath(path);
732                         if (topValue != null) {
733                             XPathParts fullParts = XPathParts.getFrozenInstance(fullPath);
734                             String draftStatus = fullParts.getAttributeValue(-1, "draft");
735                             if (draftStatus != null && !draftStatus.equals("contributed")) {
736                                 rawProvisionalCount++;
737                                 rawStatus = RawStatus.provisional;
738                             }
739                         }
740                         if (showMissing && rawStatus != RawStatus.present) {
741                             String englishValue = englishFile.getStringValue(path);
742                             if (englishValue == null) {
743                                 englishValue = "n/a";
744                             }
745                             System.out.println(
746                                     getLocaleAndName(localeID)
747                                             + "\tRaw "
748                                             + rawStatus
749                                             + "\t"
750                                             + pathHeader
751                                             + "\t"
752                                             + englishValue
753                                             + "\t"
754                                             + path);
755                         }
756 
757                         String example = "";
758                         if (SHOW_EXAMPLES && exampleGenerator != null) {
759                             example =
760                                     ExampleGenerator.simplify(
761                                             exampleGenerator.getExampleHtml(path, value));
762                             showExamples(
763                                     file, prettyPath, localeID, path, value, fullPath, example);
764                         }
765                         if (checkFlexibleDates) {
766                             fset.checkFlexibles(path, value, fullPath);
767                         }
768                         int limit = 1;
769                         for (int jj = 0; jj < limit; ++jj) {
770                             final List<CheckStatus> result = new ArrayList<>();
771                             if (jj == 0) {
772                                 bundle.check(path, result, value);
773                             } else {
774                                 bundle.getExamples(path, value, result);
775                             }
776 
777                             boolean showedOne = false;
778                             for (Iterator<CheckStatus> it3 = result.iterator(); it3.hasNext(); ) {
779                                 CheckStatus status = it3.next();
780                                 if (status.getEntireLocale()) {
781                                     possibleProblems.add(status);
782                                     continue;
783                                 }
784                                 String statusString =
785                                         status.toString(); // com.ibm.icu.impl.Utility.escape(
786                                 CheckStatus.Type statusType = status.getType();
787                                 Object[] parameters = status.getParameters();
788 
789                                 if (parameters != null) {
790                                     if (parameters.length >= 1
791                                             && status.getCause().getClass()
792                                                     == CheckForExemplars.class) {
793                                         try {
794                                             UnicodeSet set =
795                                                     new UnicodeSet(parameters[0].toString());
796                                             if (status.getMessage().contains("currency")) {
797                                                 missingCurrencyExemplars.addAll(set);
798                                             } else {
799                                                 missingExemplars.addAll(set);
800                                             }
801                                         } catch (RuntimeException e) {
802                                         } // skip if not parseable as set
803                                     }
804                                 }
805 
806                                 if (errorsOnly && !statusType.equals(CheckStatus.errorType)) {
807                                     continue;
808                                 }
809 
810                                 if (subtypeFilter != null) {
811                                     if (!subtypeFilter.contains(status.getSubtype())) {
812                                         continue;
813                                     }
814                                 }
815                                 if (checkOnSubmit) {
816                                     if (!status.isCheckOnSubmit()
817                                             || !statusType.equals(CheckStatus.errorType)) continue;
818                                 }
819 
820                                 if (statusType.equals(CheckStatus.demoType)) {
821                                     SimpleDemo d = status.getDemo();
822                                     if (d != null && d instanceof FormatDemo) {
823                                         FormatDemo fd = (FormatDemo) d;
824                                         m.clear();
825                                         if (d.processPost(m))
826                                             System.out.println("\tDemo:\t" + fd.getPlainText(m));
827                                     }
828                                     continue;
829                                 }
830 
831                                 if (parameters != null) {
832                                     for (int i = 0; i < parameters.length; ++i) {
833                                         if (showStackTrace && parameters[i] instanceof Throwable) {
834                                             ((Throwable) parameters[i]).printStackTrace();
835                                         }
836                                     }
837                                 }
838 
839                                 showValue(
840                                         file,
841                                         prettyPath,
842                                         localeID,
843                                         example,
844                                         path,
845                                         value,
846                                         fullPath,
847                                         statusString,
848                                         status.getSubtype());
849                                 showedOne = true;
850                             }
851                             if (!showedOne && phase != Phase.FINAL_TESTING) {
852                                 if (!showedOne && showAll) {
853                                     showValue(
854                                             file,
855                                             prettyPath,
856                                             localeID,
857                                             example,
858                                             path,
859                                             value,
860                                             fullPath,
861                                             "ok",
862                                             Subtype.none);
863                                     showedOne = true;
864                                 }
865                             }
866                         }
867                     }
868 
869                     // handle possibleProblems (non path-based errors)
870                     {
871                         for (Iterator<CheckStatus> it3 = possibleProblems.iterator();
872                                 it3.hasNext(); ) {
873                             CheckStatus status = it3.next();
874                             String statusString =
875                                     status.toString(); // com.ibm.icu.impl.Utility.escape(
876                             CheckStatus.Type statusType = status.getType();
877 
878                             if (errorsOnly) {
879                                 if (!statusType.equals(CheckStatus.errorType)) continue;
880                             }
881 
882                             if (subtypeFilter != null) {
883                                 if (!subtypeFilter.contains(status.getSubtype())) {
884                                     continue;
885                                 }
886                             }
887 
888                             if (checkOnSubmit) {
889                                 if (!status.isCheckOnSubmit()
890                                         || !statusType.equals(CheckStatus.errorType)) continue;
891                             }
892                             showValue(
893                                     file,
894                                     null,
895                                     localeID,
896                                     null,
897                                     null,
898                                     null,
899                                     null,
900                                     statusString,
901                                     status.getSubtype());
902                         }
903                     }
904 
905                     if (resolveVotesDirectory != null) {
906                         LocaleVotingData.resolveErrors(localeID);
907                     }
908 
909                     showSummary(
910                             localeID,
911                             level,
912                             "Items:\t"
913                                     + pathCount
914                                     + "\tRaw Missing:\t"
915                                     + rawMissingCount
916                                     + "\tRaw Provisional:\t"
917                                     + rawProvisionalCount);
918 
919                     if (missingExemplars.size() != 0) {
920                         missingExemplars.removeAll(
921                                 new UnicodeSet("[[:Uppercase:]-[İ]]")); // remove uppercase #4670
922                         if (missingExemplars.size() != 0) {
923                             showSummary(
924                                     localeID,
925                                     level,
926                                     "Total missing from general exemplars:\t"
927                                             + missingExemplars.size()
928                                             + "\t"
929                                             + UnicodeSetPrettyPrinter.fromIcuLocale(localeID)
930                                                     .format(missingExemplars));
931                         }
932                     }
933                     if (missingCurrencyExemplars.size() != 0) {
934                         Collator col = Collator.getInstance(new ULocale(localeID));
935                         showSummary(
936                                 localeID,
937                                 level,
938                                 "Total missing from currency exemplars:\t"
939                                         + UnicodeSetPrettyPrinter.fromIcuLocale(localeID)
940                                                 .format(missingCurrencyExemplars));
941                     }
942                     for (ErrorType type : subtotalCount.keySet()) {
943                         showSummary(
944                                 localeID,
945                                 level,
946                                 "Subtotal " + type + ":\t" + subtotalCount.getCount(type));
947                     }
948 
949                     if (checkFlexibleDates) {
950                         fset.showFlexibles();
951                     }
952                     if (SHOW_EXAMPLES && exampleGenerator != null) {
953                         // ldml/dates/timeZoneNames/zone[@type="America/Argentina/San_Juan"]/exemplarCity
954                         for (String zone : StandardCodes.make().getGoodAvailableCodes("tzid")) {
955                             String path =
956                                     "//ldml/dates/timeZoneNames/zone[@type=\""
957                                             + zone
958                                             + "\"]/exemplarCity";
959                             PathHeader pathHeader = pathHeaderFactory.fromPath(path);
960                             String prettyPath =
961                                     pathHeader.toString().replace('\t', '|').replace(' ', '_');
962                             if (pathFilter != null && !pathFilter.reset(path).matches()) {
963                                 continue;
964                             }
965                             String fullPath = file.getStringValue(path);
966                             if (fullPath != null) {
967                                 continue;
968                             }
969                             /*
970                              * TODO: fix this code. Calling getExampleHtml with value = null will always return null,
971                              * so what's this supposed to accomplish?
972                              */
973                             String example =
974                                     ExampleGenerator.simplify(
975                                             exampleGenerator.getExampleHtml(
976                                                     path, null /* value */));
977                             showExamples(file, prettyPath, localeID, path, null, fullPath, example);
978                         }
979                     }
980                     System.out.println("# " + localeID + " Elapsed time: " + timer);
981                     System.out.flush();
982                 });
983 
984         if (ErrorFile.errorFileWriter != null) {
985             ErrorFile.closeErrorFile();
986         }
987 
988         // an error occurred opening HTML, rethrow it.
989         if (ErrorFile.writeError != null) {
990             throw ErrorFile.writeError;
991         }
992 
993         if (ErrorFile.generated_html_directory != null) {
994             ErrorFile.writeErrorCountsText();
995             ErrorFile.writeErrorFileIndex();
996         }
997         System.out.println();
998         for (ErrorType type : totalCount.keySet()) {
999             System.out.println("# Total " + type + ":\t" + totalCount.getCount(type));
1000         }
1001 
1002         System.out.println();
1003         System.out.println("# Total elapsed time: " + totalTimer);
1004         if (fatalErrors.size() != 0) {
1005             System.out.println("# FATAL ERRORS:");
1006         }
1007         long errorCount = totalCount.getCount(ErrorType.error) + fatalErrors.size();
1008         if (errorCount != 0) {
1009             System.out.println();
1010             System.out.println("<< FAILURE - Error count is " + errorCount + " . >>");
1011             System.exit(-1);
1012         } else {
1013             System.out.println();
1014             System.out.println("<< SUCCESS - No errors found. >>");
1015         }
1016         if (LogicalGrouping.GET_TYPE_COUNTS) {
1017             for (String s : LogicalGrouping.typeCount.keySet()) {
1018                 System.out.println(s + "=" + LogicalGrouping.typeCount.get(s));
1019             }
1020         }
1021     } // end of main()
1022 
1023     static class LocaleVotingData {
1024         private int disputedCount = 0;
1025         Counter<Organization> missingOrganizationCounter = new Counter<>(true);
1026         Counter<Organization> goodOrganizationCounter = new Counter<>(true);
1027         Counter<Organization> conflictedOrganizations = new Counter<>(true);
1028         Counter<VoteResolver.Status> winningStatusCounter = new Counter<>(true);
1029 
1030         static Map<String, LocaleVotingData> localeToErrors = new HashMap<>();
1031         private static Map<Integer, String> idToPath;
1032 
resolveErrors(String locale)1033         public static void resolveErrors(String locale) {
1034             localeToErrors.put(locale, new LocaleVotingData(locale));
1035         }
1036 
LocaleVotingData(String locale)1037         public LocaleVotingData(String locale) {
1038 
1039             Map<Organization, VoteResolver.Level> orgToMaxVote =
1040                     voterInfoList.getOrganizationToMaxVote(locale);
1041 
1042             Map<Integer, Map<Integer, CandidateInfo>> info =
1043                     VoteResolver.getBaseToAlternateToInfo(
1044                             resolveVotesDirectory + locale + ".xml", voterInfoList);
1045 
1046             Map<String, Integer> valueToItem = new HashMap<>();
1047 
1048             for (int basePath : info.keySet()) {
1049                 final Map<Integer, CandidateInfo> itemInfo = info.get(basePath);
1050 
1051                 // find the last release status and value
1052                 voteResolver.clear();
1053                 valueToItem.clear();
1054 
1055                 for (int item : itemInfo.keySet()) {
1056                     String itemValue = getValue(item);
1057                     valueToItem.put(itemValue, item);
1058 
1059                     CandidateInfo candidateInfo = itemInfo.get(item);
1060                     if (candidateInfo.oldStatus != null) {
1061                         voteResolver.setBaseline(itemValue, candidateInfo.oldStatus);
1062                     }
1063                     voteResolver.add(itemValue);
1064                     for (int voter : candidateInfo.voters) {
1065                         try {
1066                             voteResolver.add(itemValue, voter);
1067                         } catch (UnknownVoterException e) {
1068                             // skip
1069                         }
1070                     }
1071                 }
1072 
1073                 EnumSet<Organization> basePathConflictedOrganizations =
1074                         voteResolver.getConflictedOrganizations();
1075                 conflictedOrganizations.addAll(basePathConflictedOrganizations, 1);
1076 
1077                 VoteResolver.Status winningStatus = voteResolver.getWinningStatus();
1078                 String winningValue = voteResolver.getWinningValue();
1079 
1080                 winningStatusCounter.add(winningStatus, 1);
1081 
1082                 if (winningStatus == VoteResolver.Status.approved) {
1083                     continue;
1084                 }
1085 
1086                 CandidateInfo candidateInfo = itemInfo.get(valueToItem.get(winningValue));
1087                 Map<Organization, VoteResolver.Level> orgToMaxVoteHere =
1088                         voterInfoList.getOrganizationToMaxVote(candidateInfo.voters);
1089 
1090                 // if the winning item is less than contributed, record the organizations that
1091                 // haven't given their
1092                 // maximum vote to the winning item.
1093                 if (winningStatus.compareTo(VoteResolver.Status.contributed) < 0) {
1094                     // showPaths(basePath, itemInfo);
1095                     for (Organization org : orgToMaxVote.keySet()) {
1096                         VoteResolver.Level maxVote = orgToMaxVote.get(org);
1097                         VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org);
1098                         if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) {
1099                             missingOrganizationCounter.add(org, 1);
1100                         }
1101                     }
1102                     if (voteResolver.isDisputed()) {
1103                         disputedCount++;
1104                         String path = getIdToPath(basePath);
1105                         ErrorFile.addDataToErrorFile(
1106                                 locale, path, ErrorType.disputed, Subtype.none);
1107                     }
1108                 } else {
1109                     for (Organization org : orgToMaxVote.keySet()) {
1110                         VoteResolver.Level maxVote = orgToMaxVote.get(org);
1111                         VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org);
1112                         if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) {
1113                         } else {
1114                             goodOrganizationCounter.add(org, 1);
1115                         }
1116                     }
1117                 }
1118             }
1119             System.out.println(
1120                     getLocaleAndName(locale) + "\tEnabled Organizations:\t" + orgToMaxVote);
1121             if (disputedCount != 0) {
1122                 System.out.println(
1123                         getLocaleAndName(locale) + "\tDisputed Items:\t" + disputedCount);
1124             }
1125 
1126             if (missingOrganizationCounter.size() > 0) {
1127                 System.out.println(
1128                         getLocaleAndName(locale)
1129                                 + "\tMIA organizations:\t"
1130                                 + missingOrganizationCounter);
1131                 System.out.println(
1132                         getLocaleAndName(locale)
1133                                 + "\tConflicted organizations:\t"
1134                                 + conflictedOrganizations);
1135                 System.out.println(
1136                         getLocaleAndName(locale)
1137                                 + "\tCool organizations!:\t"
1138                                 + goodOrganizationCounter);
1139             }
1140             System.out.println(
1141                     getLocaleAndName(locale) + "\tOptimal Status:\t" + winningStatusCounter);
1142         }
1143 
getIdToPath(int basePath)1144         private static String getIdToPath(int basePath) {
1145             if (idToPath == null) {
1146                 idToPath = VoteResolver.getIdToPath(resolveVotesDirectory + "xpathTable.xml");
1147             }
1148             return idToPath.get(basePath);
1149         }
1150 
get(String locale)1151         public static LocaleVotingData get(String locale) {
1152             return localeToErrors.get(locale);
1153         }
1154 
getDisputedCount()1155         int getDisputedCount() {
1156             return disputedCount;
1157         }
1158 
getConflictedHTML()1159         String getConflictedHTML() {
1160             String result = conflictedOrganizations.toString();
1161             if (result.length() == 0) {
1162                 return "";
1163             }
1164             result = result.substring(1, result.length() - 1);
1165             result = result.replace(", ", "<br>");
1166             return result;
1167         }
1168     }
1169 
getValue(int item)1170     private static String getValue(int item) {
1171         return String.valueOf(item);
1172     }
1173 
1174     static Matcher draftStatusMatcher =
1175             PatternCache.get("\\[@draft=\"(provisional|unconfirmed)\"]").matcher("");
1176 
1177     enum ErrorType {
1178         ok,
1179         error,
1180         disputed,
1181         warning,
1182         core,
1183         posix,
1184         minimal,
1185         basic,
1186         moderate,
1187         modern,
1188         comprehensive,
1189         optional,
1190         contributed,
1191         provisional,
1192         unconfirmed,
1193         unknown;
1194         static EnumSet<ErrorType> unapproved =
1195                 EnumSet.range(ErrorType.contributed, ErrorType.unconfirmed);
1196         static EnumSet<ErrorType> coverage = EnumSet.range(ErrorType.posix, ErrorType.optional);
1197         static EnumSet<ErrorType> showInSummary =
1198                 EnumSet.of(
1199                         ErrorType.error,
1200                         ErrorType.warning,
1201                         ErrorType.posix,
1202                         ErrorType.minimal,
1203                         ErrorType.basic);
1204 
fromStatusString(String statusString)1205         static ErrorType fromStatusString(String statusString) {
1206             ErrorType shortStatus =
1207                     statusString.equals("ok")
1208                             ? ErrorType.ok
1209                             : statusString.startsWith("Error")
1210                                     ? ErrorType.error
1211                                     : statusString.equals("disputed")
1212                                             ? ErrorType.disputed
1213                                             : statusString.startsWith("Warning")
1214                                                     ? ErrorType.warning
1215                                                     : statusString.equals("contributed")
1216                                                             ? ErrorType.contributed
1217                                                             : statusString.equals("provisional")
1218                                                                     ? ErrorType.provisional
1219                                                                     : statusString.equals(
1220                                                                                     "unconfirmed")
1221                                                                             ? ErrorType.unconfirmed
1222                                                                             : ErrorType.unknown;
1223             if (shortStatus == ErrorType.unknown) {
1224                 throw new IllegalArgumentException("Unknown error type: " + statusString);
1225             } else if (shortStatus == ErrorType.warning) {
1226                 if (coverageMatcher.reset(statusString).find()) {
1227                     shortStatus = ErrorType.valueOf(coverageMatcher.group(1));
1228                 }
1229             }
1230             return shortStatus;
1231         }
1232     }
1233 
1234     static class ErrorFile {
1235         /** cached error for later rethrow */
1236         public static Throwable writeError = null;
1237 
1238         public static String lastBaseLanguage = "";
1239         private static final boolean SHOW_VOTING_INFO = false;
1240         public static CLDRFile voteFile;
1241         public static Factory voteFactory;
1242 
openErrorFile(String localeID, String baseLanguage)1243         private static void openErrorFile(String localeID, String baseLanguage) throws IOException {
1244             if (ErrorFile.errorFileWriter != null) {
1245                 ErrorFile.closeErrorFile();
1246             }
1247             ErrorFile.errorFileWriter =
1248                     FileUtilities.openUTF8Writer(
1249                             ErrorFile.generated_html_directory, baseLanguage + ".html");
1250             ErrorFile.errorFileTable = new TablePrinter();
1251             errorFileCounter.clear();
1252             ErrorFile.errorFileTable
1253                     .setCaption("Problem Details")
1254                     .addColumn("Problem")
1255                     .setCellAttributes("align=\"left\" class=\"{0}\"")
1256                     .setSortPriority(0)
1257                     .setSpanRows(true)
1258                     .setBreakSpans(true)
1259                     .setRepeatHeader(true)
1260                     .setHeaderCell(true)
1261                     .addColumn("Subtype")
1262                     .setCellAttributes("align=\"left\" class=\"{1}\"")
1263                     .setSortPriority(1)
1264                     .setSpanRows(true)
1265                     .setBreakSpans(true)
1266                     .setRepeatHeader(true)
1267                     .setHeaderCell(true)
1268                     .addColumn("Locale")
1269                     .setCellAttributes("class=\"{1}\"")
1270                     .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={0}\">{0}</a>")
1271                     .setSortPriority(2)
1272                     .setSpanRows(true)
1273                     .setBreakSpans(true) // .setRepeatDivider(true)
1274                     .addColumn("Name")
1275                     .setCellAttributes("class=\"{1}\"")
1276                     .setSpanRows(true)
1277                     .setBreakSpans(true)
1278                     .addColumn("Section")
1279                     .setCellAttributes("class=\"{1}\"")
1280                     .setSortPriority(3)
1281                     .setCellPattern(
1282                             "<a href=\"http://unicode.org/cldr/apps/survey?_={3}&x={0}\">{0}</a>")
1283                     .setSpanRows(true)
1284                     .addColumn("Count")
1285                     .setCellAttributes("class=\"{1}\" align=\"right\"");
1286 
1287             showIndexHead("", localeID, ErrorFile.errorFileWriter);
1288         }
1289 
1290         static TablePrinter errorFileTable = new TablePrinter();
1291         static Counter<Row.R4<String, String, ErrorType, Subtype>> errorFileCounter =
1292                 new Counter<>(true);
1293 
addDataToErrorFile( String localeID, String path, ErrorType shortStatus, Subtype subtype)1294         private static void addDataToErrorFile(
1295                 String localeID, String path, ErrorType shortStatus, Subtype subtype) {
1296             String section = path == null ? null : XPathToMenu.xpathToMenu(path);
1297             if (section == null) {
1298                 section = "general";
1299             }
1300             errorFileCounter.add(new Row.R4<>(localeID, section, shortStatus, subtype), 1);
1301             ErrorFile.sectionToProblemsToLocaleToCount.add(
1302                     new Row.R4<>(section, shortStatus, subtype, localeID), 1);
1303         }
1304 
closeErrorFile()1305         private static void closeErrorFile() {
1306             Set<String> locales = new TreeSet<>();
1307             for (Row.R4<String, String, ErrorType, Subtype> item : errorFileCounter.keySet()) {
1308                 String localeID = item.get0();
1309                 locales.add(localeID);
1310                 String section = item.get1();
1311                 ErrorType shortStatus = item.get2();
1312                 Subtype subtype = item.get3();
1313                 errorFileTable
1314                         .addRow()
1315                         .addCell(shortStatus)
1316                         .addCell(subtype)
1317                         .addCell(localeID)
1318                         .addCell(ConsoleCheckCLDR.getLocaleName(localeID))
1319                         .addCell(section) // menuPath == null ? "" : "<a href='" + link + "'>" +
1320                         // menuPath + "</a>"
1321                         .addCell(errorFileCounter.getCount(item))
1322                         .finishRow();
1323             }
1324 
1325             if (SHOW_VOTING_INFO) {
1326                 TablePrinter data =
1327                         new TablePrinter()
1328                                 .setCaption("Voting Information")
1329                                 .addColumn("Locale")
1330                                 .setHeaderCell(true)
1331                                 .addColumn("Name")
1332                                 .setHeaderCell(true)
1333                                 .addColumn("Organization")
1334                                 .addColumn("Missing")
1335                                 .addColumn("Conflicted");
1336                 for (String localeID : locales) {
1337                     // now the voting info
1338                     LocaleVotingData localeVotingData =
1339                             LocaleVotingData.localeToErrors.get(localeID);
1340                     if (localeVotingData != null) {
1341                         // find all the orgs with data
1342                         EnumSet<Organization> orgs = EnumSet.noneOf(Organization.class);
1343                         orgs.addAll(localeVotingData.missingOrganizationCounter.keySet());
1344                         orgs.addAll(localeVotingData.conflictedOrganizations.keySet());
1345                         orgs.addAll(localeVotingData.goodOrganizationCounter.keySet());
1346                         for (Organization org : orgs) {
1347                             data.addRow()
1348                                     .addCell(ConsoleCheckCLDR.getLinkedLocale(localeID))
1349                                     .addCell(ConsoleCheckCLDR.getLocaleName(localeID))
1350                                     .addCell(org)
1351                                     .addCell(
1352                                             localeVotingData.missingOrganizationCounter.getCount(
1353                                                     org))
1354                                     .addCell(localeVotingData.conflictedOrganizations.getCount(org))
1355                                     .finishRow();
1356                         }
1357                     }
1358                 }
1359                 ErrorFile.errorFileWriter.println(data.toTable());
1360                 ErrorFile.errorFileWriter.println("<p></p>");
1361             }
1362 
1363             // generated_html.println("<table border='1' style='border-collapse: collapse'
1364             // bordercolor='#CCCCFF'>");
1365             // Locale Group Error Warning Missing Votes: Contributed Missing Votes: Provisional
1366             // Missing Votes:
1367             // Unconfirmed Missing Coverage: Posix Missing Coverage: Minimal Missing Coverage: Basic
1368             // Missing Coverage:
1369             // Moderate Missing Coverage: Modern
1370             ErrorFile.errorFileWriter.println(ErrorFile.errorFileTable.toTable());
1371             ErrorFile.errorFileWriter.println(ShowData.dateFooter());
1372             ErrorFile.errorFileWriter.println(CldrUtility.ANALYTICS);
1373             ErrorFile.errorFileWriter.println("</body></html>");
1374             ErrorFile.errorFileWriter.close();
1375             ErrorFile.errorFileTable = null;
1376         }
1377 
1378         // ================ Index File ===================
1379 
showErrorFileIndex(PrintWriter generated_html_index)1380         static void showErrorFileIndex(PrintWriter generated_html_index) {
1381 
1382             // get organizations
1383             Relation<Organization, String> orgToLocales = getOrgToLocales();
1384 
1385             TablePrinter indexTablePrinter =
1386                     new TablePrinter()
1387                             .setCaption("Problem Summary")
1388                             .setTableAttributes(
1389                                     "border='1' style='border-collapse: collapse' bordercolor='blue'")
1390                             .addColumn("BASE")
1391                             .setHidden(true) // .setRepeatDivider(true)
1392                             .addColumn("Locale")
1393                             .setCellPattern(
1394                                     "<a name=\"{0}\" href=\"{1}.html\">{0}</a>") // link to base,
1395                             // anchor
1396                             // with full
1397                             .addColumn("Name");
1398             if (SHOW_VOTING_INFO) {
1399                 indexTablePrinter.addColumn("Summary").addColumn("Missing");
1400             }
1401             for (Organization org : orgToLocales.keySet()) {
1402                 indexTablePrinter.addColumn(org.toString().substring(0, 2));
1403             }
1404             indexTablePrinter
1405                     .addColumn("Disputed")
1406                     .setHeaderAttributes("class='disputed'")
1407                     .setCellAttributes("class='disputed'")
1408                     .addColumn("Conflicted")
1409                     .setHeaderAttributes("class='conflicted'")
1410                     .setCellAttributes("class='conflicted'");
1411 
1412             for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) {
1413                 String columnTitle = UCharacter.toTitleCase(type.toString(), null);
1414                 final boolean coverage = ConsoleCheckCLDR.ErrorType.coverage.contains(type);
1415                 if (coverage) {
1416                     columnTitle = "MC: " + columnTitle;
1417                 } else if (ConsoleCheckCLDR.ErrorType.unapproved.contains(type)) {
1418                     columnTitle = "MV: " + columnTitle;
1419                 }
1420                 indexTablePrinter
1421                         .addColumn(columnTitle)
1422                         .setHeaderAttributes("class='" + type + "'")
1423                         .setCellAttributes("class='" + type + "'");
1424             }
1425 
1426             // now fill in the data
1427             LanguageTagParser ltp = new LanguageTagParser();
1428             for (String key : ErrorFile.errorFileIndexData.keySet()) {
1429                 Pair<String, Counter<ErrorType>> pair = ErrorFile.errorFileIndexData.get(key);
1430                 String htmlOpenedFileLanguage = pair.getFirst();
1431                 Counter<ErrorType> counts = pair.getSecond();
1432                 LocaleVotingData votingData = LocaleVotingData.get(htmlOpenedFileLanguage);
1433                 if (counts.getTotal() == 0) {
1434                     continue;
1435                 }
1436                 final String baseLanguage = ltp.set(htmlOpenedFileLanguage).getLanguage();
1437                 indexTablePrinter
1438                         .addRow()
1439                         .addCell(baseLanguage)
1440                         .addCell(htmlOpenedFileLanguage)
1441                         .addCell(ConsoleCheckCLDR.getLocaleName(htmlOpenedFileLanguage));
1442                 if (SHOW_VOTING_INFO) {
1443                     indexTablePrinter
1444                             .addCell(
1445                                     votingData == null
1446                                             ? ""
1447                                             : votingData.winningStatusCounter.toString())
1448                             .addCell(
1449                                     votingData == null
1450                                             ? ""
1451                                             : votingData.missingOrganizationCounter.toString());
1452                 }
1453                 for (Organization org : orgToLocales.keySet()) {
1454                     indexTablePrinter.addCell(
1455                             orgToLocales.getAll(org).contains(htmlOpenedFileLanguage)
1456                                     ? org.toString().substring(0, 2)
1457                                     : "");
1458                 }
1459                 indexTablePrinter
1460                         .addCell(
1461                                 votingData == null
1462                                         ? ""
1463                                         : formatSkippingZero(votingData.getDisputedCount()))
1464                         .addCell(votingData == null ? "" : votingData.getConflictedHTML());
1465                 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) {
1466                     indexTablePrinter.addCell(formatSkippingZero(counts.getCount(type)));
1467                 }
1468                 indexTablePrinter.finishRow();
1469             }
1470             generated_html_index.println(indexTablePrinter.toTable());
1471             generated_html_index.println(ShowData.dateFooter());
1472             generated_html_index.println(CldrUtility.ANALYTICS);
1473             generated_html_index.println("</body></html>");
1474         }
1475 
1476         static Relation<Organization, String> orgToLocales;
1477 
getOrgToLocales()1478         private static Relation<Organization, String> getOrgToLocales() {
1479             if (orgToLocales == null) {
1480                 orgToLocales = Relation.of(new TreeMap<Organization, Set<String>>(), TreeSet.class);
1481                 StandardCodes sc = StandardCodes.make();
1482                 for (Organization org : sc.getLocaleCoverageOrganizations()) {
1483                     for (String locale : sc.getLocaleCoverageLocales(org)) {
1484                         Level x = sc.getLocaleCoverageLevel(org, locale);
1485                         if (x.compareTo(Level.BASIC) > 0) {
1486                             orgToLocales.put(org, locale);
1487                         }
1488                     }
1489                 }
1490             }
1491             return orgToLocales;
1492         }
1493 
showSections()1494         static void showSections() throws IOException {
1495             Relation<Organization, String> orgToLocales = getOrgToLocales();
1496             TablePrinter indexTablePrinter =
1497                     new TablePrinter()
1498                             .setCaption("Problem Summary")
1499                             .setTableAttributes(
1500                                     "border='1' style='border-collapse: collapse' bordercolor='blue'")
1501                             .addColumn("Section")
1502                             .setSpanRows(true)
1503                             .setBreakSpans(true) // .setRepeatDivider(true)
1504                             .addColumn("Problems")
1505                             .setCellAttributes("style=\"text-align:left\" class=\"{2}\"")
1506                             .setSpanRows(true)
1507                             .addColumn("Subtype")
1508                             .setCellAttributes("style=\"text-align:left\" class=\"{2}\"")
1509                             .setSpanRows(true)
1510                             .addColumn("Locale")
1511                             .setCellAttributes("class=\"{2}\"")
1512                             .addColumn("Code")
1513                             .setCellAttributes("class=\"{2}\"")
1514                             .setCellPattern(
1515                                     "<a href=\"http://unicode.org/cldr/apps/survey?_={0}&x={1}\">{0}</a>") // TODO: use CLDRConfig.urls()
1516                             .addColumn("Count")
1517                             .setCellAttributes("class=\"{2}\"");
1518             for (Organization org : orgToLocales.keySet()) {
1519                 indexTablePrinter.addColumn(org.toString().substring(0, 2));
1520             }
1521 
1522             for (Row.R4<String, ErrorType, Subtype, String> sectionAndProblemsAndLocale :
1523                     ErrorFile.sectionToProblemsToLocaleToCount.getKeysetSortedByKey()) {
1524                 final ErrorType problem = sectionAndProblemsAndLocale.get1();
1525                 final Subtype subtype = sectionAndProblemsAndLocale.get2();
1526                 if (!ConsoleCheckCLDR.ErrorType.showInSummary.contains(problem)) {
1527                     continue;
1528                 }
1529                 final String locale = sectionAndProblemsAndLocale.get3();
1530                 if (problem != ErrorType.error
1531                         && problem != ErrorType.disputed
1532                         && !orgToLocales.containsValue(locale)) {
1533                     continue;
1534                 }
1535                 long count =
1536                         ErrorFile.sectionToProblemsToLocaleToCount.getCount(
1537                                 sectionAndProblemsAndLocale);
1538                 final String section = sectionAndProblemsAndLocale.get0();
1539                 indexTablePrinter
1540                         .addRow()
1541                         .addCell(section)
1542                         .addCell(problem)
1543                         .addCell(subtype)
1544                         .addCell(ConsoleCheckCLDR.getLocaleName(locale))
1545                         .addCell(locale)
1546                         .addCell(count);
1547                 for (Organization org : orgToLocales.keySet()) {
1548                     indexTablePrinter.addCell(
1549                             orgToLocales.getAll(org).contains(locale)
1550                                     ? org.toString().substring(0, 2)
1551                                     : "");
1552                 }
1553                 indexTablePrinter.finishRow();
1554             }
1555             PrintWriter generated_html_index =
1556                     FileUtilities.openUTF8Writer(
1557                             ErrorFile.generated_html_directory, "sections.html");
1558             ConsoleCheckCLDR.ErrorFile.showIndexHead(
1559                     "Error Report Index by Section", "", generated_html_index);
1560             generated_html_index.println(indexTablePrinter.toTable());
1561             generated_html_index.println(ShowData.dateFooter());
1562             generated_html_index.println(CldrUtility.ANALYTICS);
1563             generated_html_index.println("</body></html>");
1564             generated_html_index.close();
1565         }
1566 
formatSkippingZero(long count)1567         static String formatSkippingZero(long count) {
1568             if (count == 0) {
1569                 return "";
1570             }
1571             return String.valueOf(count);
1572         }
1573 
showIndexHead(String title, String localeID, PrintWriter generated_html_index)1574         static void showIndexHead(String title, String localeID, PrintWriter generated_html_index) {
1575             final boolean notLocaleSpecific = localeID.length() == 0;
1576             if ((!notLocaleSpecific)) {
1577                 title = "Errors in " + ConsoleCheckCLDR.getNameAndLocale(localeID, false);
1578             }
1579             generated_html_index.println(
1580                     "<html>"
1581                             + "<head><meta http-equiv='Content-Type' content='text/html; charset=utf-8'>"
1582                             + CldrUtility.LINE_SEPARATOR
1583                             + "<title>"
1584                             + title
1585                             + "</title>"
1586                             + CldrUtility.LINE_SEPARATOR
1587                             + "<link rel='stylesheet' href='errors.css' type='text/css'>"
1588                             + CldrUtility.LINE_SEPARATOR
1589                             + "<base target='_blank'>"
1590                             + CldrUtility.LINE_SEPARATOR
1591                             + "</head><body>"
1592                             + CldrUtility.LINE_SEPARATOR
1593                             + "<h1>"
1594                             + title
1595                             + "</h1>"
1596                             + CldrUtility.LINE_SEPARATOR
1597                             + "<p>"
1598                             + "<a href='index.html"
1599                             + (notLocaleSpecific ? "" : "#" + localeID)
1600                             + "'>Index</a>"
1601                             + " | "
1602                             + "<a href='sections.html"
1603                             + (notLocaleSpecific ? "" : "#" + localeID)
1604                             + "'>Index by Section</a>"
1605                             + " | "
1606                             + "<a href='http://unicode.org/cldr/data/docs/survey/vetting.html'><b style='background-color: yellow;'><i>Help: How to Vet</i></b></a>"
1607                             + "</p>"
1608                             + "<p>The following errors have been detected in the locale"
1609                             + (notLocaleSpecific
1610                                     ? "s. "
1611                                             + org.unicode.cldr.test.HelpMessages.getChartMessages(
1612                                                     "error_index_header")
1613                                     : " "
1614                                             + ConsoleCheckCLDR.getNameAndLocale(localeID, false)
1615                                             + ". "
1616                                             + ErrorFile.ERROR_CHART_HEADER));
1617         }
1618 
writeErrorFileIndex()1619         private static void writeErrorFileIndex() throws IOException {
1620             PrintWriter generated_html_index =
1621                     FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "index.html");
1622             ConsoleCheckCLDR.ErrorFile.showIndexHead(
1623                     "Error Report Index", "", generated_html_index);
1624             ConsoleCheckCLDR.ErrorFile.showErrorFileIndex(generated_html_index);
1625             generated_html_index.close();
1626             showSections();
1627         }
1628 
writeErrorCountsText()1629         private static void writeErrorCountsText() {
1630             // do the plain text file
1631             ErrorFile.generated_html_count.print(ConsoleCheckCLDR.lastHtmlLocaleID + ";\tcounts");
1632             for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) {
1633                 ErrorFile.generated_html_count.print(
1634                         ";\t" + type + "=" + ErrorFile.htmlErrorsPerLocale.getCount(type));
1635             }
1636             ErrorFile.generated_html_count.println();
1637             ErrorFile.generated_html_count.flush();
1638 
1639             // now store the data for the index
1640             ErrorFile.errorFileIndexData.put(
1641                     ConsoleCheckCLDR.lastHtmlLocaleID,
1642                     new Pair<>(ConsoleCheckCLDR.lastHtmlLocaleID, ErrorFile.htmlErrorsPerLocale));
1643             ErrorFile.htmlErrorsPerLocale = new Counter<>();
1644         }
1645 
1646         static Counter<ErrorType> htmlErrorsPerLocale =
1647                 new Counter<>(); // ConsoleCheckCLDR.ErrorCount();
1648         static PrintWriter generated_html_count = null;
1649         private static TreeMap<String, Pair<String, Counter<ErrorType>>> errorFileIndexData =
1650                 new TreeMap<>();
1651 
1652         static PrintWriter errorFileWriter = null;
1653         private static final String ERROR_CHART_HEADER =
1654                 org.unicode.cldr.test.HelpMessages.getChartMessages("error_locale_header");
1655         static String generated_html_directory = null;
1656         public static Counter<Row.R4<String, ErrorType, Subtype, String>>
1657                 sectionToProblemsToLocaleToCount = new Counter<>();
1658     }
1659 
showSummary(String localeID, Level level, String value)1660     private static void showSummary(String localeID, Level level, String value) {
1661         String line = "# " + getLocaleAndName(localeID) + "\tSummary\t" + level + "\t" + value;
1662         System.out.println(line);
1663     }
1664 
showExamples( CLDRFile cldrFile, String prettyPath, String localeID, String path, String value, String fullPath, String example)1665     private static void showExamples(
1666             CLDRFile cldrFile,
1667             String prettyPath,
1668             String localeID,
1669             String path,
1670             String value,
1671             String fullPath,
1672             String example) {
1673         if (example != null) {
1674             showValue(
1675                     cldrFile,
1676                     prettyPath,
1677                     localeID,
1678                     example,
1679                     path,
1680                     value,
1681                     fullPath,
1682                     "ok",
1683                     Subtype.none);
1684         }
1685     }
1686 
addPrettyPaths( CLDRFile file, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target)1687     private static void addPrettyPaths(
1688             CLDRFile file,
1689             Matcher pathFilter,
1690             PathHeader.Factory pathHeaderFactory,
1691             boolean noaliases,
1692             boolean filterDraft,
1693             Collection<String> target) {
1694         for (Iterator<String> pit = file.iterator(pathFilter); pit.hasNext(); ) {
1695             String path = pit.next();
1696             addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path);
1697         }
1698     }
1699 
addPrettyPaths( CLDRFile file, Collection<String> paths, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target)1700     private static void addPrettyPaths(
1701             CLDRFile file,
1702             Collection<String> paths,
1703             Matcher pathFilter,
1704             PathHeader.Factory pathHeaderFactory,
1705             boolean noaliases,
1706             boolean filterDraft,
1707             Collection<String> target) {
1708         for (String path : paths) {
1709             if (pathFilter != null && !pathFilter.reset(path).matches()) continue;
1710             addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path);
1711         }
1712     }
1713 
addPrettyPath( CLDRFile file, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target, String path)1714     private static void addPrettyPath(
1715             CLDRFile file,
1716             PathHeader.Factory pathHeaderFactory,
1717             boolean noaliases,
1718             boolean filterDraft,
1719             Collection<String> target,
1720             String path) {
1721         if (noaliases
1722                 && XMLSource.Alias.isAliasPath(
1723                         path)) { // this is just for console testing, the survey tool
1724             // shouldn't do it.
1725             return;
1726         }
1727         if (filterDraft) {
1728             String newPath = CLDRFile.getNondraftNonaltXPath(path);
1729             if (!newPath.equals(path)) {
1730                 String value = file.getStringValue(newPath);
1731                 if (value != null) {
1732                     return;
1733                 }
1734             }
1735         }
1736         String prettyPath = pathHeaderFactory.fromPath(path).toString();
1737         target.add(prettyPath);
1738     }
1739 
setExampleGenerator(ExampleGenerator inputExampleGenerator)1740     private static synchronized void setExampleGenerator(ExampleGenerator inputExampleGenerator) {
1741         englishExampleGenerator = inputExampleGenerator;
1742     }
1743 
getExampleGenerator()1744     private static synchronized ExampleGenerator getExampleGenerator() {
1745         return englishExampleGenerator;
1746     }
1747 
1748     private static ExampleGenerator englishExampleGenerator;
1749 
1750     static Matcher coverageMatcher =
1751             PatternCache.get("meet ([a-z]*) coverage").matcher(""); // HACK TODO fix
1752 
showHeaderLine()1753     private static void showHeaderLine() {
1754         if (SHOW_LOCALE) {
1755             if (idView) {
1756                 System.out.println(
1757                         "Locale\tID\tDesc.\t〈Eng.Value〉\t【Eng.Ex.】\t〈Loc.Value〉\t【Loc.Ex】\t⁅error/warning type⁆\t❮Error/Warning Msg❯");
1758             } else {
1759                 System.out.println(
1760                         "Locale\tStatus\t▸PPath◂\t〈Eng.Value〉\t【Eng.Ex.】\t〈Loc.Value〉\t«fill-in»\t【Loc.Ex】\t⁅error/warning type⁆\t❮Error/Warning Msg❯\tFull Path\tAliasedSource/Path?");
1761             }
1762         }
1763     }
1764 
1765     private static PathDescription pathDescription = null;
1766 
getIdString(String path, String value)1767     private static String getIdString(String path, String value) {
1768         if (pathDescription == null) {
1769             pathDescription =
1770                     new PathDescription(
1771                             supplementalDataInfo,
1772                             english,
1773                             null,
1774                             null,
1775                             PathDescription.ErrorHandling.CONTINUE);
1776         }
1777         final String description = pathDescription.getDescription(path, value, null);
1778         return "\t" + StringId.getId(path) + "" + "\t" + description + "";
1779     }
1780 
showValue( CLDRFile cldrFile, String prettyPath, final String localeID, String example, final String path, String value, String fullPath, String statusString, Subtype subtype)1781     private static void showValue(
1782             CLDRFile cldrFile,
1783             String prettyPath,
1784             final String localeID,
1785             String example,
1786             final String path,
1787             String value,
1788             String fullPath,
1789             String statusString,
1790             Subtype subtype) {
1791         ErrorType shortStatus = ErrorType.fromStatusString(statusString);
1792         subtotalCount.add(shortStatus, 1);
1793         totalCount.add(shortStatus, 1);
1794         if (subtype == null) {
1795             subtype = Subtype.none;
1796         }
1797         final SourceLocation location =
1798                 fullPath == null ? null : cldrFile.getSourceLocation(fullPath);
1799 
1800         if (ErrorFile.errorFileWriter == null) {
1801             example = example == null ? "" : example;
1802             String englishExample = null;
1803             final String englishPathValue = path == null ? null : getEnglishPathValue(path);
1804             if (SHOW_EXAMPLES && path != null) {
1805                 englishExample =
1806                         ExampleGenerator.simplify(
1807                                 getExampleGenerator().getExampleHtml(path, englishPathValue));
1808             }
1809             englishExample = englishExample == null ? "" : englishExample;
1810             String cleanPrettyPath = path == null ? null : prettyPath;
1811             Status status = new Status();
1812             String sourceLocaleID = path == null ? null : cldrFile.getSourceLocaleID(path, status);
1813             String fillinValue = path == null ? null : cldrFile.getFillInValue(path);
1814             fillinValue = fillinValue == null ? "" : fillinValue.equals(value) ? "=" : fillinValue;
1815 
1816             String pathLink = CLDR_CONFIG.urls().forXpath(localeID, path);
1817 
1818             final String otherSource =
1819                     path == null
1820                             ? null
1821                             : (sourceLocaleID.equals(localeID) ? "" : "\t" + sourceLocaleID);
1822             final String otherPath =
1823                     path == null
1824                             ? null
1825                             : (status.pathWhereFound.equals(path)
1826                                     ? ""
1827                                     : "\t" + status.pathWhereFound);
1828             if (location != null) {
1829                 System.err.println(location.toString() + shortStatus); // print full path here
1830             }
1831             String idViewString =
1832                     idView ? (path == null ? "\tNO_ID" : getIdString(path, value)) : "";
1833             System.out.println(
1834                     getLocaleAndName(localeID)
1835                             + (idViewString.isEmpty()
1836                                     ?
1837                                     // + "\t" + subtotalCount.getCount(shortStatus)
1838                                     "\t"
1839                                             + shortStatus
1840                                             + "\t▸"
1841                                             + cleanPrettyPath
1842                                             + "◂"
1843                                             + "\t〈"
1844                                             + englishPathValue
1845                                             + "〉"
1846                                             + "\t【"
1847                                             + englishExample
1848                                             + "】"
1849                                             + "\t〈"
1850                                             + value
1851                                             + "〉"
1852                                             + "\t«"
1853                                             + fillinValue
1854                                             + "»"
1855                                             + "\t【"
1856                                             + example
1857                                             + "】"
1858                                             + "\t⁅"
1859                                             + subtype
1860                                             + "⁆"
1861                                             + "\t❮"
1862                                             + statusString
1863                                             + "❯"
1864                                             + "\t"
1865                                             + pathLink
1866                                             + otherSource
1867                                             + otherPath
1868                                     : idViewString
1869                                             + "\t〈"
1870                                             + englishPathValue
1871                                             + "〉"
1872                                             + "\t【"
1873                                             + englishExample
1874                                             + "】"
1875                                             + "\t"
1876                                             + value
1877                                             + "〉"
1878                                             + "\t【"
1879                                             + example
1880                                             + "】"
1881                                             + "\t⁅"
1882                                             + subtype
1883                                             + "⁆"
1884                                             + "\t❮"
1885                                             + statusString
1886                                             + "❯"));
1887         } else if (ErrorFile.errorFileWriter != null) {
1888             if (shortStatus == ErrorType.contributed) {
1889                 return;
1890             }
1891             if (shortStatus == ErrorType.posix) {
1892                 shortStatus = ErrorType.minimal;
1893             }
1894             if (!localeID.equals(lastHtmlLocaleID)) {
1895                 ErrorFile.writeErrorCountsText();
1896                 lastHtmlLocaleID = localeID;
1897             }
1898             addError(shortStatus);
1899             ErrorFile.addDataToErrorFile(localeID, path, shortStatus, subtype);
1900         }
1901         if (CLDR_GITHUB_ANNOTATIONS) {
1902             // Annotate anything that needs annotation
1903             if (shortStatus == ErrorType.error || shortStatus == ErrorType.warning) {
1904                 String filePath = null;
1905                 if (location != null) {
1906                     // Use accurate location
1907                     filePath =
1908                             location.forGitHub(
1909                                     CLDRPaths.BASE_DIRECTORY); // Trim to CLDR_DIR for GitHub
1910                 } else {
1911                     // Fallback if SourceLocation fails
1912                     filePath =
1913                             "file="
1914                                     + localeXpathToFilePath.computeIfAbsent(
1915                                             Pair.of(localeID, path),
1916                                             locPath -> guessFilePath(locPath));
1917                 }
1918                 System.out.println(
1919                         "::"
1920                                 + shortStatus
1921                                 + " "
1922                                 + filePath.trim()
1923                                 + ",title="
1924                                 + subtype
1925                                 + ":: "
1926                                 + statusString);
1927             }
1928         }
1929         if (PATH_IN_COUNT && ErrorFile.generated_html_count != null) {
1930             ErrorFile.generated_html_count.println(lastHtmlLocaleID + ";\tpath:\t" + path);
1931         }
1932     }
1933 
addError(ErrorType shortStatus)1934     private static void addError(ErrorType shortStatus) {
1935         if (ErrorType.showInSummary.contains(shortStatus)) {
1936             ErrorFile.htmlErrorsPerLocale.increment(shortStatus);
1937         }
1938     }
1939 
1940     static String lastHtmlLocaleID = "";
1941     private static VoterInfoList voterInfoList;
1942     private static VoteResolver<String> voteResolver;
1943     private static String resolveVotesDirectory;
1944     private static boolean idView;
1945     private static SupplementalDataInfo supplementalDataInfo;
1946     private static CLDRFile english;
1947 
1948     public static class PathShower {
1949         String localeID;
1950         boolean newLocale = true;
1951         String lastPath;
1952         String[] lastSplitPath;
1953         boolean showEnglish;
1954         String splitChar = "/";
1955 
1956         static final String lead = "****************************************";
1957 
set(String localeID)1958         public void set(String localeID) {
1959             this.localeID = localeID;
1960             newLocale = true;
1961             LocaleIDParser localeIDParser = new LocaleIDParser();
1962             showEnglish = !localeIDParser.set(localeID).getLanguageScript().equals("en");
1963             lastPath = null;
1964             lastSplitPath = null;
1965         }
1966 
getSplitChar()1967         public String getSplitChar() {
1968             return splitChar;
1969         }
1970 
setSplitChar(String splitChar)1971         public PathShower setSplitChar(String splitChar) {
1972             this.splitChar = splitChar;
1973             return this;
1974         }
1975     }
1976 
getEnglishPathValue(String path)1977     private static String getEnglishPathValue(String path) {
1978         String englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path);
1979         if (englishValue == null) {
1980             String path2 = CLDRFile.getNondraftNonaltXPath(path);
1981             englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path2);
1982         }
1983         return englishValue;
1984     }
1985 
1986     /**
1987      * Utility for getting information.
1988      *
1989      * @param locale
1990      * @return
1991      */
getLocaleAndName(String locale)1992     private static String getLocaleAndName(String locale) {
1993         String localizedName = CheckCLDR.getDisplayInformation().getName(locale);
1994         if (localizedName == null || localizedName.equals(locale)) return locale;
1995         return locale + " [" + localizedName + "]";
1996     }
1997 
1998     /**
1999      * Utility for getting information.
2000      *
2001      * @param locale
2002      * @param linkToXml TODO
2003      * @return
2004      */
getNameAndLocale(String locale, boolean linkToXml)2005     private static String getNameAndLocale(String locale, boolean linkToXml) {
2006         String localizedName = CheckCLDR.getDisplayInformation().getName(locale);
2007         if (localizedName == null || localizedName.equals(locale)) return locale;
2008         if (linkToXml) {
2009             locale =
2010                     "<a href='https://github.com/unicode-org/cldr/tree/main/common/main/"
2011                             + locale
2012                             + ".xml'>"
2013                             + locale
2014                             + "</a>";
2015         }
2016         return localizedName + " [" + locale + "]";
2017     }
2018 
getLocaleName(String locale)2019     private static String getLocaleName(String locale) {
2020         String localizedName = CheckCLDR.getDisplayInformation().getName(locale);
2021         if (localizedName == null || localizedName.equals(locale)) return locale;
2022         return localizedName;
2023     }
2024 
getLinkedLocale(String locale)2025     private static String getLinkedLocale(String locale) {
2026         return "<a href='http://unicode.org/cldr/apps/survey?_=" + locale + "'>" + locale + "</a>";
2027     }
2028 
2029     /** Approximate xml path */
guessFilePath(Pair<String, String> locPath)2030     private static String guessFilePath(Pair<String, String> locPath) {
2031         final File base = new File(CLDRPaths.BASE_DIRECTORY);
2032         final String loc = locPath.getFirst();
2033         final String path = locPath.getSecond();
2034         if (path != null) {
2035             String subdir = "main";
2036             if (path.startsWith("//ldml/annotations")) {
2037                 subdir = "annotations";
2038             } else if (path.startsWith("//ldml/subdivisions")) {
2039                 subdir = "subdivisions";
2040             }
2041             File inCommon = new File(base, "common");
2042             File subsub = new File(inCommon, subdir);
2043             if (subsub.isDirectory()) {
2044                 File subFile = new File(subsub, loc + ".xml");
2045                 if (subFile.canRead())
2046                     return subFile.getAbsolutePath().substring(base.getAbsolutePath().length() + 1);
2047             }
2048 
2049             File inSeed = new File(base, "seed");
2050             subsub = new File(inSeed, subdir);
2051             if (subsub.isDirectory()) {
2052                 File subFile = new File(subsub, loc + ".xml");
2053                 if (subFile.canRead())
2054                     return subFile.getAbsolutePath().substring(base.getAbsolutePath().length() + 1);
2055             }
2056         }
2057         // no XPath - could be an entire-locale error.
2058         return loc + ".xml";
2059     }
2060 
2061     static final ConcurrentHashMap<Pair<String, String>, String> localeXpathToFilePath =
2062             new ConcurrentHashMap<>();
2063 }
2064