1 package org.unicode.cldr.test; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.PrintWriter; 6 import java.util.ArrayList; 7 import java.util.Arrays; 8 import java.util.Collection; 9 import java.util.Collections; 10 import java.util.Comparator; 11 import java.util.EnumSet; 12 import java.util.HashMap; 13 import java.util.HashSet; 14 import java.util.Iterator; 15 import java.util.List; 16 import java.util.Map; 17 import java.util.Set; 18 import java.util.TreeMap; 19 import java.util.TreeSet; 20 import java.util.concurrent.ConcurrentHashMap; 21 import java.util.regex.Matcher; 22 23 import org.unicode.cldr.draft.FileUtilities; 24 import org.unicode.cldr.test.CheckCLDR.CheckStatus; 25 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 26 import org.unicode.cldr.test.CheckCLDR.CompoundCheckCLDR; 27 import org.unicode.cldr.test.CheckCLDR.FormatDemo; 28 import org.unicode.cldr.test.CheckCLDR.Options; 29 import org.unicode.cldr.test.CheckCLDR.Phase; 30 import org.unicode.cldr.test.CheckCLDR.SimpleDemo; 31 import org.unicode.cldr.tool.Option; 32 import org.unicode.cldr.tool.Option.Params; 33 import org.unicode.cldr.tool.ShowData; 34 import org.unicode.cldr.tool.TablePrinter; 35 import org.unicode.cldr.util.CLDRConfig; 36 import org.unicode.cldr.util.CLDRConfig.Environment; 37 import org.unicode.cldr.util.CLDRFile; 38 import org.unicode.cldr.util.CLDRFile.Status; 39 import org.unicode.cldr.util.CLDRPaths; 40 import org.unicode.cldr.util.CLDRTool; 41 import org.unicode.cldr.util.CldrUtility; 42 import org.unicode.cldr.util.Counter; 43 import org.unicode.cldr.util.CoverageInfo; 44 import org.unicode.cldr.util.Factory; 45 import org.unicode.cldr.util.LanguageTagParser; 46 import org.unicode.cldr.util.Level; 47 import org.unicode.cldr.util.LocaleIDParser; 48 import org.unicode.cldr.util.LogicalGrouping; 49 import org.unicode.cldr.util.Organization; 50 import org.unicode.cldr.util.Pair; 51 import org.unicode.cldr.util.PathDescription; 52 import org.unicode.cldr.util.PathHeader; 53 import org.unicode.cldr.util.PathUtilities; 54 import org.unicode.cldr.util.PatternCache; 55 import org.unicode.cldr.util.SimpleFactory; 56 import org.unicode.cldr.util.StandardCodes; 57 import org.unicode.cldr.util.StringId; 58 import org.unicode.cldr.util.SupplementalDataInfo; 59 import org.unicode.cldr.util.UnicodeSetPrettyPrinter; 60 import org.unicode.cldr.util.VoteResolver; 61 import org.unicode.cldr.util.VoteResolver.CandidateInfo; 62 import org.unicode.cldr.util.VoteResolver.UnknownVoterException; 63 import org.unicode.cldr.util.XMLSource; 64 65 import com.ibm.icu.dev.tool.UOption; 66 import com.ibm.icu.dev.util.ElapsedTimer; 67 import com.ibm.icu.impl.Relation; 68 import com.ibm.icu.impl.Row; 69 import com.ibm.icu.lang.UCharacter; 70 import com.ibm.icu.text.Collator; 71 import com.ibm.icu.text.UnicodeSet; 72 import com.ibm.icu.util.ULocale; 73 74 /** 75 * Console test for CheckCLDR. <br> 76 * Some common source directories: 77 * 78 * <pre> 79 * -s C:/cvsdata/unicode/cldr/incoming/vetted/main 80 * -s C:/cvsdata/unicode/cldr/incoming/proposed/main 81 * -s C:/cvsdata/unicode/cldr/incoming/proposed/main 82 * -s C:/cvsdata/unicode/cldr/testdata/main 83 * </pre> 84 * 85 * @author markdavis 86 * 87 */ 88 @CLDRTool(alias = "check", 89 description = "Run CheckCLDR against CLDR data") 90 public class ConsoleCheckCLDR { 91 private static final CLDRConfig CLDR_CONFIG = CLDRConfig.getInstance(); 92 private static final PathHeader.Factory PATH_HEADER_FACTORY = PathHeader.getFactory(); 93 public static boolean showStackTrace = false; 94 public static boolean errorsOnly = false; 95 static boolean SHOW_LOCALE = true; 96 static boolean SHOW_EXAMPLES = false; 97 // static PrettyPath prettyPathMaker = new PrettyPath(); 98 private static boolean CLDR_GITHUB_ANNOTATIONS = (Boolean.parseBoolean(System.getProperty("CLDR_GITHUB_ANNOTATIONS", "false"))); 99 100 private static final int HELP1 = 0, 101 HELP2 = 1, 102 COVERAGE = 2, 103 EXAMPLES = 3, 104 FILE_FILTER = 4, 105 TEST_FILTER = 5, 106 DATE_FORMATS = 6, 107 ORGANIZATION = 7, 108 SHOWALL = 8, 109 PATH_FILTER = 9, 110 ERRORS_ONLY = 10, 111 CHECK_ON_SUBMIT = 11, 112 NO_ALIASES = 12, 113 SOURCE_DIRECTORY = 13, 114 USER = 14, 115 PHASE = 15, 116 GENERATE_HTML = 16, 117 VOTE_RESOLVE = 17, 118 ID_VIEW = 18, 119 SUBTYPE_FILTER = 19, 120 SOURCE_ALL = 20, 121 BAILEY = 21 122 // VOTE_RESOLVE2 = 21 123 ; 124 125 static final String SOURCE_DIRS = CLDRPaths.MAIN_DIRECTORY + "," + CLDRPaths.ANNOTATIONS_DIRECTORY + "," + CLDRPaths.SEED_DIRECTORY; 126 127 enum MyOptions { 128 coverage(new Params().setHelp("Set the coverage: eg -c comprehensive") 129 .setMatch("comprehensive|modern|moderate|basic")), // UOption.REQUIRES_ARG 130 examples(new Params().setHelp("Turn on examples (actually a summary of the demo)") 131 .setFlag('x')), //, 'x', UOption.NO_ARG), 132 file_filter(new Params().setHelp("Pick the locales (files) to check: arg is a regular expression, eg -f fr, or -f fr.*, or -f (fr|en-.*)") 133 .setDefault(".*").setMatch(".*")), //, 'f', UOption.REQUIRES_ARG).setDefault(".*"), 134 test_filter(new Params() 135 .setHelp("Filter the Checks: arg is a regular expression, eg -t.*number.*. To check all BUT a given test, use the style -t ((?!.*CheckZones).*)") 136 .setDefault(".*").setMatch(".*")), //, 't', UOption.REQUIRES_ARG).setDefault(".*"), 137 date_formats(new Params().setHelp("Turn on special date format checks")), //, 'd', UOption.NO_ARG), 138 organization(new Params().setHelp("Organization: ibm, google, ....; Uses Locales.txt for to filter locales and set coverage levels") 139 .setDefault(".*").setMatch(".*")), //, 'o', UOption.REQUIRES_ARG), 140 showall(new Params().setHelp("Show all paths, including aliased").setFlag('a')), //, 'a', UOption.NO_ARG), 141 path_filter(new Params().setHelp("Pick the paths to check, eg -p.*languages.*") 142 .setDefault(".*").setMatch(".*")), //, 'p', UOption.REQUIRES_ARG).setDefault(".*"), 143 errors_only(new Params().setHelp("Show errors only (with -ef, only final processing errors)")), //, 'e', UOption.NO_ARG), 144 check_on_submit(new Params().setHelp("") 145 .setFlag('k')), //, 'k', UOption.NO_ARG), 146 noaliases(new Params().setHelp("No aliases")), //, 'n', UOption.NO_ARG), 147 source_directory(new Params().setHelp("Fully qualified source directories. (Conflicts with -S.)") 148 .setDefault(SOURCE_DIRS).setMatch(".*")), //, 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS), 149 user(new Params().setHelp("User, eg -uu148") 150 .setMatch(".*")), //, 'u', UOption.REQUIRES_ARG), 151 phase(new Params().setHelp("?") 152 .setMatch(Phase.class).setFlag('z')), //, 'z', UOption.REQUIRES_ARG), 153 generate_html(new Params().setHelp("Generate HTML-style chart in directory.") 154 .setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/").setMatch(".*")), //, 'g', UOption.OPTIONAL_ARG).setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"), 155 vote_resolution(new Params().setHelp("")), //, 'v', UOption.NO_ARG), 156 id_view(new Params().setHelp("")), //, 'i', UOption.NO_ARG), 157 subtype_filter(new Params().setHelp("error/warning subtype filter, eg unexpectedOrderOfEraYear") 158 .setDefault(".*").setMatch(".*").setFlag('y')), //, 'y', UOption.REQUIRES_ARG), 159 source_all(new Params().setHelp( 160 "Partially qualified directories. Standard subdirectories added if not specified (/main, /annotations, /subdivisions). (Conflicts with -s.)") 161 .setMatch(".*").setFlag('S').setDefault("common,seed,exemplars")), //, 'S', <changed>), 162 bailey(new Params().setHelp("check bailey values (" + CldrUtility.INHERITANCE_MARKER + ")")), //, 'b', UOption.NO_ARG) 163 exemplarError(new Params().setFlag('E').setHelp("include to force strict Exemplar check")); 164 165 // BOILERPLATE TO COPY 166 final Option option; 167 MyOptions(Params params)168 private MyOptions(Params params) { 169 option = new Option(this, params); 170 } 171 172 private static Option.Options myOptions = new Option.Options(); 173 static { 174 for (MyOptions option : MyOptions.values()) { myOptions.add(option, option.option)175 myOptions.add(option, option.option); 176 } 177 } 178 parse(String[] args, boolean showArguments)179 private static Set<String> parse(String[] args, boolean showArguments) { 180 return myOptions.parse(MyOptions.values()[0], args, true); 181 } 182 } 183 184 private static final UOption[] options = { 185 UOption.HELP_H(), 186 UOption.HELP_QUESTION_MARK(), 187 UOption.create("coverage", 'c', UOption.REQUIRES_ARG), 188 UOption.create("examples", 'x', UOption.NO_ARG), 189 UOption.create("file_filter", 'f', UOption.REQUIRES_ARG).setDefault(".*"), 190 UOption.create("test_filter", 't', UOption.REQUIRES_ARG).setDefault(".*"), 191 UOption.create("date_formats", 'd', UOption.NO_ARG), 192 UOption.create("organization", 'o', UOption.REQUIRES_ARG), 193 UOption.create("showall", 'a', UOption.NO_ARG), 194 UOption.create("path_filter", 'p', UOption.REQUIRES_ARG).setDefault(".*"), 195 UOption.create("errors_only", 'e', UOption.NO_ARG), 196 UOption.create("check-on-submit", 'k', UOption.NO_ARG), 197 UOption.create("noaliases", 'n', UOption.NO_ARG), 198 UOption.create("source_directory", 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS), 199 UOption.create("user", 'u', UOption.REQUIRES_ARG), 200 UOption.create("phase", 'z', UOption.REQUIRES_ARG), 201 UOption.create("generate_html", 'g', UOption.OPTIONAL_ARG).setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"), 202 UOption.create("vote resolution", 'v', UOption.NO_ARG), 203 UOption.create("id view", 'i', UOption.NO_ARG), 204 UOption.create("subtype_filter", 'y', UOption.REQUIRES_ARG), 205 UOption.create("source_all", 'S', UOption.OPTIONAL_ARG).setDefault("common,seed,exemplars"), 206 UOption.create("bailey", 'b', UOption.NO_ARG), 207 UOption.create("exemplarError", 'E', UOption.NO_ARG) 208 // UOption.create("vote resolution2", 'w', UOption.OPTIONAL_ARG).setDefault(Utility.BASE_DIRECTORY + 209 // "incoming/vetted/main/votes/"), 210 }; 211 212 private static final Comparator<String> baseFirstCollator = new Comparator<String>() { 213 LanguageTagParser languageTagParser1 = new LanguageTagParser(); 214 LanguageTagParser languageTagParser2 = new LanguageTagParser(); 215 216 @Override 217 public int compare(String o1, String o2) { 218 String ls1 = languageTagParser1.set(o1).getLanguageScript(); 219 String ls2 = languageTagParser2.set(o2).getLanguageScript(); 220 int result = ls1.compareTo(ls2); 221 if (result != 0) return result; 222 return o1.compareTo(o2); 223 } 224 }; 225 private static final boolean PATH_IN_COUNT = false; 226 227 /* 228 * TODO: unused? Should be used? 229 */ 230 private static String[] HelpMessage = { 231 "-h \t This message", 232 "-s \t Source directory, default = " + SOURCE_DIRS, 233 "-S common,seed\t Use common AND seed directories. ( Set CLDR_DIR, don't use this with -s. )\n", 234 "-fxxx \t Pick the locales (files) to check: xxx is a regular expression, eg -f fr, or -f fr.*, or -f (fr|en-.*)", 235 "-pxxx \t Pick the paths to check, eg -p(.*languages.*)", 236 "-cxxx \t Set the coverage: eg -c comprehensive or -c modern or -c moderate or -c basic", 237 "-txxx \t Filter the Checks: xxx is a regular expression, eg -t.*number.*. To check all BUT a given test, use the style -t ((?!.*CheckZones).*)", 238 "-oxxx \t Organization: ibm, google, ....; filters locales and uses Locales.txt for coverage tests", 239 "-x \t Turn on examples (actually a summary of the demo).", 240 "-d \t Turn on special date format checks", 241 "-a \t Show all paths", 242 "-e \t Show errors only (with -ef, only final processing errors)", 243 "-n \t No aliases", 244 "-u \t User, eg -uu148", 245 "-y \t error/warning subtype filter, eg unexpectedOrderOfEraYear", 246 "-b \t check bailey values (" + CldrUtility.INHERITANCE_MARKER + ")", 247 }; 248 249 static Counter<ErrorType> subtotalCount = new Counter<>(true); // new ErrorCount(); 250 static Counter<ErrorType> totalCount = new Counter<>(true); 251 252 /** 253 * This will be the test framework way of using these tests. It is preliminary for now. 254 * The Survey Tool will call setDisplayInformation, and getCheckAll. 255 * For each cldrfile, it will set the cldrFile. 256 * Then on each path in the file it will call check. 257 * Right now it doesn't work with resolved files, so just use unresolved ones. 258 * 259 * @param args 260 * @throws IOException 261 */ main(String[] args)262 public static void main(String[] args) throws IOException { 263 MyOptions.parse(args, true); 264 ElapsedTimer totalTimer = new ElapsedTimer(); 265 //CldrUtility.showOptions(args); 266 UOption.parseArgs(args, options); 267 // if (options[HELP1].doesOccur || options[HELP2].doesOccur) { 268 // for (int i = 0; i < HelpMessage.length; ++i) { 269 // System.out.println(HelpMessage[i]); 270 // } 271 // return; 272 // } 273 String factoryFilter = options[FILE_FILTER].value; 274 if (factoryFilter.equals("key")) { 275 factoryFilter = "(en|ru|nl|fr|de|it|pl|es|tr|th|ja|zh|ko|ar|bg|sr|uk|ca|hr|cs|da|fil|fi|hu|id|lv|lt|no|pt|ro|sk|sl|sv|vi|el|he|fa|hi|am|af|et|is|ms|sw|zu|bn|mr|ta|eu|gl|ur|gu|kn|ml|te|zh_Hant|pt_PT|en_GB)"; 276 } 277 String checkFilter = options[TEST_FILTER].value; 278 String subtypeFilterString = options[SUBTYPE_FILTER].value; 279 EnumSet<Subtype> subtypeFilter = null; 280 if (subtypeFilterString != null) { 281 subtypeFilter = EnumSet.noneOf(Subtype.class); 282 Matcher m = PatternCache.get(subtypeFilterString).matcher(""); 283 for (Subtype value : Subtype.values()) { 284 if (m.reset(value.toString()).find() || m.reset(value.name()).find()) { 285 subtypeFilter.add(value); 286 } 287 } 288 if (subtypeFilter.size() == 0) { 289 System.err.println("No subtype match for " + subtypeFilterString); 290 return; 291 } 292 } 293 294 errorsOnly = options[ERRORS_ONLY].doesOccur; 295 // if ("f".equals(options[ERRORS_ONLY].value)) { 296 // CheckCLDR.finalErrorType = CheckStatus.warningType; 297 // } 298 299 SHOW_EXAMPLES = options[EXAMPLES].doesOccur; 300 boolean showAll = options[SHOWALL].doesOccur; 301 boolean checkFlexibleDates = options[DATE_FORMATS].doesOccur; 302 String pathFilterString = options[PATH_FILTER].value; 303 Matcher pathFilter = null; 304 if (!pathFilterString.equals(".*")) { 305 pathFilter = PatternCache.get(pathFilterString).matcher(""); 306 } 307 boolean checkOnSubmit = options[CHECK_ON_SUBMIT].doesOccur; 308 boolean noaliases = options[NO_ALIASES].doesOccur; 309 310 Level coverageLevel = null; 311 String coverageLevelInput = options[COVERAGE].value; 312 if (coverageLevelInput != null) { 313 coverageLevel = Level.get(coverageLevelInput); 314 if (coverageLevel == Level.UNDETERMINED) { 315 throw new IllegalArgumentException("-c" + coverageLevelInput + "\t is invalid: must be one of: " 316 + "basic,moderate,..."); 317 } 318 } 319 320 Organization organization = options[ORGANIZATION].value == null ? null : Organization.fromString(options[ORGANIZATION].value); 321 if (organization != null) { 322 Set<Organization> organizations = StandardCodes.make().getLocaleCoverageOrganizations(); 323 if (!organizations.contains(organization)) { 324 throw new IllegalArgumentException("-o" + organization + "\t is invalid: must be one of: " 325 + organizations); 326 } 327 } 328 final CLDRConfig cldrConf = CLDR_CONFIG; 329 // set the envronment to UNITTEST as suggested 330 cldrConf.setEnvironment(Environment.UNITTEST); 331 // get the Phase from CLDRConfig object 332 final Phase phase; 333 // Phase phase = Phase.BUILD; 334 if (options[PHASE].doesOccur) { 335 String phaseVal = options[PHASE].value; 336 try { 337 // no null check for argument; if it is is null, Phase.forString would return the one from CLDRConfig 338 phase = Phase.forString(phaseVal); 339 } catch (IllegalArgumentException e) { 340 StringBuilder sb = new StringBuilder("Incorrect Phase value"); 341 if (phaseVal != null && !phaseVal.isEmpty()) { 342 sb.append(" '"); 343 sb.append(phaseVal); 344 sb.append("'"); 345 } 346 sb.append(": should be one of "); 347 for (Phase curPhase : Phase.values()) { 348 // implicitly does a toString; 349 sb.append(curPhase); 350 sb.append(", "); 351 } 352 int lastIdx = sb.lastIndexOf(","); 353 // remove the last comma, if it occurs 354 if (lastIdx > -1) { 355 String tmpBuf = sb.substring(0, lastIdx); 356 sb.setLength(0); 357 sb.append(tmpBuf); 358 } 359 sb.append("."); 360 // TODO: Reporting should be similar to an error (wrong parameter...), and not actually an Exception 361 throw new IllegalArgumentException(sb.toString(), e); 362 } 363 } else { 364 phase = cldrConf.getPhase(); 365 } 366 367 boolean baileyTest = options[BAILEY].doesOccur; 368 369 File sourceDirectories[] = null; 370 371 if (MyOptions.source_all.option.doesOccur()) { 372 if (MyOptions.source_directory.option.doesOccur()) { 373 throw new IllegalArgumentException("Don't use -s and -S together."); 374 } 375 sourceDirectories = cldrConf.addStandardSubdirectories(cldrConf.getCLDRDataDirectories(MyOptions.source_all.option.getValue())); 376 } else { 377 String[] sdirs = options[SOURCE_DIRECTORY].value.split(",\\s*"); 378 sourceDirectories = new File[sdirs.length]; 379 for (int i = 0; i < sdirs.length; ++i) { 380 sourceDirectories[i] = new File(CldrUtility.checkValidDirectory(sdirs[i], 381 "Fix with -s. Use -h for help.")); 382 } 383 } 384 385 if (options[GENERATE_HTML].doesOccur) { 386 coverageLevel = Level.MODERN; // reset 387 ErrorFile.generated_html_directory = options[GENERATE_HTML].value; 388 ErrorFile.generated_html_count = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "count.txt"); 389 // try { 390 // ErrorFile.voteFactory = CLDRFile.Factory.make(sourceDirectory + "../../proposed/main/", ".*"); 391 // } catch (RuntimeException e) { 392 // ErrorFile.voteFactory = null; 393 // } 394 // PrintWriter cssFile = FileUtilities.openUTF8Writer(generated_html_directory, "index.css"); 395 // Utility; 396 } 397 398 idView = options[ID_VIEW].doesOccur; 399 400 if (options[VOTE_RESOLVE].doesOccur) { 401 resolveVotesDirectory = CldrUtility.checkValidFile(CLDRPaths.BASE_DIRECTORY + "incoming/vetted/votes/", 402 true, null); 403 VoteResolver.setVoterToInfo(CldrUtility.checkValidFile(CLDRPaths.BASE_DIRECTORY 404 + "incoming/vetted/usersa/usersa.xml", false, null)); 405 voteResolver = new VoteResolver<>(); 406 } 407 408 // check stuff 409 // Comparator cc = StandardCodes.make().getTZIDComparator(); 410 // System.out.println(cc.compare("Antarctica/Rothera", "America/Cordoba")); 411 // System.out.println(cc.compare("Antarctica/Rothera", "America/Indianapolis")); 412 413 String user = options[USER].value; 414 415 System.out.println("Source directories:\n"); 416 for (File f : sourceDirectories) { 417 System.out.println(" " + f.getPath() + "\t(" 418 + PathUtilities.getNormalizedPathString(f) + ")"); 419 } 420 // System.out.println("factoryFilter: " + factoryFilter); 421 // System.out.println("test filter: " + checkFilter); 422 // System.out.println("organization: " + organization); 423 // System.out.println("show examples: " + SHOW_EXAMPLES); 424 // System.out.println("phase: " + phase); 425 // System.out.println("path filter: " + pathFilterString); 426 // System.out.println("coverage level: " + coverageLevel); 427 // System.out.println("checking dates: " + checkFlexibleDates); 428 // System.out.println("only check-on-submit: " + checkOnSubmit); 429 // System.out.println("show all: " + showAll); 430 // System.out.println("errors only?: " + errorsOnly); 431 // System.out.println("generate error counts: " + ErrorFile.generated_html_directory); 432 // // System.out.println("vote directory: " + (ErrorFile.voteFactory == null ? null : 433 // // ErrorFile.voteFactory.getSourceDirectory())); 434 // System.out.println("resolve votes: " + resolveVotesDirectory); 435 // System.out.println("id view: " + idView); 436 // System.out.println("subtype filter: " + subtypeFilter); 437 438 // set up the test 439 Factory cldrFactory = SimpleFactory.make(sourceDirectories, factoryFilter) 440 .setSupplementalDirectory(new File(CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 441 CompoundCheckCLDR checkCldr = CheckCLDR.getCheckAll(cldrFactory, checkFilter); 442 if (checkCldr.getFilteredTestList().size() == 0) { 443 throw new IllegalArgumentException("The filter doesn't match any tests."); 444 } 445 System.out.println("filtered tests: " + checkCldr.getFilteredTests()); 446 Factory backCldrFactory = CLDRConfig.getInstance().getMainAndAnnotationsFactory(); 447 english = backCldrFactory.make("en", true); 448 449 CheckCLDR.setDisplayInformation(english); 450 checkCldr.setEnglishFile(english); 451 setExampleGenerator(new ExampleGenerator(english, english, CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 452 PathShower pathShower = new PathShower(); 453 454 // call on the files 455 Set<String> locales = new TreeSet<>(baseFirstCollator); 456 locales.addAll(cldrFactory.getAvailable()); 457 458 List<CheckStatus> result = new ArrayList<>(); 459 Set<PathHeader> paths = new TreeSet<>(); // CLDRFile.ldmlComparator); 460 Map m = new TreeMap(); 461 // double testNumber = 0; 462 Map<String, String> options = new HashMap<>(); 463 FlexibleDateFromCLDR fset = new FlexibleDateFromCLDR(); 464 Set<String> englishPaths = null; 465 466 Set<String> fatalErrors = new TreeSet<>(); 467 468 showHeaderLine(); 469 470 supplementalDataInfo = SupplementalDataInfo.getInstance(CLDRPaths.SUPPLEMENTAL_DIRECTORY); 471 472 LocaleIDParser localeIDParser = new LocaleIDParser(); 473 String lastBaseLanguage = ""; 474 PathHeader.Factory pathHeaderFactory = PathHeader.getFactory(english); 475 476 final List<String> specialPurposeLocales = new ArrayList<>(Arrays.asList("en_US_POSIX", "en_ZZ")); 477 for (String localeID : locales) { 478 if (CLDRFile.isSupplementalName(localeID)) continue; 479 if (supplementalDataInfo.getDefaultContentLocales().contains(localeID)) { 480 System.out.println("# Skipping default content locale: " + localeID); 481 continue; 482 } 483 484 // We don't really need to check the POSIX locale, as it is a special purpose locale 485 if (specialPurposeLocales.contains(localeID)) { 486 System.out.println("# Skipping special purpose locale: " + localeID); 487 continue; 488 } 489 490 boolean isLanguageLocale = localeID.equals(localeIDParser.set(localeID).getLanguageScript()); 491 options.clear(); 492 493 if (MyOptions.exemplarError.option.doesOccur()) { 494 options.put(Options.Option.exemplarErrors.toString(), "true"); 495 } 496 497 // if the organization is set, skip any locale that doesn't have a value in Locales.txt 498 Level level = coverageLevel; 499 if (level == null) { 500 level = Level.BASIC; 501 } 502 if (organization != null) { 503 Map<String, Level> locale_status = StandardCodes.make().getLocaleToLevel(organization); 504 if (locale_status == null) continue; 505 level = locale_status.get(localeID); 506 if (level == null) continue; 507 if (level.compareTo(Level.BASIC) <= 0) continue; 508 } else if (!isLanguageLocale) { 509 // otherwise, skip all language locales 510 options.put(Options.Option.CheckCoverage_skip.getKey(), "true"); 511 } 512 513 // if (coverageLevel != null) options.put("CoverageLevel.requiredLevel", coverageLevel.toString()); 514 if (organization != null) options.put(Options.Option.CoverageLevel_localeType.getKey(), organization.toString()); 515 options.put(Options.Option.phase.getKey(), phase.toString()); 516 //options.put(Options.Option.SHOW_TIMES.getKey(), "true"); 517 518 if (SHOW_LOCALE) System.out.println(); 519 520 // options.put("CheckCoverage.requiredLevel","comprehensive"); 521 522 CLDRFile file; 523 CLDRFile englishFile = english; 524 CLDRFile parent = null; 525 526 ElapsedTimer timer = new ElapsedTimer(); 527 try { 528 file = cldrFactory.make(localeID, true); 529 if (ErrorFile.voteFactory != null) { 530 ErrorFile.voteFile = ErrorFile.voteFactory.make(localeID, true); 531 } 532 final String parentID = LocaleIDParser.getParent(localeID); 533 if (parentID != null) { 534 parent = cldrFactory.make(parentID, true); 535 } 536 //englishFile = cldrFactory.make("en", true); 537 } catch (RuntimeException e) { 538 fatalErrors.add(localeID); 539 System.out.println("FATAL ERROR: " + localeID); 540 e.printStackTrace(System.out); 541 continue; 542 } 543 544 // generate HTML if asked for 545 if (ErrorFile.generated_html_directory != null) { 546 String baseLanguage = localeIDParser.set(localeID).getLanguageScript(); 547 548 if (!baseLanguage.equals(lastBaseLanguage)) { 549 lastBaseLanguage = baseLanguage; 550 ErrorFile.openErrorFile(localeID, baseLanguage); 551 } 552 553 } 554 555 if (user != null) { 556 file = new CLDRFile.TestUser(file, user, isLanguageLocale); 557 if (parent != null) { 558 parent = new CLDRFile.TestUser(parent, user, isLanguageLocale); 559 } 560 } 561 checkCldr.setCldrFileToCheck(file, options, result); 562 563 subtotalCount.clear(); 564 565 for (Iterator<CheckStatus> it3 = result.iterator(); it3.hasNext();) { 566 CheckStatus status = it3.next(); 567 String statusString = status.toString(); // com.ibm.icu.impl.Utility.escape( 568 CheckStatus.Type statusType = status.getType(); 569 570 if (errorsOnly) { 571 if (!statusType.equals(CheckStatus.errorType)) continue; 572 } 573 574 if (subtypeFilter != null) { 575 if (!subtypeFilter.contains(status.getSubtype())) { 576 continue; 577 } 578 } 579 580 if (checkOnSubmit) { 581 if (!status.isCheckOnSubmit() || !statusType.equals(CheckStatus.errorType)) continue; 582 } 583 showValue(file, null, localeID, null, null, null, null, statusString, status.getSubtype()); 584 } 585 paths.clear(); 586 587 CoverageInfo covInfo = cldrConf.getCoverageInfo(); 588 for (String path : file.fullIterable()) { 589 if (pathFilter != null && !pathFilter.reset(path).find()) { 590 continue; 591 } 592 if (coverageLevel != null) { 593 Level currentLevel = covInfo.getCoverageLevel(path, localeID); 594 if (currentLevel.compareTo(coverageLevel) > 0) { 595 continue; 596 } 597 } 598 paths.add(pathHeaderFactory.fromPath(path)); 599 } 600 // addPrettyPaths(file, pathFilter, prettyPathMaker, noaliases, false, paths); 601 // addPrettyPaths(file, file.getExtraPaths(), pathFilter, prettyPathMaker, noaliases, false, paths); 602 603 // also add the English paths 604 // initialize the first time in. 605 if (englishPaths == null) { 606 englishPaths = new HashSet<>(); 607 final CLDRFile displayFile = CheckCLDR.getDisplayInformation(); 608 addPrettyPaths(displayFile, pathFilter, pathHeaderFactory, noaliases, true, englishPaths); 609 addPrettyPaths(displayFile, displayFile.getExtraPaths(), pathFilter, pathHeaderFactory, noaliases, 610 true, englishPaths); 611 englishPaths = Collections.unmodifiableSet(englishPaths); // for robustness 612 } 613 // paths.addAll(englishPaths); 614 615 UnicodeSet missingExemplars = new UnicodeSet(); 616 UnicodeSet missingCurrencyExemplars = new UnicodeSet(); 617 if (checkFlexibleDates) { 618 fset.set(file); 619 } 620 pathShower.set(localeID); 621 622 // only create if we are going to use 623 ExampleGenerator exampleGenerator = SHOW_EXAMPLES ? new ExampleGenerator(file, englishFile, 624 CLDRPaths.DEFAULT_SUPPLEMENTAL_DIRECTORY) : null; 625 626 // Status pathStatus = new Status(); 627 int pathCount = 0; 628 Status otherPath = new Status(); 629 630 for (PathHeader pathHeader : paths) { 631 pathCount++; 632 String path = pathHeader.getOriginalPath(); 633 String prettyPath = pathHeader.toString().replace('\t', '|').replace(' ', '_'); 634 // String prettyPath = it2.next(); 635 // String path = prettyPathMaker.getOriginal(prettyPath); 636 // if (path == null) { 637 // prettyPathMaker.getOriginal(prettyPath); 638 // } 639 640 if (!showAll && !file.isWinningPath(path)) { 641 continue; 642 } 643 if (!isLanguageLocale && !baileyTest) { 644 final String sourceLocaleID = file.getSourceLocaleID(path, otherPath); 645 if (!localeID.equals(sourceLocaleID)) { 646 continue; 647 } 648 // also skip aliases 649 if (!path.equals(otherPath.pathWhereFound)) { 650 continue; 651 } 652 } 653 654 if (path.contains("@alt")) { 655 if (path.contains("proposed")) continue; 656 } 657 String value = file.getStringValue(path); 658 if (baileyTest) { 659 value = CldrUtility.INHERITANCE_MARKER; 660 } 661 String fullPath = file.getFullXPath(path); 662 663 String example = ""; 664 665 if (SHOW_EXAMPLES) { 666 example = ExampleGenerator.simplify(exampleGenerator.getExampleHtml(path, value)); 667 showExamples(checkCldr, prettyPath, localeID, path, value, fullPath, example); 668 } 669 670 if (checkFlexibleDates) { 671 fset.checkFlexibles(path, value, fullPath); 672 } 673 674 if (path.contains("duration-century")) { 675 int debug = 0; 676 } 677 678 int limit = 1; 679 for (int jj = 0; jj < limit; ++jj) { 680 if (jj == 0) { 681 checkCldr.check(path, fullPath, value, new Options(options), result); 682 } else { 683 checkCldr.getExamples(path, fullPath, value, new Options(options), result); 684 } 685 686 boolean showedOne = false; 687 for (Iterator<CheckStatus> it3 = result.iterator(); it3.hasNext();) { 688 CheckStatus status = it3.next(); 689 String statusString = status.toString(); // com.ibm.icu.impl.Utility.escape( 690 CheckStatus.Type statusType = status.getType(); 691 Object[] parameters = status.getParameters(); 692 693 if (parameters != null) { 694 if (parameters.length >= 1 && status.getCause().getClass() == CheckForExemplars.class) { 695 try { 696 UnicodeSet set = new UnicodeSet(parameters[0].toString()); 697 if (status.getMessage().contains("currency")) { 698 missingCurrencyExemplars.addAll(set); 699 } else { 700 missingExemplars.addAll(set); 701 } 702 } catch (RuntimeException e) { 703 } // skip if not parseable as set 704 } 705 } 706 707 if (errorsOnly && !statusType.equals(CheckStatus.errorType)) { 708 continue; 709 } 710 711 if (subtypeFilter != null) { 712 if (!subtypeFilter.contains(status.getSubtype())) { 713 continue; 714 } 715 } 716 if (checkOnSubmit) { 717 if (!status.isCheckOnSubmit() || !statusType.equals(CheckStatus.errorType)) continue; 718 } 719 720 // System.out.print("Locale:\t" + getLocaleAndName(localeID) + "\t"); 721 if (statusType.equals(CheckStatus.demoType)) { 722 SimpleDemo d = status.getDemo(); 723 if (d != null && d instanceof FormatDemo) { 724 FormatDemo fd = (FormatDemo) d; 725 m.clear(); 726 // m.put("pattern", fd.getPattern()); 727 // m.put("input", fd.getRandomInput()); 728 if (d.processPost(m)) System.out.println("\tDemo:\t" + fd.getPlainText(m)); 729 } 730 continue; 731 } 732 733 if (parameters != null) { 734 for (int i = 0; i < parameters.length; ++i) { 735 if (showStackTrace && parameters[i] instanceof Throwable) { 736 ((Throwable) parameters[i]).printStackTrace(); 737 } 738 } 739 } 740 741 showValue(file, prettyPath, localeID, example, path, value, fullPath, statusString, 742 status.getSubtype()); 743 showedOne = true; 744 745 // survey tool will use: if (status.hasHTMLMessage()) 746 // System.out.println(status.getHTMLMessage()); 747 } 748 if (!showedOne && phase != Phase.FINAL_TESTING) { 749 if (!showedOne && showAll) { 750 showValue(file, prettyPath, localeID, example, path, value, fullPath, "ok", Subtype.none); 751 showedOne = true; 752 } 753 } 754 755 } 756 } 757 758 if (resolveVotesDirectory != null) { 759 LocaleVotingData.resolveErrors(localeID); 760 } 761 762 showSummary(localeID, level, "Items (including inherited):\t" + pathCount); 763 if (missingExemplars.size() != 0) { 764 missingExemplars.removeAll(new UnicodeSet("[[:Uppercase:]-[İ]]")); // remove uppercase #4670 765 if (missingExemplars.size() != 0) { 766 Collator col = Collator.getInstance(new ULocale(localeID)); 767 showSummary(localeID, level, "Total missing from general exemplars:\t" + 768 missingExemplars.size() 769 + "\t" + new UnicodeSetPrettyPrinter() 770 .setOrdering(col != null ? col : Collator.getInstance(ULocale.ROOT)) 771 .setSpaceComparator(col != null ? col : Collator.getInstance(ULocale.ROOT) 772 .setStrength2(Collator.PRIMARY)) 773 .setCompressRanges(false) 774 .format(missingExemplars)); 775 } 776 } 777 if (missingCurrencyExemplars.size() != 0) { 778 Collator col = Collator.getInstance(new ULocale(localeID)); 779 showSummary(localeID, level, "Total missing from currency exemplars:\t" 780 + new UnicodeSetPrettyPrinter() 781 .setOrdering(col != null ? col : Collator.getInstance(ULocale.ROOT)) 782 .setSpaceComparator(col != null ? col : Collator.getInstance(ULocale.ROOT) 783 .setStrength2(Collator.PRIMARY)) 784 .setCompressRanges(true) 785 .format(missingCurrencyExemplars)); 786 } 787 for (ErrorType type : subtotalCount.keySet()) { 788 showSummary(localeID, level, "Subtotal " + type + ":\t" + subtotalCount.getCount(type)); 789 } 790 if (checkFlexibleDates) { 791 fset.showFlexibles(); 792 } 793 if (SHOW_EXAMPLES) { 794 // ldml/dates/timeZoneNames/zone[@type="America/Argentina/San_Juan"]/exemplarCity 795 for (String zone : StandardCodes.make().getGoodAvailableCodes("tzid")) { 796 String path = "//ldml/dates/timeZoneNames/zone[@type=\"" + zone + "\"]/exemplarCity"; 797 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 798 String prettyPath = pathHeader.toString().replace('\t', '|').replace(' ', '_'); 799 if (pathFilter != null && !pathFilter.reset(path).matches()) { 800 continue; 801 } 802 String fullPath = file.getStringValue(path); 803 if (fullPath != null) { 804 continue; 805 } 806 /* 807 * TODO: fix this code. Calling getExampleHtml with value = null will always return null, 808 * so what's this supposed to accomplish? 809 */ 810 String example = ExampleGenerator.simplify(exampleGenerator.getExampleHtml(path, null /* value */)); 811 showExamples(checkCldr, prettyPath, localeID, path, null, fullPath, example); 812 } 813 } 814 System.out.println("# Elapsed time: " + timer); 815 System.out.flush(); 816 } 817 818 if (ErrorFile.errorFileWriter != null) { 819 ErrorFile.closeErrorFile(); 820 } 821 822 if (ErrorFile.generated_html_directory != null) { 823 ErrorFile.writeErrorCountsText(); 824 ErrorFile.writeErrorFileIndex(); 825 } 826 System.out.println(); 827 for (ErrorType type : totalCount.keySet()) { 828 System.out.println("# Total " + type + ":\t" + totalCount.getCount(type)); 829 } 830 831 System.out.println(); 832 System.out.println("# Total elapsed time: " + totalTimer); 833 if (fatalErrors.size() != 0) { 834 System.out.println("# FATAL ERRORS:"); 835 } 836 long errorCount = totalCount.getCount(ErrorType.error) + fatalErrors.size(); 837 if (errorCount != 0) { 838 // System.exit((int) errorCount); // cast is safe; we'll never have that many errors 839 System.out.println(); 840 System.out.println("<< FAILURE - Error count is " + errorCount + " . >>"); 841 System.exit(-1); 842 } else { 843 System.out.println(); 844 System.out.println("<< SUCCESS - No errors found. >>"); 845 } 846 if (LogicalGrouping.GET_TYPE_COUNTS) { 847 for (String s : LogicalGrouping.typeCount.keySet()) { 848 System.out.println(s + "=" + LogicalGrouping.typeCount.get(s)); 849 } 850 } 851 checkCldr.handleFinish(); 852 } 853 854 static class LocaleVotingData { 855 private int disputedCount = 0; 856 Counter<Organization> missingOrganizationCounter = new Counter<>(true); 857 Counter<Organization> goodOrganizationCounter = new Counter<>(true); 858 Counter<Organization> conflictedOrganizations = new Counter<>(true); 859 Counter<VoteResolver.Status> winningStatusCounter = new Counter<>(true); 860 861 static Map<String, LocaleVotingData> localeToErrors = new HashMap<>(); 862 private static Map<Integer, String> idToPath; 863 resolveErrors(String locale)864 public static void resolveErrors(String locale) { 865 localeToErrors.put(locale, new LocaleVotingData(locale)); 866 } 867 LocaleVotingData(String locale)868 public LocaleVotingData(String locale) { 869 870 Map<Organization, VoteResolver.Level> orgToMaxVote = VoteResolver.getOrganizationToMaxVote(locale); 871 872 Map<Integer, Map<Integer, CandidateInfo>> info = VoteResolver 873 .getBaseToAlternateToInfo(resolveVotesDirectory + locale + ".xml"); 874 875 Map<String, Integer> valueToItem = new HashMap<>(); 876 877 for (int basePath : info.keySet()) { 878 final Map<Integer, CandidateInfo> itemInfo = info.get(basePath); 879 880 // find the last release status and value 881 voteResolver.clear(); 882 valueToItem.clear(); 883 884 for (int item : itemInfo.keySet()) { 885 String itemValue = getValue(item); 886 valueToItem.put(itemValue, item); 887 888 CandidateInfo candidateInfo = itemInfo.get(item); 889 if (candidateInfo.oldStatus != null) { 890 voteResolver.setBaseline(itemValue, candidateInfo.oldStatus); 891 } 892 voteResolver.add(itemValue); 893 for (int voter : candidateInfo.voters) { 894 try { 895 voteResolver.add(itemValue, voter); 896 } catch (UnknownVoterException e) { 897 // skip 898 } 899 } 900 } 901 902 EnumSet<Organization> basePathConflictedOrganizations = voteResolver.getConflictedOrganizations(); 903 conflictedOrganizations.addAll(basePathConflictedOrganizations, 1); 904 905 VoteResolver.Status winningStatus = voteResolver.getWinningStatus(); 906 String winningValue = voteResolver.getWinningValue(); 907 908 winningStatusCounter.add(winningStatus, 1); 909 910 if (winningStatus == VoteResolver.Status.approved) { 911 continue; 912 } 913 914 CandidateInfo candidateInfo = itemInfo.get(valueToItem.get(winningValue)); 915 Map<Organization, VoteResolver.Level> orgToMaxVoteHere = VoteResolver 916 .getOrganizationToMaxVote(candidateInfo.voters); 917 918 // if the winning item is less than contributed, record the organizations that haven't given their 919 // maximum vote to the winning item. 920 if (winningStatus.compareTo(VoteResolver.Status.contributed) < 0) { 921 // showPaths(basePath, itemInfo); 922 for (Organization org : orgToMaxVote.keySet()) { 923 VoteResolver.Level maxVote = orgToMaxVote.get(org); 924 VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org); 925 if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) { 926 missingOrganizationCounter.add(org, 1); 927 } 928 } 929 if (voteResolver.isDisputed()) { 930 disputedCount++; 931 String path = getIdToPath(basePath); 932 ErrorFile.addDataToErrorFile(locale, path, ErrorType.disputed, Subtype.none); 933 } 934 } else { 935 for (Organization org : orgToMaxVote.keySet()) { 936 VoteResolver.Level maxVote = orgToMaxVote.get(org); 937 VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org); 938 if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) { 939 } else { 940 goodOrganizationCounter.add(org, 1); 941 } 942 } 943 } 944 } 945 System.out.println(getLocaleAndName(locale) + "\tEnabled Organizations:\t" + orgToMaxVote); 946 if (disputedCount != 0) { 947 System.out.println(getLocaleAndName(locale) + "\tDisputed Items:\t" + disputedCount); 948 } 949 950 if (missingOrganizationCounter.size() > 0) { 951 System.out.println(getLocaleAndName(locale) + "\tMIA organizations:\t" + missingOrganizationCounter); 952 System.out 953 .println(getLocaleAndName(locale) + "\tConflicted organizations:\t" + conflictedOrganizations); 954 System.out.println(getLocaleAndName(locale) + "\tCool organizations!:\t" + goodOrganizationCounter); 955 } 956 System.out.println(getLocaleAndName(locale) + "\tOptimal Status:\t" + winningStatusCounter); 957 } 958 getIdToPath(int basePath)959 private static String getIdToPath(int basePath) { 960 if (idToPath == null) { 961 idToPath = VoteResolver.getIdToPath(resolveVotesDirectory + "xpathTable.xml"); 962 } 963 return idToPath.get(basePath); 964 } 965 get(String locale)966 public static LocaleVotingData get(String locale) { 967 return localeToErrors.get(locale); 968 } 969 getDisputedCount()970 int getDisputedCount() { 971 return disputedCount; 972 } 973 getConflictedHTML()974 String getConflictedHTML() { 975 String result = conflictedOrganizations.toString(); 976 if (result.length() == 0) { 977 return ""; 978 } 979 result = result.substring(1, result.length() - 1); 980 result = result.replace(", ", "<br>"); 981 return result; 982 } 983 } 984 getValue(int item)985 private static String getValue(int item) { 986 return String.valueOf(item); 987 } 988 989 static Matcher draftStatusMatcher = PatternCache.get("\\[@draft=\"(provisional|unconfirmed)\"]").matcher(""); 990 991 enum ErrorType { 992 ok, error, disputed, warning, core, posix, minimal, basic, moderate, modern, comprehensive, optional, contributed, provisional, unconfirmed, unknown; 993 static EnumSet<ErrorType> unapproved = EnumSet.range(ErrorType.contributed, ErrorType.unconfirmed); 994 static EnumSet<ErrorType> coverage = EnumSet.range(ErrorType.posix, ErrorType.optional); 995 static EnumSet<ErrorType> showInSummary = EnumSet.of( 996 ErrorType.error, ErrorType.warning, ErrorType.posix, ErrorType.minimal, ErrorType.basic); 997 fromStatusString(String statusString)998 static ErrorType fromStatusString(String statusString) { 999 ErrorType shortStatus = statusString.equals("ok") ? ErrorType.ok 1000 : statusString.startsWith("Error") ? ErrorType.error 1001 : statusString.equals("disputed") ? ErrorType.disputed 1002 : statusString.startsWith("Warning") ? ErrorType.warning 1003 : statusString.equals("contributed") ? ErrorType.contributed 1004 : statusString.equals("provisional") ? ErrorType.provisional 1005 : statusString.equals("unconfirmed") ? ErrorType.unconfirmed 1006 : ErrorType.unknown; 1007 if (shortStatus == ErrorType.unknown) { 1008 throw new IllegalArgumentException("Unknown error type: " + statusString); 1009 } else if (shortStatus == ErrorType.warning) { 1010 if (coverageMatcher.reset(statusString).find()) { 1011 shortStatus = ErrorType.valueOf(coverageMatcher.group(1)); 1012 } 1013 } 1014 return shortStatus; 1015 } 1016 } 1017 1018 /* 1019 * static class ErrorCount implements Comparable<ErrorCount> { 1020 * private Counter<ErrorType> counter = new Counter<ErrorType>(); 1021 * 1022 * public int compareTo(ErrorCount o) { 1023 * // we don't really need a good comparison - aren't going to be sorting 1024 * return total() < o.total() ? -1 : total() > o.total() ? 1 : 0; 1025 * } 1026 * public long total() { 1027 * return counter.getTotal(); 1028 * } 1029 * public void clear() { 1030 * counter.clear(); 1031 * } 1032 * public Set<ErrorType> keySet() { 1033 * return counter.getKeysetSortedByKey(); 1034 * } 1035 * public long getCount(ErrorType input) { 1036 * return counter.getCount(input); 1037 * } 1038 * public void increment(ErrorType errorType) { 1039 * counter.add(errorType, 1); 1040 * } 1041 * } 1042 */ 1043 1044 static class ErrorFile { 1045 1046 private static final boolean SHOW_VOTING_INFO = false; 1047 public static CLDRFile voteFile; 1048 public static Factory voteFactory; 1049 openErrorFile(String localeID, String baseLanguage)1050 private static void openErrorFile(String localeID, String baseLanguage) throws IOException { 1051 if (ErrorFile.errorFileWriter != null) { 1052 ErrorFile.closeErrorFile(); 1053 } 1054 ErrorFile.errorFileWriter = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, baseLanguage + ".html"); 1055 ErrorFile.errorFileTable = new TablePrinter(); 1056 errorFileCounter.clear(); 1057 ErrorFile.errorFileTable.setCaption("Problem Details") 1058 .addColumn("Problem").setCellAttributes("align=\"left\" class=\"{0}\"").setSortPriority(0) 1059 .setSpanRows(true) 1060 .setBreakSpans(true).setRepeatHeader(true).setHeaderCell(true) 1061 .addColumn("Subtype").setCellAttributes("align=\"left\" class=\"{1}\"").setSortPriority(1) 1062 .setSpanRows(true) 1063 .setBreakSpans(true).setRepeatHeader(true).setHeaderCell(true) 1064 .addColumn("Locale").setCellAttributes("class=\"{1}\"") 1065 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={0}\">{0}</a>").setSortPriority(2) 1066 .setSpanRows(true).setBreakSpans(true)//.setRepeatDivider(true) 1067 .addColumn("Name").setCellAttributes("class=\"{1}\"").setSpanRows(true) 1068 .setBreakSpans(true) 1069 .addColumn("Section").setCellAttributes("class=\"{1}\"").setSortPriority(3) 1070 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={3}&x={0}\">{0}</a>") 1071 .setSpanRows(true) 1072 .addColumn("Count").setCellAttributes("class=\"{1}\" align=\"right\""); 1073 1074 showIndexHead("", localeID, ErrorFile.errorFileWriter); 1075 } 1076 1077 static TablePrinter errorFileTable = new TablePrinter(); 1078 static Counter<Row.R4<String, String, ErrorType, Subtype>> errorFileCounter = new Counter<>( 1079 true); 1080 addDataToErrorFile(String localeID, String path, ErrorType shortStatus, Subtype subType)1081 private static void addDataToErrorFile(String localeID, String path, ErrorType shortStatus, 1082 Subtype subType) { 1083 String section = path == null ? null : XPathToMenu.xpathToMenu(path); 1084 if (section == null) { 1085 section = "general"; 1086 } 1087 errorFileCounter.add( 1088 new Row.R4<>(localeID, section, shortStatus, subType), 1); 1089 ErrorFile.sectionToProblemsToLocaleToCount.add( 1090 new Row.R4<>(section, shortStatus, subType, localeID), 1); 1091 } 1092 closeErrorFile()1093 private static void closeErrorFile() { 1094 Set<String> locales = new TreeSet<>(); 1095 for (Row.R4<String, String, ErrorType, Subtype> item : errorFileCounter.keySet()) { 1096 String localeID = item.get0(); 1097 locales.add(localeID); 1098 String section = item.get1(); 1099 ErrorType shortStatus = item.get2(); 1100 Subtype subtype = item.get3(); 1101 // final String prettyPath = path == null ? "general" : prettyPathMaker.getPrettyPath(path, true); 1102 // final String outputForm = path == null ? "general" : prettyPathMaker.getOutputForm(prettyPath); 1103 errorFileTable.addRow() 1104 .addCell(shortStatus) 1105 .addCell(subtype) 1106 .addCell(localeID) 1107 .addCell(ConsoleCheckCLDR.getLocaleName(localeID)) 1108 // .addCell(prettyPath) // menuPath == null ? "" : "<a href='" + link + "'>" + menuPath + "</a>" 1109 .addCell(section) // menuPath == null ? "" : "<a href='" + link + "'>" + menuPath + "</a>" 1110 .addCell(errorFileCounter.getCount(item)) 1111 // .addCell(ConsoleCheckCLDR.safeForHtml(path == null ? null : 1112 // ConsoleCheckCLDR.getEnglishPathValue(path))) 1113 // .addCell(ConsoleCheckCLDR.safeForHtml(value)) 1114 .finishRow(); 1115 } 1116 1117 if (SHOW_VOTING_INFO) { 1118 TablePrinter data = new TablePrinter().setCaption("Voting Information") 1119 .addColumn("Locale").setHeaderCell(true) 1120 .addColumn("Name").setHeaderCell(true) 1121 .addColumn("Organization") 1122 .addColumn("Missing") 1123 .addColumn("Conflicted") 1124 // .addColumn("Good") 1125 ; 1126 for (String localeID : locales) { 1127 // now the voting info 1128 LocaleVotingData localeVotingData = LocaleVotingData.localeToErrors.get(localeID); 1129 if (localeVotingData != null) { 1130 // find all the orgs with data 1131 EnumSet<Organization> orgs = EnumSet.noneOf(Organization.class); 1132 orgs.addAll(localeVotingData.missingOrganizationCounter.keySet()); 1133 orgs.addAll(localeVotingData.conflictedOrganizations.keySet()); 1134 orgs.addAll(localeVotingData.goodOrganizationCounter.keySet()); 1135 for (Organization org : orgs) { 1136 data.addRow() 1137 .addCell(ConsoleCheckCLDR.getLinkedLocale(localeID)) 1138 .addCell(ConsoleCheckCLDR.getLocaleName(localeID)) 1139 .addCell(org) 1140 .addCell(localeVotingData.missingOrganizationCounter.getCount(org)) 1141 .addCell(localeVotingData.conflictedOrganizations.getCount(org)) 1142 // .addCell(localeVotingData.goodOrganizationCounter.getCount(org)) 1143 .finishRow(); 1144 } 1145 } 1146 } 1147 ErrorFile.errorFileWriter.println(data.toTable()); 1148 ErrorFile.errorFileWriter.println("<p></p>"); 1149 } 1150 1151 // generated_html.println("<table border='1' style='border-collapse: collapse' bordercolor='#CCCCFF'>"); 1152 // Locale Group Error Warning Missing Votes: Contributed Missing Votes: Provisional Missing Votes: 1153 // Unconfirmed Missing Coverage: Posix Missing Coverage: Minimal Missing Coverage: Basic Missing Coverage: 1154 // Moderate Missing Coverage: Modern 1155 ErrorFile.errorFileWriter.println(ErrorFile.errorFileTable.toTable()); 1156 ErrorFile.errorFileWriter.println(ShowData.dateFooter()); 1157 ErrorFile.errorFileWriter.println(CldrUtility.ANALYTICS); 1158 ErrorFile.errorFileWriter.println("</body></html>"); 1159 ErrorFile.errorFileWriter.close(); 1160 ErrorFile.errorFileTable = null; 1161 } 1162 1163 // ================ Index File =================== 1164 showErrorFileIndex(PrintWriter generated_html_index)1165 static void showErrorFileIndex(PrintWriter generated_html_index) { 1166 1167 // get organizations 1168 Relation<Organization, String> orgToLocales = getOrgToLocales(); 1169 1170 TablePrinter indexTablePrinter = new TablePrinter().setCaption("Problem Summary") 1171 .setTableAttributes("border='1' style='border-collapse: collapse' bordercolor='blue'") 1172 .addColumn("BASE").setHidden(true)//.setRepeatDivider(true) 1173 .addColumn("Locale").setCellPattern("<a name=\"{0}\" href=\"{1}.html\">{0}</a>") // link to base, anchor 1174 // with full 1175 .addColumn("Name"); 1176 if (SHOW_VOTING_INFO) { 1177 indexTablePrinter.addColumn("Summary") 1178 .addColumn("Missing"); 1179 } 1180 for (Organization org : orgToLocales.keySet()) { 1181 indexTablePrinter.addColumn(org.toString().substring(0, 2)); 1182 } 1183 indexTablePrinter 1184 .addColumn("Disputed").setHeaderAttributes("class='disputed'").setCellAttributes("class='disputed'") 1185 .addColumn("Conflicted").setHeaderAttributes("class='conflicted'") 1186 .setCellAttributes("class='conflicted'"); 1187 1188 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1189 String columnTitle = UCharacter.toTitleCase(type.toString(), null); 1190 final boolean coverage = ConsoleCheckCLDR.ErrorType.coverage.contains(type); 1191 if (coverage) { 1192 columnTitle = "MC: " + columnTitle; 1193 } else if (ConsoleCheckCLDR.ErrorType.unapproved.contains(type)) { 1194 columnTitle = "MV: " + columnTitle; 1195 } 1196 indexTablePrinter.addColumn(columnTitle).setHeaderAttributes("class='" + type + "'") 1197 .setCellAttributes("class='" + type + "'"); 1198 } 1199 1200 // now fill in the data 1201 LanguageTagParser ltp = new LanguageTagParser(); 1202 for (String key : ErrorFile.errorFileIndexData.keySet()) { 1203 Pair<String, Counter<ErrorType>> pair = ErrorFile.errorFileIndexData.get(key); 1204 String htmlOpenedFileLanguage = pair.getFirst(); 1205 Counter<ErrorType> counts = pair.getSecond(); 1206 LocaleVotingData votingData = LocaleVotingData.get(htmlOpenedFileLanguage); 1207 if (counts.getTotal() == 0) { 1208 continue; 1209 } 1210 final String baseLanguage = ltp.set(htmlOpenedFileLanguage).getLanguage(); 1211 indexTablePrinter.addRow() 1212 .addCell(baseLanguage) 1213 .addCell(htmlOpenedFileLanguage) 1214 .addCell(ConsoleCheckCLDR.getLocaleName(htmlOpenedFileLanguage)); 1215 if (SHOW_VOTING_INFO) { 1216 indexTablePrinter.addCell(votingData == null ? "" : votingData.winningStatusCounter.toString()) 1217 .addCell(votingData == null ? "" : votingData.missingOrganizationCounter.toString()); 1218 } 1219 for (Organization org : orgToLocales.keySet()) { 1220 indexTablePrinter.addCell(orgToLocales.getAll(org).contains(htmlOpenedFileLanguage) ? org.toString() 1221 .substring(0, 2) : ""); 1222 } 1223 indexTablePrinter 1224 .addCell(votingData == null ? "" : formatSkippingZero(votingData.getDisputedCount())) 1225 .addCell(votingData == null ? "" : votingData.getConflictedHTML()); 1226 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1227 indexTablePrinter.addCell(formatSkippingZero(counts.getCount(type))); 1228 } 1229 indexTablePrinter.finishRow(); 1230 } 1231 generated_html_index.println(indexTablePrinter.toTable()); 1232 generated_html_index.println(ShowData.dateFooter()); 1233 generated_html_index.println(CldrUtility.ANALYTICS); 1234 generated_html_index.println("</body></html>"); 1235 } 1236 1237 static Relation<Organization, String> orgToLocales; 1238 getOrgToLocales()1239 private static Relation<Organization, String> getOrgToLocales() { 1240 if (orgToLocales == null) { 1241 orgToLocales = Relation.of(new TreeMap<Organization, Set<String>>(), TreeSet.class); 1242 StandardCodes sc = StandardCodes.make(); 1243 for (Organization org : sc.getLocaleCoverageOrganizations()) { 1244 for (String locale : sc.getLocaleCoverageLocales(org)) { 1245 Level x = sc.getLocaleCoverageLevel(org, locale); 1246 if (x.compareTo(Level.BASIC) > 0) { 1247 orgToLocales.put(org, locale); 1248 } 1249 } 1250 } 1251 } 1252 return orgToLocales; 1253 } 1254 showSections()1255 static void showSections() throws IOException { 1256 Relation<Organization, String> orgToLocales = getOrgToLocales(); 1257 TablePrinter indexTablePrinter = new TablePrinter().setCaption("Problem Summary") 1258 .setTableAttributes("border='1' style='border-collapse: collapse' bordercolor='blue'") 1259 .addColumn("Section").setSpanRows(true).setBreakSpans(true)//.setRepeatDivider(true) 1260 .addColumn("Problems").setCellAttributes("style=\"text-align:left\" class=\"{2}\"").setSpanRows(true) 1261 .addColumn("Subtype").setCellAttributes("style=\"text-align:left\" class=\"{2}\"").setSpanRows(true) 1262 .addColumn("Locale").setCellAttributes("class=\"{2}\"") 1263 .addColumn("Code").setCellAttributes("class=\"{2}\"") 1264 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={0}&x={1}\">{0}</a>") // TODO: use CLDRConfig.urls() 1265 .addColumn("Count").setCellAttributes("class=\"{2}\""); 1266 for (Organization org : orgToLocales.keySet()) { 1267 indexTablePrinter.addColumn(org.toString().substring(0, 2)); 1268 } 1269 1270 for (Row.R4<String, ErrorType, Subtype, String> sectionAndProblemsAndLocale : ErrorFile.sectionToProblemsToLocaleToCount 1271 .getKeysetSortedByKey()) { 1272 final ErrorType problem = sectionAndProblemsAndLocale.get1(); 1273 final Subtype subtype = sectionAndProblemsAndLocale.get2(); 1274 if (!ConsoleCheckCLDR.ErrorType.showInSummary.contains(problem)) { 1275 continue; 1276 } 1277 final String locale = sectionAndProblemsAndLocale.get3(); 1278 if (problem != ErrorType.error && problem != ErrorType.disputed && !orgToLocales.containsValue(locale)) { 1279 continue; 1280 } 1281 long count = ErrorFile.sectionToProblemsToLocaleToCount.getCount(sectionAndProblemsAndLocale); 1282 final String section = sectionAndProblemsAndLocale.get0(); 1283 indexTablePrinter.addRow() 1284 .addCell(section) 1285 .addCell(problem) 1286 .addCell(subtype) 1287 .addCell(ConsoleCheckCLDR.getLocaleName(locale)) 1288 .addCell(locale) 1289 .addCell(count); 1290 for (Organization org : orgToLocales.keySet()) { 1291 indexTablePrinter.addCell(orgToLocales.getAll(org).contains(locale) ? org.toString().substring(0, 2) : ""); 1292 } 1293 indexTablePrinter.finishRow(); 1294 } 1295 PrintWriter generated_html_index = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "sections.html"); 1296 ConsoleCheckCLDR.ErrorFile.showIndexHead("Error Report Index by Section", "", generated_html_index); 1297 generated_html_index.println(indexTablePrinter.toTable()); 1298 generated_html_index.println(ShowData.dateFooter()); 1299 generated_html_index.println(CldrUtility.ANALYTICS); 1300 generated_html_index.println("</body></html>"); 1301 generated_html_index.close(); 1302 } 1303 formatSkippingZero(long count)1304 static String formatSkippingZero(long count) { 1305 if (count == 0) { 1306 return ""; 1307 } 1308 return String.valueOf(count); 1309 } 1310 showIndexHead(String title, String localeID, PrintWriter generated_html_index)1311 static void showIndexHead(String title, String localeID, PrintWriter generated_html_index) { 1312 final boolean notLocaleSpecific = localeID.length() == 0; 1313 if ((!notLocaleSpecific)) { 1314 title = "Errors in " + ConsoleCheckCLDR.getNameAndLocale(localeID, false); 1315 } 1316 generated_html_index 1317 .println("<html>" + 1318 "<head><meta http-equiv='Content-Type' content='text/html; charset=utf-8'>" 1319 + CldrUtility.LINE_SEPARATOR 1320 + 1321 "<title>" 1322 + title 1323 + "</title>" 1324 + CldrUtility.LINE_SEPARATOR 1325 + 1326 "<link rel='stylesheet' href='errors.css' type='text/css'>" 1327 + CldrUtility.LINE_SEPARATOR 1328 + 1329 "<base target='_blank'>" 1330 + CldrUtility.LINE_SEPARATOR 1331 + 1332 "</head><body>" 1333 + CldrUtility.LINE_SEPARATOR 1334 + 1335 "<h1>" 1336 + title 1337 + "</h1>" 1338 + CldrUtility.LINE_SEPARATOR 1339 + 1340 "<p>" 1341 + 1342 "<a href='index.html" 1343 + (notLocaleSpecific ? "" : "#" + localeID) 1344 + "'>Index</a>" 1345 + 1346 " | " 1347 + 1348 "<a href='sections.html" 1349 + (notLocaleSpecific ? "" : "#" + localeID) 1350 + "'>Index by Section</a>" 1351 + 1352 " | " 1353 + 1354 "<a href='http://unicode.org/cldr/data/docs/survey/vetting.html'><b style='background-color: yellow;'><i>Help: How to Vet</i></b></a>" 1355 + 1356 "</p>" 1357 + 1358 "<p>The following errors have been detected in the locale" 1359 + 1360 (notLocaleSpecific 1361 ? "s. " + org.unicode.cldr.test.HelpMessages.getChartMessages("error_index_header") 1362 : " " + ConsoleCheckCLDR.getNameAndLocale(localeID, false) + ". " 1363 + ErrorFile.ERROR_CHART_HEADER)); 1364 } 1365 writeErrorFileIndex()1366 private static void writeErrorFileIndex() throws IOException { 1367 PrintWriter generated_html_index = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "index.html"); 1368 ConsoleCheckCLDR.ErrorFile.showIndexHead("Error Report Index", "", generated_html_index); 1369 ConsoleCheckCLDR.ErrorFile.showErrorFileIndex(generated_html_index); 1370 generated_html_index.close(); 1371 showSections(); 1372 } 1373 writeErrorCountsText()1374 private static void writeErrorCountsText() { 1375 // if (ErrorFile.htmlErrorsPerLocale.total() != 0) { 1376 1377 // do the plain text file 1378 ErrorFile.generated_html_count.print(ConsoleCheckCLDR.lastHtmlLocaleID + ";\tcounts"); 1379 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1380 ErrorFile.generated_html_count.print(";\t" + type + "=" + ErrorFile.htmlErrorsPerLocale.getCount(type)); 1381 } 1382 ErrorFile.generated_html_count.println(); 1383 ErrorFile.generated_html_count.flush(); 1384 1385 // now store the data for the index 1386 ErrorFile.errorFileIndexData.put(ConsoleCheckCLDR.lastHtmlLocaleID, 1387 new Pair<>(ConsoleCheckCLDR.lastHtmlLocaleID, ErrorFile.htmlErrorsPerLocale)); 1388 ErrorFile.htmlErrorsPerLocale = new Counter<>(); 1389 // } 1390 } 1391 1392 /* 1393 * static Counter<Organization> missingOrganizationCounter = new Counter<Organization>(true); 1394 * static Counter<Organization> goodOrganizationCounter = new Counter<Organization>(true); 1395 * static Counter<Organization> conflictedOrganizations = new Counter<Organization>(true); 1396 * static Counter<VoteResolver.Status> winningStatusCounter = new Counter<VoteResolver.Status>(true); 1397 */ 1398 1399 static Counter<ErrorType> htmlErrorsPerLocale = new Counter<>(); // ConsoleCheckCLDR.ErrorCount(); 1400 static PrintWriter generated_html_count = null; 1401 private static TreeMap<String, Pair<String, Counter<ErrorType>>> errorFileIndexData = new TreeMap<>(); 1402 1403 // private static ConsoleCheckCLDR.ErrorCount htmlErrorsPerBaseLanguage = new ConsoleCheckCLDR.ErrorCount(); 1404 static PrintWriter errorFileWriter = null; 1405 private static final String ERROR_CHART_HEADER = org.unicode.cldr.test.HelpMessages 1406 .getChartMessages("error_locale_header"); 1407 // "Please review and correct them. " + 1408 // "Note that errors in <i>sublocales</i> are often fixed by fixing the main locale.</p>" + 1409 // Utility.LINE_SEPARATOR + 1410 // "<p><i>This list is only generated daily, and so may not reflect fixes you have made until tomorrow. " + 1411 // "(There were production problems in integrating it fully into the Survey tool. " + 1412 // "However, it should let you see the problems and make sure that they get taken care of.)</i></p>" + 1413 // "<p>Coverage depends on your organizations goals: the highest tier languages should include up through all Modern values.</p>" 1414 // + Utility.LINE_SEPARATOR; 1415 static String generated_html_directory = null; 1416 public static Counter<Row.R4<String, ErrorType, Subtype, String>> sectionToProblemsToLocaleToCount = new Counter<>(); 1417 } 1418 showSummary(String localeID, Level level, String value)1419 private static void showSummary(String localeID, Level level, String value) { 1420 String line = "# " + getLocaleAndName(localeID) + "\tSummary\t" + level + "\t" + value; 1421 System.out.println(line); 1422 } 1423 showExamples(CheckCLDR checkCldr, String prettyPath, String localeID, String path, String value, String fullPath, String example)1424 private static void showExamples(CheckCLDR checkCldr, String prettyPath, String localeID, 1425 String path, String value, String fullPath, String example) { 1426 if (example != null) { 1427 showValue(checkCldr.getCldrFileToCheck(), prettyPath, localeID, example, path, value, fullPath, "ok", 1428 Subtype.none); 1429 } 1430 } 1431 addPrettyPaths(CLDRFile file, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target)1432 private static void addPrettyPaths(CLDRFile file, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, 1433 boolean noaliases, boolean filterDraft, Collection<String> target) { 1434 for (Iterator<String> pit = file.iterator(pathFilter); pit.hasNext();) { 1435 String path = pit.next(); 1436 addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path); 1437 } 1438 } 1439 addPrettyPaths(CLDRFile file, Collection<String> paths, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target)1440 private static void addPrettyPaths(CLDRFile file, Collection<String> paths, Matcher pathFilter, 1441 PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target) { 1442 // Status pathStatus = new Status(); 1443 for (String path : paths) { 1444 if (pathFilter != null && !pathFilter.reset(path).matches()) continue; 1445 addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path); 1446 } 1447 } 1448 addPrettyPath(CLDRFile file, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target, String path)1449 private static void addPrettyPath(CLDRFile file, PathHeader.Factory pathHeaderFactory, boolean noaliases, 1450 boolean filterDraft, Collection<String> target, String path) { 1451 if (noaliases && XMLSource.Alias.isAliasPath(path)) { // this is just for console testing, the survey tool 1452 // shouldn't do it. 1453 return; 1454 // file.getSourceLocaleID(path, pathStatus); 1455 // if (!path.equals(pathStatus.pathWhereFound)) { 1456 // continue; 1457 // } 1458 } 1459 if (filterDraft) { 1460 String newPath = CLDRFile.getNondraftNonaltXPath(path); 1461 if (!newPath.equals(path)) { 1462 String value = file.getStringValue(newPath); 1463 if (value != null) { 1464 return; 1465 } 1466 } 1467 } 1468 String prettyPath = pathHeaderFactory.fromPath(path).toString(); // prettyPathMaker.getPrettyPath(path, true); 1469 // // get sortable version 1470 target.add(prettyPath); 1471 } 1472 setDisplayInformation(CLDRFile inputDisplayInformation, ExampleGenerator inputExampleGenerator)1473 public static synchronized void setDisplayInformation(CLDRFile inputDisplayInformation, 1474 ExampleGenerator inputExampleGenerator) { 1475 CheckCLDR.setDisplayInformation(inputDisplayInformation); 1476 englishExampleGenerator = inputExampleGenerator; 1477 } 1478 setExampleGenerator(ExampleGenerator inputExampleGenerator)1479 public static synchronized void setExampleGenerator(ExampleGenerator inputExampleGenerator) { 1480 englishExampleGenerator = inputExampleGenerator; 1481 } 1482 getExampleGenerator()1483 public static synchronized ExampleGenerator getExampleGenerator() { 1484 return englishExampleGenerator; 1485 } 1486 1487 private static ExampleGenerator englishExampleGenerator; 1488 1489 static Matcher coverageMatcher = PatternCache.get("meet ([a-z]*) coverage").matcher(""); // HACK TODO fix 1490 showHeaderLine()1491 private static void showHeaderLine() { 1492 if (SHOW_LOCALE) { 1493 if (idView) { 1494 System.out 1495 .println("Locale\tID\tDesc.\t〈Eng.Value〉\t【Eng.Ex.】\t〈Loc.Value〉\t【Loc.Ex】\t⁅error/warning type⁆\t❮Error/Warning Msg❯"); 1496 } else { 1497 System.out 1498 .println( 1499 "Locale\tStatus\t▸PPath◂\t〈Eng.Value〉\t【Eng.Ex.】\t〈Loc.Value〉\t«fill-in»\t【Loc.Ex】\t⁅error/warning type⁆\t❮Error/Warning Msg❯\tFull Path\tAliasedSource/Path?"); 1500 } 1501 } 1502 } 1503 1504 private static PathDescription pathDescription = null; 1505 getIdString(String path, String value)1506 private static String getIdString(String path, String value) { 1507 if (pathDescription == null) { 1508 pathDescription = new PathDescription(supplementalDataInfo, english, null, null, 1509 PathDescription.ErrorHandling.CONTINUE); 1510 } 1511 final String description = pathDescription.getDescription(path, value, null, null); 1512 return "\t" + StringId.getId(path) + "" + "\t" + description + ""; 1513 } 1514 showValue(CLDRFile cldrFile, String prettyPath, String localeID, String example, String path, String value, String fullPath, String statusString, Subtype subType)1515 private static void showValue(CLDRFile cldrFile, String prettyPath, String localeID, String example, 1516 String path, String value, String fullPath, String statusString, Subtype subType) { 1517 ErrorType shortStatus = ErrorType.fromStatusString(statusString); 1518 subtotalCount.add(shortStatus, 1); 1519 totalCount.add(shortStatus, 1); 1520 if (subType == null) { 1521 subType = Subtype.none; 1522 } 1523 1524 if (ErrorFile.errorFileWriter == null) { 1525 example = example == null ? "" : example; 1526 String englishExample = null; 1527 final String englishPathValue = path == null ? null : getEnglishPathValue(path); 1528 if (SHOW_EXAMPLES && path != null) { 1529 englishExample = ExampleGenerator.simplify(getExampleGenerator().getExampleHtml(path, englishPathValue)); 1530 } 1531 englishExample = englishExample == null ? "" : englishExample; 1532 String cleanPrettyPath = path == null ? null : prettyPath; // prettyPathMaker.getOutputForm(prettyPath); 1533 Status status = new Status(); 1534 String sourceLocaleID = path == null ? null : cldrFile.getSourceLocaleID(path, status); 1535 String fillinValue = path == null ? null : cldrFile.getFillInValue(path); 1536 fillinValue = fillinValue == null ? "" : fillinValue.equals(value) ? "=" : fillinValue; 1537 1538 String pathLink = CLDR_CONFIG.urls().forXpath(localeID, path); 1539 1540 final String otherSource = path == null ? null 1541 : (sourceLocaleID.equals(localeID) ? "" 1542 : "\t" + sourceLocaleID); 1543 final String otherPath = path == null ? null 1544 : (status.pathWhereFound.equals(path) ? "" 1545 : "\t" + status.pathWhereFound); 1546 1547 String idViewString = idView ? (path == null ? "\tNO_ID" : getIdString(path, value)) : ""; 1548 System.out.println( 1549 getLocaleAndName(localeID) 1550 + (idViewString.isEmpty() ? 1551 // + "\t" + subtotalCount.getCount(shortStatus) 1552 "\t" + shortStatus 1553 + "\t▸" + cleanPrettyPath + "◂" 1554 + "\t〈" + englishPathValue + "〉" 1555 + "\t【" + englishExample + "】" 1556 + "\t〈" + value + "〉" 1557 + "\t«" + fillinValue + "»" 1558 + "\t【" + example + "】" 1559 + "\t⁅" + subType + "⁆" 1560 + "\t❮" + statusString + "❯" 1561 + "\t" + pathLink 1562 + otherSource 1563 + otherPath 1564 : idViewString 1565 + "\t〈" + englishPathValue + "〉" 1566 + "\t【" + englishExample + "】" 1567 + "\t" + value + "〉" 1568 + "\t【" + example + "】" 1569 + "\t⁅" + subType + "⁆" 1570 + "\t❮" + statusString + "❯")); 1571 } else if (ErrorFile.errorFileWriter != null) { 1572 if (shortStatus == ErrorType.contributed) { 1573 return; 1574 } 1575 if (shortStatus == ErrorType.posix) { 1576 shortStatus = ErrorType.minimal; 1577 } 1578 if (!localeID.equals(lastHtmlLocaleID)) { 1579 ErrorFile.writeErrorCountsText(); 1580 // startGeneratedTable(generated_html, generated_html_table); 1581 lastHtmlLocaleID = localeID; 1582 } 1583 addError(shortStatus); 1584 ErrorFile.addDataToErrorFile(localeID, path, shortStatus, subType); 1585 } 1586 if (CLDR_GITHUB_ANNOTATIONS) { 1587 final String filePath = localeXpathToFilePath.computeIfAbsent(Pair.of(localeID, path), locPath -> guessFilePath(locPath)); 1588 // Annotate anything that needs annotation 1589 if (shortStatus == ErrorType.error || shortStatus == ErrorType.warning) { 1590 String cleanPrettyPath = path == null ? "—" : prettyPath; // prettyPathMaker.getOutputForm(prettyPath); 1591 System.out.println("::" + shortStatus + " file=" + filePath + ":: " + localeID + ": " + subType + " @ " + cleanPrettyPath); 1592 } 1593 } 1594 if (PATH_IN_COUNT && ErrorFile.generated_html_count != null) { 1595 ErrorFile.generated_html_count.println(lastHtmlLocaleID + ";\tpath:\t" + path); 1596 } 1597 } 1598 addError(ErrorType shortStatus)1599 private static void addError(ErrorType shortStatus) { 1600 if (ErrorType.showInSummary.contains(shortStatus)) { 1601 ErrorFile.htmlErrorsPerLocale.increment(shortStatus); 1602 } 1603 } 1604 1605 static String lastHtmlLocaleID = ""; 1606 private static VoteResolver<String> voteResolver; 1607 private static String resolveVotesDirectory; 1608 private static boolean idView; 1609 private static SupplementalDataInfo supplementalDataInfo; 1610 private static CLDRFile english; 1611 1612 public static class PathShower { 1613 String localeID; 1614 boolean newLocale = true; 1615 String lastPath; 1616 String[] lastSplitPath; 1617 boolean showEnglish; 1618 String splitChar = "/"; 1619 1620 static final String lead = "****************************************"; 1621 set(String localeID)1622 public void set(String localeID) { 1623 this.localeID = localeID; 1624 newLocale = true; 1625 LocaleIDParser localeIDParser = new LocaleIDParser(); 1626 showEnglish = !localeIDParser.set(localeID).getLanguageScript().equals("en"); 1627 // localeID.equals(CheckCLDR.displayInformation.getLocaleID()); 1628 lastPath = null; 1629 lastSplitPath = null; 1630 } 1631 getSplitChar()1632 public String getSplitChar() { 1633 return splitChar; 1634 } 1635 setSplitChar(String splitChar)1636 public PathShower setSplitChar(String splitChar) { 1637 this.splitChar = splitChar; 1638 return this; 1639 } 1640 } 1641 showValue(String path, String value, boolean showEnglish, String localeID)1642 private static void showValue(String path, String value, boolean showEnglish, String localeID) { 1643 System.out.println("\tValue:\t" + value + (showEnglish ? "\t" + getEnglishPathValue(path) : "") + "\tLocale:\t" 1644 + localeID); 1645 } 1646 getEnglishPathValue(String path)1647 private static String getEnglishPathValue(String path) { 1648 String englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path); 1649 if (englishValue == null) { 1650 String path2 = CLDRFile.getNondraftNonaltXPath(path); 1651 englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path2); 1652 } 1653 return englishValue; 1654 } 1655 1656 /** 1657 * Utility for getting information. 1658 * 1659 * @param locale 1660 * @return 1661 */ getLocaleAndName(String locale)1662 public static String getLocaleAndName(String locale) { 1663 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1664 if (localizedName == null || localizedName.equals(locale)) return locale; 1665 return locale + " [" + localizedName + "]"; 1666 } 1667 1668 /** 1669 * Utility for getting information. 1670 * 1671 * @param locale 1672 * @param linkToXml 1673 * TODO 1674 * @return 1675 */ getNameAndLocale(String locale, boolean linkToXml)1676 public static String getNameAndLocale(String locale, boolean linkToXml) { 1677 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1678 if (localizedName == null || localizedName.equals(locale)) return locale; 1679 if (linkToXml) { 1680 locale = "<a href='http://unicode.org/cldr/data/common/main/" + locale + ".xml'>" + locale + "</a>"; 1681 } 1682 return localizedName + " [" + locale + "]"; 1683 } 1684 getLocaleName(String locale)1685 public static String getLocaleName(String locale) { 1686 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1687 if (localizedName == null || localizedName.equals(locale)) return locale; 1688 return localizedName; 1689 } 1690 getLinkedLocale(String locale)1691 public static String getLinkedLocale(String locale) { 1692 return "<a href='http://unicode.org/cldr/apps/survey?_=" + locale + "'>" + locale + "</a>"; 1693 } 1694 1695 1696 /** 1697 * Approximate xml path 1698 */ guessFilePath(Pair<String, String> locPath)1699 public static String guessFilePath(Pair<String, String> locPath) { 1700 final File base = new File(CLDRPaths.BASE_DIRECTORY); 1701 final String loc = locPath.getFirst(); 1702 final String path = locPath.getSecond(); 1703 String subdir = "main"; 1704 if (path.startsWith("//ldml/annotations")) { 1705 subdir = "annotations"; 1706 } else if(path.startsWith("//ldml/subdivisions")) { 1707 subdir = "subdivisions"; 1708 } 1709 File inCommon = new File(base, "common"); 1710 File subsub = new File(inCommon, subdir); 1711 if (subsub.isDirectory()) { 1712 File subFile = new File(subsub, loc+".xml"); 1713 if (subFile.canRead()) return subFile.getAbsolutePath().substring(base.getAbsolutePath().length() + 1); 1714 } 1715 1716 File inSeed = new File(base, "seed"); 1717 subsub = new File(inSeed, subdir); 1718 if (subsub.isDirectory()) { 1719 File subFile = new File(subsub, loc+".xml"); 1720 if (subFile.canRead()) return subFile.getAbsolutePath().substring(base.getAbsolutePath().length() + 1); 1721 } 1722 return loc+".xml"; 1723 } 1724 final static ConcurrentHashMap<Pair<String, String>, String> localeXpathToFilePath = new ConcurrentHashMap<>(); 1725 } 1726