1 package org.unicode.cldr.unittest; 2 3 import com.google.common.collect.ImmutableList; 4 import com.ibm.icu.dev.test.TestFmwk; 5 import com.ibm.icu.impl.Row.R2; 6 import com.ibm.icu.impl.Row.R5; 7 import com.ibm.icu.text.UnicodeSet; 8 import java.util.ArrayList; 9 import java.util.Arrays; 10 import java.util.Collection; 11 import java.util.Collections; 12 import java.util.HashMap; 13 import java.util.HashSet; 14 import java.util.LinkedHashMap; 15 import java.util.List; 16 import java.util.Map; 17 import java.util.Map.Entry; 18 import java.util.Set; 19 import java.util.TreeMap; 20 import java.util.TreeSet; 21 import java.util.regex.Matcher; 22 import org.unicode.cldr.test.CheckCLDR; 23 import org.unicode.cldr.test.CheckCLDR.CheckStatus; 24 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 25 import org.unicode.cldr.test.CheckCLDR.InputMethod; 26 import org.unicode.cldr.test.CheckCLDR.Options; 27 import org.unicode.cldr.test.CheckCLDR.Phase; 28 import org.unicode.cldr.test.CheckCLDR.StatusAction; 29 import org.unicode.cldr.test.CheckConsistentCasing; 30 import org.unicode.cldr.test.CheckDates; 31 import org.unicode.cldr.test.CheckForExemplars; 32 import org.unicode.cldr.test.CheckNames; 33 import org.unicode.cldr.test.CheckNew; 34 import org.unicode.cldr.test.OutdatedPaths; 35 import org.unicode.cldr.test.SubmissionLocales; 36 import org.unicode.cldr.test.TestCache; 37 import org.unicode.cldr.test.TestCache.TestResultBundle; 38 import org.unicode.cldr.tool.LikelySubtags; 39 import org.unicode.cldr.util.CLDRConfig; 40 import org.unicode.cldr.util.CLDRFile; 41 import org.unicode.cldr.util.CLDRInfo.CandidateInfo; 42 import org.unicode.cldr.util.CLDRInfo.PathValueInfo; 43 import org.unicode.cldr.util.CLDRInfo.UserInfo; 44 import org.unicode.cldr.util.CLDRLocale; 45 import org.unicode.cldr.util.Counter; 46 import org.unicode.cldr.util.DayPeriodInfo; 47 import org.unicode.cldr.util.DayPeriodInfo.DayPeriod; 48 import org.unicode.cldr.util.DayPeriodInfo.Type; 49 import org.unicode.cldr.util.Factory; 50 import org.unicode.cldr.util.GrammarInfo; 51 import org.unicode.cldr.util.LanguageTagParser; 52 import org.unicode.cldr.util.Level; 53 import org.unicode.cldr.util.Organization; 54 import org.unicode.cldr.util.Pair; 55 import org.unicode.cldr.util.PathHeader; 56 import org.unicode.cldr.util.PathHeader.SurveyToolStatus; 57 import org.unicode.cldr.util.PatternPlaceholders; 58 import org.unicode.cldr.util.PatternPlaceholders.PlaceholderInfo; 59 import org.unicode.cldr.util.PatternPlaceholders.PlaceholderStatus; 60 import org.unicode.cldr.util.SimpleXMLSource; 61 import org.unicode.cldr.util.StandardCodes; 62 import org.unicode.cldr.util.StandardCodes.LstrType; 63 import org.unicode.cldr.util.StringId; 64 import org.unicode.cldr.util.SupplementalDataInfo; 65 import org.unicode.cldr.util.Validity; 66 import org.unicode.cldr.util.Validity.Status; 67 import org.unicode.cldr.util.VoteResolver.VoterInfo; 68 import org.unicode.cldr.util.XMLSource; 69 70 public class TestCheckCLDR extends TestFmwk { 71 72 private static final boolean SHOW_LIMITED = 73 System.getProperty("TestCheckCLDR:SHOW_LIMITED") != null; 74 75 static CLDRConfig testInfo = CLDRConfig.getInstance(); 76 private final Set<String> eightPointLocales = 77 new TreeSet<>( 78 Arrays.asList( 79 "ar ca cs da de el es fi fr he hi hr hu id it ja ko lt lv nl no pl pt pt_PT ro ru sk sl sr sv th tr uk vi zh zh_Hant" 80 .split(" "))); 81 main(String[] args)82 public static void main(String[] args) { 83 new TestCheckCLDR().run(args); 84 } 85 86 static class MyCheckCldr extends org.unicode.cldr.test.CheckCLDR { doTest()87 CheckStatus doTest() { 88 try { 89 throw new IllegalArgumentException("hi"); 90 } catch (Exception e) { 91 return new CheckStatus() 92 .setCause(this) 93 .setMainType(CheckStatus.warningType) 94 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 95 .setMessage("An exception {0}, and a number {1}", e, 1.5); 96 } 97 } 98 99 @Override handleCheck( String path, String fullPath, String value, Options options, List<CheckStatus> result)100 public CheckCLDR handleCheck( 101 String path, 102 String fullPath, 103 String value, 104 Options options, 105 List<CheckStatus> result) { 106 return null; 107 } 108 } 109 TestExceptions()110 public void TestExceptions() { 111 CheckStatus status = new MyCheckCldr().doTest(); 112 Exception[] exceptions = status.getExceptionParameters(); 113 assertEquals("Number of exceptions:", exceptions.length, 1); 114 assertEquals("Exception message:", "hi", exceptions[0].getMessage()); 115 logln(Arrays.asList(exceptions[0].getStackTrace()).toString()); 116 logln(status.getMessage()); 117 } 118 TestCheckConsistentCasing()119 public static void TestCheckConsistentCasing() { 120 CheckConsistentCasing c = new CheckConsistentCasing(testInfo.getCldrFactory()); 121 Map<String, String> options = new LinkedHashMap<>(); 122 List<CheckStatus> possibleErrors = new ArrayList<>(); 123 final CLDRFile english = testInfo.getEnglish(); 124 c.setCldrFileToCheck(english, new CheckCLDR.Options(options), possibleErrors); 125 for (String path : english) { 126 c.check( 127 path, 128 english.getFullXPath(path), 129 english.getStringValue(path), 130 new CheckCLDR.Options(options), 131 possibleErrors); 132 } 133 } 134 135 /** Test the TestCache and TestResultBundle objects */ TestTestCache()136 public void TestTestCache() { 137 String localeString = "en"; 138 CLDRLocale locale = CLDRLocale.getInstance(localeString); 139 CheckCLDR.Options checkCldrOptions = 140 new Options(locale, Phase.SUBMISSION, "default", "basic"); 141 TestCache testCache = testInfo.getCldrFactory().getTestCache(); 142 testCache.setNameMatcher(".*"); // will clear the cache 143 TestResultBundle bundle = testCache.getBundle(checkCldrOptions); 144 final CLDRFile cldrFile = testInfo.getCLDRFile(localeString, true); 145 /* 146 * Loop through the set of paths twice. The second time should be much faster. 147 * Measured times for the two passes, without pathCache in TestResultBundle, 148 * 4017 and 3293 milliseconds. With pathCache, 4125 and 46 milliseconds. 149 * That's with locale "en", all 19698 paths. Results for "fr" were similar. 150 * To save time, limit the number of paths if getInclusion() is small. 151 * A thousand paths take about half a second to loop through twice. 152 */ 153 int maxPathCount = (getInclusion() < 5) ? 1000 : 100000; 154 double[] deltaTime = {0, 0}; 155 for (int i = 0; i < 2; i++) { 156 List<CheckStatus> possibleErrors = new ArrayList<>(); 157 int pathCount = 0; 158 double startTime = System.currentTimeMillis(); 159 for (String path : cldrFile) { 160 String fullPath = cldrFile.getFullXPath(path); 161 String value = cldrFile.getStringValue(path); 162 bundle.check(fullPath, possibleErrors, value); 163 if (++pathCount == maxPathCount) { 164 break; 165 } 166 } 167 deltaTime[i] = System.currentTimeMillis() - startTime; 168 /* 169 * Expect possibleErrors to have size zero. 170 * A future enhancement of this test could modify some values to force errors, 171 * and confirm that the errors are returned identically the first and second times. 172 */ 173 assertEquals("possibleErrors, loop index " + i, possibleErrors.size(), 0); 174 } 175 /* 176 * Expect second time to be about a hundredth of first time; error if more than a tenth. 177 * On one occasion, smoketest had times 171.0 and 5.0. 178 */ 179 if (deltaTime[1] > deltaTime[0] / 10) { 180 errln( 181 "TestResultBundle cache should yield more benefit: times " 182 + deltaTime[0] 183 + " and " 184 + deltaTime[1]); 185 } 186 } 187 188 /** Test the "collisionless" error/warning messages. */ 189 public static final String INDIVIDUAL_TESTS = 190 ".*(CheckCasing|CheckCurrencies|CheckDates|CheckExemplars|CheckForCopy|CheckForExemplars|CheckMetazones|CheckNumbers)"; 191 192 static final Factory factory = testInfo.getCldrFactory(); 193 static final CLDRFile english = testInfo.getEnglish(); 194 195 private static final boolean DEBUG = true; 196 197 static final Factory cldrFactory = CLDRConfig.getInstance().getCldrFactory(); 198 static final Factory cldrFactoryWithSeed = 199 CLDRConfig.getInstance().getCommonAndSeedAndMainAndAnnotationsFactory(); 200 testPlaceholderSamples()201 public void testPlaceholderSamples() { 202 CLDRFile root = cldrFactory.make("root", true); 203 String[][] tests = { 204 {"he", "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"one\"]", "שנה"}, 205 // test edge cases 206 // locale, path, value, 0..n Subtype errors 207 {"en", "//ldml/localeDisplayNames/localeDisplayPattern/localePattern", "{0}huh?{1}"}, 208 { 209 "en", 210 "//ldml/localeDisplayNames/localeDisplayPattern/localePattern", 211 "huh?", 212 "missingPlaceholders" 213 }, 214 { 215 "en", 216 "//ldml/localeDisplayNames/localeDisplayPattern/localePattern", 217 "huh?{0}", 218 "missingPlaceholders" 219 }, 220 { 221 "en", 222 "//ldml/localeDisplayNames/localeDisplayPattern/localePattern", 223 "huh?{1}", 224 "missingPlaceholders", 225 "gapsInPlaceholderNumbers" 226 }, 227 { 228 "en", 229 "//ldml/localeDisplayNames/localeDisplayPattern/localePattern", 230 "{0}huh?{1}{2}", 231 "extraPlaceholders" 232 }, 233 { 234 "en", 235 "//ldml/localeDisplayNames/localeDisplayPattern/localePattern", 236 "{0}huh?{1}{0}", 237 "duplicatePlaceholders" 238 }, 239 { 240 "fr", 241 "//ldml/numbers/minimalPairs/ordinalMinimalPairs[@ordinal=\"other\"]", 242 "Prenez la e à droite.", 243 "missingPlaceholders" 244 }, 245 { 246 "fr", 247 "//ldml/numbers/minimalPairs/ordinalMinimalPairs[@ordinal=\"other\"]", 248 "Prenez la {0}e à droite." 249 }, 250 { 251 "fr", 252 "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"other\"]", 253 "jours", 254 "missingPlaceholders" 255 }, 256 {"fr", "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"other\"]", "{0} jours"}, 257 { 258 "cy", 259 "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"other\"]", 260 "ci cath", 261 "missingPlaceholders" 262 }, 263 {"cy", "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"other\"]", "{0} ci"}, 264 { 265 "cy", 266 "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"other\"]", 267 "{0} ci, {0} cath" 268 }, 269 { 270 "pl", 271 "//ldml/numbers/minimalPairs/caseMinimalPairs[@case=\"accusative\"]", 272 "biernik", 273 "missingPlaceholders" 274 }, 275 { 276 "pl", 277 "//ldml/numbers/minimalPairs/caseMinimalPairs[@case=\"accusative\"]", 278 "{0} biernik" 279 }, 280 { 281 "fr", 282 "//ldml/numbers/minimalPairs/genderMinimalPairs[@gender=\"feminine\"]", 283 "de genre féminin", 284 "missingPlaceholders" 285 }, 286 { 287 "fr", 288 "//ldml/numbers/minimalPairs/genderMinimalPairs[@gender=\"feminine\"]", 289 "la {0}" 290 }, 291 { 292 "ar", 293 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"duration-hour\"]/unitPattern[@count=\"one\"]", 294 "ساعة" 295 }, 296 { 297 "ar", 298 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"duration-hour\"]/unitPattern[@count=\"one\"]", 299 "{0} ساعة" 300 }, 301 { 302 "ar", 303 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"duration-hour\"]/unitPattern[@count=\"one\"]", 304 "{1}{0} ساعة", 305 "extraPlaceholders" 306 }, 307 {"he", "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"one\"]", "שנה"}, 308 {"he", "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"two\"]", "שנתיים"}, 309 { 310 "he", 311 "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"many\"]", 312 "שנה", 313 "missingPlaceholders" 314 }, 315 { 316 "he", 317 "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"other\"]", 318 "שנים", 319 "missingPlaceholders" 320 }, 321 }; 322 for (String[] row : tests) { 323 String localeId = row[0]; 324 String path = row[1]; 325 String value = row[2]; 326 Set<Subtype> expected = new TreeSet<>(); 327 for (int i = 3; i < row.length; ++i) { 328 expected.add(Subtype.valueOf(row[i])); 329 } 330 List<CheckStatus> possibleErrors = new ArrayList<>(); 331 checkPathValue(root, localeId, path, value, possibleErrors); 332 Set<Subtype> actual = new TreeSet<>(); 333 for (CheckStatus item : possibleErrors) { 334 if (PatternPlaceholders.PLACEHOLDER_SUBTYPES.contains(item.getSubtype())) { 335 actual.add(item.getSubtype()); 336 } 337 } 338 if (!assertEquals(Arrays.asList(row).toString(), expected, actual)) { 339 int debug = 0; 340 } 341 } 342 } 343 checkPathValue( CLDRFile root, String localeId, String path, String value, List<CheckStatus> possibleErrors)344 public void checkPathValue( 345 CLDRFile root, 346 String localeId, 347 String path, 348 String value, 349 List<CheckStatus> possibleErrors) { 350 XMLSource localeSource = new SimpleXMLSource(localeId); 351 localeSource.putValueAtPath(path, value); 352 353 TestFactory currFactory = makeTestFactory(root, localeSource); 354 CLDRFile cldrFile = currFactory.make(localeSource.getLocaleID(), true); 355 CheckForExemplars check = new CheckForExemplars(currFactory); 356 357 Options options = new Options(); 358 check.setCldrFileToCheck(cldrFile, options, possibleErrors); 359 check.handleCheck(path, path, value, options, possibleErrors); 360 } 361 TestPlaceholders()362 public void TestPlaceholders() { 363 CheckCLDR.setDisplayInformation(english); 364 checkPlaceholders(english); 365 checkPlaceholders(factory.make("de", true)); 366 } 367 checkPlaceholders(CLDRFile cldrFileToTest)368 public void checkPlaceholders(CLDRFile cldrFileToTest) { 369 // verify that every item with {0} has a pattern in pattern 370 // placeholders, 371 // and that every one generates an error in CheckCDLR for patterns when 372 // given "?" 373 // and that every non-pattern doesn't have an error in CheckCLDR for 374 // patterns when given "?" 375 // 376 // For the following: traditional placeholders just have {0}, {1}, {2}, ... 377 // But personName namePattern placeHolders start with [a-z], then continue with 378 // [0-9a-zA-Z-]+ 379 // They need to be distinguished from non-placeholder patterns using {} in UnicodeSets 380 Matcher messagePlaceholder = CheckForExemplars.PLACEHOLDER.matcher(""); 381 PatternPlaceholders patternPlaceholders = PatternPlaceholders.getInstance(); 382 383 CheckCLDR test = CheckCLDR.getCheckAll(factory, ".*"); 384 List<CheckStatus> possibleErrors = new ArrayList<>(); 385 Options options = new Options(); 386 test.setCldrFileToCheck(cldrFileToTest, options, possibleErrors); 387 List<CheckStatus> result = new ArrayList<>(); 388 389 PathHeader.Factory pathHeaderFactory = PathHeader.getFactory(cldrFileToTest); 390 Set<PathHeader> sorted = new TreeSet<>(); 391 for (String path : cldrFileToTest.fullIterable()) { 392 sorted.add(pathHeaderFactory.fromPath(path)); 393 } 394 // test actual example with count=<digits> 395 final String testPath = 396 "//ldml/numbers/decimalFormats[@numberSystem=\"latn\"]/decimalFormatLength[@type=\"long\"]/decimalFormat[@type=\"standard\"]/pattern[@type=\"1000\"][@count=\"1\"]"; 397 sorted.add(pathHeaderFactory.fromPath(testPath)); 398 399 for (PathHeader pathHeader : sorted) { 400 String path = pathHeader.getOriginalPath(); 401 if (path.contains("/exemplarCharacters") || path.contains("/parseLenients")) { 402 // skip some paths with UnicodeSets that may include {} constructs 403 // that should not be interpreted as placeholders 404 continue; 405 } 406 String value = cldrFileToTest.getStringValue(path); 407 if (value == null) { 408 continue; 409 } 410 boolean containsMessagePattern = messagePlaceholder.reset(value).find(); 411 final Map<String, PlaceholderInfo> placeholderInfo = patternPlaceholders.get(path); 412 final PlaceholderStatus placeholderStatus = patternPlaceholders.getStatus(path); 413 if (placeholderStatus == PlaceholderStatus.DISALLOWED) { 414 if (containsMessagePattern) { 415 errln( 416 cldrFileToTest.getLocaleID() 417 + " Value (" 418 + value 419 + ") contains placeholder, but placeholder info = «" 420 + placeholderStatus 421 + "»\t" 422 + path); 423 continue; 424 } 425 } else { // not disallowed 426 if (!containsMessagePattern) { 427 // The following error seems wrong if placeholderStatus is LOCALE_DEPENDENT 428 // in which case some entries might not have placeholders; see CLDR-17820 429 errln( 430 cldrFileToTest.getLocaleID() 431 + " Value (" 432 + value 433 + ") does not contain placeholder, but placeholder info = «" 434 + placeholderStatus 435 + "»\t" 436 + path); 437 continue; 438 } 439 // get the set of placeholders 440 HashSet<String> found = new HashSet<>(); 441 do { 442 found.add(messagePlaceholder.group()); // we loaded first one up above 443 } while (messagePlaceholder.find()); 444 445 if (!found.equals(placeholderInfo.keySet())) { 446 if (placeholderStatus != PlaceholderStatus.LOCALE_DEPENDENT 447 && placeholderStatus != PlaceholderStatus.OPTIONAL) { 448 errln( 449 cldrFileToTest.getLocaleID() 450 + " Value (" 451 + value 452 + ") has different placeholders than placeholder info «" 453 + placeholderInfo.keySet() 454 + "»\t" 455 + path); 456 } 457 } else { 458 logln("placeholder info = " + placeholderInfo + "\t" + path); 459 } 460 } 461 } 462 } 463 TestFullErrors()464 public void TestFullErrors() { 465 CheckCLDR test = CheckCLDR.getCheckAll(factory, INDIVIDUAL_TESTS); 466 CheckCLDR.setDisplayInformation(english); 467 468 final String localeID = "fr"; 469 checkLocale(test, localeID, "?", null); 470 } 471 TestAllLocales()472 public void TestAllLocales() { 473 CheckCLDR test = CheckCLDR.getCheckAll(factory, INDIVIDUAL_TESTS); 474 CheckCLDR.setDisplayInformation(english); 475 Set<String> unique = new HashSet<>(); 476 LanguageTagParser ltp = new LanguageTagParser(); 477 Set<String> locales = new HashSet<>(); 478 for (String locale : getInclusion() <= 5 ? eightPointLocales : factory.getAvailable()) { 479 /* 480 * Only test locales without regions. E.g., test "pt", skip "pt_PT" 481 */ 482 if (ltp.set(locale).getRegion().isEmpty()) { 483 locales.add(locale); 484 } 485 } 486 // With ICU4J libs of 2020-03-23, using locales.parallelStream().forEach below 487 // hangs, or crashes with NPE. Likely an ICU4J issue, but we don't really need 488 // parallelStream() here anyway since we are only handling around 35 locales. 489 // (And in fact this test seems faster without it) 490 locales.forEach(locale -> checkLocale(test, locale, null, unique)); 491 logln("Count:\t" + locales.size()); 492 } 493 TestA()494 public void TestA() { 495 CheckCLDR test = CheckCLDR.getCheckAll(factory, INDIVIDUAL_TESTS); 496 CheckCLDR.setDisplayInformation(english); 497 Set<String> unique = new HashSet<>(); 498 499 checkLocale(test, "ko", null, unique); 500 } 501 checkLocale( CheckCLDR test, String localeID, String dummyValue, Set<String> unique)502 public void checkLocale( 503 CheckCLDR test, String localeID, String dummyValue, Set<String> unique) { 504 checkLocale(test, testInfo.getCLDRFile(localeID, false), dummyValue, unique); 505 } 506 checkLocale( CheckCLDR test, CLDRFile nativeFile, String dummyValue, Set<String> unique)507 public void checkLocale( 508 CheckCLDR test, CLDRFile nativeFile, String dummyValue, Set<String> unique) { 509 String localeID = nativeFile.getLocaleID(); 510 List<CheckStatus> possibleErrors = new ArrayList<>(); 511 CheckCLDR.Options options = new CheckCLDR.Options(); 512 test.setCldrFileToCheck(nativeFile, options, possibleErrors); 513 List<CheckStatus> result = new ArrayList<>(); 514 515 CLDRFile patched = nativeFile; // new CLDRFile(override); 516 PathHeader.Factory pathHeaderFactory = PathHeader.getFactory(english); 517 Set<PathHeader> sorted = new TreeSet<>(); 518 for (String path : patched) { 519 final PathHeader pathHeader = pathHeaderFactory.fromPath(path); 520 if (pathHeader != null) { 521 sorted.add(pathHeader); 522 } 523 } 524 525 logln("Checking: " + localeID); 526 UnicodeSet missingCurrencyExemplars = new UnicodeSet(); 527 UnicodeSet missingExemplars = new UnicodeSet(); 528 529 for (PathHeader pathHeader : sorted) { 530 String path = pathHeader.getOriginalPath(); 531 // override.overridePath = path; 532 final String resolvedValue = 533 dummyValue == null ? patched.getStringValueWithBailey(path) : dummyValue; 534 test.handleCheck(path, patched.getFullXPath(path), resolvedValue, options, result); 535 if (result.size() != 0) { 536 for (CheckStatus item : result) { 537 addExemplars(item, missingCurrencyExemplars, missingExemplars); 538 final String mainMessage = 539 StringId.getId(path) 540 + "\t" 541 + pathHeader 542 + "\t" 543 + english.getStringValue(path) 544 + "\t" 545 + item.getType() 546 + "\t" 547 + item.getSubtype(); 548 if (unique != null) { 549 if (unique.contains(mainMessage)) { 550 continue; 551 } else { 552 unique.add(mainMessage); 553 } 554 } 555 logln( 556 localeID 557 + "\t" 558 + mainMessage 559 + "\t" 560 + resolvedValue 561 + "\t" 562 + item.getMessage() 563 + "\t" 564 + pathHeader.getOriginalPath()); 565 } 566 } 567 } 568 if (missingCurrencyExemplars.size() != 0) { 569 logln( 570 localeID 571 + "\tMissing Exemplars (Currency):\t" 572 + missingCurrencyExemplars.toPattern(false)); 573 } 574 if (missingExemplars.size() != 0) { 575 logln(localeID + "\tMissing Exemplars:\t" + missingExemplars.toPattern(false)); 576 } 577 } 578 addExemplars( CheckStatus status, UnicodeSet missingCurrencyExemplars, UnicodeSet missingExemplars)579 void addExemplars( 580 CheckStatus status, UnicodeSet missingCurrencyExemplars, UnicodeSet missingExemplars) { 581 Object[] parameters = status.getParameters(); 582 if (parameters != null) { 583 if (parameters.length >= 1 && status.getCause().getClass() == CheckForExemplars.class) { 584 try { 585 UnicodeSet set = new UnicodeSet(parameters[0].toString()); 586 if (status.getMessage().contains("currency")) { 587 missingCurrencyExemplars.addAll(set); 588 } else { 589 missingExemplars.addAll(set); 590 } 591 } catch (RuntimeException e) { 592 } // skip if not parseable as set 593 } 594 } 595 } 596 TestCheckNames()597 public void TestCheckNames() { 598 CheckCLDR c = new CheckNames(); 599 Options options = new CheckCLDR.Options(new LinkedHashMap<>()); 600 List<CheckStatus> possibleErrors = new ArrayList<>(); 601 final CLDRFile english = testInfo.getEnglish(); 602 c.setCldrFileToCheck(english, options, possibleErrors); 603 String xpath = "//ldml/localeDisplayNames/languages/language[@type=\"mga\"]"; 604 c.check(xpath, xpath, "Middle Irish (900-1200) ", options, possibleErrors); 605 assertEquals("There should be an error", 1, possibleErrors.size()); 606 607 possibleErrors.clear(); 608 xpath = "//ldml/localeDisplayNames/currencies/currency[@type=\"afa\"]/name"; 609 c.check(xpath, xpath, "Afghan Afghani (1927-2002)", options, possibleErrors); 610 assertEquals("Currencies are allowed to have dates", 0, possibleErrors.size()); 611 } 612 613 /** 614 * Check that at least one path in a locale is outdated and one path is not. That may change 615 * each time. This needs to be a <locale,path> that is currently outdated (birth older than 616 * English's) if the test fails with "no failure message" run GenerateBirths (if you haven't 617 * done so) look at readable results in the log file in 618 * https://github.com/unicode-org/cldr-staging/blob/main/births/41.0/fr.txt (for the current 619 * version, not nec. 41.0) for a reasonable locale ( may change locale to something other than 620 * fr) find a path that is outdated. To work on both limited and full submissions, choose one 621 * with English = trunk Sometimes the English change is suppressed in a limited release if the 622 * change is small. Pick another in that case. check the data files to ensure that it is in fact 623 * outdated. change the path to that value the 3rd parameter is the message displayed to the 624 * user, or "" if not 'English Changed' So the first group of tests are for items that should 625 * not be outdated And the second group is ones that should be outdated. 626 */ TestCheckNew()627 public void TestCheckNew() { 628 // Not outdated 629 checkCheckNew("de", "//ldml/localeDisplayNames/languages/language[@type=\"en\"]", ""); 630 631 // Outdated 632 checkCheckNew( 633 "de", 634 "//ldml/localeDisplayNames/territories/territory[@type=\"001\"]", 635 "In CLDR 39.0 the English value for this field changed from “World” to “world”, but the corresponding value for your locale didn't change."); 636 checkCheckNew( 637 "el", 638 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"mass-grain\"]/displayName", 639 "In CLDR 40.0 the English value for this field changed from “grain” to “grains”, but the corresponding value for your locale didn't change."); 640 } 641 checkCheckNew(String locale, String path, String expectedMessage)642 public void checkCheckNew(String locale, String path, String expectedMessage) { 643 final String title = "CheckNew " + locale + ", " + path; 644 645 OutdatedPaths outdatedPaths = OutdatedPaths.getInstance(); 646 boolean isOutdated = outdatedPaths.isOutdated(locale, path); 647 648 // 649 String oldEnglishValue = outdatedPaths.getPreviousEnglish(path); 650 if (!OutdatedPaths.NO_VALUE.equals(oldEnglishValue)) { 651 assertEquals(title, expectedMessage.isEmpty(), !isOutdated); 652 } 653 654 CheckCLDR c = new CheckNew(testInfo.getCommonAndSeedAndMainAndAnnotationsFactory()); 655 List<CheckStatus> result = new ArrayList<>(); 656 final CheckCLDR.Options options = new CheckCLDR.Options(new HashMap<>()); 657 c.setCldrFileToCheck(testInfo.getCLDRFile(locale, true), options, result); 658 c.check(path, path, "foobar", options, result); 659 String actualMessage = ""; 660 for (CheckStatus status : result) { 661 if (status.getSubtype() != Subtype.modifiedEnglishValue) { 662 continue; 663 } 664 actualMessage = status.getMessage(); 665 break; 666 } 667 assertEquals(title, expectedMessage, actualMessage); 668 } 669 TestCheckNewRootFailure()670 public void TestCheckNewRootFailure() { 671 // Check that we get an error with the root value for an emoji. 672 final Factory annotationsFactory = testInfo.getAnnotationsFactory(); 673 String locale = "yo"; // the name doesn't matter, since we're going to create a new one 674 String path = "//ldml/annotations/annotation[@cp=\"��\"][@type=\"tts\"]"; 675 CheckCLDR c = new CheckNew(annotationsFactory); 676 List<CheckStatus> result = new ArrayList<>(); 677 Map<String, String> options = new HashMap<>(); 678 for (Phase phase : Phase.values()) { 679 options.put(Options.Option.phase.getKey(), phase.toString()); 680 String value = "E10-836"; 681 // The following code used to check values of both "E10-836" and 682 // CldrUtility.INHERITANCE_MARKER="↑↑↑", 683 // but the latter does not make sense; "↑↑↑" will be followed up through its parent 684 // chain, either 685 // yielding a real value or the root value (but never "↑↑↑"). In regular CLDR data the 686 // root value will 687 // be like "E10-836" but in production data those root entreis are stripped and the root 688 // value will be 689 // null. Hence CheckNew.handleCheck only checks for entries like "E10-836", not "↑↑↑", 690 // and the test 691 // should only check those as well. 692 { 693 // make a fake locale, starting with real root 694 695 CLDRFile root = annotationsFactory.make("root", false); 696 XMLSource localeSource = new SimpleXMLSource(locale); 697 localeSource.putValueAtPath(path, value); 698 699 TestFactory currFactory = makeTestFactory(root, localeSource); 700 CLDRFile cldrFile = currFactory.make(localeSource.getLocaleID(), true); 701 702 c.setCldrFileToCheck(cldrFile, new CheckCLDR.Options(options), result); 703 c.check(path, path, value, new CheckCLDR.Options(options), result); 704 boolean gotOne = false; 705 for (CheckStatus status : result) { 706 if (status.getSubtype() == Subtype.valueMustBeOverridden) { 707 gotOne = true; 708 assertEquals( 709 phase + " Error message check", 710 "This value must be a real translation, NOT the name/keyword placeholder.", 711 status.getMessage()); 712 } 713 } 714 if (!gotOne) { 715 errln(phase + " Missing failure message for value=" + value + "; path=" + path); 716 } 717 } 718 } 719 } 720 makeTestFactory(CLDRFile root, XMLSource localeSource)721 public TestFactory makeTestFactory(CLDRFile root, XMLSource localeSource) { 722 CLDRFile localeCldr = new CLDRFile(localeSource); 723 724 TestFactory factory = new TestFactory(); 725 factory.addFile(root); 726 factory.addFile(localeCldr); 727 return factory; 728 } 729 TestCheckDates()730 public void TestCheckDates() { 731 CheckCLDR.setDisplayInformation(testInfo.getEnglish()); // just in case 732 String prefix = 733 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\""; 734 String infix = "\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod[@type=\""; 735 String suffix = "\"]"; 736 737 TestFactory testFactory = new TestFactory(); 738 739 List<CheckStatus> result = new ArrayList<>(); 740 Options options = new Options(); 741 final String collidingValue = "foobar"; 742 743 // Selection has stricter collision rules, because is is used to select different messages. 744 // So two types with the same localization do collide unless they have exactly the same 745 // rules. 746 747 Object[][] tests = { 748 {"en"}, // set locale 749 750 // nothing collides with itself 751 {Type.format, DayPeriod.night1, Type.format, DayPeriod.night1, Subtype.none}, 752 {Type.format, DayPeriod.morning1, Type.format, DayPeriod.morning1, Subtype.none}, 753 {Type.format, DayPeriod.afternoon1, Type.format, DayPeriod.afternoon1, Subtype.none}, 754 {Type.format, DayPeriod.evening1, Type.format, DayPeriod.evening1, Subtype.none}, 755 {Type.format, DayPeriod.am, Type.format, DayPeriod.am, Subtype.none}, 756 {Type.format, DayPeriod.pm, Type.format, DayPeriod.pm, Subtype.none}, 757 {Type.format, DayPeriod.noon, Type.format, DayPeriod.noon, Subtype.none}, 758 {Type.format, DayPeriod.midnight, Type.format, DayPeriod.midnight, Subtype.none}, 759 {Type.selection, DayPeriod.night1, Type.selection, DayPeriod.night1, Subtype.none}, 760 {Type.selection, DayPeriod.morning1, Type.selection, DayPeriod.morning1, Subtype.none}, 761 { 762 Type.selection, 763 DayPeriod.afternoon1, 764 Type.selection, 765 DayPeriod.afternoon1, 766 Subtype.none 767 }, 768 {Type.selection, DayPeriod.evening1, Type.selection, DayPeriod.evening1, Subtype.none}, 769 {Type.selection, DayPeriod.am, Type.selection, DayPeriod.am, Subtype.none}, 770 {Type.selection, DayPeriod.pm, Type.selection, DayPeriod.pm, Subtype.none}, 771 {Type.selection, DayPeriod.noon, Type.selection, DayPeriod.noon, Subtype.none}, 772 {Type.selection, DayPeriod.midnight, Type.selection, DayPeriod.midnight, Subtype.none}, 773 774 // fixed classes always collide 775 {Type.format, DayPeriod.am, Type.format, DayPeriod.pm, Subtype.dateSymbolCollision}, 776 {Type.format, DayPeriod.am, Type.format, DayPeriod.noon, Subtype.dateSymbolCollision}, 777 { 778 Type.format, 779 DayPeriod.am, 780 Type.format, 781 DayPeriod.midnight, 782 Subtype.dateSymbolCollision 783 }, 784 {Type.format, DayPeriod.pm, Type.format, DayPeriod.noon, Subtype.dateSymbolCollision}, 785 { 786 Type.format, 787 DayPeriod.pm, 788 Type.format, 789 DayPeriod.midnight, 790 Subtype.dateSymbolCollision 791 }, 792 { 793 Type.format, 794 DayPeriod.noon, 795 Type.format, 796 DayPeriod.midnight, 797 Subtype.dateSymbolCollision 798 }, 799 { 800 Type.selection, 801 DayPeriod.am, 802 Type.selection, 803 DayPeriod.pm, 804 Subtype.dateSymbolCollision 805 }, 806 { 807 Type.selection, 808 DayPeriod.am, 809 Type.selection, 810 DayPeriod.noon, 811 Subtype.dateSymbolCollision 812 }, 813 { 814 Type.selection, 815 DayPeriod.am, 816 Type.selection, 817 DayPeriod.midnight, 818 Subtype.dateSymbolCollision 819 }, 820 { 821 Type.selection, 822 DayPeriod.pm, 823 Type.selection, 824 DayPeriod.noon, 825 Subtype.dateSymbolCollision 826 }, 827 { 828 Type.selection, 829 DayPeriod.pm, 830 Type.selection, 831 DayPeriod.midnight, 832 Subtype.dateSymbolCollision 833 }, 834 { 835 Type.selection, 836 DayPeriod.noon, 837 Type.selection, 838 DayPeriod.midnight, 839 Subtype.dateSymbolCollision 840 }, 841 842 // 00-06 night1 843 // 06-12 morning1 844 // 12-18 afternoon1 845 // 18-21 evening1 846 // 21-24 night1 847 // 848 // So for a 12hour time, we have: 849 // 850 // 12 1 2 3 4 5 6 7 8 9 10 11 851 // n n n n n n m m m m m m 852 // a a a a a a e e e n n n 853 854 // Formatting has looser collision rules, because it is always paired with a time. 855 // That is, it is not a problem if two items collide, 856 // if it doesn't cause a collision when paired with a time. 857 // But if 11:00 has the same format (eg 11 X) as 23:00, there IS a collision. 858 // So we see if there is an overlap mod 12. 859 860 { 861 Type.format, 862 DayPeriod.night1, 863 Type.format, 864 DayPeriod.morning1, 865 Subtype.dateSymbolCollision 866 }, 867 { 868 Type.format, 869 DayPeriod.night1, 870 Type.format, 871 DayPeriod.afternoon1, 872 Subtype.dateSymbolCollision 873 }, 874 {Type.format, DayPeriod.night1, Type.format, DayPeriod.evening1, Subtype.none}, 875 {Type.format, DayPeriod.morning1, Type.format, DayPeriod.afternoon1, Subtype.none}, 876 { 877 Type.format, 878 DayPeriod.morning1, 879 Type.format, 880 DayPeriod.evening1, 881 Subtype.dateSymbolCollision 882 }, 883 {Type.format, DayPeriod.afternoon1, Type.format, DayPeriod.evening1, Subtype.none}, 884 885 // Selection has stricter collision rules, because is is used to select different 886 // messages. 887 // So two types with the same localization do collide unless they have exactly the same 888 // rules. 889 // We use chr to test the "unless they have exactly the same rules" below. 890 891 { 892 Type.selection, 893 DayPeriod.morning1, 894 Type.selection, 895 DayPeriod.night1, 896 Subtype.dateSymbolCollision 897 }, 898 { 899 Type.selection, 900 DayPeriod.morning1, 901 Type.selection, 902 DayPeriod.afternoon1, 903 Subtype.dateSymbolCollision 904 }, 905 { 906 Type.selection, DayPeriod.morning1, Type.selection, DayPeriod.am, Subtype.none 907 }, // morning1 and am is allowable 908 { 909 Type.selection, 910 DayPeriod.morning1, 911 Type.selection, 912 DayPeriod.pm, 913 Subtype.dateSymbolCollision 914 }, 915 {"fr"}, 916 917 // nothing collides with itself 918 {Type.format, DayPeriod.night1, Type.format, DayPeriod.night1, Subtype.none}, 919 {Type.format, DayPeriod.morning1, Type.format, DayPeriod.morning1, Subtype.none}, 920 {Type.format, DayPeriod.afternoon1, Type.format, DayPeriod.afternoon1, Subtype.none}, 921 {Type.format, DayPeriod.evening1, Type.format, DayPeriod.evening1, Subtype.none}, 922 923 // French has different rules 924 // 00-04 night1 925 // 04-12 morning1 926 // 12-18 afternoon1 927 // 18-00 evening1 928 // 929 // So for a 12hour time, we have: 930 // 931 // 12 1 2 3 4 5 6 7 8 9 10 11 932 // n n n n m m m m m m m m 933 // a a a a a a e e e e e e 934 935 {Type.format, DayPeriod.night1, Type.format, DayPeriod.morning1, Subtype.none}, 936 { 937 Type.format, 938 DayPeriod.night1, 939 Type.format, 940 DayPeriod.afternoon1, 941 Subtype.dateSymbolCollision 942 }, 943 {Type.format, DayPeriod.night1, Type.format, DayPeriod.evening1, Subtype.none}, 944 { 945 Type.format, 946 DayPeriod.morning1, 947 Type.format, 948 DayPeriod.afternoon1, 949 Subtype.dateSymbolCollision 950 }, 951 { 952 Type.format, 953 DayPeriod.morning1, 954 Type.format, 955 DayPeriod.evening1, 956 Subtype.dateSymbolCollision 957 }, 958 {Type.format, DayPeriod.afternoon1, Type.format, DayPeriod.evening1, Subtype.none}, 959 {"chr"}, 960 // Chr lets use test that same rules don't collide in selection 961 // <dayPeriodRule type="morning1" from="0:00" before="12:00" /> 962 // <dayPeriodRule type="noon" at="12:00" /> 963 // <dayPeriodRule type="afternoon1" after="12:00" before="24:00" /> 964 {Type.selection, DayPeriod.morning1, Type.selection, DayPeriod.am, Subtype.none}, 965 {Type.selection, DayPeriod.afternoon1, Type.selection, DayPeriod.pm, Subtype.none}, 966 }; 967 CLDRFile testFile = null; 968 for (Object[] test : tests) { 969 // set locale 970 if (test.length == 1) { 971 if (testFile != null) { 972 logln(""); 973 } 974 testFile = new CLDRFile(new SimpleXMLSource((String) test[0])); 975 testFactory.addFile(testFile); 976 continue; 977 } 978 final DayPeriodInfo.Type type1 = (Type) test[0]; 979 final DayPeriodInfo.DayPeriod period1 = (DayPeriod) test[1]; 980 final DayPeriodInfo.Type type2 = (Type) test[2]; 981 final DayPeriodInfo.DayPeriod period2 = (DayPeriod) test[3]; 982 final Subtype expectedSubtype = (Subtype) test[4]; 983 984 final String path1 = prefix + type1.pathValue + infix + period1 + suffix; 985 final String path2 = prefix + type2.pathValue + infix + period2 + suffix; 986 987 testFile.add(path1, collidingValue); 988 testFile.add(path2, collidingValue); 989 990 CheckCLDR c = new CheckDates(testFactory); 991 c.setCldrFileToCheck(testFile, options, result); 992 993 result.clear(); 994 c.check(path1, path1, collidingValue, options, result); 995 Subtype actualSubtype = Subtype.none; 996 String message = null; 997 for (CheckStatus status : result) { 998 actualSubtype = status.getSubtype(); 999 message = status.getMessage(); 1000 break; 1001 } 1002 assertEquals( 1003 testFile.getLocaleID() 1004 + " " 1005 + type1 1006 + "/" 1007 + period1 1008 + " vs " 1009 + type2 1010 + "/" 1011 + period2 1012 + (message == null ? "" : " [" + message + "]"), 1013 expectedSubtype, 1014 actualSubtype); 1015 1016 testFile.remove(path1); 1017 testFile.remove(path2); 1018 } 1019 // Test for CLDR-14865 1020 testFile = new CLDRFile(new SimpleXMLSource("fi")); 1021 testFactory.addFile(testFile); 1022 String availableFormatTestPath = 1023 "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\"GyMd\"]"; 1024 String availableFormatValue = "d.m.y G"; // erroneous, should have M not m 1025 testFile.add(availableFormatTestPath, availableFormatValue); 1026 CheckCLDR c = new CheckDates(testFactory); 1027 c.setCldrFileToCheck(testFile, options, result); 1028 result.clear(); 1029 c.check( 1030 availableFormatTestPath, 1031 availableFormatTestPath, 1032 availableFormatValue, 1033 options, 1034 result); 1035 Subtype actualSubtype = Subtype.none; 1036 String message = null; 1037 for (CheckStatus status : result) { 1038 actualSubtype = status.getSubtype(); 1039 message = status.getMessage(); 1040 break; 1041 } 1042 if (actualSubtype != Subtype.incorrectDatePattern 1043 || message == null 1044 || !message.contains("d.M.y G")) { 1045 String errorMessage = 1046 "fi generic availableFormat for id=GyMd with value " 1047 + availableFormatValue 1048 + ":"; 1049 if (actualSubtype != Subtype.incorrectDatePattern) { 1050 errorMessage += 1051 " expected Subtype.incorrectDatePattern, got " + actualSubtype + " ;"; 1052 } 1053 if (message == null || !message.contains("d.M.y G")) { 1054 errorMessage += " expected message should contain suggested d.M.y G, got message: "; 1055 errorMessage += (message == null) ? "(null)" : message; 1056 } 1057 errln(errorMessage); 1058 } 1059 } 1060 1061 /** Should be some CLDR locales, plus a locale specially allowed in limited submission */ 1062 final List<String> localesForRowAction = ImmutableList.of("cs", "fr"); 1063 1064 /** Needs adjustment for Limited Submission! */ TestShowRowAction()1065 public void TestShowRowAction() { 1066 Map<Key, Pair<Boolean, String>> actionToExamplePath = new TreeMap<>(); 1067 Counter<Key> counter = new Counter<>(); 1068 1069 for (String locale : localesForRowAction) { 1070 DummyPathValueInfo dummyPathValueInfo = new DummyPathValueInfo(); 1071 dummyPathValueInfo.setLocale(CLDRLocale.getInstance(locale)); 1072 CLDRFile cldrFile = testInfo.getCldrFactory().make(locale, true); 1073 CLDRFile cldrFileUnresolved = testInfo.getCldrFactory().make(locale, false); 1074 1075 Set<PathHeader> sorted = new TreeSet<>(); 1076 for (String path : cldrFile) { 1077 PathHeader ph = pathHeaderFactory.fromPath(path); 1078 sorted.add(ph); 1079 } 1080 1081 for (Phase phase : Arrays.asList(Phase.SUBMISSION, Phase.VETTING)) { 1082 for (CheckStatus.Type status : 1083 Arrays.asList(CheckStatus.warningType, CheckStatus.errorType)) { 1084 dummyPathValueInfo.checkStatusType = status; 1085 1086 for (PathHeader ph : sorted) { 1087 String path = ph.getOriginalPath(); 1088 SurveyToolStatus surveyToolStatus = ph.getSurveyToolStatus(); 1089 dummyPathValueInfo.setXpath(path); 1090 dummyPathValueInfo.setBaselineValue( 1091 cldrFileUnresolved.getStringValue(path)); 1092 StatusAction action = 1093 phase.getShowRowAction( 1094 dummyPathValueInfo, InputMethod.DIRECT, ph, dummyUserInfo); 1095 1096 if (ph.shouldHide()) { 1097 assertEquals( 1098 "HIDE ==> FORBID_READONLY", 1099 StatusAction.FORBID_READONLY, 1100 action); 1101 } else if (CheckCLDR.LIMITED_SUBMISSION) { 1102 if (status == CheckStatus.Type.Error) { 1103 assertEquals("ERROR ==> ALLOW", StatusAction.ALLOW, action); 1104 } else if (locale.equalsIgnoreCase("vo")) { 1105 assertEquals( 1106 "vo ==> FORBID_READONLY", 1107 StatusAction.FORBID_READONLY, 1108 action); 1109 } else if (dummyPathValueInfo.getBaselineValue() == null) { 1110 if (!assertEquals( 1111 "missing ==> ALLOW", StatusAction.ALLOW, action)) { 1112 warnln("\t\t" + locale + "\t" + ph); 1113 } 1114 } 1115 } 1116 1117 if (isVerbose()) { 1118 Key key = new Key(locale, phase, status, surveyToolStatus, action); 1119 counter.add(key, 1); 1120 1121 if (!actionToExamplePath.containsKey(key)) { 1122 // for debugging 1123 if (locale.equals("vo") && action == StatusAction.ALLOW) { 1124 StatusAction action2 = 1125 phase.getShowRowAction( 1126 dummyPathValueInfo, 1127 InputMethod.DIRECT, 1128 ph, 1129 dummyUserInfo); 1130 } 1131 actionToExamplePath.put( 1132 key, 1133 Pair.of( 1134 dummyPathValueInfo.getBaselineValue() != null, 1135 path)); 1136 } 1137 } 1138 } 1139 } 1140 } 1141 } 1142 if (isVerbose()) { 1143 for (Entry<Key, Pair<Boolean, String>> entry : actionToExamplePath.entrySet()) { 1144 System.out.print( 1145 "\n" 1146 + entry.getKey() 1147 + "\t" 1148 + entry.getValue().getFirst() 1149 + "\t" 1150 + entry.getValue().getSecond()); 1151 } 1152 System.out.println(); 1153 for (R2<Long, Key> entry : counter.getEntrySetSortedByCount(false, null)) { 1154 System.out.println(entry.get0() + "\t" + entry.get1()); 1155 } 1156 } 1157 } 1158 1159 static class Key extends R5<Phase, CheckStatus.Type, SurveyToolStatus, StatusAction, String> { Key( String locale, Phase phase, CheckStatus.Type status, SurveyToolStatus stStatus, StatusAction action)1160 public Key( 1161 String locale, 1162 Phase phase, 1163 CheckStatus.Type status, 1164 SurveyToolStatus stStatus, 1165 StatusAction action) { 1166 super(phase, status, stStatus, action, locale); 1167 } 1168 1169 @Override toString()1170 public String toString() { 1171 return get0() + "\t" + get1() + "\t" + get2() + "\t" + get3() + "\t" + get4(); 1172 } 1173 } 1174 1175 // private static CLDRURLS URLS = testInfo.urls(); 1176 1177 private static final PathHeader.Factory pathHeaderFactory = 1178 PathHeader.getFactory(testInfo.getEnglish()); 1179 1180 // private static final CoverageInfo coverageInfo = new 1181 // CoverageInfo(testInfo.getSupplementalDataInfo()); 1182 1183 private static final VoterInfo dummyVoterInfo = 1184 new VoterInfo( 1185 Organization.cldr, org.unicode.cldr.util.VoteResolver.Level.vetter, "somename"); 1186 1187 private static final UserInfo dummyUserInfo = 1188 new UserInfo() { 1189 @Override 1190 public VoterInfo getVoterInfo() { 1191 return dummyVoterInfo; 1192 } 1193 }; 1194 1195 public static class DummyPathValueInfo implements PathValueInfo { 1196 private CLDRLocale locale; 1197 private String xpath; 1198 private String baselineValue; 1199 private CheckStatus.Type checkStatusType; 1200 1201 private CandidateInfo candidateInfo = 1202 new CandidateInfo() { 1203 @Override 1204 public String getValue() { 1205 return null; 1206 } 1207 1208 @Override 1209 public Collection<UserInfo> getUsersVotingOn() { 1210 throw new UnsupportedOperationException(); 1211 } 1212 1213 @Override 1214 public List<CheckStatus> getCheckStatusList() { 1215 return checkStatusType == null 1216 ? Collections.emptyList() 1217 : Collections.singletonList( 1218 new CheckStatus().setMainType(checkStatusType)); 1219 } 1220 }; 1221 1222 @Override getValues()1223 public Collection<? extends CandidateInfo> getValues() { 1224 throw new UnsupportedOperationException(); 1225 } 1226 1227 @Override getCurrentItem()1228 public CandidateInfo getCurrentItem() { 1229 return candidateInfo; 1230 } 1231 1232 @Override getBaselineValue()1233 public String getBaselineValue() { 1234 return baselineValue; 1235 } 1236 1237 @Override getCoverageLevel()1238 public Level getCoverageLevel() { 1239 return Level.MODERN; 1240 } 1241 1242 @Override hadVotesSometimeThisRelease()1243 public boolean hadVotesSometimeThisRelease() { 1244 throw new UnsupportedOperationException(); 1245 } 1246 1247 @Override getLocale()1248 public CLDRLocale getLocale() { 1249 return locale; 1250 } 1251 1252 @Override getXpath()1253 public String getXpath() { 1254 return xpath; 1255 } 1256 setLocale(CLDRLocale locale)1257 public void setLocale(CLDRLocale locale) { 1258 this.locale = locale; 1259 } 1260 setXpath(String xpath)1261 public void setXpath(String xpath) { 1262 this.xpath = xpath; 1263 } 1264 setBaselineValue(String baselineValue)1265 public void setBaselineValue(String baselineValue) { 1266 this.baselineValue = baselineValue; 1267 } 1268 } 1269 1270 final Set<String> cldrLocales = 1271 StandardCodes.make().getLocaleCoverageLocales(Organization.cldr); 1272 final Map<String, Status> validity = Validity.getInstance().getCodeToStatus(LstrType.language); 1273 final Map<String, R2<List<String>, String>> langAliases = 1274 CLDRConfig.getInstance().getSupplementalDataInfo().getLocaleAliasInfo().get("language"); 1275 final Set<String> existingLocales = 1276 CLDRConfig.getInstance().getCommonAndSeedAndMainAndAnnotationsFactory().getAvailable(); 1277 final LikelySubtags likely = new LikelySubtags(); 1278 1279 /** Simple check on locales and paths for limited submissions */ TestSubmissionLocales()1280 public void TestSubmissionLocales() { 1281 1282 for (String locale : SubmissionLocales.ALLOW_ALL_PATHS_BASIC) { 1283 checkLocaleOk(locale, false); 1284 } 1285 for (String locale : SubmissionLocales.LOCALES_FOR_LIMITED) { 1286 checkLocaleOk(locale, true); 1287 } 1288 } 1289 1290 /** 1291 * Check that the locale is valid, that it is either in or out of allowed locales, and that the 1292 * locale is not deprecated 1293 * 1294 * @param language 1295 * @param expectedInCLDRLocales 1296 */ checkLocaleOk(final String locale, final boolean expectedInCLDRLocales)1297 private void checkLocaleOk(final String locale, final boolean expectedInCLDRLocales) { 1298 final LanguageTagParser ltp = new LanguageTagParser().set(locale); 1299 String language = ltp.getLanguage(); 1300 1301 Status status = validity.get(language); 1302 assertTrue( 1303 language + " valid?", status == Status.regular); // || status == Status.macroregion 1304 1305 final R2<List<String>, String> alias = langAliases.get(language); 1306 if (!assertNull(language + " language is not deprecated", alias)) { 1307 errln(language + ": " + alias); 1308 } 1309 1310 // locale tests 1311 if (!assertTrue( 1312 locale + " locale is in common or seed", existingLocales.contains(locale))) { 1313 return; 1314 } 1315 if (expectedInCLDRLocales) { 1316 assertTrue(locale + " is in cldrLocales", cldrLocales.contains(locale)); 1317 } 1318 } 1319 1320 final String UNIT_PATH = "//ldml/units/unitLength[@type=\"long\"]"; 1321 1322 enum LimitedStatus { 1323 allowedUnitMissing, 1324 allowedUnitNotMissing, 1325 allowedOtherMissing, 1326 allowedOtherNotMissing, 1327 disallowed; 1328 of(boolean unit, boolean missing)1329 static LimitedStatus of(boolean unit, boolean missing) { 1330 if (unit && missing) { 1331 return allowedUnitMissing; 1332 } else if (unit && !missing) { 1333 return allowedUnitNotMissing; 1334 } else if (!unit && missing) { 1335 return allowedOtherMissing; 1336 } else { 1337 return allowedOtherNotMissing; 1338 } 1339 } 1340 } 1341 1342 /** Depends on correct values in the above constants. */ TestALLOWED_IN_LIMITED_PATHS()1343 public void TestALLOWED_IN_LIMITED_PATHS() { 1344 if (!CheckCLDR.LIMITED_SUBMISSION) { 1345 return; 1346 } 1347 1348 /** 1349 * Note: Constants moved from here to data driven test. 1350 * 1351 * <p>see org.unicode.cldr.test.TestSubmissionLocales and TestSubmissionLocales.csv 1352 */ 1353 if (SHOW_LIMITED) { 1354 System.out.println(); 1355 for (String locale : cldrFactoryWithSeed.getAvailable()) { 1356 LanguageTagParser ltp = new LanguageTagParser(); 1357 if (!ltp.set(locale).getRegion().isEmpty() 1358 || !ltp.set(locale).getVariants().isEmpty() 1359 || locale.equals("root")) { 1360 continue; 1361 } 1362 CLDRFile cldrFile = cldrFactoryWithSeed.make(locale, false); 1363 Level cldrLevel = 1364 StandardCodes.make().getLocaleCoverageLevel(Organization.cldr, locale); 1365 // patch until Rohingya is added 1366 if (cldrLevel == Level.UNDETERMINED && locale.equals("rhg")) { 1367 cldrLevel = Level.BASIC; 1368 } 1369 Counter<LimitedStatus> counter = new Counter<>(); 1370 for (String path : cldrFile.fullIterable()) { 1371 Level coverage = 1372 SupplementalDataInfo.getInstance().getCoverageLevel(path, locale); 1373 if (coverage.compareTo(cldrLevel) > 0) { 1374 continue; 1375 } 1376 String value = cldrFile.getStringValue(path); 1377 boolean isMissing = value == null; 1378 boolean allowed = 1379 SubmissionLocales.allowEvenIfLimited(locale, path, false, isMissing); 1380 if (allowed) { 1381 boolean isUnit = path.startsWith(UNIT_PATH); 1382 counter.add(LimitedStatus.of(isUnit, isMissing), 1); 1383 } else { 1384 counter.add(LimitedStatus.disallowed, 1); 1385 } 1386 } 1387 System.out.print(locale + "\t" + english.getName(locale) + "\t" + cldrLevel); 1388 for (LimitedStatus limitedStatus : LimitedStatus.values()) { 1389 System.out.print("\t" + limitedStatus + ":\t" + counter.get(limitedStatus)); 1390 } 1391 System.out.println(); 1392 } 1393 } else { 1394 warnln("Set -DTestCheckCLDR:SHOW_LIMITED to see information about affected paths."); 1395 } 1396 } 1397 TestInfohubLinks13979()1398 public void TestInfohubLinks13979() { 1399 CLDRFile root = cldrFactory.make("root", true); 1400 List<CheckStatus> possibleErrors = new ArrayList<>(); 1401 String[][] tests = { 1402 // test edge cases 1403 // locale, path, value, expected 1404 { 1405 "fr", 1406 "//ldml/numbers/minimalPairs/genderMinimalPairs[@gender=\"feminine\"]", 1407 "de genre féminin", 1408 "Need at least 1 placeholder(s), but only have 0. Placeholders are: {{0}={GENDER}, e.g. “‹noun phrase in this gender›”}; see <a href='http://cldr.unicode.org/translation/error-codes#missingPlaceholders' target='cldr_error_codes'>missing placeholders</a>." 1409 }, 1410 }; 1411 for (String[] row : tests) { 1412 String localeId = row[0]; 1413 String path = row[1]; 1414 String value = row[2]; 1415 String expected = row[3]; 1416 1417 checkPathValue(root, localeId, path, value, possibleErrors); 1418 for (CheckStatus error : possibleErrors) { 1419 if (error.getSubtype() == Subtype.missingPlaceholders) { 1420 assertEquals("message", expected, error.getMessage()); 1421 } 1422 } 1423 } 1424 } 1425 Test14866()1426 public void Test14866() { 1427 final SupplementalDataInfo supplementalDataInfo = SupplementalDataInfo.getInstance(); 1428 String locale = "pl"; 1429 int expectedCount = 14; 1430 1431 GrammarInfo grammarInfo = supplementalDataInfo.getGrammarInfo(locale); 1432 logln("Locale:\t" + locale + "\n\tGrammarInfo:\t" + grammarInfo); 1433 CLDRFile pl = factory.make(locale, true); 1434 System.out.println(""); 1435 Collection<PathHeader> pathHeaders = new TreeSet<>(); // new ArrayList(); // 1436 for (String path : pl.fullIterable()) { 1437 if (path.startsWith( 1438 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"duration-century\"]")) { 1439 PathHeader pathHeader = PathHeader.getFactory().fromPath(path); 1440 boolean added = pathHeaders.add(pathHeader); 1441 } 1442 } 1443 int count = 0; 1444 for (PathHeader pathHeader : pathHeaders) { 1445 String path = pathHeader.getOriginalPath(); 1446 String value = pl.getStringValue(path); 1447 CLDRFile.Status status = new CLDRFile.Status(); 1448 String localeFound = pl.getSourceLocaleID(path, status); 1449 Level level = supplementalDataInfo.getCoverageLevel(path, locale); 1450 logln( 1451 "\n\t" 1452 + ++count 1453 + " Locale:\t" 1454 + locale 1455 + "\n\tLocaleFound:\t" 1456 + (locale.equals(localeFound) ? "«same»" : localeFound) 1457 + "\n\tPathHeader:\t" 1458 + pathHeader 1459 + "\n\tPath: \t" 1460 + path 1461 + "\n\tPathFound:\t" 1462 + (path.equals(status.pathWhereFound) 1463 ? "«same»" 1464 : status.pathWhereFound) 1465 + "\n\tValue:\t" 1466 + value 1467 + "\n\tLevel:\t" 1468 + level); 1469 } 1470 assertEquals("right number of elements found", expectedCount, count); 1471 } 1472 } 1473