1 package org.unicode.cldr.unittest; 2 3 import java.io.File; 4 import java.util.ArrayList; 5 import java.util.Arrays; 6 import java.util.Collection; 7 import java.util.EnumMap; 8 import java.util.EnumSet; 9 import java.util.HashMap; 10 import java.util.HashSet; 11 import java.util.LinkedHashMap; 12 import java.util.LinkedHashSet; 13 import java.util.List; 14 import java.util.Map; 15 import java.util.Map.Entry; 16 import java.util.Set; 17 import java.util.TreeMap; 18 import java.util.TreeSet; 19 import java.util.regex.Matcher; 20 21 import org.unicode.cldr.test.CoverageLevel2; 22 import org.unicode.cldr.test.ExampleGenerator; 23 import org.unicode.cldr.util.CLDRConfig; 24 import org.unicode.cldr.util.CLDRFile; 25 import org.unicode.cldr.util.CLDRFile.Status; 26 import org.unicode.cldr.util.CLDRPaths; 27 import org.unicode.cldr.util.CldrUtility; 28 import org.unicode.cldr.util.Containment; 29 import org.unicode.cldr.util.Counter; 30 import org.unicode.cldr.util.DtdData; 31 import org.unicode.cldr.util.DtdType; 32 import org.unicode.cldr.util.Factory; 33 import org.unicode.cldr.util.LanguageTagParser; 34 import org.unicode.cldr.util.Level; 35 import org.unicode.cldr.util.Organization; 36 import org.unicode.cldr.util.Pair; 37 import org.unicode.cldr.util.PathDescription; 38 import org.unicode.cldr.util.PathHeader; 39 import org.unicode.cldr.util.PathHeader.PageId; 40 import org.unicode.cldr.util.PathHeader.SectionId; 41 import org.unicode.cldr.util.PathHeader.SurveyToolStatus; 42 import org.unicode.cldr.util.PathStarrer; 43 import org.unicode.cldr.util.PatternCache; 44 import org.unicode.cldr.util.PatternPlaceholders; 45 import org.unicode.cldr.util.PatternPlaceholders.PlaceholderInfo; 46 import org.unicode.cldr.util.PatternPlaceholders.PlaceholderStatus; 47 import org.unicode.cldr.util.PrettyPath; 48 import org.unicode.cldr.util.StandardCodes; 49 import org.unicode.cldr.util.SupplementalDataInfo; 50 import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo; 51 import org.unicode.cldr.util.SupplementalDataInfo.PluralType; 52 import org.unicode.cldr.util.With; 53 import org.unicode.cldr.util.XMLFileReader; 54 import org.unicode.cldr.util.XPathParts; 55 56 import com.google.common.collect.HashMultimap; 57 import com.google.common.collect.ImmutableSet; 58 import com.google.common.collect.LinkedListMultimap; 59 import com.google.common.collect.Multimap; 60 import com.google.common.collect.TreeMultimap; 61 import com.ibm.icu.dev.util.CollectionUtilities; 62 import com.ibm.icu.impl.Relation; 63 import com.ibm.icu.impl.Row; 64 import com.ibm.icu.impl.Row.R2; 65 66 public class TestPathHeader extends TestFmwkPlus { 67 private static final DtdType DEBUG_DTD_TYPE = null; // DtdType.supplementalData; 68 private static final String COMMON_DIR = CLDRPaths.BASE_DIRECTORY + "common/"; 69 private static final boolean DEBUG = false; 70 main(String[] args)71 public static void main(String[] args) { 72 new TestPathHeader().run(args); 73 } 74 75 static final CLDRConfig info = CLDRConfig.getInstance(); 76 static final Factory factory = info.getCommonAndSeedAndMainAndAnnotationsFactory(); 77 static final CLDRFile english = factory.make("en", true); 78 static final SupplementalDataInfo supplemental = info 79 .getSupplementalDataInfo(); 80 static PathHeader.Factory pathHeaderFactory = PathHeader 81 .getFactory(english); 82 private EnumSet<PageId> badZonePages = EnumSet.of(PageId.UnknownT); 83 tempTestAnnotation()84 public void tempTestAnnotation() { 85 // NEW: <annotation cp="��">face | grin</annotation> 86 // <annotation cp="��" type="tts">grinning face</annotation> 87 88 final String path1 = "//ldml/annotations/annotation[@cp=\"��\"]"; 89 PathHeader ph1 = pathHeaderFactory.fromPath(path1); 90 logln(ph1.toString() + "\t" + path1); 91 final String path2 = "//ldml/annotations/annotation[@cp=\"��\"][@type=\"tts\"]"; 92 PathHeader ph2 = pathHeaderFactory.fromPath(path2); 93 logln(ph2.toString() + "\t" + path2); 94 final String path3 = "//ldml/annotations/annotation[@cp=\"��\"]"; 95 PathHeader ph3 = pathHeaderFactory.fromPath(path2); 96 logln(ph3.toString() + "\t" + path3); 97 98 assertNotEquals("pathheader", ph1, ph2); 99 assertNotEquals("pathheader", ph1.toString(), ph2.toString()); 100 assertRelation("pathheader", true, ph1, TestFmwkPlus.LEQ, ph3); 101 assertRelation("pathheader", true, ph3, TestFmwkPlus.LEQ, ph2); 102 } 103 tempTestCompletenessLdmlDtd()104 public void tempTestCompletenessLdmlDtd() { 105 // List<String> failures = null; 106 pathHeaderFactory.clearCache(); 107 PathChecker pathChecker = new PathChecker(); 108 for (String directory : DtdType.ldml.directories) { 109 Factory factory2 = CLDRConfig.getInstance().getMainAndAnnotationsFactory(); 110 Set<String> source = factory2.getAvailable(); 111 for (String file : getFilesToTest(source, "root", "en", "da")) { 112 if (DEBUG) warnln(" TestCompletenessLdmlDtd: " + directory + ", " + file); 113 DtdData dtdData = null; 114 CLDRFile cldrFile = factory2.make(file, true); 115 for (String path : cldrFile.fullIterable()) { 116 pathChecker.checkPathHeader(cldrFile.getDtdData(), path); 117 } 118 } 119 } 120 Set<String> missing = pathHeaderFactory.getUnmatchedRegexes(); 121 if (missing.size() != 0) { 122 for (String e : missing) { 123 errln("Path Regex never matched:\t" + e); 124 } 125 } 126 } 127 getFilesToTest(Collection<String> source, String... doFirst)128 private Collection<String> getFilesToTest(Collection<String> source, String... doFirst) { 129 LinkedHashSet<String> files = new LinkedHashSet<>(Arrays.asList(doFirst)); 130 files.retainAll(source); // put first 131 files.addAll(new HashSet<>(source)); // now add others semi-randomly 132 int max = Math.min(30, files.size()); 133 if (getInclusion() == 10 || files.size() <= max) { 134 return files; 135 } 136 ArrayList<String> shortFiles = new ArrayList<>(files); 137 if (getInclusion() > 5) { 138 max += (files.size() - 30) * (getInclusion() - 5) / 10; // use proportional amount 139 } 140 return shortFiles.subList(0, max); 141 } 142 TestCompleteness()143 public void TestCompleteness() { 144 PathHeader.Factory pathHeaderFactory2 = PathHeader.getFactory(english); 145 // List<String> failures = null; 146 pathHeaderFactory2.clearCache(); 147 Multimap<PathHeader.PageId, PathHeader.SectionId> pageUniqueness = TreeMultimap.create(); 148 Multimap<String, Pair<PathHeader.SectionId, PathHeader.PageId>> headerUniqueness = TreeMultimap.create(); 149 Set<String> toTest; 150 switch (getInclusion()) { 151 default: 152 toTest = StandardCodes.make().getLocaleCoverageLocales(Organization.cldr); 153 break; 154 case 10: 155 toTest = factory.getAvailable(); 156 break; 157 } 158 toTest = ImmutableSet.<String> builder().add("en").addAll(toTest).build(); 159 Set<String> seenPaths = new HashSet<>(); 160 Set<String> localSeenPaths = new TreeSet<>(); 161 for (String locale : toTest) { 162 localSeenPaths.clear(); 163 for (String p : factory.make(locale, true).fullIterable()) { 164 if (p.startsWith("//ldml/identity/")) { 165 continue; 166 } 167 if (seenPaths.contains(p)) { 168 continue; 169 } 170 seenPaths.add(p); 171 localSeenPaths.add(p); 172 // if (p.contains("symbol[@alt") && failures == null) { 173 // PathHeader result = pathHeaderFactory2.fromPath(p, failures = new 174 // ArrayList<String>()); 175 // logln("Matching " + p + ": " + result + "\t" + 176 // result.getSurveyToolStatus()); 177 // for (String failure : failures) { 178 // logln("\t" + failure); 179 // } 180 // } 181 PathHeader ph = pathHeaderFactory2.fromPath(p); 182 if (ph == null) { 183 errln("Failed to create path from: " + p); 184 continue; 185 } 186 final SectionId sectionId = ph.getSectionId(); 187 if (sectionId != SectionId.Special) { 188 pageUniqueness.put(ph.getPageId(), sectionId); 189 headerUniqueness.put(ph.getHeader(), new Pair<>(sectionId, ph.getPageId())); 190 } 191 } 192 if (!localSeenPaths.isEmpty()) { 193 logln(locale + ": checked " + localSeenPaths.size() + " new paths"); 194 } 195 } 196 Set<String> missing = pathHeaderFactory2.getUnmatchedRegexes(); 197 if (missing.size() != 0) { 198 for (String e : missing) { 199 if (e.contains("//ldml/")) { 200 if (e.contains("//ldml/rbnf/") || e.contains("//ldml/segmentations/") || e.contains("//ldml/collations/")) { 201 continue; 202 } 203 logln("Path Regex never matched:\t" + e); 204 } 205 } 206 } 207 208 for (Entry<PageId, Collection<SectionId>> e : pageUniqueness.asMap().entrySet()) { 209 Collection<SectionId> values = e.getValue(); 210 if (values.size() != 1) { 211 warnln("Duplicate page in section: " + CldrUtility.toString(e)); 212 } 213 } 214 215 for (Entry<String, Collection<Pair<SectionId, PageId>>> e : headerUniqueness.asMap().entrySet()) { 216 Collection<Pair<SectionId, PageId>> values = e.getValue(); 217 if (values.size() != 1) { 218 warnln("Duplicate header in (section,page): " + CldrUtility.toString(e)); 219 } 220 } 221 } 222 Test6170()223 public void Test6170() { 224 String p1 = "//ldml/units/unitLength[@type=\"narrow\"]/unit[@type=\"speed-kilometer-per-hour\"]/unitPattern[@count=\"other\"]"; 225 String p2 = "//ldml/units/unitLength[@type=\"narrow\"]/unit[@type=\"area-square-meter\"]/unitPattern[@count=\"other\"]"; 226 PathHeader ph1 = pathHeaderFactory.fromPath(p1); 227 PathHeader ph2 = pathHeaderFactory.fromPath(p2); 228 int comp12 = ph1.compareTo(ph2); 229 int comp21 = ph2.compareTo(ph1); 230 assertEquals("comp ph", comp12, -comp21); 231 } 232 TestVariant()233 public void TestVariant() { 234 PathHeader p1 = pathHeaderFactory 235 .fromPath("//ldml/localeDisplayNames/languages/language[@type=\"ug\"][@alt=\"variant\"]"); 236 PathHeader p2 = pathHeaderFactory 237 .fromPath("//ldml/localeDisplayNames/languages/language[@type=\"ug\"]"); 238 assertNotEquals("variants", p1, p2); 239 assertNotEquals("variants", p1.toString(), p2.toString()); 240 // Code Lists Languages Arabic Script ug-variant 241 } 242 Test4587()243 public void Test4587() { 244 String test = "//ldml/dates/timeZoneNames/metazone[@type=\"Pacific/Wallis\"]/short/standard"; 245 PathHeader ph = pathHeaderFactory.fromPath(test); 246 if (ph == null) { 247 errln("Failure with " + test); 248 } else { 249 logln(ph + "\t" + test); 250 } 251 } 252 TestMiscPatterns()253 public void TestMiscPatterns() { 254 String test = "//ldml/numbers/miscPatterns[@numberSystem=\"arab\"]/pattern[@type=\"atLeast\"]"; 255 PathHeader ph = pathHeaderFactory.fromPath(test); 256 assertNotNull("MiscPatterns path not found", ph); 257 if (false) 258 System.out.println(english.getStringValue(test)); 259 } 260 TestPluralOrder()261 public void TestPluralOrder() { 262 Set<PathHeader> sorted = new TreeSet<PathHeader>(); 263 for (String locale : new String[] { "ru", "ar", "ja" }) { 264 sorted.clear(); 265 CLDRFile cldrFile = info.getCLDRFile(locale, true); 266 CoverageLevel2 coverageLevel = CoverageLevel2.getInstance(locale); 267 for (String path : cldrFile.fullIterable()) { 268 if (!path.contains("@count")) { 269 continue; 270 } 271 Level level = coverageLevel.getLevel(path); 272 if (Level.MODERN.compareTo(level) < 0) { 273 continue; 274 } 275 PathHeader p = pathHeaderFactory.fromPath(path); 276 sorted.add(p); 277 } 278 for (PathHeader p : sorted) { 279 logln(locale + "\t" + p + "\t" + p.getOriginalPath()); 280 } 281 } 282 } 283 284 static final String APPEND_TIMEZONE = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/appendItems/appendItem[@request=\"Timezone\"]"; 285 static final String APPEND_TIMEZONE_END = "/dateTimeFormats/appendItems/appendItem[@request=\"Timezone\"]"; 286 static final String BEFORE_PH = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\"ms\"]"; 287 static final String AFTER_PH = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"d\"]/greatestDifference[@id=\"d\"]"; 288 TestAppendTimezone()289 public void TestAppendTimezone() { 290 CLDRFile cldrFile = info.getEnglish(); 291 CoverageLevel2 coverageLevel = CoverageLevel2.getInstance("en"); 292 assertEquals("appendItem:Timezone", Level.MODERATE, 293 coverageLevel.getLevel(APPEND_TIMEZONE)); 294 295 PathHeader ph = pathHeaderFactory.fromPath(APPEND_TIMEZONE); 296 assertEquals("appendItem:Timezone pathheader", "Timezone", ph.getCode()); 297 // check that they are in the right place (they weren't before!) 298 PathHeader phBefore = pathHeaderFactory.fromPath(BEFORE_PH); 299 PathHeader phAfter = pathHeaderFactory.fromPath(AFTER_PH); 300 assertTrue(phBefore, LEQ, ph); 301 assertTrue(ph, LEQ, phAfter); 302 303 PathDescription pathDescription = new PathDescription(supplemental, 304 english, null, null, PathDescription.ErrorHandling.CONTINUE); 305 String description = pathDescription.getDescription(APPEND_TIMEZONE, 306 "tempvalue", null, null); 307 assertTrue("appendItem:Timezone pathDescription", 308 description.contains("“Timezone”")); 309 310 PatternPlaceholders patternPlaceholders = PatternPlaceholders 311 .getInstance(); 312 PlaceholderStatus status = patternPlaceholders 313 .getStatus(APPEND_TIMEZONE); 314 assertEquals("appendItem:Timezone placeholders", 315 PlaceholderStatus.REQUIRED, status); 316 317 Map<String, PlaceholderInfo> placeholderInfo = patternPlaceholders 318 .get(APPEND_TIMEZONE); 319 PlaceholderInfo placeholderInfo2 = placeholderInfo.get("{1}"); 320 if (assertNotNull("appendItem:Timezone placeholders", placeholderInfo2)) { 321 assertEquals("appendItem:Timezone placeholders", 322 "APPEND_FIELD_FORMAT", placeholderInfo2.name); 323 assertEquals("appendItem:Timezone placeholders", "Pacific Time", 324 placeholderInfo2.example); 325 } 326 ExampleGenerator eg = new ExampleGenerator(cldrFile, cldrFile, 327 CLDRPaths.SUPPLEMENTAL_DIRECTORY); 328 String example = eg.getExampleHtml(APPEND_TIMEZONE, 329 cldrFile.getStringValue(APPEND_TIMEZONE)); 330 String result = ExampleGenerator.simplify(example, false); 331 assertEquals("", "〖❬6:25:59 PM❭ ❬GMT❭〗", result); 332 } 333 TestOptional()334 public void TestOptional() { 335 if (true) return; 336 Map<PathHeader, String> sorted = new TreeMap<PathHeader, String>(); 337 XPathParts parts = new XPathParts(); 338 for (String locale : new String[] { "af" }) { 339 sorted.clear(); 340 CLDRFile cldrFile = info.getCLDRFile(locale, true); 341 CoverageLevel2 coverageLevel = CoverageLevel2.getInstance(locale); 342 for (String path : cldrFile.fullIterable()) { 343 // if (!path.contains("@count")) { 344 // continue; 345 // } 346 Level level = coverageLevel.getLevel(path); 347 if (supplemental.isDeprecated(DtdType.ldml, path)) { 348 continue; 349 } 350 351 if (Level.OPTIONAL.compareTo(level) != 0) { 352 continue; 353 } 354 355 PathHeader p = pathHeaderFactory.fromPath(path); 356 final SurveyToolStatus status = p.getSurveyToolStatus(); 357 if (status == SurveyToolStatus.DEPRECATED) { 358 continue; 359 } 360 sorted.put( 361 p, 362 locale + "\t" + status + "\t" + p + "\t" 363 + p.getOriginalPath()); 364 } 365 Set<String> codes = new LinkedHashSet<String>(); 366 PathHeader old = null; 367 String line = null; 368 for (Entry<PathHeader, String> s : sorted.entrySet()) { 369 PathHeader p = s.getKey(); 370 String v = s.getValue(); 371 if (old == null) { 372 line = v; 373 codes.add(p.getCode()); 374 } else if (p.getSectionId() == old.getSectionId() 375 && p.getPageId() == old.getPageId() 376 && p.getHeader().equals(old.getHeader())) { 377 codes.add(p.getCode()); 378 } else { 379 logln(line + "\t" + codes.toString()); 380 codes.clear(); 381 line = v; 382 codes.add(p.getCode()); 383 } 384 old = p; 385 } 386 logln(line + "\t" + codes.toString()); 387 } 388 } 389 TestPluralCanonicals()390 public void TestPluralCanonicals() { 391 Relation<String, String> data = Relation.of( 392 new LinkedHashMap<String, Set<String>>(), TreeSet.class); 393 for (String locale : factory.getAvailable()) { 394 if (locale.contains("_")) { 395 continue; 396 } 397 PluralInfo info = supplemental.getPlurals(PluralType.cardinal, 398 locale); 399 Set<String> keywords = info.getCanonicalKeywords(); 400 data.put(keywords.toString(), locale); 401 } 402 for (Entry<String, Set<String>> entry : data.keyValuesSet()) { 403 logln(entry.getKey() + "\t" + entry.getValue()); 404 } 405 } 406 TestPluralPaths()407 public void TestPluralPaths() { 408 // do the following line once, when the file is opened 409 Set<String> filePaths = pathHeaderFactory.pathsForFile(english); 410 411 // check that English doesn't contain few or many 412 verifyContains(PageId.Duration, filePaths, "few", false); 413 verifyContains(PageId.C_NAmerica, filePaths, "many", false); 414 verifyContains(PageId.C_SAmerica, filePaths, "many", false); 415 verifyContains(PageId.C_NWEurope, filePaths, "many", false); 416 verifyContains(PageId.C_SEEurope, filePaths, "many", false); 417 verifyContains(PageId.C_NAfrica, filePaths, "many", false); 418 verifyContains(PageId.C_WAfrica, filePaths, "many", false); 419 verifyContains(PageId.C_SAfrica, filePaths, "many", false); 420 verifyContains(PageId.C_EAfrica, filePaths, "many", false); 421 verifyContains(PageId.C_CAsia, filePaths, "many", false); 422 verifyContains(PageId.C_WAsia, filePaths, "many", false); 423 verifyContains(PageId.C_SEAsia, filePaths, "many", false); 424 verifyContains(PageId.C_Oceania, filePaths, "many", false); 425 verifyContains(PageId.C_Unknown, filePaths, "many", false); 426 427 // check that Arabic does contain few and many 428 filePaths = pathHeaderFactory.pathsForFile(info.getCLDRFile("ar", true)); 429 430 verifyContains(PageId.Duration, filePaths, "few", true); 431 verifyContains(PageId.C_NAmerica, filePaths, "many", true); 432 verifyContains(PageId.C_SAmerica, filePaths, "many", true); 433 verifyContains(PageId.C_NWEurope, filePaths, "many", true); 434 verifyContains(PageId.C_SEEurope, filePaths, "many", true); 435 verifyContains(PageId.C_NAfrica, filePaths, "many", true); 436 verifyContains(PageId.C_WAfrica, filePaths, "many", true); 437 verifyContains(PageId.C_SAfrica, filePaths, "many", true); 438 verifyContains(PageId.C_EAfrica, filePaths, "many", true); 439 verifyContains(PageId.C_CAsia, filePaths, "many", true); 440 verifyContains(PageId.C_WAsia, filePaths, "many", true); 441 verifyContains(PageId.C_SEAsia, filePaths, "many", true); 442 verifyContains(PageId.C_Oceania, filePaths, "many", true); 443 verifyContains(PageId.C_Unknown, filePaths, "many", true); 444 } 445 TestCoverage()446 public void TestCoverage() { 447 Map<Row.R2<SectionId, PageId>, Counter<Level>> data = new TreeMap<Row.R2<SectionId, PageId>, Counter<Level>>(); 448 CLDRFile cldrFile = english; 449 for (String path : cldrFile.fullIterable()) { 450 if (supplemental.isDeprecated(DtdType.ldml, path)) { 451 errln("Deprecated path in English: " + path); 452 continue; 453 } 454 Level level = supplemental.getCoverageLevel(path, 455 cldrFile.getLocaleID()); 456 PathHeader p = pathHeaderFactory.fromPath(path); 457 SurveyToolStatus status = p.getSurveyToolStatus(); 458 459 boolean hideCoverage = level == Level.OPTIONAL; 460 boolean hidePathHeader = status == SurveyToolStatus.DEPRECATED 461 || status == SurveyToolStatus.HIDE; 462 if (hidePathHeader != hideCoverage) { 463 String message = "PathHeader: " + status + ", Coverage: " 464 + level + ": " + path; 465 if (hidePathHeader && !hideCoverage) { 466 errln(message); 467 } else if (!hidePathHeader && hideCoverage) { 468 logln(message); 469 } 470 } 471 final R2<SectionId, PageId> key = Row.of(p.getSectionId(), 472 p.getPageId()); 473 Counter<Level> counter = data.get(key); 474 if (counter == null) { 475 data.put(key, counter = new Counter<Level>()); 476 } 477 counter.add(level, 1); 478 } 479 StringBuffer b = new StringBuffer("\t"); 480 for (Level level : Level.values()) { 481 b.append("\t" + level); 482 } 483 logln(b.toString()); 484 for (Entry<R2<SectionId, PageId>, Counter<Level>> entry : data 485 .entrySet()) { 486 b.setLength(0); 487 b.append(entry.getKey().get0() + "\t" + entry.getKey().get1()); 488 Counter<Level> counter = entry.getValue(); 489 long total = 0; 490 for (Level level : Level.values()) { 491 total += counter.getCount(level); 492 b.append("\t" + total); 493 } 494 logln(b.toString()); 495 } 496 } 497 Test00AFile()498 public void Test00AFile() { 499 final String localeId = "en"; 500 Counter<Level> counter = new Counter<Level>(); 501 Map<String, PathHeader> uniqueness = new HashMap<String, PathHeader>(); 502 Set<String> alreadySeen = new HashSet<String>(); 503 check(localeId, true, uniqueness, alreadySeen); 504 // check paths 505 for (Entry<SectionId, Set<PageId>> sectionAndPages : PathHeader.Factory 506 .getSectionIdsToPageIds().keyValuesSet()) { 507 final SectionId section = sectionAndPages.getKey(); 508 if (section == SectionId.Supplemental || section == SectionId.BCP47) { 509 continue; 510 } 511 logln(section.toString()); 512 for (PageId page : sectionAndPages.getValue()) { 513 final Set<String> cachedPaths = PathHeader.Factory 514 .getCachedPaths(section, page); 515 if (cachedPaths == null) { 516 if (!badZonePages.contains(page) && page != PageId.Unknown) { 517 errln("Null pages for: " + section + "\t" + page); 518 } 519 } else if (section == SectionId.Special 520 && page == PageId.Unknown) { 521 // skip 522 } else if (section == SectionId.Timezones 523 && page == PageId.UnknownT) { 524 // skip 525 } else if (section == SectionId.Misc 526 && page == PageId.Transforms) { 527 // skip 528 } else { 529 530 int count2 = cachedPaths.size(); 531 if (count2 == 0) { 532 warnln("Missing pages for: " + section + "\t" + page); 533 } else { 534 counter.clear(); 535 for (String s : cachedPaths) { 536 Level coverage = supplemental.getCoverageLevel(s, 537 localeId); 538 counter.add(coverage, 1); 539 } 540 String countString = ""; 541 int total = 0; 542 for (Level item : Level.values()) { 543 long count = counter.get(item); 544 if (count != 0) { 545 if (!countString.isEmpty()) { 546 countString += ",\t+"; 547 } 548 total += count; 549 countString += item + "=" + total; 550 } 551 } 552 logln("\t" + page + "\t" + countString); 553 if (page.toString().startsWith("Unknown")) { 554 logln("\t\t" + cachedPaths); 555 } 556 } 557 } 558 } 559 } 560 } 561 TestMetazones()562 public void TestMetazones() { 563 564 CLDRFile nativeFile = info.getEnglish(); 565 Set<PathHeader> pathHeaders = getPathHeaders(nativeFile); 566 // String oldPage = ""; 567 String oldHeader = ""; 568 for (PathHeader entry : pathHeaders) { 569 final String page = entry.getPage(); 570 // if (!oldPage.equals(page)) { 571 // logln(page); 572 // oldPage = page; 573 // } 574 String header = entry.getHeader(); 575 if (!oldHeader.equals(header)) { 576 logln(page + "\t" + header); 577 oldHeader = header; 578 } 579 } 580 } 581 getPathHeaders(CLDRFile nativeFile)582 public Set<PathHeader> getPathHeaders(CLDRFile nativeFile) { 583 Set<PathHeader> pathHeaders = new TreeSet<PathHeader>(); 584 for (String path : nativeFile.fullIterable()) { 585 PathHeader p = pathHeaderFactory.fromPath(path); 586 pathHeaders.add(p); 587 } 588 return pathHeaders; 589 } 590 verifyContains(PageId pageId, Set<String> filePaths, String substring, boolean contains)591 public void verifyContains(PageId pageId, Set<String> filePaths, 592 String substring, boolean contains) { 593 String path; 594 path = findOneContaining(allPaths(pageId, filePaths), substring); 595 if (contains) { 596 if (path == null) { 597 errln("No path contains <" + substring + ">"); 598 } 599 } else { 600 if (path != null) { 601 errln("Path contains <" + substring + ">\t" + path); 602 } 603 } 604 } 605 findOneContaining(Collection<String> allPaths, String substring)606 private String findOneContaining(Collection<String> allPaths, 607 String substring) { 608 for (String path : allPaths) { 609 if (path.contains(substring)) { 610 return path; 611 } 612 } 613 return null; 614 } 615 allPaths(PageId pageId, Set<String> filePaths)616 public Set<String> allPaths(PageId pageId, Set<String> filePaths) { 617 Set<String> result = PathHeader.Factory.getCachedPaths( 618 pageId.getSectionId(), pageId); 619 result.retainAll(filePaths); 620 return result; 621 } 622 TestUniqueness()623 public void TestUniqueness() { 624 CLDRFile nativeFile = info.getEnglish(); 625 Map<PathHeader, String> headerToPath = new HashMap<PathHeader, String>(); 626 Map<String, String> headerVisibleToPath = new HashMap<String, String>(); 627 for (String path : nativeFile.fullIterable()) { 628 PathHeader p = pathHeaderFactory.fromPath(path); 629 if (p.getSectionId() == SectionId.Special) { 630 continue; 631 } 632 String old = headerToPath.get(p); 633 if (old == null) { 634 headerToPath.put(p, path); 635 } else if (!old.equals(path)) { 636 if (true) { // for debugging 637 pathHeaderFactory.clearCache(); 638 List<String> failuresOld = new ArrayList<>(); 639 pathHeaderFactory.fromPath(old, failuresOld); 640 List<String> failuresPath = new ArrayList<>(); 641 pathHeaderFactory.fromPath(path, failuresPath); 642 } 643 errln("Collision with path " + p + "\t" + old + "\t" + path); 644 } 645 final String visible = p.toString(); 646 old = headerVisibleToPath.get(visible); 647 if (old == null) { 648 headerVisibleToPath.put(visible, path); 649 } else if (!old.equals(path)) { 650 errln("Collision with path " + visible + "\t" + old + "\t" 651 + path); 652 } 653 } 654 } 655 TestStatus()656 public void TestStatus() { 657 CLDRFile nativeFile = info.getEnglish(); 658 PathStarrer starrer = new PathStarrer(); 659 EnumMap<SurveyToolStatus, Relation<String, String>> info2 = new EnumMap<SurveyToolStatus, Relation<String, String>>( 660 SurveyToolStatus.class); 661 Set<String> nuked = new HashSet<String>(); 662 PrettyPath pp = new PrettyPath(); 663 XPathParts parts = new XPathParts(); 664 Set<String> deprecatedStar = new HashSet<String>(); 665 Set<String> differentStar = new HashSet<String>(); 666 667 for (String path : nativeFile.fullIterable()) { 668 669 PathHeader p = pathHeaderFactory.fromPath(path); 670 final SurveyToolStatus surveyToolStatus = p.getSurveyToolStatus(); 671 672 if (p.getSectionId() == SectionId.Special 673 && surveyToolStatus == SurveyToolStatus.READ_WRITE) { 674 errln("SurveyToolStatus should not be " + surveyToolStatus 675 + ": " + p); 676 } 677 678 final SurveyToolStatus tempSTS = surveyToolStatus == SurveyToolStatus.DEPRECATED ? SurveyToolStatus.HIDE 679 : surveyToolStatus; 680 String starred = starrer.set(path); 681 List<String> attr = starrer.getAttributes(); 682 if (surveyToolStatus != SurveyToolStatus.READ_WRITE) { 683 nuked.add(starred); 684 } 685 686 // check against old 687 SurveyToolStatus oldStatus = SurveyToolStatus.READ_WRITE; 688 689 if (tempSTS != oldStatus 690 && oldStatus != SurveyToolStatus.READ_WRITE 691 && !path.endsWith(APPEND_TIMEZONE_END)) { 692 if (!differentStar.contains(starred)) { 693 errln("Different from old:\t" + oldStatus + "\tnew:\t" 694 + surveyToolStatus + "\t" + path); 695 differentStar.add(starred); 696 } 697 } 698 699 // check against deprecated 700 boolean isDeprecated = supplemental.isDeprecated(DtdType.ldml, path); 701 if (isDeprecated != (surveyToolStatus == SurveyToolStatus.DEPRECATED)) { 702 if (!deprecatedStar.contains(starred)) { 703 errln("Different from DtdData deprecated:\t" 704 + isDeprecated + "\t" + surveyToolStatus + "\t" 705 + path); 706 deprecatedStar.add(starred); 707 } 708 } 709 710 Relation<String, String> data = info2.get(surveyToolStatus); 711 if (data == null) { 712 info2.put( 713 surveyToolStatus, 714 data = Relation.of(new TreeMap<String, Set<String>>(), 715 TreeSet.class)); 716 } 717 data.put(starred, CollectionUtilities.join(attr, "|")); 718 } 719 for (Entry<SurveyToolStatus, Relation<String, String>> entry : info2 720 .entrySet()) { 721 final SurveyToolStatus status = entry.getKey(); 722 for (Entry<String, Set<String>> item : entry.getValue() 723 .keyValuesSet()) { 724 final String starred = item.getKey(); 725 if (status == SurveyToolStatus.READ_WRITE 726 && !nuked.contains(starred)) { 727 continue; 728 } 729 logln(status + "\t" + starred + "\t" + item.getValue()); 730 } 731 } 732 } 733 TestPathsNotInEnglish()734 public void TestPathsNotInEnglish() { 735 Set<String> englishPaths = new HashSet<String>(); 736 for (String path : english.fullIterable()) { 737 englishPaths.add(path); 738 } 739 Set<String> alreadySeen = new HashSet<String>(englishPaths); 740 741 for (String locale : factory.getAvailable()) { 742 CLDRFile nativeFile = info.getCLDRFile(locale, false); 743 CoverageLevel2 coverageLevel2 = null; 744 for (String path : nativeFile.fullIterable()) { 745 if (alreadySeen.contains(path) || path.contains("@count")) { 746 continue; 747 } 748 if (coverageLevel2 == null) { 749 coverageLevel2 = CoverageLevel2.getInstance(locale); 750 } 751 Level level = coverageLevel2.getLevel(path); 752 if (Level.COMPREHENSIVE.compareTo(level) < 0) { 753 continue; 754 } 755 logln("Path not in English\t" + locale + "\t" + path); 756 alreadySeen.add(path); 757 } 758 } 759 } 760 TestPathDescriptionCompleteness()761 public void TestPathDescriptionCompleteness() { 762 PathDescription pathDescription = new PathDescription(supplemental, 763 english, null, null, PathDescription.ErrorHandling.CONTINUE); 764 Matcher normal = PatternCache.get( 765 "http://cldr.org/translation/[a-zA-Z0-9_]").matcher(""); 766 // http://cldr.unicode.org/translation/plurals#TOC-Minimal-Pairs 767 Set<String> alreadySeen = new HashSet<String>(); 768 PathStarrer starrer = new PathStarrer(); 769 770 checkPathDescriptionCompleteness(pathDescription, normal, 771 "//ldml/numbers/defaultNumberingSystem", alreadySeen, starrer); 772 for (PathHeader pathHeader : getPathHeaders(english)) { 773 final SurveyToolStatus surveyToolStatus = pathHeader 774 .getSurveyToolStatus(); 775 if (surveyToolStatus == SurveyToolStatus.DEPRECATED 776 || surveyToolStatus == SurveyToolStatus.HIDE) { 777 continue; 778 } 779 String path = pathHeader.getOriginalPath(); 780 checkPathDescriptionCompleteness(pathDescription, normal, path, 781 alreadySeen, starrer); 782 } 783 } 784 checkPathDescriptionCompleteness( PathDescription pathDescription, Matcher normal, String path, Set<String> alreadySeen, PathStarrer starrer)785 public void checkPathDescriptionCompleteness( 786 PathDescription pathDescription, Matcher normal, String path, 787 Set<String> alreadySeen, PathStarrer starrer) { 788 String value = english.getStringValue(path); 789 String description = pathDescription.getDescription(path, value, null, 790 null); 791 String starred = starrer.set(path); 792 if (alreadySeen.contains(starred)) { 793 return; 794 } else if (description == null) { 795 errln("Path has no description:\t" + value + "\t" + path); 796 } else if (!description.contains("http://")) { 797 errln("Description has no URL:\t" + description + "\t" + value 798 + "\t" + path); 799 } else if (!normal.reset(description).find()) { 800 errln("Description has generic URL, fix to be specific:\t" 801 + description + "\t" + value + "\t" + path); 802 } else if (description == PathDescription.MISSING_DESCRIPTION) { 803 errln("Fallback Description:\t" + value + "\t" + path); 804 } else { 805 return; 806 } 807 // Add if we had a problem, keeping us from being overwhelmed with 808 // errors. 809 alreadySeen.add(starred); 810 } 811 TestTerritoryOrder()812 public void TestTerritoryOrder() { 813 final Set<String> goodAvailableCodes = CLDRConfig.getInstance() 814 .getStandardCodes().getGoodAvailableCodes("territory"); 815 Set<String> results = showContained("001", 0, new HashSet<String>( 816 goodAvailableCodes)); 817 results.remove("ZZ"); 818 for (String territory : results) { 819 String sub = Containment.getSubcontinent(territory); 820 String cont = Containment.getContinent(territory); 821 errln("Missing\t" + getNameAndOrder(territory) + "\t" 822 + getNameAndOrder(sub) + "\t" + getNameAndOrder(cont)); 823 } 824 } 825 showContained(String territory, int level, Set<String> soFar)826 private Set<String> showContained(String territory, int level, 827 Set<String> soFar) { 828 if (!soFar.contains(territory)) { 829 return soFar; 830 } 831 soFar.remove(territory); 832 Set<String> contained = supplemental.getContained(territory); 833 if (contained == null) { 834 return soFar; 835 } 836 for (String containedItem : contained) { 837 logln(level + "\t" + getNameAndOrder(territory) + "\t" 838 + getNameAndOrder(containedItem)); 839 } 840 for (String containedItem : contained) { 841 showContained(containedItem, level + 1, soFar); 842 } 843 return soFar; 844 } 845 getNameAndOrder(String territory)846 private String getNameAndOrder(String territory) { 847 return territory + "\t" 848 + english.getName(CLDRFile.TERRITORY_NAME, territory) + "\t" 849 + Containment.getOrder(territory); 850 } 851 TestZCompleteness()852 public void TestZCompleteness() { 853 Map<String, PathHeader> uniqueness = new HashMap<String, PathHeader>(); 854 Set<String> alreadySeen = new HashSet<String>(); 855 LanguageTagParser ltp = new LanguageTagParser(); 856 int count = 0; 857 for (String locale : factory.getAvailable()) { 858 if (!ltp.set(locale).getRegion().isEmpty()) { 859 continue; 860 } 861 check(locale, false, uniqueness, alreadySeen); 862 ++count; 863 } 864 logln("Count:\t" + count); 865 } 866 check(String localeID, boolean resolved, Map<String, PathHeader> uniqueness, Set<String> alreadySeen)867 public void check(String localeID, boolean resolved, 868 Map<String, PathHeader> uniqueness, Set<String> alreadySeen) { 869 CLDRFile nativeFile = info.getCLDRFile(localeID, resolved); 870 int count = 0; 871 for (String path : nativeFile) { 872 if (alreadySeen.contains(path)) { 873 continue; 874 } 875 alreadySeen.add(path); 876 final PathHeader pathHeader = pathHeaderFactory.fromPath(path); 877 ++count; 878 if (pathHeader == null) { 879 errln("Null pathheader for " + path); 880 } else { 881 String visible = pathHeader.toString(); 882 PathHeader old = uniqueness.get(visible); 883 if (pathHeader.getSectionId() == SectionId.Timezones) { 884 final PageId pageId = pathHeader.getPageId(); 885 if (badZonePages.contains(pageId) 886 && !pathHeader.getCode().equals("Unknown")) { 887 String msg = "Bad page ID:\t" + pageId + "\t" + pathHeader + "\t" + path; 888 if (!logKnownIssue("cldrbug:7802", "ICU/CLDR time zone data sync problem - " + msg)) { 889 errln("Bad page ID:\t" + pageId + "\t" + pathHeader 890 + "\t" + path); 891 } 892 } 893 } 894 if (old == null) { 895 if (pathHeader.getSection().equals("Special")) { 896 if (pathHeader.getSection().equals("Unknown")) { 897 errln("PathHeader has fallback: " + visible + "\t" 898 + pathHeader.getOriginalPath()); 899 // } else { 900 // logln("Special:\t" + visible + "\t" + 901 // pathHeader.getOriginalPath()); 902 } 903 } 904 uniqueness.put(visible, pathHeader); 905 } else if (!old.equals(pathHeader)) { 906 if (pathHeader.getSectionId() == SectionId.Special) { 907 logln("Special PathHeader not unique: " + visible 908 + "\t" + pathHeader.getOriginalPath() + "\t" 909 + old.getOriginalPath()); 910 } else { 911 errln("PathHeader not unique: " + visible + "\t" 912 + pathHeader.getOriginalPath() + "\t" 913 + old.getOriginalPath()); 914 } 915 } 916 } 917 } 918 logln(localeID + "\t" + count); 919 } 920 TestContainment()921 public void TestContainment() { 922 Map<String, Map<String, String>> metazoneToRegionToZone = supplemental 923 .getMetazoneToRegionToZone(); 924 Map<String, String> metazoneToContinent = supplemental 925 .getMetazoneToContinentMap(); 926 for (String metazone : metazoneToRegionToZone.keySet()) { 927 Map<String, String> regionToZone = metazoneToRegionToZone 928 .get(metazone); 929 String worldZone = regionToZone.get("001"); 930 String territory = Containment.getRegionFromZone(worldZone); 931 if (territory == null) { 932 territory = "ZZ"; 933 } 934 String cont = Containment.getContinent(territory); 935 int order = Containment.getOrder(territory); 936 String sub = Containment.getSubcontinent(territory); 937 String revision = PathHeader.getMetazonePageTerritory(metazone); 938 String continent = metazoneToContinent.get(metazone); 939 if (continent == null) { 940 continent = "UnknownT"; 941 } 942 // Russia, Antarctica => territory 943 // in Australasia, Asia, S. America => subcontinent 944 // in N. America => N. America (grouping of 3 subcontinents) 945 // in everything else => continent 946 947 if (territory.equals("RU")) { 948 assertEquals("Russia special case", "RU", revision); 949 } else if (territory.equals("US")) { 950 assertEquals("N. America special case", "003", revision); 951 } else if (territory.equals("BR")) { 952 assertEquals("S. America special case", "005", revision); 953 } 954 if (isVerbose()) { 955 String name = english.getName(CLDRFile.TERRITORY_NAME, cont); 956 String name2 = english.getName(CLDRFile.TERRITORY_NAME, sub); 957 String name3 = english.getName(CLDRFile.TERRITORY_NAME, 958 territory); 959 String name4 = english.getName(CLDRFile.TERRITORY_NAME, 960 revision); 961 962 logln(metazone + "\t" + continent + "\t" + name + "\t" + name2 963 + "\t" + name3 + "\t" + order + "\t" + name4); 964 } 965 } 966 } 967 TestZ()968 public void TestZ() { 969 PathStarrer pathStarrer = new PathStarrer(); 970 pathStarrer.setSubstitutionPattern("%A"); 971 972 Set<PathHeader> sorted = new TreeSet<PathHeader>(); 973 Map<String, String> missing = new TreeMap<String, String>(); 974 Map<String, String> skipped = new TreeMap<String, String>(); 975 Map<String, String> collide = new TreeMap<String, String>(); 976 977 logln("Traversing Paths"); 978 for (String path : english) { 979 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 980 String value = english.getStringValue(path); 981 if (pathHeader == null) { 982 final String starred = pathStarrer.set(path); 983 missing.put(starred, value + "\t" + path); 984 continue; 985 } 986 if (pathHeader.getSection().equalsIgnoreCase("skip")) { 987 final String starred = pathStarrer.set(path); 988 skipped.put(starred, value + "\t" + path); 989 continue; 990 } 991 sorted.add(pathHeader); 992 } 993 logln("\nConverted:\t" + sorted.size()); 994 String lastHeader = ""; 995 String lastPage = ""; 996 String lastSection = ""; 997 List<String> threeLevel = new ArrayList<String>(); 998 Status status = new Status(); 999 CoverageLevel2 coverageLevel2 = CoverageLevel2.getInstance("en"); 1000 1001 for (PathHeader pathHeader : sorted) { 1002 String original = pathHeader.getOriginalPath(); 1003 if (!original.equals(status.pathWhereFound)) { 1004 continue; 1005 } 1006 if (!lastSection.equals(pathHeader.getSection())) { 1007 logln(""); 1008 threeLevel.add(pathHeader.getSection()); 1009 threeLevel.add("\t" + pathHeader.getPage()); 1010 threeLevel.add("\t\t" + pathHeader.getHeader()); 1011 lastSection = pathHeader.getSection(); 1012 lastPage = pathHeader.getPage(); 1013 lastHeader = pathHeader.getHeader(); 1014 } else if (!lastPage.equals(pathHeader.getPage())) { 1015 logln(""); 1016 threeLevel.add("\t" + pathHeader.getPage()); 1017 threeLevel.add("\t\t" + pathHeader.getHeader()); 1018 lastPage = pathHeader.getPage(); 1019 lastHeader = pathHeader.getHeader(); 1020 } else if (!lastHeader.equals(pathHeader.getHeader())) { 1021 logln(""); 1022 threeLevel.add("\t\t" + pathHeader.getHeader()); 1023 lastHeader = pathHeader.getHeader(); 1024 } 1025 logln(pathHeader + "\t" + coverageLevel2.getLevel(original) + "\t" 1026 + english.getStringValue(pathHeader.getOriginalPath()) 1027 + "\t" + pathHeader.getOriginalPath()); 1028 } 1029 if (collide.size() != 0) { 1030 errln("\nCollide:\t" + collide.size()); 1031 for (Entry<String, String> item : collide.entrySet()) { 1032 errln("\t" + item); 1033 } 1034 } 1035 if (missing.size() != 0) { 1036 errln("\nMissing:\t" + missing.size()); 1037 for (Entry<String, String> item : missing.entrySet()) { 1038 errln("\t" + item.getKey() + "\tvalue:\t" + item.getValue()); 1039 } 1040 } 1041 if (skipped.size() != 0) { 1042 errln("\nSkipped:\t" + skipped.size()); 1043 for (Entry<String, String> item : skipped.entrySet()) { 1044 errln("\t" + item); 1045 } 1046 } 1047 Counter<PathHeader.Factory.CounterData> counterData = pathHeaderFactory 1048 .getInternalCounter(); 1049 logln("\nInternal Counter:\t" + counterData.size()); 1050 for (PathHeader.Factory.CounterData item : counterData.keySet()) { 1051 logln("\t" + counterData.getCount(item) + "\t" + item.get2() // externals 1052 + "\t" + item.get3() + "\t" + item.get0() // internals 1053 + "\t" + item.get1()); 1054 } 1055 logln("\nMenus/Headers:\t" + threeLevel.size()); 1056 for (String item : threeLevel) { 1057 logln(item); 1058 } 1059 LinkedHashMap<String, Set<String>> sectionsToPages = pathHeaderFactory 1060 .getSectionsToPages(); 1061 logln("\nMenus:\t" + sectionsToPages.size()); 1062 for (Entry<String, Set<String>> item : sectionsToPages.entrySet()) { 1063 final String section = item.getKey(); 1064 for (String page : item.getValue()) { 1065 logln("\t" + section + "\t" + page); 1066 int count = 0; 1067 for (String path : pathHeaderFactory.filterCldr(section, page, 1068 english)) { 1069 count += 1; // just count them. 1070 } 1071 logln("\t" + count); 1072 } 1073 } 1074 } 1075 TestOrder()1076 public void TestOrder() { 1077 String[] paths = { 1078 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod[@type=\"noon\"]", 1079 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod[@type=\"afternoon1\"]", 1080 }; 1081 PathHeader pathHeaderLast = null; 1082 for (String path : paths) { 1083 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 1084 if (pathHeaderLast != null) { 1085 assertRelation("ordering", true, pathHeaderLast, LEQ, pathHeader); 1086 } 1087 pathHeaderLast = pathHeader; 1088 } 1089 1090 } 1091 Test8414()1092 public void Test8414() { 1093 PathDescription pathDescription = new PathDescription(supplemental, 1094 english, null, null, PathDescription.ErrorHandling.CONTINUE); 1095 1096 String prefix = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\""; 1097 String suffix = "\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod[@type=\"morning1\"]"; 1098 1099 final String path0 = prefix + "format" + suffix; 1100 final String path1 = prefix + "stand-alone" + suffix; 1101 String v0 = english.getStringValue(path0); 1102 String v1 = english.getStringValue(path1); 1103 String p0 = pathDescription.getDescription(path0, v0, null, null); 1104 String p1 = pathDescription.getDescription(path1, v1, null, null); 1105 assertTrue("Check pd for format", p0.contains("in the morning")); 1106 assertTrue("Check pd for stand-alone", !p1.contains("in the morning")); 1107 } 1108 TestCompletenessNonLdmlDtd()1109 public void TestCompletenessNonLdmlDtd() { 1110 PathChecker pathChecker = new PathChecker(); 1111 Set<String> directories = new LinkedHashSet<>(); 1112 XPathParts parts = new XPathParts(); 1113 Multimap<String, String> pathValuePairs = LinkedListMultimap.create(); 1114 // get all the directories containing non-Ldml dtd files 1115 for (DtdType dtdType : DtdType.values()) { 1116 if (dtdType == DtdType.ldml || dtdType == DtdType.ldmlICU) { 1117 continue; 1118 } 1119 DtdData dtdData = DtdData.getInstance(dtdType); 1120 for (String dir : dtdType.directories) { 1121 if (DEBUG_DTD_TYPE != null && !DEBUG_DTD_TYPE.directories.contains(dir)) { 1122 continue; 1123 } 1124 File dir2 = new File(COMMON_DIR + dir); 1125 logln(dir2.getName()); 1126 for (String file : dir2.list()) { 1127 // don't need to restrict with getFilesToTest(Arrays.asList(dir2.list()), "root", "en")) { 1128 if (!file.endsWith(".xml")) { 1129 continue; 1130 } 1131 if (DEBUG) warnln(" TestCompletenessNonLdmlDtd: " + dir + ", " + file); 1132 logln(" \t" + file); 1133 for (Pair<String, String> pathValue : XMLFileReader.loadPathValues( 1134 dir2 + "/" + file, new ArrayList<Pair<String, String>>(), true)) { 1135 final String path = pathValue.getFirst(); 1136 final String value = pathValue.getSecond(); 1137 // logln("\t\t" + path); 1138 if (path.startsWith("//supplementalData/weekData/weekOf")) { 1139 int debug = 0; 1140 } 1141 pathChecker.checkPathHeader(dtdData, path); 1142 } 1143 ; 1144 } 1145 } 1146 } 1147 } 1148 1149 private class PathChecker { 1150 PathHeader.Factory phf = pathHeaderFactory; 1151 PathStarrer starrer = new PathStarrer().setSubstitutionPattern("%A"); 1152 1153 Set<String> badHeaders = new TreeSet<>(); 1154 Map<PathHeader, PathHeader> goodHeaders = new HashMap<>(); 1155 Set<PathHeader> seenBad = new HashSet<>(); 1156 { phf.clearCache()1157 phf.clearCache(); 1158 } 1159 checkPathHeader(DtdData dtdData, String rawPath)1160 public void checkPathHeader(DtdData dtdData, String rawPath) { 1161 XPathParts pathPlain = XPathParts.getFrozenInstance(rawPath); 1162 if (dtdData.isMetadata(pathPlain)) { 1163 return; 1164 } 1165 if (dtdData.isDeprecated(pathPlain)) { 1166 return; 1167 } 1168 Multimap<String, String> extras = HashMultimap.create(); 1169 Set<String> fixedPaths = dtdData.getRegularizedPaths(pathPlain, extras); 1170 if (fixedPaths != null) { 1171 for (String fixedPath : fixedPaths) { 1172 checkSubpath(fixedPath); 1173 } 1174 } 1175 for (String path : extras.keySet()) { 1176 checkSubpath(path); 1177 } 1178 } 1179 checkSubpath(String path)1180 public void checkSubpath(String path) { 1181 String message = ": Can't compute path header"; 1182 PathHeader ph = null; 1183 try { 1184 ph = phf.fromPath(path); 1185 if (seenBad.contains(ph)) { 1186 return; 1187 } 1188 if (ph.getPageId() == PageId.Deprecated) { 1189 return; // don't care 1190 } 1191 if (ph.getPageId() != PageId.Unknown) { 1192 PathHeader old = goodHeaders.put(ph, ph); 1193 if (old != null && !path.equals(old.getOriginalPath())) { 1194 errln("Duplicate path header for: " + ph 1195 + "\n\t\t " + path 1196 + "\n\t\t≠" + old.getOriginalPath()); 1197 seenBad.add(ph); 1198 } 1199 return; 1200 } 1201 // for debugging 1202 phf.clearCache(); 1203 List<String> failures = new ArrayList<>(); 1204 ph = phf.fromPath(path, failures); 1205 message = ": Unknown path header" + failures; 1206 } catch (Exception e) { 1207 message = ": Exception in path header: " + e.getMessage(); 1208 } 1209 String star = starrer.set(path); 1210 if (badHeaders.add(star)) { 1211 errln(star + message + ", " + ph); 1212 } 1213 } 1214 } 1215 TestSupplementalItems()1216 public void TestSupplementalItems() { 1217 // <weekOfPreference ordering="weekOfYear weekOfMonth" locales="am az bs cs cy da el et hi ky lt mk sk ta th"/> 1218 // logln(pathHeaderFactory.getRegexInfo()); 1219 CLDRFile supplementalFile = CLDRConfig.getInstance().getSupplementalFactory().make("supplementalData", false); 1220 List<String> failures = new ArrayList<>(); 1221 Multimap<String, String> pathValuePairs = LinkedListMultimap.create(); 1222 XPathParts parts = new XPathParts(); 1223 for (String test : With.in(supplementalFile.iterator("//supplementalData/weekData"))) { 1224 failures.clear(); 1225 supplementalFile.getDtdData().getRegularizedPaths(parts.set(supplementalFile.getFullXPath(test)), pathValuePairs); 1226 for (Entry<String, Collection<String>> entry : pathValuePairs.asMap().entrySet()) { 1227 final String normalizedPath = entry.getKey(); 1228 final Collection<String> normalizedValue = entry.getValue(); 1229 PathHeader ph = pathHeaderFactory.fromPath(normalizedPath, failures); 1230 if (ph == null || ph.getSectionId() == SectionId.Special) { 1231 errln("Failure with " + test + " => " + normalizedPath + " = " + normalizedValue); 1232 } else { 1233 logln(ph + "\t" + test + " = " + normalizedValue); 1234 } 1235 } 1236 } 1237 } 1238 test10232()1239 public void test10232() { 1240 String[][] tests = { 1241 { "MMM", "Formats - Flexible - Date Formats" }, 1242 { "dMM", "Formats - Flexible - Date Formats" }, 1243 { "h", "Formats - Flexible - 12 Hour Time Formats" }, 1244 { "hm", "Formats - Flexible - 12 Hour Time Formats" }, 1245 { "Ehm", "Formats - Flexible - 12 Hour Time Formats" }, 1246 { "H", "Formats - Flexible - 24 Hour Time Formats" }, 1247 { "Hm", "Formats - Flexible - 24 Hour Time Formats" }, 1248 { "EHm", "Formats - Flexible - 24 Hour Time Formats" }, 1249 }; 1250 for (String[] test : tests) { 1251 String path = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\"" 1252 + test[0] + "\"]"; 1253 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 1254 assertEquals("flexible formats", test[1] + "|" + test[0], pathHeader.getHeader() + "|" + pathHeader.getCode()); 1255 } 1256 } 1257 1258 // Moved from TestAnnotations and generalized testPathHeaderSize()1259 public void testPathHeaderSize() { 1260 String locale = "ar"; // choose one with lots of plurals 1261 int maxSize = 700; 1262 boolean showTable = false; // only printed if test fails or verbose 1263 1264 Factory factory = CLDRConfig.getInstance().getCommonAndSeedAndMainAndAnnotationsFactory(); 1265 CLDRFile english = factory.make(locale, true); 1266 1267 PathHeader.Factory phf = PathHeader.getFactory(CLDRConfig.getInstance().getEnglish()); 1268 Counter<PageId> counterPageId = new Counter<>(); 1269 Counter<PageId> counterPageIdAll = new Counter<>(); 1270 for (String path : english) { 1271 Level level = CLDRConfig.getInstance().getSupplementalDataInfo().getCoverageLevel(path, locale); 1272 PathHeader ph = phf.fromPath(path); 1273 if (level.compareTo(Level.MODERN) <= 0) { 1274 counterPageId.add(ph.getPageId(), 1); 1275 } 1276 counterPageIdAll.add(ph.getPageId(), 1); 1277 } 1278 Set<R2<Long, PageId>> entrySetSortedByCount = counterPageId.getEntrySetSortedByCount(false, null); 1279 for (R2<Long, PageId> sizeAndPageId : entrySetSortedByCount) { 1280 long size = sizeAndPageId.get0(); 1281 PageId pageId = sizeAndPageId.get1(); 1282 if (!assertTrue(pageId.getSectionId() + "/" + pageId + " size (" + size 1283 + ") < " + maxSize + "?", size < maxSize)) { 1284 showTable = true; 1285 } 1286 // System.out.println(pageId + "\t" + size); 1287 } 1288 if (showTable || isVerbose()) { 1289 for (R2<Long, PageId> sizeAndPageId : entrySetSortedByCount) { 1290 PageId pageId = sizeAndPageId.get1(); 1291 System.out.println(pageId.getSectionId() + "\t" + pageId + "\t" + sizeAndPageId.get0() + "\t" + counterPageIdAll.get(pageId)); 1292 } 1293 } 1294 } 1295 } 1296