1 package org.unicode.cldr.unittest; 2 3 import java.io.File; 4 import java.util.ArrayList; 5 import java.util.Arrays; 6 import java.util.Collection; 7 import java.util.EnumMap; 8 import java.util.EnumSet; 9 import java.util.HashMap; 10 import java.util.HashSet; 11 import java.util.LinkedHashMap; 12 import java.util.LinkedHashSet; 13 import java.util.List; 14 import java.util.Map; 15 import java.util.Map.Entry; 16 import java.util.Set; 17 import java.util.TreeMap; 18 import java.util.TreeSet; 19 import java.util.regex.Matcher; 20 21 import org.unicode.cldr.test.CoverageLevel2; 22 import org.unicode.cldr.test.ExampleGenerator; 23 import org.unicode.cldr.util.CLDRConfig; 24 import org.unicode.cldr.util.CLDRFile; 25 import org.unicode.cldr.util.CLDRFile.Status; 26 import org.unicode.cldr.util.CLDRPaths; 27 import org.unicode.cldr.util.CLDRURLS; 28 import org.unicode.cldr.util.CldrUtility; 29 import org.unicode.cldr.util.Containment; 30 import org.unicode.cldr.util.Counter; 31 import org.unicode.cldr.util.DtdData; 32 import org.unicode.cldr.util.DtdType; 33 import org.unicode.cldr.util.Emoji; 34 import org.unicode.cldr.util.Factory; 35 import org.unicode.cldr.util.GrammarInfo; 36 import org.unicode.cldr.util.GrammarInfo.CaseValues; 37 import org.unicode.cldr.util.GrammarInfo.GenderValues; 38 import org.unicode.cldr.util.LanguageTagParser; 39 import org.unicode.cldr.util.Level; 40 import org.unicode.cldr.util.Organization; 41 import org.unicode.cldr.util.Pair; 42 import org.unicode.cldr.util.PathDescription; 43 import org.unicode.cldr.util.PathHeader; 44 import org.unicode.cldr.util.PathHeader.PageId; 45 import org.unicode.cldr.util.PathHeader.SectionId; 46 import org.unicode.cldr.util.PathHeader.SurveyToolStatus; 47 import org.unicode.cldr.util.PathHeader.Width; 48 import org.unicode.cldr.util.PathStarrer; 49 import org.unicode.cldr.util.PatternCache; 50 import org.unicode.cldr.util.PatternPlaceholders; 51 import org.unicode.cldr.util.PatternPlaceholders.PlaceholderInfo; 52 import org.unicode.cldr.util.PatternPlaceholders.PlaceholderStatus; 53 import org.unicode.cldr.util.StandardCodes; 54 import org.unicode.cldr.util.SupplementalDataInfo; 55 import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo; 56 import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo.Count; 57 import org.unicode.cldr.util.SupplementalDataInfo.PluralType; 58 import org.unicode.cldr.util.With; 59 import org.unicode.cldr.util.XMLFileReader; 60 import org.unicode.cldr.util.XPathParts; 61 62 import com.google.common.base.Joiner; 63 import com.google.common.collect.HashMultimap; 64 import com.google.common.collect.ImmutableSet; 65 import com.google.common.collect.LinkedListMultimap; 66 import com.google.common.collect.Multimap; 67 import com.google.common.collect.TreeMultimap; 68 import com.ibm.icu.impl.Relation; 69 import com.ibm.icu.impl.Row; 70 import com.ibm.icu.impl.Row.R2; 71 72 public class TestPathHeader extends TestFmwkPlus { 73 private static final DtdType DEBUG_DTD_TYPE = null; // DtdType.supplementalData; 74 private static final String COMMON_DIR = CLDRPaths.BASE_DIRECTORY + "common/"; 75 private static final boolean DEBUG = false; 76 main(String[] args)77 public static void main(String[] args) { 78 new TestPathHeader().run(args); 79 } 80 81 static final CLDRConfig info = CLDRConfig.getInstance(); 82 static final Factory factory = info.getCommonAndSeedAndMainAndAnnotationsFactory(); 83 static final CLDRFile english = factory.make("en", true); 84 static final SupplementalDataInfo supplemental = info 85 .getSupplementalDataInfo(); 86 static PathHeader.Factory pathHeaderFactory = PathHeader 87 .getFactory(english); 88 private EnumSet<PageId> badZonePages = EnumSet.of(PageId.UnknownT); 89 tempTestAnnotation()90 public void tempTestAnnotation() { 91 // NEW: <annotation cp="��">face | grin</annotation> 92 // <annotation cp="��" type="tts">grinning face</annotation> 93 94 final String path1 = "//ldml/annotations/annotation[@cp=\"��\"]"; 95 PathHeader ph1 = pathHeaderFactory.fromPath(path1); 96 logln(ph1.toString() + "\t" + path1); 97 final String path2 = "//ldml/annotations/annotation[@cp=\"��\"][@type=\"tts\"]"; 98 PathHeader ph2 = pathHeaderFactory.fromPath(path2); 99 logln(ph2.toString() + "\t" + path2); 100 final String path3 = "//ldml/annotations/annotation[@cp=\"��\"]"; 101 PathHeader ph3 = pathHeaderFactory.fromPath(path2); 102 logln(ph3.toString() + "\t" + path3); 103 104 assertNotEquals("pathheader", ph1, ph2); 105 assertNotEquals("pathheader", ph1.toString(), ph2.toString()); 106 assertRelation("pathheader", true, ph1, TestFmwkPlus.LEQ, ph3); 107 assertRelation("pathheader", true, ph3, TestFmwkPlus.LEQ, ph2); 108 } 109 110 static final String[] MIN_LOCALES = {"root", "en", "de", "ru", "ko"}; // choose locales with range of case/gender structures 111 tempTestCompletenessLdmlDtd()112 public void tempTestCompletenessLdmlDtd() { 113 // List<String> failures = null; 114 pathHeaderFactory.clearCache(); 115 PathChecker pathChecker = new PathChecker(); 116 for (String directory : DtdType.ldml.directories) { 117 Factory factory2 = CLDRConfig.getInstance().getMainAndAnnotationsFactory(); 118 Set<String> source = factory2.getAvailable(); 119 for (String file : getFilesToTest(source, MIN_LOCALES)) { 120 if (DEBUG) warnln(" TestCompletenessLdmlDtd: " + directory + ", " + file); 121 DtdData dtdData = null; 122 CLDRFile cldrFile = factory2.make(file, true); 123 for (String path : cldrFile.fullIterable()) { 124 pathChecker.checkPathHeader(cldrFile.getDtdData(), path); 125 } 126 } 127 } 128 Set<String> missing = pathHeaderFactory.getUnmatchedRegexes(); 129 if (missing.size() != 0) { 130 for (String e : missing) { 131 errln("Path Regex never matched:\t" + e); 132 } 133 } 134 if (!pathChecker.badHeaders.isEmpty()) { 135 System.out.println("For help with DTD updates: " + CLDRURLS.CLDR_UPDATINGDTD_URL); 136 } 137 } 138 getFilesToTest(Collection<String> source, String... doFirst)139 private Collection<String> getFilesToTest(Collection<String> source, String... doFirst) { 140 LinkedHashSet<String> files = new LinkedHashSet<>(Arrays.asList(doFirst)); 141 files.retainAll(source); // put first 142 files.addAll(new HashSet<>(source)); // now add others semi-randomly 143 int max = Math.min(30, files.size()); 144 if (getInclusion() == 10 || files.size() <= max) { 145 return files; 146 } 147 ArrayList<String> shortFiles = new ArrayList<>(files); 148 if (getInclusion() > 5) { 149 max += (files.size() - 30) * (getInclusion() - 5) / 10; // use proportional amount 150 } 151 return shortFiles.subList(0, max); 152 } 153 TestCompleteness()154 public void TestCompleteness() { 155 PathHeader.Factory pathHeaderFactory2 = PathHeader.getFactory(english); 156 // List<String> failures = null; 157 pathHeaderFactory2.clearCache(); 158 Multimap<PathHeader.PageId, PathHeader.SectionId> pageUniqueness = TreeMultimap.create(); 159 Multimap<String, Pair<PathHeader.SectionId, PathHeader.PageId>> headerUniqueness = TreeMultimap.create(); 160 Set<String> toTest; 161 switch (getInclusion()) { 162 default: 163 toTest = StandardCodes.make().getLocaleCoverageLocales(Organization.cldr); 164 break; 165 case 10: 166 toTest = factory.getAvailable(); 167 break; 168 } 169 toTest = ImmutableSet.<String> builder().add("en").addAll(toTest).build(); 170 Set<String> seenPaths = new HashSet<>(); 171 Set<String> localSeenPaths = new TreeSet<>(); 172 for (String locale : toTest) { 173 localSeenPaths.clear(); 174 for (String p : factory.make(locale, true).fullIterable()) { 175 if (p.startsWith("//ldml/identity/")) { 176 continue; 177 } 178 if (seenPaths.contains(p)) { 179 continue; 180 } 181 seenPaths.add(p); 182 localSeenPaths.add(p); 183 // if (p.contains("symbol[@alt") && failures == null) { 184 // PathHeader result = pathHeaderFactory2.fromPath(p, failures = new 185 // ArrayList<String>()); 186 // logln("Matching " + p + ": " + result + "\t" + 187 // result.getSurveyToolStatus()); 188 // for (String failure : failures) { 189 // logln("\t" + failure); 190 // } 191 // } 192 PathHeader ph; 193 try { 194 ph = pathHeaderFactory2.fromPath(p); 195 } catch (Exception e1) { 196 try { 197 ph = pathHeaderFactory2.fromPath(p); 198 } catch (Exception e2) { 199 throw new IllegalArgumentException(locale + ":\t" + p, e2); 200 } 201 } 202 if (ph == null) { 203 errln("Failed to create path from: " + p); 204 continue; 205 } 206 final SectionId sectionId = ph.getSectionId(); 207 if (sectionId != SectionId.Special) { 208 pageUniqueness.put(ph.getPageId(), sectionId); 209 headerUniqueness.put(ph.getHeader(), new Pair<>(sectionId, ph.getPageId())); 210 } 211 } 212 if (!localSeenPaths.isEmpty()) { 213 logln(locale + ": checked " + localSeenPaths.size() + " new paths"); 214 } 215 } 216 Set<String> missing = pathHeaderFactory2.getUnmatchedRegexes(); 217 if (missing.size() != 0) { 218 for (String e : missing) { 219 if (e.contains("//ldml/")) { 220 if (e.contains("//ldml/rbnf/") || e.contains("//ldml/segmentations/") || e.contains("//ldml/collations/")) { 221 continue; 222 } 223 logln("Path Regex never matched:\t" + e); 224 } 225 } 226 } 227 228 for (Entry<PageId, Collection<SectionId>> e : pageUniqueness.asMap().entrySet()) { 229 Collection<SectionId> values = e.getValue(); 230 if (values.size() != 1) { 231 warnln("Duplicate page in section: " + CldrUtility.toString(e)); 232 } 233 } 234 235 for (Entry<String, Collection<Pair<SectionId, PageId>>> e : headerUniqueness.asMap().entrySet()) { 236 Collection<Pair<SectionId, PageId>> values = e.getValue(); 237 if (values.size() != 1) { 238 warnln("Duplicate header in (section,page): " + CldrUtility.toString(e)); 239 } 240 } 241 } 242 Test6170()243 public void Test6170() { 244 String p1 = "//ldml/units/unitLength[@type=\"narrow\"]/unit[@type=\"speed-kilometer-per-hour\"]/unitPattern[@count=\"other\"]"; 245 String p2 = "//ldml/units/unitLength[@type=\"narrow\"]/unit[@type=\"area-square-meter\"]/unitPattern[@count=\"other\"]"; 246 PathHeader ph1 = pathHeaderFactory.fromPath(p1); 247 PathHeader ph2 = pathHeaderFactory.fromPath(p2); 248 int comp12 = ph1.compareTo(ph2); 249 int comp21 = ph2.compareTo(ph1); 250 assertEquals("comp ph", comp12, -comp21); 251 } 252 TestVariant()253 public void TestVariant() { 254 PathHeader p1 = pathHeaderFactory 255 .fromPath("//ldml/localeDisplayNames/languages/language[@type=\"ug\"][@alt=\"variant\"]"); 256 PathHeader p2 = pathHeaderFactory 257 .fromPath("//ldml/localeDisplayNames/languages/language[@type=\"ug\"]"); 258 assertNotEquals("variants", p1, p2); 259 assertNotEquals("variants", p1.toString(), p2.toString()); 260 // Code Lists Languages Arabic Script ug-variant 261 } 262 Test4587()263 public void Test4587() { 264 String test = "//ldml/dates/timeZoneNames/metazone[@type=\"Pacific/Wallis\"]/short/standard"; 265 PathHeader ph = pathHeaderFactory.fromPath(test); 266 if (ph == null) { 267 errln("Failure with " + test); 268 } else { 269 logln(ph + "\t" + test); 270 } 271 } 272 TestMiscPatterns()273 public void TestMiscPatterns() { 274 String test = "//ldml/numbers/miscPatterns[@numberSystem=\"arab\"]/pattern[@type=\"atLeast\"]"; 275 PathHeader ph = pathHeaderFactory.fromPath(test); 276 assertNotNull("MiscPatterns path not found", ph); 277 if (false) 278 System.out.println(english.getStringValue(test)); 279 } 280 TestPluralOrder()281 public void TestPluralOrder() { 282 Set<PathHeader> sorted = new TreeSet<>(); 283 for (String locale : new String[] { "ru", "ar", "ja" }) { 284 sorted.clear(); 285 CLDRFile cldrFile = info.getCLDRFile(locale, true); 286 CoverageLevel2 coverageLevel = CoverageLevel2.getInstance(locale); 287 for (String path : cldrFile.fullIterable()) { 288 if (!path.contains("@count")) { 289 continue; 290 } 291 Level level = coverageLevel.getLevel(path); 292 if (Level.MODERN.compareTo(level) < 0) { 293 continue; 294 } 295 PathHeader p = pathHeaderFactory.fromPath(path); 296 sorted.add(p); 297 } 298 for (PathHeader p : sorted) { 299 logln(locale + "\t" + p + "\t" + p.getOriginalPath()); 300 } 301 } 302 } 303 304 static final String APPEND_TIMEZONE = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/appendItems/appendItem[@request=\"Timezone\"]"; 305 static final String APPEND_TIMEZONE_END = "/dateTimeFormats/appendItems/appendItem[@request=\"Timezone\"]"; 306 static final String BEFORE_PH = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\"ms\"]"; 307 static final String AFTER_PH = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"d\"]/greatestDifference[@id=\"d\"]"; 308 TestAppendTimezone()309 public void TestAppendTimezone() { 310 CLDRFile cldrFile = info.getEnglish(); 311 CoverageLevel2 coverageLevel = CoverageLevel2.getInstance("en"); 312 assertEquals("appendItem:Timezone", Level.MODERATE, 313 coverageLevel.getLevel(APPEND_TIMEZONE)); 314 315 PathHeader ph = pathHeaderFactory.fromPath(APPEND_TIMEZONE); 316 assertEquals("appendItem:Timezone pathheader", "Timezone", ph.getCode()); 317 // check that they are in the right place (they weren't before!) 318 PathHeader phBefore = pathHeaderFactory.fromPath(BEFORE_PH); 319 PathHeader phAfter = pathHeaderFactory.fromPath(AFTER_PH); 320 assertTrue(phBefore, LEQ, ph); 321 assertTrue(ph, LEQ, phAfter); 322 323 PathDescription pathDescription = new PathDescription(supplemental, 324 english, null, null, PathDescription.ErrorHandling.CONTINUE); 325 String description = pathDescription.getDescription(APPEND_TIMEZONE, 326 "tempvalue", null, null); 327 assertTrue("appendItem:Timezone pathDescription", 328 description.contains("“Timezone”")); 329 330 PatternPlaceholders patternPlaceholders = PatternPlaceholders 331 .getInstance(); 332 PlaceholderStatus status = patternPlaceholders 333 .getStatus(APPEND_TIMEZONE); 334 assertEquals("appendItem:Timezone placeholders", 335 PlaceholderStatus.REQUIRED, status); 336 337 Map<String, PlaceholderInfo> placeholderInfo = patternPlaceholders 338 .get(APPEND_TIMEZONE); 339 PlaceholderInfo placeholderInfo2 = placeholderInfo.get("{1}"); 340 if (assertNotNull("appendItem:Timezone placeholders", placeholderInfo2)) { 341 assertEquals("appendItem:Timezone placeholders", 342 "APPEND_FIELD_FORMAT", placeholderInfo2.name); 343 assertEquals("appendItem:Timezone placeholders", "Pacific Time", 344 placeholderInfo2.example); 345 } 346 ExampleGenerator eg = new ExampleGenerator(cldrFile, cldrFile, CLDRPaths.SUPPLEMENTAL_DIRECTORY); 347 String example = eg.getExampleHtml(APPEND_TIMEZONE, cldrFile.getStringValue(APPEND_TIMEZONE)); 348 String result = ExampleGenerator.simplify(example, false); 349 assertEquals("", "〖❬6:25:59 PM❭ ❬GMT❭〗", result); 350 } 351 TestOptional()352 public void TestOptional() { 353 if (true) return; 354 Map<PathHeader, String> sorted = new TreeMap<>(); 355 for (String locale : new String[] { "af" }) { 356 sorted.clear(); 357 CLDRFile cldrFile = info.getCLDRFile(locale, true); 358 CoverageLevel2 coverageLevel = CoverageLevel2.getInstance(locale); 359 for (String path : cldrFile.fullIterable()) { 360 Level level = coverageLevel.getLevel(path); 361 if (supplemental.isDeprecated(DtdType.ldml, path)) { 362 continue; 363 } 364 365 if (Level.COMPREHENSIVE.compareTo(level) != 0) { 366 continue; 367 } 368 369 PathHeader ph = pathHeaderFactory.fromPath(path); 370 if (ph == null || ph.shouldHide()) { 371 continue; 372 } 373 final SurveyToolStatus status = ph.getSurveyToolStatus(); 374 sorted.put( 375 ph, 376 locale + "\t" + status + "\t" + ph + "\t" 377 + ph.getOriginalPath()); 378 } 379 Set<String> codes = new LinkedHashSet<>(); 380 PathHeader old = null; 381 String line = null; 382 for (Entry<PathHeader, String> s : sorted.entrySet()) { 383 PathHeader p = s.getKey(); 384 String v = s.getValue(); 385 if (old == null) { 386 line = v; 387 codes.add(p.getCode()); 388 } else if (p.getSectionId() == old.getSectionId() 389 && p.getPageId() == old.getPageId() 390 && p.getHeader().equals(old.getHeader())) { 391 codes.add(p.getCode()); 392 } else { 393 logln(line + "\t" + codes.toString()); 394 codes.clear(); 395 line = v; 396 codes.add(p.getCode()); 397 } 398 old = p; 399 } 400 logln(line + "\t" + codes.toString()); 401 } 402 } 403 TestPluralCanonicals()404 public void TestPluralCanonicals() { 405 Relation<String, String> data = Relation.of( 406 new LinkedHashMap<String, Set<String>>(), TreeSet.class); 407 for (String locale : factory.getAvailable()) { 408 if (locale.contains("_")) { 409 continue; 410 } 411 PluralInfo info = supplemental.getPlurals(PluralType.cardinal, 412 locale); 413 Set<String> keywords = info.getCanonicalKeywords(); 414 data.put(keywords.toString(), locale); 415 } 416 for (Entry<String, Set<String>> entry : data.keyValuesSet()) { 417 logln(entry.getKey() + "\t" + entry.getValue()); 418 } 419 } 420 TestPluralPaths()421 public void TestPluralPaths() { 422 // do the following line once, when the file is opened 423 Set<String> filePaths = pathHeaderFactory.pathsForFile(english); 424 425 // check that English doesn't contain few or many 426 verifyContains(PageId.Duration, filePaths, "few", false); 427 verifyContains(PageId.C_NAmerica, filePaths, "many", false); 428 verifyContains(PageId.C_SAmerica, filePaths, "many", false); 429 verifyContains(PageId.C_NWEurope, filePaths, "many", false); 430 verifyContains(PageId.C_SEEurope, filePaths, "many", false); 431 verifyContains(PageId.C_NAfrica, filePaths, "many", false); 432 verifyContains(PageId.C_WAfrica, filePaths, "many", false); 433 verifyContains(PageId.C_SAfrica, filePaths, "many", false); 434 verifyContains(PageId.C_EAfrica, filePaths, "many", false); 435 verifyContains(PageId.C_CAsia, filePaths, "many", false); 436 verifyContains(PageId.C_WAsia, filePaths, "many", false); 437 verifyContains(PageId.C_SEAsia, filePaths, "many", false); 438 verifyContains(PageId.C_Oceania, filePaths, "many", false); 439 verifyContains(PageId.C_Unknown, filePaths, "many", false); 440 441 // check that Arabic does contain few and many 442 filePaths = pathHeaderFactory.pathsForFile(info.getCLDRFile("ar", true)); 443 444 verifyContains(PageId.Duration, filePaths, "few", true); 445 verifyContains(PageId.C_NAmerica, filePaths, "many", true); 446 verifyContains(PageId.C_SAmerica, filePaths, "many", true); 447 verifyContains(PageId.C_NWEurope, filePaths, "many", true); 448 verifyContains(PageId.C_SEEurope, filePaths, "many", true); 449 verifyContains(PageId.C_NAfrica, filePaths, "many", true); 450 verifyContains(PageId.C_WAfrica, filePaths, "many", true); 451 verifyContains(PageId.C_SAfrica, filePaths, "many", true); 452 verifyContains(PageId.C_EAfrica, filePaths, "many", true); 453 verifyContains(PageId.C_CAsia, filePaths, "many", true); 454 verifyContains(PageId.C_WAsia, filePaths, "many", true); 455 verifyContains(PageId.C_SEAsia, filePaths, "many", true); 456 verifyContains(PageId.C_Oceania, filePaths, "many", true); 457 verifyContains(PageId.C_Unknown, filePaths, "many", true); 458 } 459 TestCoverage()460 public void TestCoverage() { 461 Map<Row.R2<SectionId, PageId>, Counter<Level>> data = new TreeMap<>(); 462 CLDRFile cldrFile = english; 463 for (String path : cldrFile.fullIterable()) { 464 if (supplemental.isDeprecated(DtdType.ldml, path)) { 465 errln("Deprecated path in English: " + path); 466 continue; 467 } 468 Level level = supplemental.getCoverageLevel(path, 469 cldrFile.getLocaleID()); 470 PathHeader p = pathHeaderFactory.fromPath(path); 471 SurveyToolStatus status = p.getSurveyToolStatus(); 472 473 boolean hideCoverage = level == Level.COMPREHENSIVE; 474 boolean hidePathHeader = p.shouldHide(); 475 if (hidePathHeader != hideCoverage) { 476 String message = "PathHeader: " + status + ", Coverage: " 477 + level + ": " + path; 478 if (hidePathHeader && !hideCoverage) { 479 errln(message); 480 } else if (!hidePathHeader && hideCoverage) { 481 logln(message); 482 } 483 } 484 final R2<SectionId, PageId> key = Row.of(p.getSectionId(), 485 p.getPageId()); 486 Counter<Level> counter = data.get(key); 487 if (counter == null) { 488 data.put(key, counter = new Counter<>()); 489 } 490 counter.add(level, 1); 491 } 492 StringBuffer b = new StringBuffer("\t"); 493 for (Level level : Level.values()) { 494 b.append("\t" + level); 495 } 496 logln(b.toString()); 497 for (Entry<R2<SectionId, PageId>, Counter<Level>> entry : data 498 .entrySet()) { 499 b.setLength(0); 500 b.append(entry.getKey().get0() + "\t" + entry.getKey().get1()); 501 Counter<Level> counter = entry.getValue(); 502 long total = 0; 503 for (Level level : Level.values()) { 504 total += counter.getCount(level); 505 b.append("\t" + total); 506 } 507 logln(b.toString()); 508 } 509 } 510 Test00AFile()511 public void Test00AFile() { 512 final String localeId = "en"; 513 Counter<Level> counter = new Counter<>(); 514 Map<String, PathHeader> uniqueness = new HashMap<>(); 515 Set<String> alreadySeen = new HashSet<>(); 516 check(localeId, true, uniqueness, alreadySeen); 517 // check paths 518 for (Entry<SectionId, Set<PageId>> sectionAndPages : PathHeader.Factory 519 .getSectionIdsToPageIds().keyValuesSet()) { 520 final SectionId section = sectionAndPages.getKey(); 521 if (section == SectionId.Supplemental || section == SectionId.BCP47) { 522 continue; 523 } 524 logln(section.toString()); 525 for (PageId page : sectionAndPages.getValue()) { 526 final Set<String> cachedPaths = PathHeader.Factory 527 .getCachedPaths(section, page); 528 if (cachedPaths == null) { 529 if (!badZonePages.contains(page) && page != PageId.Unknown) { 530 errln("Null pages for: " + section + "\t" + page); 531 } 532 } else if (section == SectionId.Special 533 && page == PageId.Unknown) { 534 // skip 535 } else if (section == SectionId.Timezones 536 && page == PageId.UnknownT) { 537 // skip 538 } else if (section == SectionId.Misc 539 && page == PageId.Transforms) { 540 // skip 541 } else { 542 543 int count2 = cachedPaths.size(); 544 if (count2 == 0) { 545 warnln("Missing pages for: " + section + "\t" + page); 546 } else { 547 counter.clear(); 548 for (String s : cachedPaths) { 549 Level coverage = supplemental.getCoverageLevel(s, 550 localeId); 551 counter.add(coverage, 1); 552 } 553 String countString = ""; 554 int total = 0; 555 for (Level item : Level.values()) { 556 long count = counter.get(item); 557 if (count != 0) { 558 if (!countString.isEmpty()) { 559 countString += ",\t+"; 560 } 561 total += count; 562 countString += item + "=" + total; 563 } 564 } 565 logln("\t" + page + "\t" + countString); 566 if (page.toString().startsWith("Unknown")) { 567 logln("\t\t" + cachedPaths); 568 } 569 } 570 } 571 } 572 } 573 } 574 TestMetazones()575 public void TestMetazones() { 576 577 CLDRFile nativeFile = info.getEnglish(); 578 Set<PathHeader> pathHeaders = getPathHeaders(nativeFile); 579 // String oldPage = ""; 580 String oldHeader = ""; 581 for (PathHeader entry : pathHeaders) { 582 final String page = entry.getPage(); 583 // if (!oldPage.equals(page)) { 584 // logln(page); 585 // oldPage = page; 586 // } 587 String header = entry.getHeader(); 588 if (!oldHeader.equals(header)) { 589 logln(page + "\t" + header); 590 oldHeader = header; 591 } 592 } 593 } 594 getPathHeaders(CLDRFile nativeFile)595 public Set<PathHeader> getPathHeaders(CLDRFile nativeFile) { 596 Set<PathHeader> pathHeaders = new TreeSet<>(); 597 for (String path : nativeFile.fullIterable()) { 598 PathHeader p = pathHeaderFactory.fromPath(path); 599 pathHeaders.add(p); 600 } 601 return pathHeaders; 602 } 603 verifyContains(PageId pageId, Set<String> filePaths, String substring, boolean contains)604 public void verifyContains(PageId pageId, Set<String> filePaths, 605 String substring, boolean contains) { 606 String path; 607 path = findOneContaining(allPaths(pageId, filePaths), substring); 608 if (contains) { 609 if (path == null) { 610 errln("No path contains <" + substring + ">"); 611 } 612 } else { 613 if (path != null) { 614 errln("Path contains <" + substring + ">\t" + path); 615 } 616 } 617 } 618 findOneContaining(Collection<String> allPaths, String substring)619 private String findOneContaining(Collection<String> allPaths, 620 String substring) { 621 for (String path : allPaths) { 622 if (path.contains(substring)) { 623 return path; 624 } 625 } 626 return null; 627 } 628 allPaths(PageId pageId, Set<String> filePaths)629 public Set<String> allPaths(PageId pageId, Set<String> filePaths) { 630 Set<String> result = PathHeader.Factory.getCachedPaths( 631 pageId.getSectionId(), pageId); 632 result.retainAll(filePaths); 633 return result; 634 } 635 TestUniqueness()636 public void TestUniqueness() { 637 Factory factory2 = CLDRConfig.getInstance().getMainAndAnnotationsFactory(); 638 Set<String> source = factory2.getAvailable(); 639 for (String file : getFilesToTest(source, MIN_LOCALES)) { 640 CLDRFile nativeFile = factory2.make(file,true); 641 Map<PathHeader, String> headerToPath = new HashMap<>(); 642 Map<String, String> headerVisibleToPath = new HashMap<>(); 643 for (String path : nativeFile.fullIterable()) { 644 PathHeader p = pathHeaderFactory.fromPath(path); 645 if (p.getSectionId() == SectionId.Special) { 646 continue; 647 } 648 String old = headerToPath.get(p); 649 if (old == null) { 650 headerToPath.put(p, path); 651 } else if (!old.equals(path)) { 652 if (true) { // for debugging 653 pathHeaderFactory.clearCache(); 654 List<String> failuresOld = new ArrayList<>(); 655 pathHeaderFactory.fromPath(old, failuresOld); 656 List<String> failuresPath = new ArrayList<>(); 657 pathHeaderFactory.fromPath(path, failuresPath); 658 } 659 errln(file + " collision with path " + p + "\t" + old + "\t" + path); 660 } 661 final String visible = p.toString(); 662 old = headerVisibleToPath.get(visible); 663 if (old == null) { 664 headerVisibleToPath.put(visible, path); 665 } else if (!old.equals(path)) { 666 errln("Collision with path " + visible + "\t" + old + "\t" 667 + path); 668 } 669 } 670 } 671 } 672 TestStatus()673 public void TestStatus() { 674 CLDRFile nativeFile = info.getEnglish(); 675 PathStarrer starrer = new PathStarrer(); 676 EnumMap<SurveyToolStatus, Relation<String, String>> info2 = new EnumMap<>( 677 SurveyToolStatus.class); 678 Set<String> nuked = new HashSet<>(); 679 Set<String> deprecatedStar = new HashSet<>(); 680 681 for (String path : nativeFile.fullIterable()) { 682 683 PathHeader p = pathHeaderFactory.fromPath(path); 684 final SurveyToolStatus surveyToolStatus = p.getSurveyToolStatus(); 685 686 if (p.getSectionId() == SectionId.Special 687 && surveyToolStatus == SurveyToolStatus.READ_WRITE) { 688 errln("SurveyToolStatus should not be " + surveyToolStatus 689 + ": " + p); 690 } 691 692 String starred = starrer.set(path); 693 List<String> attr = starrer.getAttributes(); 694 if (surveyToolStatus != SurveyToolStatus.READ_WRITE) { 695 nuked.add(starred); 696 } 697 698 // check against deprecated 699 boolean isDeprecated = supplemental.isDeprecated(DtdType.ldml, path); 700 if (isDeprecated != (surveyToolStatus == SurveyToolStatus.DEPRECATED)) { 701 if (!deprecatedStar.contains(starred)) { 702 errln("Different from DtdData deprecated:\t" 703 + isDeprecated + "\t" + surveyToolStatus + "\t" 704 + path); 705 deprecatedStar.add(starred); 706 } 707 } 708 709 Relation<String, String> data = info2.get(surveyToolStatus); 710 if (data == null) { 711 info2.put( 712 surveyToolStatus, 713 data = Relation.of(new TreeMap<String, Set<String>>(), 714 TreeSet.class)); 715 } 716 data.put(starred, Joiner.on("|").join(attr)); 717 } 718 for (Entry<SurveyToolStatus, Relation<String, String>> entry : info2 719 .entrySet()) { 720 final SurveyToolStatus status = entry.getKey(); 721 for (Entry<String, Set<String>> item : entry.getValue() 722 .keyValuesSet()) { 723 final String starred = item.getKey(); 724 if (status == SurveyToolStatus.READ_WRITE 725 && !nuked.contains(starred)) { 726 continue; 727 } 728 logln(status + "\t" + starred + "\t" + item.getValue()); 729 } 730 } 731 } 732 TestPathsNotInEnglish()733 public void TestPathsNotInEnglish() { 734 Set<String> englishPaths = new HashSet<>(); 735 for (String path : english.fullIterable()) { 736 englishPaths.add(path); 737 } 738 Set<String> alreadySeen = new HashSet<>(englishPaths); 739 740 for (String locale : factory.getAvailable()) { 741 CLDRFile nativeFile = info.getCLDRFile(locale, false); 742 CoverageLevel2 coverageLevel2 = null; 743 for (String path : nativeFile.fullIterable()) { 744 if (alreadySeen.contains(path) || path.contains("@count")) { 745 continue; 746 } 747 if (coverageLevel2 == null) { 748 coverageLevel2 = CoverageLevel2.getInstance(locale); 749 } 750 Level level = coverageLevel2.getLevel(path); 751 if (Level.COMPREHENSIVE.compareTo(level) < 0) { 752 continue; 753 } 754 logln("Path not in English\t" + locale + "\t" + path); 755 alreadySeen.add(path); 756 } 757 } 758 } 759 TestPathDescriptionCompleteness()760 public void TestPathDescriptionCompleteness() { 761 PathDescription pathDescription = new PathDescription(supplemental, 762 english, null, null, PathDescription.ErrorHandling.CONTINUE); 763 Matcher normal = PatternCache.get( 764 "https://cldr.unicode.org/translation/[-a-zA-Z0-9_]").matcher(""); 765 // https://cldr.unicode.org/translation/plurals#TOC-Minimal-Pairs 766 Set<String> alreadySeen = new HashSet<>(); 767 PathStarrer starrer = new PathStarrer(); 768 769 checkPathDescriptionCompleteness(pathDescription, normal, 770 "//ldml/numbers/defaultNumberingSystem", alreadySeen, starrer); 771 for (PathHeader pathHeader : getPathHeaders(english)) { 772 if (pathHeader.shouldHide()) { 773 continue; 774 } 775 String path = pathHeader.getOriginalPath(); 776 checkPathDescriptionCompleteness(pathDescription, normal, path, 777 alreadySeen, starrer); 778 } 779 } 780 checkPathDescriptionCompleteness( PathDescription pathDescription, Matcher normal, String path, Set<String> alreadySeen, PathStarrer starrer)781 public void checkPathDescriptionCompleteness( 782 PathDescription pathDescription, Matcher normal, String path, 783 Set<String> alreadySeen, PathStarrer starrer) { 784 String value = english.getStringValue(path); 785 String description = pathDescription.getDescription(path, value, null, 786 null); 787 String starred = starrer.set(path); 788 if (alreadySeen.contains(starred)) { 789 return; 790 } else if (description == null) { 791 errln("Path has no description:\t" + value + "\t" + path); 792 } else if (!description.contains("https://")) { 793 errln("Description has no URL:\t" + description + "\t" + value 794 + "\t" + path); 795 } else if (!normal.reset(description).find()) { 796 errln("Description has generic URL, fix to be specific:\t" 797 + description + "\t" + value + "\t" + path); 798 } else if (description == PathDescription.MISSING_DESCRIPTION) { 799 errln("Fallback Description:\t" + value + "\t" + path); 800 } else { 801 return; 802 } 803 // Add if we had a problem, keeping us from being overwhelmed with 804 // errors. 805 alreadySeen.add(starred); 806 } 807 TestTerritoryOrder()808 public void TestTerritoryOrder() { 809 final Set<String> goodAvailableCodes = StandardCodes.make().getGoodAvailableCodes("territory"); 810 Set<String> results = showContained("001", 0, new HashSet<>( 811 goodAvailableCodes)); 812 results.remove("ZZ"); 813 for (String territory : results) { 814 String sub = Containment.getSubcontinent(territory); 815 String cont = Containment.getContinent(territory); 816 errln("Missing\t" + getNameAndOrder(territory) + "\t" 817 + getNameAndOrder(sub) + "\t" + getNameAndOrder(cont)); 818 } 819 } 820 showContained(String territory, int level, Set<String> soFar)821 private Set<String> showContained(String territory, int level, 822 Set<String> soFar) { 823 if (!soFar.contains(territory)) { 824 return soFar; 825 } 826 soFar.remove(territory); 827 Set<String> contained = supplemental.getContained(territory); 828 if (contained == null) { 829 return soFar; 830 } 831 for (String containedItem : contained) { 832 logln(level + "\t" + getNameAndOrder(territory) + "\t" 833 + getNameAndOrder(containedItem)); 834 } 835 for (String containedItem : contained) { 836 showContained(containedItem, level + 1, soFar); 837 } 838 return soFar; 839 } 840 getNameAndOrder(String territory)841 private String getNameAndOrder(String territory) { 842 return territory + "\t" 843 + english.getName(CLDRFile.TERRITORY_NAME, territory) + "\t" 844 + Containment.getOrder(territory); 845 } 846 TestZCompleteness()847 public void TestZCompleteness() { 848 Map<String, PathHeader> uniqueness = new HashMap<>(); 849 Set<String> alreadySeen = new HashSet<>(); 850 LanguageTagParser ltp = new LanguageTagParser(); 851 int count = 0; 852 for (String locale : factory.getAvailable()) { 853 if (!ltp.set(locale).getRegion().isEmpty()) { 854 continue; 855 } 856 check(locale, false, uniqueness, alreadySeen); 857 ++count; 858 } 859 logln("Count:\t" + count); 860 } 861 check(String localeID, boolean resolved, Map<String, PathHeader> uniqueness, Set<String> alreadySeen)862 public void check(String localeID, boolean resolved, 863 Map<String, PathHeader> uniqueness, Set<String> alreadySeen) { 864 CLDRFile nativeFile = info.getCLDRFile(localeID, resolved); 865 int count = 0; 866 for (String path : nativeFile) { 867 if (alreadySeen.contains(path)) { 868 continue; 869 } 870 alreadySeen.add(path); 871 final PathHeader pathHeader = pathHeaderFactory.fromPath(path); 872 ++count; 873 if (pathHeader == null) { 874 errln("Null pathheader for " + path); 875 } else { 876 String visible = pathHeader.toString(); 877 PathHeader old = uniqueness.get(visible); 878 if (pathHeader.getSectionId() == SectionId.Timezones) { 879 final PageId pageId = pathHeader.getPageId(); 880 if (badZonePages.contains(pageId) 881 && !pathHeader.getCode().equals("Unknown")) { 882 String msg = "Bad page ID:\t" + pageId + "\t" + pathHeader + "\t" + path; 883 if (!logKnownIssue("cldrbug:7802", "ICU/CLDR time zone data sync problem - " + msg)) { 884 errln("Bad page ID:\t" + pageId + "\t" + pathHeader 885 + "\t" + path); 886 } 887 } 888 } 889 if (old == null) { 890 if (pathHeader.getSection().equals("Special")) { 891 if (pathHeader.getSection().equals("Unknown")) { 892 errln("PathHeader has fallback: " + visible + "\t" 893 + pathHeader.getOriginalPath()); 894 // } else { 895 // logln("Special:\t" + visible + "\t" + 896 // pathHeader.getOriginalPath()); 897 } 898 } 899 uniqueness.put(visible, pathHeader); 900 } else if (!old.equals(pathHeader)) { 901 if (pathHeader.getSectionId() == SectionId.Special) { 902 logln("Special PathHeader not unique: " + visible 903 + "\t" + pathHeader.getOriginalPath() + "\t" 904 + old.getOriginalPath()); 905 } else { 906 errln("PathHeader not unique: " + visible + "\t" 907 + pathHeader.getOriginalPath() + "\t" 908 + old.getOriginalPath()); 909 } 910 } 911 } 912 } 913 logln(localeID + "\t" + count); 914 } 915 TestContainment()916 public void TestContainment() { 917 Map<String, Map<String, String>> metazoneToRegionToZone = supplemental 918 .getMetazoneToRegionToZone(); 919 Map<String, String> metazoneToContinent = supplemental 920 .getMetazoneToContinentMap(); 921 for (String metazone : metazoneToRegionToZone.keySet()) { 922 Map<String, String> regionToZone = metazoneToRegionToZone 923 .get(metazone); 924 String worldZone = regionToZone.get("001"); 925 String territory = Containment.getRegionFromZone(worldZone); 926 if (territory == null) { 927 territory = "ZZ"; 928 } 929 String cont = Containment.getContinent(territory); 930 int order = Containment.getOrder(territory); 931 String sub = Containment.getSubcontinent(territory); 932 String revision = PathHeader.getMetazonePageTerritory(metazone); 933 String continent = metazoneToContinent.get(metazone); 934 if (continent == null) { 935 continent = "UnknownT"; 936 } 937 // Russia, Antarctica => territory 938 // in Australasia, Asia, S. America => subcontinent 939 // in N. America => N. America (grouping of 3 subcontinents) 940 // in everything else => continent 941 942 if (territory.equals("RU")) { 943 assertEquals("Russia special case", "RU", revision); 944 } else if (territory.equals("US")) { 945 assertEquals("N. America special case", "003", revision); 946 } else if (territory.equals("BR")) { 947 assertEquals("S. America special case", "005", revision); 948 } 949 if (isVerbose()) { 950 String name = english.getName(CLDRFile.TERRITORY_NAME, cont); 951 String name2 = english.getName(CLDRFile.TERRITORY_NAME, sub); 952 String name3 = english.getName(CLDRFile.TERRITORY_NAME, 953 territory); 954 String name4 = english.getName(CLDRFile.TERRITORY_NAME, 955 revision); 956 957 logln(metazone + "\t" + continent + "\t" + name + "\t" + name2 958 + "\t" + name3 + "\t" + order + "\t" + name4); 959 } 960 } 961 } 962 TestZ()963 public void TestZ() { 964 PathStarrer pathStarrer = new PathStarrer(); 965 pathStarrer.setSubstitutionPattern("%A"); 966 967 Set<PathHeader> sorted = new TreeSet<>(); 968 Map<String, String> missing = new TreeMap<>(); 969 Map<String, String> skipped = new TreeMap<>(); 970 Map<String, String> collide = new TreeMap<>(); 971 972 logln("Traversing Paths"); 973 for (String path : english) { 974 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 975 String value = english.getStringValue(path); 976 if (pathHeader == null) { 977 final String starred = pathStarrer.set(path); 978 missing.put(starred, value + "\t" + path); 979 continue; 980 } 981 if (pathHeader.getSection().equalsIgnoreCase("skip")) { 982 final String starred = pathStarrer.set(path); 983 skipped.put(starred, value + "\t" + path); 984 continue; 985 } 986 sorted.add(pathHeader); 987 } 988 logln("\nConverted:\t" + sorted.size()); 989 String lastHeader = ""; 990 String lastPage = ""; 991 String lastSection = ""; 992 List<String> threeLevel = new ArrayList<>(); 993 Status status = new Status(); 994 CoverageLevel2 coverageLevel2 = CoverageLevel2.getInstance("en"); 995 996 for (PathHeader pathHeader : sorted) { 997 String original = pathHeader.getOriginalPath(); 998 if (!original.equals(status.pathWhereFound)) { 999 continue; 1000 } 1001 if (!lastSection.equals(pathHeader.getSection())) { 1002 logln(""); 1003 threeLevel.add(pathHeader.getSection()); 1004 threeLevel.add("\t" + pathHeader.getPage()); 1005 threeLevel.add("\t\t" + pathHeader.getHeader()); 1006 lastSection = pathHeader.getSection(); 1007 lastPage = pathHeader.getPage(); 1008 lastHeader = pathHeader.getHeader(); 1009 } else if (!lastPage.equals(pathHeader.getPage())) { 1010 logln(""); 1011 threeLevel.add("\t" + pathHeader.getPage()); 1012 threeLevel.add("\t\t" + pathHeader.getHeader()); 1013 lastPage = pathHeader.getPage(); 1014 lastHeader = pathHeader.getHeader(); 1015 } else if (!lastHeader.equals(pathHeader.getHeader())) { 1016 logln(""); 1017 threeLevel.add("\t\t" + pathHeader.getHeader()); 1018 lastHeader = pathHeader.getHeader(); 1019 } 1020 logln(pathHeader + "\t" + coverageLevel2.getLevel(original) + "\t" 1021 + english.getStringValue(pathHeader.getOriginalPath()) 1022 + "\t" + pathHeader.getOriginalPath()); 1023 } 1024 if (collide.size() != 0) { 1025 errln("\nCollide:\t" + collide.size()); 1026 for (Entry<String, String> item : collide.entrySet()) { 1027 errln("\t" + item); 1028 } 1029 } 1030 if (missing.size() != 0) { 1031 errln("\nMissing:\t" + missing.size()); 1032 for (Entry<String, String> item : missing.entrySet()) { 1033 errln("\t" + item.getKey() + "\tvalue:\t" + item.getValue()); 1034 } 1035 } 1036 if (skipped.size() != 0) { 1037 errln("\nSkipped:\t" + skipped.size()); 1038 for (Entry<String, String> item : skipped.entrySet()) { 1039 errln("\t" + item); 1040 } 1041 } 1042 Counter<PathHeader.Factory.CounterData> counterData = pathHeaderFactory 1043 .getInternalCounter(); 1044 logln("\nInternal Counter:\t" + counterData.size()); 1045 for (PathHeader.Factory.CounterData item : counterData.keySet()) { 1046 logln("\t" + counterData.getCount(item) + "\t" + item.get2() // externals 1047 + "\t" + item.get3() + "\t" + item.get0() // internals 1048 + "\t" + item.get1()); 1049 } 1050 logln("\nMenus/Headers:\t" + threeLevel.size()); 1051 for (String item : threeLevel) { 1052 logln(item); 1053 } 1054 LinkedHashMap<String, Set<String>> sectionsToPages = org.unicode.cldr.util.PathHeader.Factory 1055 .getSectionsToPages(); 1056 logln("\nMenus:\t" + sectionsToPages.size()); 1057 for (Entry<String, Set<String>> item : sectionsToPages.entrySet()) { 1058 final String section = item.getKey(); 1059 for (String page : item.getValue()) { 1060 logln("\t" + section + "\t" + page); 1061 int count = 0; 1062 for (String path : pathHeaderFactory.filterCldr(section, page, 1063 english)) { 1064 count += 1; // just count them. 1065 } 1066 logln("\t" + count); 1067 } 1068 } 1069 } 1070 1071 public static final Set<String> GERMAN_UNIT_ORDER = ImmutableSet.of( 1072 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]", 1073 "//ldml/units/unitLength[@type=\"short\"]/compoundUnit[@type=\"power2\"]", 1074 "//ldml/units/unitLength[@type=\"narrow\"]/compoundUnit[@type=\"power2\"]", 1075 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]", 1076 "//ldml/units/unitLength[@type=\"short\"]/unit[@type=\"volume-liter\"]", 1077 "//ldml/units/unitLength[@type=\"narrrow\"]/unit[@type=\"volume-liter\"]", 1078 "//ldml/numbers/minimalPairs/caseMinimalPairs", 1079 "//ldml/numbers/minimalPairs/genderMinimalPairs" 1080 ); TestOrder()1081 public void TestOrder() { 1082 String[] paths = { 1083 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod[@type=\"noon\"]", 1084 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod[@type=\"afternoon1\"]", 1085 }; 1086 PathHeader pathHeaderLast = null; 1087 for (String path : paths) { 1088 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 1089 if (pathHeaderLast != null) { 1090 assertRelation("ordering", true, pathHeaderLast, LEQ, pathHeader); 1091 } 1092 pathHeaderLast = pathHeader; 1093 } 1094 CLDRFile german = factory.make("de", true); 1095 Multimap<PathHeader, String> pathHeaderToPaths = TreeMultimap.create(); 1096 for (String path : german.fullIterable()) { 1097 for (String prefix : GERMAN_UNIT_ORDER) { 1098 if (path.startsWith(prefix)) { 1099 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 1100 pathHeaderToPaths.put(pathHeader, path); 1101 } 1102 } 1103 } 1104 String[] germanExpected = { 1105 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/gender", // Units Volume liter long-gender 1106 1107 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/displayName", // Units Volume liter long-displayName 1108 "//ldml/units/unitLength[@type=\"short\"]/unit[@type=\"volume-liter\"]/displayName", // Units Volume liter short-displayName 1109 1110 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/perUnitPattern", // Units Volume liter long-per 1111 "//ldml/units/unitLength[@type=\"short\"]/unit[@type=\"volume-liter\"]/perUnitPattern", // Units Volume liter short-per 1112 1113 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"one\"]", // Units Volume liter long-one-nominative 1114 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"one\"][@case=\"accusative\"]", // Units Volume liter long-one-accusative 1115 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"one\"][@case=\"genitive\"]", // Units Volume liter long-one-genitive 1116 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"one\"][@case=\"dative\"]", // Units Volume liter long-one-dative 1117 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"other\"]", // Units Volume liter long-other-nominative 1118 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"other\"][@case=\"accusative\"]", // Units Volume liter long-other-accusative 1119 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"other\"][@case=\"genitive\"]", // Units Volume liter long-other-genitive 1120 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"other\"][@case=\"dative\"]", // Units Volume liter long-other-dative 1121 "//ldml/units/unitLength[@type=\"short\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"one\"]", // Units Volume liter short-one-nominative 1122 "//ldml/units/unitLength[@type=\"short\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"other\"]", // Units Volume liter short-other-nominative 1123 1124 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@gender=\"masculine\"]", // Units Compound Units power2 long-one-nominative-masculine 1125 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@gender=\"feminine\"]", // Units Compound Units power2 long-one-nominative-feminine 1126 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"]", // Units Compound Units power2 long-one-nominative-dgender 1127 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@gender=\"masculine\"][@case=\"accusative\"]", // Units Compound Units power2 long-one-accusative-masculine 1128 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@gender=\"feminine\"][@case=\"accusative\"]", // Units Compound Units power2 long-one-accusative-feminine 1129 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@case=\"accusative\"]", // Units Compound Units power2 long-one-accusative-dgender 1130 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@gender=\"masculine\"][@case=\"genitive\"]", // Units Compound Units power2 long-one-genitive-masculine 1131 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@gender=\"feminine\"][@case=\"genitive\"]", // Units Compound Units power2 long-one-genitive-feminine 1132 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@case=\"genitive\"]", // Units Compound Units power2 long-one-genitive-dgender 1133 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@gender=\"masculine\"][@case=\"dative\"]", // Units Compound Units power2 long-one-dative-masculine 1134 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@gender=\"feminine\"][@case=\"dative\"]", // Units Compound Units power2 long-one-dative-feminine 1135 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@case=\"dative\"]", // Units Compound Units power2 long-one-dative-dgender 1136 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@gender=\"masculine\"]", // Units Compound Units power2 long-other-nominative-masculine 1137 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@gender=\"feminine\"]", // Units Compound Units power2 long-other-nominative-feminine 1138 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"]", // Units Compound Units power2 long-other-nominative-dgender 1139 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@gender=\"masculine\"][@case=\"accusative\"]", // Units Compound Units power2 long-other-accusative-masculine 1140 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@gender=\"feminine\"][@case=\"accusative\"]", // Units Compound Units power2 long-other-accusative-feminine 1141 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@case=\"accusative\"]", // Units Compound Units power2 long-other-accusative-dgender 1142 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@gender=\"masculine\"][@case=\"genitive\"]", // Units Compound Units power2 long-other-genitive-masculine 1143 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@gender=\"feminine\"][@case=\"genitive\"]", // Units Compound Units power2 long-other-genitive-feminine 1144 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@case=\"genitive\"]", // Units Compound Units power2 long-other-genitive-dgender 1145 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@gender=\"masculine\"][@case=\"dative\"]", // Units Compound Units power2 long-other-dative-masculine 1146 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@gender=\"feminine\"][@case=\"dative\"]", // Units Compound Units power2 long-other-dative-feminine 1147 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"][@case=\"dative\"]", // Units Compound Units power2 long-other-dative-dgender 1148 "//ldml/units/unitLength[@type=\"short\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"]", // Units Compound Units power2 short-one-nominative-dgender 1149 "//ldml/units/unitLength[@type=\"short\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"]", // Units Compound Units power2 short-other-nominative-dgender 1150 "//ldml/units/unitLength[@type=\"narrow\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"]", // Units Compound Units power2 narrow-one-nominative-dgender 1151 "//ldml/units/unitLength[@type=\"narrow\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"other\"]", // Units Compound Units power2 narrow-other-nominative-dgender 1152 1153 "//ldml/numbers/minimalPairs/caseMinimalPairs[@case=\"nominative\"]", // Miscellaneous Minimal Pairs Case nominative 1154 "//ldml/numbers/minimalPairs/caseMinimalPairs[@case=\"accusative\"]", // Miscellaneous Minimal Pairs Case accusative 1155 "//ldml/numbers/minimalPairs/caseMinimalPairs[@case=\"genitive\"]", // Miscellaneous Minimal Pairs Case genitive 1156 "//ldml/numbers/minimalPairs/caseMinimalPairs[@case=\"dative\"]", // Miscellaneous Minimal Pairs Case dative 1157 "//ldml/numbers/minimalPairs/genderMinimalPairs[@gender=\"masculine\"]", // Miscellaneous Minimal Pairs Gender masculine 1158 "//ldml/numbers/minimalPairs/genderMinimalPairs[@gender=\"feminine\"]", // Miscellaneous Minimal Pairs Gender feminine 1159 "//ldml/numbers/minimalPairs/genderMinimalPairs[@gender=\"neuter\"]", // Miscellaneous Minimal Pairs Gender neuter 1160 1161 // we don't care about order here. 1162 "//ldml/units/unitLength[@type=\"long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1", // Special Suppress compound-UnitPattern1-power2 long 1163 "//ldml/units/unitLength[@type=\"narrow\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1", // Special Suppress compound-UnitPattern1-power2 narrow 1164 "//ldml/units/unitLength[@type=\"short\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1", // Special Suppress compound-UnitPattern1-power2 short 1165 }; 1166 1167 int germanExpectedIndex = 0; 1168 int errorCount = 0; 1169 int item = 0; 1170 for (Entry<PathHeader, Collection<String>> entry : pathHeaderToPaths.asMap().entrySet()) { 1171 PathHeader ph = entry.getKey(); 1172 Collection<String> epaths = entry.getValue(); 1173 if (!assertEquals(entry.toString(), 1, epaths.size())) { 1174 ++errorCount; 1175 } 1176 if (!assertEquals(++item + ") PathHeader order", germanExpected[germanExpectedIndex++], epaths.iterator().next())) { 1177 ++errorCount; 1178 } 1179 } 1180 if (errorCount != 0) { 1181 for (Entry<PathHeader, Collection<String>> entry : pathHeaderToPaths.asMap().entrySet()) { 1182 PathHeader ph = entry.getKey(); 1183 Collection<String> epaths = entry.getValue(); 1184 System.out.println("\"" + epaths.iterator().next().replace("\"", "\\\"") + "\",\t// " + ph); 1185 } 1186 } 1187 } 1188 Test8414()1189 public void Test8414() { 1190 PathDescription pathDescription = new PathDescription(supplemental, 1191 english, null, null, PathDescription.ErrorHandling.CONTINUE); 1192 1193 String prefix = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\""; 1194 String suffix = "\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod[@type=\"morning1\"]"; 1195 1196 final String path0 = prefix + "format" + suffix; 1197 final String path1 = prefix + "stand-alone" + suffix; 1198 String v0 = english.getStringValue(path0); 1199 String v1 = english.getStringValue(path1); 1200 String p0 = pathDescription.getDescription(path0, v0, null, null); 1201 String p1 = pathDescription.getDescription(path1, v1, null, null); 1202 assertTrue("Check pd for format", p0.contains("in the morning")); 1203 assertTrue("Check pd for stand-alone", !p1.contains("in the morning")); 1204 } 1205 TestCompletenessNonLdmlDtd()1206 public void TestCompletenessNonLdmlDtd() { 1207 PathChecker pathChecker = new PathChecker(); 1208 Set<String> directories = new LinkedHashSet<>(); 1209 Multimap<String, String> pathValuePairs = LinkedListMultimap.create(); 1210 // get all the directories containing non-Ldml dtd files 1211 for (DtdType dtdType : DtdType.values()) { 1212 if (dtdType == DtdType.ldml || dtdType == DtdType.ldmlICU) { 1213 continue; 1214 } 1215 DtdData dtdData = DtdData.getInstance(dtdType); 1216 for (String dir : dtdType.directories) { 1217 if (DEBUG_DTD_TYPE != null && !DEBUG_DTD_TYPE.directories.contains(dir)) { 1218 continue; 1219 } 1220 File dir2 = new File(COMMON_DIR + dir); 1221 logln(dir2.getName()); 1222 for (String file : dir2.list()) { 1223 // don't need to restrict with getFilesToTest(Arrays.asList(dir2.list()), "root", "en")) { 1224 if (!file.endsWith(".xml")) { 1225 continue; 1226 } 1227 if (DEBUG) warnln(" TestCompletenessNonLdmlDtd: " + dir + ", " + file); 1228 logln(" \t" + file); 1229 for (Pair<String, String> pathValue : XMLFileReader.loadPathValues( 1230 dir2 + "/" + file, new ArrayList<Pair<String, String>>(), true)) { 1231 final String path = pathValue.getFirst(); 1232 final String value = pathValue.getSecond(); 1233 // logln("\t\t" + path); 1234 if (path.startsWith("//supplementalData/unitPreferenceData/unitPreferences") 1235 && path.contains("skeleton")) { 1236 int debug = 0; 1237 } 1238 pathChecker.checkPathHeader(dtdData, path); 1239 } 1240 } 1241 } 1242 } 1243 if (!pathChecker.badHeaders.isEmpty()) { 1244 System.out.println("For help with DTD updates: " + CLDRURLS.CLDR_UPDATINGDTD_URL); 1245 } 1246 } 1247 1248 private class PathChecker { 1249 PathHeader.Factory phf = pathHeaderFactory; 1250 PathStarrer starrer = new PathStarrer().setSubstitutionPattern("%A"); 1251 1252 Set<String> badHeaders = new TreeSet<>(); 1253 Map<PathHeader, PathHeader> goodHeaders = new HashMap<>(); 1254 Set<PathHeader> seenBad = new HashSet<>(); 1255 { phf.clearCache()1256 phf.clearCache(); 1257 } 1258 checkPathHeader(DtdData dtdData, String rawPath)1259 public void checkPathHeader(DtdData dtdData, String rawPath) { 1260 XPathParts pathPlain = XPathParts.getFrozenInstance(rawPath); 1261 if (dtdData.isMetadata(pathPlain)) { 1262 return; 1263 } 1264 if (dtdData.isDeprecated(pathPlain)) { 1265 return; 1266 } 1267 Multimap<String, String> extras = HashMultimap.create(); 1268 Set<String> fixedPaths = dtdData.getRegularizedPaths(pathPlain, extras); 1269 if (fixedPaths != null) { 1270 for (String fixedPath : fixedPaths) { 1271 checkSubpath(fixedPath); 1272 } 1273 } 1274 for (String path : extras.keySet()) { 1275 checkSubpath(path); 1276 } 1277 } 1278 checkSubpath(String path)1279 public void checkSubpath(String path) { 1280 String message = ": Can't compute path header"; 1281 if (path.contentEquals("//supplementalData/grammaticalData/grammaticalFeatures[@targets=\"nominal\"][@locales=\"it\"]/grammaticalGender/_values") ) { 1282 int debug = 0; 1283 } 1284 PathHeader ph = null; 1285 try { 1286 ph = phf.fromPath(path); 1287 if (seenBad.contains(ph)) { 1288 return; 1289 } 1290 if (ph.getPageId() == PageId.Deprecated) { 1291 return; // don't care 1292 } 1293 if (ph.getPageId() != PageId.Unknown) { 1294 PathHeader old = goodHeaders.put(ph, ph); 1295 if (old != null && !path.equals(old.getOriginalPath())) { 1296 errln("Duplicate path header for: " + ph 1297 + "\n\t\t " + path 1298 + "\n\t\t≠" + old.getOriginalPath()); 1299 seenBad.add(ph); 1300 } 1301 return; 1302 } 1303 // for debugging 1304 phf.clearCache(); 1305 List<String> failures = new ArrayList<>(); 1306 ph = phf.fromPath(path, failures); 1307 message = ": Unknown path header" + failures; 1308 } catch (Exception e) { 1309 message = ": Exception in path header: " + e.getMessage(); 1310 } 1311 String star = starrer.set(path); 1312 if (badHeaders.add(star)) { 1313 errln(star + message + ", " + ph); 1314 System.out.println("\tNo match in PathHeader.txt for " + path 1315 + "\n\tYou get only one message for all paths matching " + star 1316 + "\n\tFor example, check to see if the field in PathHeader.txt is in PathHeader.PageId." 1317 + "\n\tIf not, either correct PathHeader.txt or add it to PageId" 1318 + "\n\tIf you have a value attribute, you will need extra _ characters. The value attribute will show at the end with prefixed _, eg [...]/_skeleton." 1319 + "If there can be a value for the path then that element will add _. " 1320 ); 1321 } 1322 } 1323 } 1324 TestSupplementalItems()1325 public void TestSupplementalItems() { 1326 // <weekOfPreference ordering="weekOfYear weekOfMonth" locales="am az bs cs cy da el et hi ky lt mk sk ta th"/> 1327 // logln(pathHeaderFactory.getRegexInfo()); 1328 CLDRFile supplementalFile = CLDRConfig.getInstance().getSupplementalFactory().make("supplementalData", false); 1329 List<String> failures = new ArrayList<>(); 1330 Multimap<String, String> pathValuePairs = LinkedListMultimap.create(); 1331 for (String test : With.in(supplementalFile.iterator("//supplementalData/weekData"))) { 1332 failures.clear(); 1333 XPathParts parts = XPathParts.getFrozenInstance(supplementalFile.getFullXPath(test)); 1334 supplementalFile.getDtdData().getRegularizedPaths(parts, pathValuePairs); 1335 for (Entry<String, Collection<String>> entry : pathValuePairs.asMap().entrySet()) { 1336 final String normalizedPath = entry.getKey(); 1337 final Collection<String> normalizedValue = entry.getValue(); 1338 PathHeader ph = pathHeaderFactory.fromPath(normalizedPath, failures); 1339 if (ph == null || ph.getSectionId() == SectionId.Special) { 1340 errln("Failure with " + test + " => " + normalizedPath + " = " + normalizedValue); 1341 } else { 1342 logln(ph + "\t" + test + " = " + normalizedValue); 1343 } 1344 } 1345 } 1346 } 1347 test10232()1348 public void test10232() { 1349 String[][] tests = { 1350 { "MMM", "Formats - Flexible - Date Formats" }, 1351 { "dMM", "Formats - Flexible - Date Formats" }, 1352 { "h", "Formats - Flexible - 12 Hour Time Formats" }, 1353 { "hm", "Formats - Flexible - 12 Hour Time Formats" }, 1354 { "Ehm", "Formats - Flexible - 12 Hour Time Formats" }, 1355 { "H", "Formats - Flexible - 24 Hour Time Formats" }, 1356 { "Hm", "Formats - Flexible - 24 Hour Time Formats" }, 1357 { "EHm", "Formats - Flexible - 24 Hour Time Formats" }, 1358 }; 1359 for (String[] test : tests) { 1360 String path = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\"" 1361 + test[0] + "\"]"; 1362 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 1363 assertEquals("flexible formats", test[1] + "|" + test[0], pathHeader.getHeader() + "|" + pathHeader.getCode()); 1364 } 1365 } 1366 1367 // Moved from TestAnnotations and generalized testPathHeaderSize()1368 public void testPathHeaderSize() { 1369 String locale = "ar"; // choose one with lots of plurals 1370 int maxSize = 750; 1371 boolean showTable = false; // only printed if test fails or verbose 1372 1373 Factory factory = CLDRConfig.getInstance().getCommonAndSeedAndMainAndAnnotationsFactory(); 1374 CLDRFile english = factory.make(locale, true); 1375 1376 PathHeader.Factory phf = PathHeader.getFactory(CLDRConfig.getInstance().getEnglish()); 1377 Counter<PageId> counterPageId = new Counter<>(); 1378 Counter<PageId> counterPageIdAll = new Counter<>(); 1379 for (String path : english) { 1380 Level level = CLDRConfig.getInstance().getSupplementalDataInfo().getCoverageLevel(path, locale); 1381 PathHeader ph = phf.fromPath(path); 1382 if (level.compareTo(Level.MODERN) <= 0) { 1383 counterPageId.add(ph.getPageId(), 1); 1384 } 1385 counterPageIdAll.add(ph.getPageId(), 1); 1386 } 1387 Set<R2<Long, PageId>> entrySetSortedByCount = counterPageId.getEntrySetSortedByCount(false, null); 1388 for (R2<Long, PageId> sizeAndPageId : entrySetSortedByCount) { 1389 long size = sizeAndPageId.get0(); 1390 PageId pageId = sizeAndPageId.get1(); 1391 if (!assertTrue(pageId.getSectionId() + "/" + pageId + " size (" + size 1392 + ") < " + maxSize + "?", size < maxSize)) { 1393 showTable = true; 1394 } 1395 // System.out.println(pageId + "\t" + size); 1396 } 1397 if (showTable || isVerbose()) { 1398 for (R2<Long, PageId> sizeAndPageId : entrySetSortedByCount) { 1399 PageId pageId = sizeAndPageId.get1(); 1400 System.out.println(pageId.getSectionId() + "\t" + pageId + "\t" + sizeAndPageId.get0() + "\t" + counterPageIdAll.get(pageId)); 1401 } 1402 } 1403 } TestCLDR_11454()1404 public void TestCLDR_11454() { 1405 PathHeader.Factory phf = PathHeader.getFactory(); 1406 PathHeader century = phf.fromPath("//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"duration-century\"]/displayName"); 1407 PathHeader decade = phf.fromPath("//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"duration-decade\"]/displayName"); 1408 assertEquals("Section", century.getSectionId(), decade.getSectionId()); 1409 assertEquals("Page", century.getPageId(), decade.getPageId()); 1410 } 1411 TestEmojiOrder()1412 public void TestEmojiOrder() { 1413 PathHeader.Factory phf = PathHeader.getFactory(); 1414 String[] desiredOrder = { 1415 "��⚕", "��⚕", "��⚕", 1416 "��⚖", "��⚖", "��⚖"}; 1417 List<PathHeader> pathHeaders = new ArrayList<>(); 1418 for (String emoji : desiredOrder) { 1419 String base = "//ldml/annotations/annotation[@cp=\"" + emoji + "\"]"; 1420 pathHeaders.add(phf.fromPath(base + "[@type=\"tts\"]")); 1421 pathHeaders.add(phf.fromPath(base)); 1422 logln(emoji 1423 + ": getEmojiMinorOrder="+ Emoji.getEmojiMinorOrder(Emoji.getMinorCategory(emoji)) 1424 + ", getEmojiToOrder="+ Emoji.getEmojiToOrder(emoji) 1425 ); 1426 } 1427 PathHeader lastItem = null; 1428 for (PathHeader item : pathHeaders) { 1429 if (lastItem != null) { 1430 assertEquals("Section", lastItem.getSectionId(), item.getSectionId()); 1431 assertEquals("Page", lastItem.getPageId(), item.getPageId()); 1432 assertEquals("Header", lastItem.getHeader(), item.getHeader()); 1433 if (!assertTrue(lastItem + " < " + item, lastItem.compareTo(item) < 0)) { 1434 lastItem.compareTo(item); // for debugging 1435 } 1436 } 1437 lastItem = item; 1438 } 1439 } 1440 TestQuotes()1441 public void TestQuotes() { 1442 // quotes should never appear in result 1443 PathHeader.Factory phf = PathHeader.getFactory(); 1444 String[] tests = { 1445 "//supplementalData/plurals[@type=\"ordinal\"]/pluralRules[@locales=\"ig\"]/pluralRule[@count=\"other\"]", 1446 "//supplementalData/transforms/transform[@source=\"und-Khmr\"][@target=\"und-Latn\"]" 1447 }; 1448 for (String test : tests) { 1449 PathHeader trial = phf.fromPath(test); 1450 assertEquals("No quotes in pathheader", false, trial.toString().contains("\"")); 1451 } 1452 } 1453 /** 1454 * Make sure that the PathHeader sort order is consistent with the grammatical feature orders 1455 * "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"volume-liter\"]/displayName" 1456 * //ldml/units/unitLength[@type=\long\"]/unit[@type=\"volume-liter\"]/unitPattern[@count=\"one\"][@case=\"genitive\"]", 1457 * //ldml/units/unitLength[@type=\long\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"one\"][@gender=\"feminine\"][@case=\"accusative\"]", 1458 */ TestUnitOrder()1459 public void TestUnitOrder() { 1460 PathHeader.Factory phf = PathHeader.getFactory(); 1461 List<PathHeader> expectedOrder = new ArrayList<>(); 1462 List<Width> widths = Arrays.asList(Width.LONG, Width.SHORT, Width.NARROW); 1463 List<CaseValues> cases = Arrays.asList(GrammarInfo.CaseValues.values()).subList(0, 3); 1464 List<GenderValues> genders = Arrays.asList(GrammarInfo.GenderValues.values()).subList(0, 3); 1465 1466 for (Width width : widths) { 1467 String path = "//ldml/units/unitLength[@type=\"" + width 1468 + "\"]/unit[@type=\"length-meter\"]/displayName"; 1469 expectedOrder.add(phf.fromPath(path)); 1470 } 1471 1472 for (Width width : widths) { 1473 for (Count count : Count.values()) { 1474 for (GrammarInfo.CaseValues gCase : cases) { 1475 if (width != Width.LONG && gCase != CaseValues.nominative) { 1476 break; 1477 } 1478 String path = "//ldml/units/unitLength[@type=\"" + width 1479 + "\"]/unit[@type=\"length-meter\"]/unitPattern[@count=\"" + count 1480 + (gCase == CaseValues.nominative ? "" : "\"][@case=\"" + gCase) 1481 + "\"]"; 1482 expectedOrder.add(phf.fromPath(path)); 1483 } 1484 } 1485 } 1486 for (Width width : widths) { 1487 for (Count count : Count.values()) { 1488 for (GrammarInfo.CaseValues gCase : cases) { 1489 if (width != Width.LONG && gCase != CaseValues.nominative) { 1490 break; 1491 } 1492 for (GrammarInfo.GenderValues gGender : genders) { 1493 if (width != Width.LONG && gGender != GenderValues.neuter) { 1494 break; 1495 } 1496 String path = "//ldml/units/unitLength[@type=\"" + width 1497 + "\"]/compoundUnit[@type=\"power2\"]/compoundUnitPattern1[@count=\"" + count 1498 + (gGender == GenderValues.neuter ? "" : "\"][@gender=\"" + gGender) 1499 + (gCase == CaseValues.nominative ? "" : "\"][@case=\"" + gCase) 1500 + "\"]"; 1501 expectedOrder.add(phf.fromPath(path)); 1502 } 1503 } 1504 } 1505 } 1506 for (Count count : Count.values()) { 1507 String path = "//ldml/numbers/minimalPairs/ordinalMinimalPairs[@ordinal=\"" + count 1508 + "\"]"; 1509 expectedOrder.add(phf.fromPath(path)); 1510 } 1511 for (Count count : Count.values()) { 1512 String path = "//ldml/numbers/minimalPairs/pluralMinimalPairs[@count=\"" + count 1513 + "\"]"; 1514 expectedOrder.add(phf.fromPath(path)); 1515 } 1516 for (GrammarInfo.CaseValues gCase : cases) { 1517 String path = "//ldml/numbers/minimalPairs/caseMinimalPairs[@case=\"" + gCase 1518 + "\"]"; 1519 expectedOrder.add(phf.fromPath(path)); 1520 } 1521 for (GrammarInfo.GenderValues gGender : genders) { 1522 String path = "//ldml/numbers/minimalPairs/genderMinimalPairs[@gender=\"" + gGender 1523 + "\"]"; 1524 expectedOrder.add(phf.fromPath(path)); 1525 } 1526 1527 PathHeader last = null; 1528 int item = 0; 1529 int errorCount = 0; 1530 for (PathHeader pathHeader : expectedOrder) { 1531 if (last != null) { 1532 if (!assertTrue(++item + ")\t" + last + "\t<\t" + pathHeader, last.compareTo(pathHeader) < 0)) { 1533 errorCount++; 1534 last.compareTo(pathHeader); 1535 } 1536 } 1537 last = pathHeader; 1538 1539 } 1540 if (errorCount != 0 || isVerbose()) { 1541 for (PathHeader pathHeader : expectedOrder) { 1542 System.out.println("\"" + pathHeader.getOriginalPath().replace("\"", "\\\"") + "\",\t// " + pathHeader); 1543 } 1544 } 1545 } 1546 } 1547