1 package org.unicode.cldr.unittest; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.InputStream; 6 import java.io.PrintWriter; 7 import java.io.StringWriter; 8 import java.util.ArrayList; 9 import java.util.Arrays; 10 import java.util.Collection; 11 import java.util.Collections; 12 import java.util.Comparator; 13 import java.util.HashSet; 14 import java.util.Iterator; 15 import java.util.LinkedHashSet; 16 import java.util.List; 17 import java.util.Map; 18 import java.util.Map.Entry; 19 import java.util.Set; 20 import java.util.TreeMap; 21 import java.util.TreeSet; 22 23 import org.unicode.cldr.test.DisplayAndInputProcessor; 24 import org.unicode.cldr.tool.CldrVersion; 25 import org.unicode.cldr.tool.LikelySubtags; 26 import org.unicode.cldr.util.Builder; 27 import org.unicode.cldr.util.CLDRConfig; 28 import org.unicode.cldr.util.CLDRFile; 29 import org.unicode.cldr.util.CLDRFile.DraftStatus; 30 import org.unicode.cldr.util.CLDRFile.Status; 31 import org.unicode.cldr.util.CLDRFile.WinningChoice; 32 import org.unicode.cldr.util.CLDRPaths; 33 import org.unicode.cldr.util.ChainedMap; 34 import org.unicode.cldr.util.ChainedMap.M4; 35 import org.unicode.cldr.util.CharacterFallbacks; 36 import org.unicode.cldr.util.CldrUtility; 37 import org.unicode.cldr.util.Counter; 38 import org.unicode.cldr.util.DiscreteComparator; 39 import org.unicode.cldr.util.DiscreteComparator.Ordering; 40 import org.unicode.cldr.util.DtdData; 41 import org.unicode.cldr.util.DtdData.Attribute; 42 import org.unicode.cldr.util.DtdData.Element; 43 import org.unicode.cldr.util.DtdData.ElementType; 44 import org.unicode.cldr.util.DtdType; 45 import org.unicode.cldr.util.ElementAttributeInfo; 46 import org.unicode.cldr.util.Factory; 47 import org.unicode.cldr.util.InputStreamFactory; 48 import org.unicode.cldr.util.LanguageTagParser; 49 import org.unicode.cldr.util.Level; 50 import org.unicode.cldr.util.LocaleIDParser; 51 import org.unicode.cldr.util.Pair; 52 import org.unicode.cldr.util.PathHeader; 53 import org.unicode.cldr.util.PathUtilities; 54 import org.unicode.cldr.util.StandardCodes; 55 import org.unicode.cldr.util.SupplementalDataInfo; 56 import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo; 57 import org.unicode.cldr.util.SupplementalDataInfo.PluralType; 58 import org.unicode.cldr.util.XMLFileReader; 59 import org.unicode.cldr.util.XPathParts; 60 import org.xml.sax.ErrorHandler; 61 import org.xml.sax.InputSource; 62 import org.xml.sax.SAXException; 63 import org.xml.sax.SAXParseException; 64 import org.xml.sax.XMLReader; 65 66 import com.google.common.base.Joiner; 67 import com.google.common.base.Objects; 68 import com.google.common.collect.ImmutableMultimap; 69 import com.google.common.collect.ImmutableSet; 70 import com.google.common.collect.Multimap; 71 import com.google.common.collect.TreeMultimap; 72 import com.ibm.icu.impl.Relation; 73 import com.ibm.icu.impl.Row; 74 import com.ibm.icu.impl.Row.R2; 75 import com.ibm.icu.impl.Row.R3; 76 import com.ibm.icu.impl.Utility; 77 import com.ibm.icu.lang.UCharacter; 78 import com.ibm.icu.text.Collator; 79 import com.ibm.icu.text.DecimalFormat; 80 import com.ibm.icu.text.Normalizer; 81 import com.ibm.icu.text.NumberFormat; 82 import com.ibm.icu.text.UTF16; 83 import com.ibm.icu.text.UnicodeSet; 84 import com.ibm.icu.text.UnicodeSetIterator; 85 import com.ibm.icu.util.Currency; 86 import com.ibm.icu.util.ULocale; 87 88 public class TestBasic extends TestFmwkPlus { 89 90 private static final boolean DEBUG = false; 91 92 static CLDRConfig testInfo = CLDRConfig.getInstance(); 93 94 private static final SupplementalDataInfo SUPPLEMENTAL_DATA_INFO = testInfo 95 .getSupplementalDataInfo(); 96 97 private static final ImmutableSet<Pair<String, String>> knownElementExceptions = ImmutableSet.of( 98 Pair.of("ldml", "usesMetazone"), 99 Pair.of("ldmlICU", "usesMetazone")); 100 101 private static final ImmutableSet<Pair<String, String>> knownAttributeExceptions = ImmutableSet.of( 102 Pair.of("ldml", "version"), 103 Pair.of("supplementalData", "version"), 104 Pair.of("ldmlICU", "version"), 105 Pair.of("layout", "standard"), 106 Pair.of("currency", "id"), // for v1.1.1 107 Pair.of("monthNames", "type"), // for v1.1.1 108 Pair.of("alias", "type") // for v1.1.1 109 ); 110 111 private static final ImmutableSet<Pair<String, String>> knownChildExceptions = ImmutableSet.of( 112 Pair.of("abbreviationFallback", "special"), 113 Pair.of("inList", "special"), 114 Pair.of("preferenceOrdering", "special")); 115 116 /** 117 * Simple test that loads each file in the cldr directory, thus verifying 118 * that the DTD works, and also checks that the PrettyPaths work. 119 * 120 * @author markdavis 121 */ 122 main(String[] args)123 public static void main(String[] args) { 124 new TestBasic().run(args); 125 } 126 127 private static final ImmutableSet<String> skipAttributes = ImmutableSet.of( 128 "alt", "draft", "references"); 129 130 private final ImmutableSet<String> eightPointLocales = ImmutableSet.of( 131 "ar", "ca", "cs", "da", "de", "el", "es", "fi", "fr", "he", "hi", "hr", "hu", "id", 132 "it", "ja", "ko", "lt", "lv", "nl", "no", "pl", "pt", "pt_PT", "ro", "ru", "sk", "sl", "sr", "sv", 133 "th", "tr", "uk", "vi", "zh", "zh_Hant"); 134 135 // private final boolean showForceZoom = Utility.getProperty("forcezoom", 136 // false); 137 138 private final boolean resolved = CldrUtility.getProperty("resolved", false); 139 140 private final Exception[] internalException = new Exception[1]; 141 TestDtds()142 public void TestDtds() throws IOException { 143 Relation<Row.R2<DtdType, String>, String> foundAttributes = Relation 144 .of(new TreeMap<Row.R2<DtdType, String>, Set<String>>(), 145 TreeSet.class); 146 final CLDRConfig config = CLDRConfig.getInstance(); 147 final File basedir = config.getCldrBaseDirectory(); 148 List<TimingInfo> data = new ArrayList<>(); 149 150 for (String subdir : config.getCLDRDataDirectories()) { 151 checkDtds(new File(basedir, subdir), 0, foundAttributes, data); 152 } 153 if (foundAttributes.size() > 0) { 154 showFoundElements(foundAttributes); 155 } 156 if (isVerbose()) { 157 long totalBytes = 0; 158 long totalNanos = 0; 159 for (TimingInfo i : data) { 160 long length = i.file.length(); 161 totalBytes += length; 162 totalNanos += i.nanos; 163 logln(i.nanos + "\t" + length + "\t" + i.file); 164 } 165 logln(totalNanos + "\t" + totalBytes); 166 } 167 } 168 checkDtds(File directoryFile, int level, Relation<R2<DtdType, String>, String> foundAttributes, List<TimingInfo> data)169 private void checkDtds(File directoryFile, int level, 170 Relation<R2<DtdType, String>, String> foundAttributes, 171 List<TimingInfo> data) throws IOException { 172 boolean deepCheck = getInclusion() >= 10; 173 File[] listFiles = directoryFile.listFiles(); 174 String normalizedPath = PathUtilities.getNormalizedPathString(directoryFile); 175 String indent = Utility.repeat("\t", level); 176 if (listFiles == null) { 177 throw new IllegalArgumentException(indent + "Empty directory: " 178 + normalizedPath); 179 } 180 logln("Checking files for DTD errors in: " + indent + normalizedPath); 181 for (File fileName : listFiles) { 182 String name = fileName.getName(); 183 if (CLDRConfig.isJunkFile(name)) { 184 continue; 185 } else if (fileName.isDirectory()) { 186 checkDtds(fileName, level + 1, foundAttributes, data); 187 } else if (name.endsWith(".xml")) { 188 data.add(check(fileName)); 189 if (deepCheck // takes too long to do all the time 190 ) { 191 CLDRFile cldrfile = CLDRFile.loadFromFile(fileName, "temp", 192 DraftStatus.unconfirmed); 193 for (String xpath : cldrfile) { 194 String fullPath = cldrfile.getFullXPath(xpath); 195 if (fullPath == null) { 196 fullPath = cldrfile.getFullXPath(xpath); 197 assertNotNull("", fullPath); 198 continue; 199 } 200 XPathParts parts = XPathParts 201 .getFrozenInstance(fullPath); 202 DtdType type = parts.getDtdData().dtdType; 203 for (int i = 0; i < parts.size(); ++i) { 204 String element = parts.getElement(i); 205 R2<DtdType, String> typeElement = Row.of(type, 206 element); 207 if (parts.getAttributeCount(i) == 0) { 208 foundAttributes.put(typeElement, "NONE"); 209 } else { 210 for (String attribute : parts 211 .getAttributeKeys(i)) { 212 foundAttributes.put(typeElement, attribute); 213 } 214 } 215 } 216 } 217 } 218 } 219 } 220 } 221 showFoundElements( Relation<Row.R2<DtdType, String>, String> foundAttributes)222 public void showFoundElements( 223 Relation<Row.R2<DtdType, String>, String> foundAttributes) { 224 Relation<Row.R2<DtdType, String>, String> theoryAttributes = Relation 225 .of(new TreeMap<Row.R2<DtdType, String>, Set<String>>(), 226 TreeSet.class); 227 for (DtdType type : DtdType.values()) { 228 DtdData dtdData = DtdData.getInstance(type); 229 for (Element element : dtdData.getElementFromName().values()) { 230 String name = element.getName(); 231 Set<Attribute> attributes = element.getAttributes().keySet(); 232 R2<DtdType, String> typeElement = Row.of(type, name); 233 if (attributes.isEmpty()) { 234 theoryAttributes.put(typeElement, "NONE"); 235 } else { 236 for (Attribute attribute : attributes) { 237 theoryAttributes.put(typeElement, attribute.name); 238 } 239 } 240 } 241 } 242 Relation<String, R3<Boolean, DtdType, String>> attributesToTypeElementUsed = Relation 243 .of(new TreeMap<String, Set<R3<Boolean, DtdType, String>>>(), 244 LinkedHashSet.class); 245 246 for (Entry<R2<DtdType, String>, Set<String>> s : theoryAttributes 247 .keyValuesSet()) { 248 R2<DtdType, String> typeElement = s.getKey(); 249 Set<String> theoryAttributeSet = s.getValue(); 250 DtdType type = typeElement.get0(); 251 String element = typeElement.get1(); 252 if (element.equals("ANY") || element.equals("#PCDATA")) { 253 continue; 254 } 255 boolean deprecatedElement = SUPPLEMENTAL_DATA_INFO.isDeprecated( 256 type, element, "*", "*"); 257 String header = type + "\t" + element + "\t" 258 + (deprecatedElement ? "X" : "") + "\t"; 259 Set<String> usedAttributes = foundAttributes.get(typeElement); 260 Set<String> unusedAttributes = new LinkedHashSet<String>( 261 theoryAttributeSet); 262 if (usedAttributes == null) { 263 logln(header 264 + "<NOT-FOUND>\t\t" 265 + siftDeprecated(type, element, unusedAttributes, 266 attributesToTypeElementUsed, false)); 267 continue; 268 } 269 unusedAttributes.removeAll(usedAttributes); 270 logln(header 271 + siftDeprecated(type, element, usedAttributes, 272 attributesToTypeElementUsed, true) 273 + "\t" 274 + siftDeprecated(type, element, unusedAttributes, 275 attributesToTypeElementUsed, false)); 276 } 277 278 logln("Undeprecated Attributes\t"); 279 for (Entry<String, R3<Boolean, DtdType, String>> s : attributesToTypeElementUsed 280 .keyValueSet()) { 281 R3<Boolean, DtdType, String> typeElementUsed = s.getValue(); 282 logln(s.getKey() + "\t" + typeElementUsed.get0() 283 + "\t" + typeElementUsed.get1() + "\t" 284 + typeElementUsed.get2()); 285 } 286 } 287 siftDeprecated( DtdType type, String element, Set<String> attributeSet, Relation<String, R3<Boolean, DtdType, String>> attributesToTypeElementUsed, boolean used)288 private String siftDeprecated( 289 DtdType type, 290 String element, 291 Set<String> attributeSet, 292 Relation<String, R3<Boolean, DtdType, String>> attributesToTypeElementUsed, 293 boolean used) { 294 StringBuilder b = new StringBuilder(); 295 StringBuilder bdep = new StringBuilder(); 296 for (String attribute : attributeSet) { 297 String attributeName = "«" 298 + attribute 299 + (!"NONE".equals(attribute) && CLDRFile.isDistinguishing(type, element, attribute) ? "*" 300 : "") 301 + "»"; 302 if (!"NONE".equals(attribute) && SUPPLEMENTAL_DATA_INFO.isDeprecated(type, element, attribute, 303 "*")) { 304 if (bdep.length() != 0) { 305 bdep.append(" "); 306 } 307 bdep.append(attributeName); 308 } else { 309 if (b.length() != 0) { 310 b.append(" "); 311 } 312 b.append(attributeName); 313 if (!"NONE".equals(attribute)) { 314 attributesToTypeElementUsed.put(attribute, 315 Row.of(used, type, element)); 316 } 317 } 318 } 319 return b.toString() + "\t" + bdep.toString(); 320 } 321 322 class MyErrorHandler implements ErrorHandler { 323 @Override error(SAXParseException exception)324 public void error(SAXParseException exception) throws SAXException { 325 errln("error: " + XMLFileReader.showSAX(exception)); 326 throw exception; 327 } 328 329 @Override fatalError(SAXParseException exception)330 public void fatalError(SAXParseException exception) throws SAXException { 331 errln("fatalError: " + XMLFileReader.showSAX(exception)); 332 throw exception; 333 } 334 335 @Override warning(SAXParseException exception)336 public void warning(SAXParseException exception) throws SAXException { 337 errln("warning: " + XMLFileReader.showSAX(exception)); 338 throw exception; 339 } 340 } 341 342 private class TimingInfo { 343 File file; 344 long nanos; 345 } 346 check(File systemID)347 public TimingInfo check(File systemID) { 348 long start = System.nanoTime(); 349 try (InputStream fis = InputStreamFactory.createInputStream(systemID)) { 350 // FileInputStream fis = new FileInputStream(systemID); 351 XMLReader xmlReader = XMLFileReader.createXMLReader(true); 352 xmlReader.setErrorHandler(new MyErrorHandler()); 353 InputSource is = new InputSource(fis); 354 is.setSystemId(systemID.toString()); 355 xmlReader.parse(is); 356 // fis.close(); 357 } catch (SAXException | IOException e) { 358 errln("\t" + "Can't read " + systemID + "\t" + e.getClass() + "\t" 359 + e.getMessage()); 360 } 361 // catch (SAXParseException e) { 362 // errln("\t" + "Can't read " + systemID + "\t" + e.getClass() + "\t" + 363 // e.getMessage()); 364 // } catch (IOException e) { 365 // errln("\t" + "Can't read " + systemID + "\t" + e.getClass() + "\t" + 366 // e.getMessage()); 367 // } 368 TimingInfo timingInfo = new TimingInfo(); 369 timingInfo.nanos = System.nanoTime() - start; 370 timingInfo.file = systemID; 371 return timingInfo; 372 } 373 TestCurrencyFallback()374 public void TestCurrencyFallback() { 375 Factory cldrFactory = testInfo.getCldrFactory(); 376 Set<String> currencies = StandardCodes.make().getAvailableCodes( 377 "currency"); 378 379 final UnicodeSet CHARACTERS_THAT_SHOULD_HAVE_FALLBACKS = new UnicodeSet( 380 "[[:sc:]-[\\u0000-\\u00FF]]").freeze(); 381 382 CharacterFallbacks fallbacks = CharacterFallbacks.make(); 383 384 for (String locale : cldrFactory.getAvailable()) { 385 CLDRFile file = testInfo.getCLDRFile(locale, false); 386 if (file.isNonInheriting()) 387 continue; 388 389 final UnicodeSet OK_CURRENCY_FALLBACK = new UnicodeSet( 390 "[\\u0000-\\u00FF]").addAll(safeExemplars(file, "")) 391 .addAll(safeExemplars(file, "auxiliary")) 392 .freeze(); 393 UnicodeSet badSoFar = new UnicodeSet(); 394 395 for (Iterator<String> it = file.iterator(); it.hasNext();) { 396 String path = it.next(); 397 if (path.endsWith("/alias")) { 398 continue; 399 } 400 String value = file.getStringValue(path); 401 402 // check for special characters 403 if (CHARACTERS_THAT_SHOULD_HAVE_FALLBACKS.containsSome(value)) { 404 XPathParts parts = XPathParts.getFrozenInstance(path); 405 if (!parts.getElement(-1).equals("symbol")) { 406 continue; 407 } 408 // We don't care about fallbacks for narrow currency symbols 409 if ("narrow".equals(parts.getAttributeValue(-1, "alt"))) { 410 continue; 411 } 412 String currencyType = parts.getAttributeValue(-2, "type"); 413 414 UnicodeSet fishy = new UnicodeSet().addAll(value) 415 .retainAll(CHARACTERS_THAT_SHOULD_HAVE_FALLBACKS) 416 .removeAll(badSoFar); 417 for (UnicodeSetIterator it2 = new UnicodeSetIterator(fishy); it2 418 .next();) { 419 final int fishyCodepoint = it2.codepoint; 420 List<String> fallbackList = fallbacks 421 .getSubstitutes(fishyCodepoint); 422 423 String nfkc = Normalizer.normalize(fishyCodepoint, Normalizer.NFKC); 424 if (!nfkc.equals(UTF16.valueOf(fishyCodepoint))) { 425 if (fallbackList == null) { 426 fallbackList = new ArrayList<String>(); 427 } else { 428 fallbackList = new ArrayList<String>( 429 fallbackList); // writable 430 } 431 fallbackList.add(nfkc); 432 } 433 // later test for all Latin-1 434 if (fallbackList == null) { 435 errln("Locale:\t" + locale 436 + ";\tCharacter with no fallback:\t" 437 + it2.getString() + "\t" 438 + UCharacter.getName(fishyCodepoint)); 439 badSoFar.add(fishyCodepoint); 440 } else { 441 String fallback = null; 442 for (String fb : fallbackList) { 443 if (OK_CURRENCY_FALLBACK.containsAll(fb)) { 444 if (!fb.equals(currencyType) 445 && currencies.contains(fb)) { 446 errln("Locale:\t" 447 + locale 448 + ";\tCurrency:\t" 449 + currencyType 450 + ";\tFallback converts to different code!:\t" 451 + fb 452 + "\t" 453 + it2.getString() 454 + "\t" 455 + UCharacter 456 .getName(fishyCodepoint)); 457 } 458 if (fallback == null) { 459 fallback = fb; 460 } 461 } 462 } 463 if (fallback == null) { 464 errln("Locale:\t" 465 + locale 466 + ";\tCharacter with no good fallback (exemplars+Latin1):\t" 467 + it2.getString() + "\t" 468 + UCharacter.getName(fishyCodepoint)); 469 badSoFar.add(fishyCodepoint); 470 } else { 471 logln("Locale:\t" + locale 472 + ";\tCharacter with good fallback:\t" 473 + it2.getString() + " " 474 + UCharacter.getName(fishyCodepoint) 475 + " => " + fallback); 476 // badSoFar.add(fishyCodepoint); 477 } 478 } 479 } 480 } 481 } 482 } 483 } 484 TestAbstractPaths()485 public void TestAbstractPaths() { 486 Factory cldrFactory = testInfo.getCldrFactory(); 487 CLDRFile english = testInfo.getEnglish(); 488 Map<String, Counter<Level>> abstactPaths = new TreeMap<String, Counter<Level>>(); 489 RegexTransform abstractPathTransform = new RegexTransform( 490 RegexTransform.Processing.ONE_PASS).add("//ldml/", "") 491 .add("\\[@alt=\"[^\"]*\"\\]", "").add("=\"[^\"]*\"", "=\"*\"") 492 .add("([^]])\\[", "$1\t[").add("([^]])/", "$1\t/") 493 .add("/", "\t"); 494 495 for (String locale : getInclusion() <= 5 ? eightPointLocales : cldrFactory.getAvailable()) { 496 CLDRFile file = testInfo.getCLDRFile(locale, resolved); 497 if (file.isNonInheriting()) 498 continue; 499 logln(locale + "\t-\t" + english.getName(locale)); 500 501 for (Iterator<String> it = file.iterator(); it.hasNext();) { 502 String path = it.next(); 503 if (path.endsWith("/alias")) { 504 continue; 505 } 506 // collect abstracted paths 507 String abstractPath = abstractPathTransform.transform(path); 508 Level level = SUPPLEMENTAL_DATA_INFO.getCoverageLevel(path, 509 locale); 510 if (level == Level.OPTIONAL) { 511 level = Level.COMPREHENSIVE; 512 } 513 Counter<Level> row = abstactPaths.get(abstractPath); 514 if (row == null) { 515 abstactPaths.put(abstractPath, row = new Counter<Level>()); 516 } 517 row.add(level, 1); 518 } 519 } 520 logln(CldrUtility.LINE_SEPARATOR + "Abstract Paths"); 521 for (Entry<String, Counter<Level>> pathInfo : abstactPaths.entrySet()) { 522 String path = pathInfo.getKey(); 523 Counter<Level> counter = pathInfo.getValue(); 524 logln(counter.getTotal() + "\t" + getCoverage(counter) + "\t" 525 + path); 526 } 527 } 528 getCoverage(Counter<Level> counter)529 private CharSequence getCoverage(Counter<Level> counter) { 530 StringBuilder result = new StringBuilder(); 531 boolean first = true; 532 for (Level level : counter.getKeysetSortedByKey()) { 533 if (first) { 534 first = false; 535 } else { 536 result.append(' '); 537 } 538 result.append("L").append(level.ordinal()).append("=") 539 .append(counter.get(level)); 540 } 541 return result; 542 } 543 544 // public void TestCLDRFileCache() { 545 // long start = System.nanoTime(); 546 // Factory cldrFactory = testInfo.getCldrFactory(); 547 // String unusualLocale = "hi"; 548 // CLDRFile file = cldrFactory.make(unusualLocale, true); 549 // long afterOne = System.nanoTime(); 550 // logln("First: " + (afterOne-start)); 551 // CLDRFile file2 = cldrFactory.make(unusualLocale, true); 552 // long afterTwo = System.nanoTime(); 553 // logln("Second: " + (afterTwo-afterOne)); 554 // } 555 // TestPaths()556 public void TestPaths() { 557 Relation<String, String> distinguishing = Relation.of( 558 new TreeMap<String, Set<String>>(), TreeSet.class); 559 Relation<String, String> nonDistinguishing = Relation.of( 560 new TreeMap<String, Set<String>>(), TreeSet.class); 561 Factory cldrFactory = testInfo.getCldrFactory(); 562 CLDRFile english = testInfo.getEnglish(); 563 564 Relation<String, String> pathToLocale = Relation.of( 565 new TreeMap<String, Set<String>>(CLDRFile 566 .getComparator(DtdType.ldml)), 567 TreeSet.class, null); 568 Set<String> localesToTest = getInclusion() <= 5 ? eightPointLocales : cldrFactory.getAvailable(); 569 for (String locale : localesToTest) { 570 CLDRFile file = testInfo.getCLDRFile(locale, resolved); 571 DtdType dtdType = null; 572 if (file.isNonInheriting()) 573 continue; 574 DisplayAndInputProcessor displayAndInputProcessor = new DisplayAndInputProcessor( 575 file, false); 576 577 logln(locale + "\t-\t" + english.getName(locale)); 578 579 for (Iterator<String> it = file.iterator(); it.hasNext();) { 580 String path = it.next(); 581 if (dtdType == null) { 582 dtdType = DtdType.fromPath(path); 583 } 584 585 if (path.endsWith("/alias")) { 586 continue; 587 } 588 String value = file.getStringValue(path); 589 if (value == null) { 590 throw new IllegalArgumentException(locale 591 + "\tError: in null value at " + path); 592 } 593 594 String displayValue = displayAndInputProcessor 595 .processForDisplay(path, value); 596 if (!displayValue.equals(value)) { 597 logln("\t" 598 + locale 599 + "\tdisplayAndInputProcessor changes display value <" 600 + value + ">\t=>\t<" + displayValue + ">\t\t" 601 + path); 602 } 603 String inputValue = displayAndInputProcessor.processInput(path, 604 value, internalException); 605 if (internalException[0] != null) { 606 errln("\t" + locale 607 + "\tdisplayAndInputProcessor internal error <" 608 + value + ">\t=>\t<" + inputValue + ">\t\t" + path); 609 internalException[0].printStackTrace(System.out); 610 } 611 if (isVerbose() && !inputValue.equals(value)) { 612 displayAndInputProcessor.processInput(path, value, 613 internalException); // for 614 // debugging 615 logln("\t" 616 + locale 617 + "\tdisplayAndInputProcessor changes input value <" 618 + value + ">\t=>\t<" + inputValue + ">\t\t" + path); 619 } 620 621 pathToLocale.put(path, locale); 622 623 // also check for non-distinguishing attributes 624 if (path.contains("/identity")) 625 continue; 626 627 String fullPath = file.getFullXPath(path); 628 XPathParts parts = XPathParts.getFrozenInstance(fullPath); 629 for (int i = 0; i < parts.size(); ++i) { 630 if (parts.getAttributeCount(i) == 0) { 631 continue; 632 } 633 String element = parts.getElement(i); 634 for (String attribute : parts.getAttributeKeys(i)) { 635 if (skipAttributes.contains(attribute)) 636 continue; 637 if (CLDRFile.isDistinguishing(dtdType, element, attribute)) { 638 distinguishing.put(element, attribute); 639 } else { 640 nonDistinguishing.put(element, attribute); 641 } 642 } 643 } 644 } 645 } 646 647 if (isVerbose()) { 648 System.out.format("Distinguishing Elements: %s" 649 + CldrUtility.LINE_SEPARATOR, distinguishing); 650 System.out.format("Nondistinguishing Elements: %s" 651 + CldrUtility.LINE_SEPARATOR, nonDistinguishing); 652 System.out.format("Skipped %s" + CldrUtility.LINE_SEPARATOR, 653 skipAttributes); 654 } 655 } 656 657 /** 658 * The verbose output shows the results of 1..3 \u00a4 signs. 659 */ checkCurrency()660 public void checkCurrency() { 661 Map<String, Set<R2<String, Integer>>> results = new TreeMap<String, Set<R2<String, Integer>>>( 662 Collator.getInstance(ULocale.ENGLISH)); 663 for (ULocale locale : ULocale.getAvailableLocales()) { 664 if (locale.getCountry().length() != 0) { 665 continue; 666 } 667 for (int i = 1; i < 4; ++i) { 668 NumberFormat format = getCurrencyInstance(locale, i); 669 for (Currency c : new Currency[] { Currency.getInstance("USD"), 670 Currency.getInstance("EUR"), 671 Currency.getInstance("INR") }) { 672 format.setCurrency(c); 673 final String formatted = format.format(12345.67); 674 Set<R2<String, Integer>> set = results.get(formatted); 675 if (set == null) { 676 results.put(formatted, 677 set = new TreeSet<R2<String, Integer>>()); 678 } 679 set.add(Row.of(locale.toString(), Integer.valueOf(i))); 680 } 681 } 682 } 683 for (String formatted : results.keySet()) { 684 logln(formatted + "\t" + results.get(formatted)); 685 } 686 } 687 getCurrencyInstance(ULocale locale, int type)688 private static NumberFormat getCurrencyInstance(ULocale locale, int type) { 689 NumberFormat format = NumberFormat.getCurrencyInstance(locale); 690 if (type > 1) { 691 DecimalFormat format2 = (DecimalFormat) format; 692 String pattern = format2.toPattern(); 693 String replacement = "\u00a4\u00a4"; 694 for (int i = 2; i < type; ++i) { 695 replacement += "\u00a4"; 696 } 697 pattern = pattern.replace("\u00a4", replacement); 698 format2.applyPattern(pattern); 699 } 700 return format; 701 } 702 safeExemplars(CLDRFile file, String string)703 private UnicodeSet safeExemplars(CLDRFile file, String string) { 704 final UnicodeSet result = file.getExemplarSet(string, 705 WinningChoice.NORMAL); 706 return result != null ? result : new UnicodeSet(); 707 } 708 TestAPath()709 public void TestAPath() { 710 // <month type="1">1</month> 711 String path = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/months/monthContext[@type=\"format\"]/monthWidth[@type=\"abbreviated\"]/month[@type=\"1\"]"; 712 CLDRFile root = testInfo.getRoot(); 713 logln("path: " + path); 714 String fullpath = root.getFullXPath(path); 715 logln("fullpath: " + fullpath); 716 String value = root.getStringValue(path); 717 logln("value: " + value); 718 Status status = new Status(); 719 String source = root.getSourceLocaleID(path, status); 720 logln("locale: " + source); 721 logln("status: " + status); 722 } 723 TestDefaultContents()724 public void TestDefaultContents() { 725 Set<String> defaultContents = Inheritance.defaultContents; 726 Multimap<String, String> parentToChildren = Inheritance.parentToChildren; 727 728 // Put a list of locales that should be default content here. 729 final String expectDC[] = { 730 "os_GE" // see CLDR-14118 731 }; 732 for(final String locale : expectDC) { 733 assertTrue("expect "+locale+" to be a default content locale", defaultContents.contains(locale)); 734 } 735 736 if (DEBUG) { 737 Inheritance.showChain("", "", "root"); 738 } 739 740 for (String locale : defaultContents) { 741 CLDRFile cldrFile; 742 try { 743 cldrFile = testInfo.getCLDRFile(locale, false); 744 } catch (RuntimeException e) { 745 logln("Can't open default content file:\t" + locale); 746 continue; 747 } 748 // we check that the default content locale is always empty 749 for (Iterator<String> it = cldrFile.iterator(); it.hasNext();) { 750 String path = it.next(); 751 if (path.contains("/identity")) { 752 continue; 753 } 754 errln("Default content file not empty:\t" + locale); 755 showDifferences(locale); 756 break; 757 } 758 } 759 760 // check that if a locale has any children, that exactly one of them is 761 // the default content. Ignore locales with variants 762 763 for (Entry<String, Collection<String>> localeAndKids : parentToChildren.asMap().entrySet()) { 764 String locale = localeAndKids.getKey(); 765 if (locale.equals("root")) { 766 continue; 767 } 768 769 Collection<String> rawChildren = localeAndKids.getValue(); 770 771 // remove variant children 772 Set<String> children = new LinkedHashSet<>(); 773 for (String child : rawChildren) { 774 if (new LocaleIDParser().set(child).getVariants().length == 0) { 775 children.add(child); 776 } 777 } 778 if (children.isEmpty()) { 779 continue; 780 } 781 782 Set<String> defaultContentChildren = new LinkedHashSet<String>(children); 783 defaultContentChildren.retainAll(defaultContents); 784 if (defaultContentChildren.size() == 1) { 785 continue; 786 // If we're already down to the region level then it's OK not to have 787 // default contents. 788 } else if (! new LocaleIDParser().set(locale).getRegion().isEmpty()) { 789 continue; 790 } else if (defaultContentChildren.isEmpty()) { 791 Object possible = highestShared(locale, children); 792 errln("Locale has children but is missing default contents locale: " 793 + locale + ", children: " + children + "; possible fixes for children:\n" + possible); 794 } else { 795 errln("Locale has too many defaultContent locales!!: " 796 + locale + ", defaultContents: " 797 + defaultContentChildren); 798 } 799 } 800 801 // check that each default content locale is likely-subtag equivalent to 802 // its parent. 803 804 for (String locale : defaultContents) { 805 String maxLocale = LikelySubtags.maximize(locale, likelyData); 806 String localeParent = LocaleIDParser.getParent(locale); 807 String maxLocaleParent = LikelySubtags.maximize(localeParent, 808 likelyData); 809 if (locale.equals("ar_001") || locale.equals("nb")) { 810 logln("Known exception to likelyMax(locale=" + locale + ")" 811 + " == " + "likelyMax(defaultContent=" + localeParent 812 + ")"); 813 continue; 814 } 815 assertEquals("likelyMax(locale=" + locale + ")" + " == " 816 + "likelyMax(defaultContent=" + localeParent + ")", 817 maxLocaleParent, maxLocale); 818 } 819 820 } 821 highestShared(String parent, Set<String> children)822 private String highestShared(String parent, Set<String> children) { 823 M4<PathHeader, String, String, Boolean> data = ChainedMap.of(new TreeMap<PathHeader, Object>(), new TreeMap<String, Object>(), 824 new TreeMap<String, Object>(), Boolean.class); 825 CLDRFile parentFile = testInfo.getCLDRFile(parent, true); 826 PathHeader.Factory phf = PathHeader.getFactory(testInfo.getEnglish()); 827 for (String child : children) { 828 CLDRFile cldrFile = testInfo.getCLDRFile(child, false); 829 for (String path : cldrFile) { 830 if (path.contains("/identity")) { 831 continue; 832 } 833 if (path.contains("provisional") || path.contains("unconfirmed")) { 834 continue; 835 } 836 String value = cldrFile.getStringValue(path); 837 // double-check 838 String parentValue = parentFile.getStringValue(path); 839 if (value.equals(parentValue)) { 840 continue; 841 } 842 PathHeader ph = phf.fromPath(path); 843 data.put(ph, value, child, Boolean.TRUE); 844 data.put(ph, parentValue == null ? "∅∅∅" : parentValue, child, Boolean.TRUE); 845 } 846 } 847 StringBuilder result = new StringBuilder(); 848 for (Entry<PathHeader, Map<String, Map<String, Boolean>>> entry : data) { 849 for (Entry<String, Map<String, Boolean>> item : entry.getValue().entrySet()) { 850 result.append("\n") 851 .append(entry.getKey()) 852 .append("\t") 853 .append(item.getKey() + "\t" + item.getValue().keySet()); 854 } 855 } 856 return result.toString(); 857 } 858 859 public static class Inheritance { 860 public static final Set<String> defaultContents = SUPPLEMENTAL_DATA_INFO 861 .getDefaultContentLocales(); 862 public static final Multimap<String, String> parentToChildren; 863 864 static { 865 Multimap<String, String> _parentToChildren = TreeMultimap.create(); 866 for (String child : testInfo.getCldrFactory().getAvailable()) { 867 if (child.equals("root")) { 868 continue; 869 } 870 String localeParent = LocaleIDParser.getParent(child); _parentToChildren.put(localeParent, child)871 _parentToChildren.put(localeParent, child); 872 } 873 parentToChildren = ImmutableMultimap.copyOf(_parentToChildren); 874 } 875 showChain(String prefix, String gparent, String current)876 public static void showChain(String prefix, String gparent, String current) { 877 Collection<String> children = parentToChildren.get(current); 878 if (children == null) { 879 throw new IllegalArgumentException(); 880 } 881 prefix += current + (defaultContents.contains(current) ? "*" : "") 882 + (isLikelyEquivalent(gparent, current) ? "~" : "") + "\t"; 883 884 // find leaves 885 Set<String> parents = new LinkedHashSet<>(children); 886 parents.retainAll(parentToChildren.keySet()); 887 Set<String> leaves = new LinkedHashSet<>(children); 888 leaves.removeAll(parentToChildren.keySet()); 889 if (!leaves.isEmpty()) { 890 List<String> presentation = new ArrayList<>(); 891 boolean gotDc = false; 892 for (String s : leaves) { 893 String shown = s; 894 if (isLikelyEquivalent(current, s)) { 895 shown += "~"; 896 } 897 if (defaultContents.contains(s)) { 898 gotDc = true; 899 shown += "*"; 900 } 901 if (!shown.equals(s)) { 902 presentation.add(0, shown); 903 } else { 904 presentation.add(shown); 905 } 906 } 907 if (!gotDc) { 908 int debug = 0; 909 } 910 if (leaves.size() == 1) { 911 System.out.println(prefix + Joiner.on(" ").join(presentation)); 912 } else { 913 System.out.println(prefix + "{" + Joiner.on(" ").join(presentation) + "}"); 914 } 915 } 916 for (String parent : parents) { 917 showChain(prefix, current, parent); 918 } 919 } 920 isLikelyEquivalent(String locale1, String locale2)921 static boolean isLikelyEquivalent(String locale1, String locale2) { 922 if (locale1.equals(locale2)) { 923 return true; 924 } 925 try { 926 String maxLocale1 = LikelySubtags.maximize(locale1, likelyData); 927 String maxLocale2 = LikelySubtags.maximize(locale2, likelyData); 928 return maxLocale1 != null && Objects.equal(maxLocale1, maxLocale2); 929 } catch (Exception e) { 930 return false; 931 } 932 } 933 } 934 935 static final Map<String, String> likelyData = SUPPLEMENTAL_DATA_INFO 936 .getLikelySubtags(); 937 TestLikelySubtagsComplete()938 public void TestLikelySubtagsComplete() { 939 LanguageTagParser ltp = new LanguageTagParser(); 940 for (String locale : testInfo.getCldrFactory().getAvailable()) { 941 if (locale.equals("root")) { 942 continue; 943 } 944 String maxLocale = LikelySubtags.maximize(locale, likelyData); 945 if (maxLocale == null) { 946 errln("Locale missing likely subtag: " + locale); 947 continue; 948 } 949 ltp.set(maxLocale); 950 if (ltp.getLanguage().isEmpty() || ltp.getScript().isEmpty() 951 || ltp.getRegion().isEmpty()) { 952 errln("Locale has defective likely subtag: " + locale + " => " 953 + maxLocale); 954 } 955 } 956 } 957 showDifferences(String locale)958 private void showDifferences(String locale) { 959 CLDRFile cldrFile = testInfo.getCLDRFile(locale, false); 960 final String localeParent = LocaleIDParser.getParent(locale); 961 CLDRFile parentFile = testInfo.getCLDRFile(localeParent, true); 962 int funnyCount = 0; 963 for (Iterator<String> it = cldrFile.iterator("", 964 cldrFile.getComparator()); it.hasNext();) { 965 String path = it.next(); 966 if (path.contains("/identity")) { 967 continue; 968 } 969 final String fullXPath = cldrFile.getFullXPath(path); 970 if (fullXPath.contains("[@draft=\"unconfirmed\"]") 971 || fullXPath.contains("[@draft=\"provisional\"]")) { 972 funnyCount++; 973 continue; 974 } 975 logln("\tpath:\t" + path); 976 logln("\t\t" + locale + " value:\t<" 977 + cldrFile.getStringValue(path) + ">"); 978 final String parentFullPath = parentFile.getFullXPath(path); 979 logln("\t\t" + localeParent + " value:\t<" 980 + parentFile.getStringValue(path) + ">"); 981 logln("\t\t" + locale + " fullpath:\t" + fullXPath); 982 logln("\t\t" + localeParent + " fullpath:\t" + parentFullPath); 983 } 984 logln("\tCount of non-approved:\t" + funnyCount); 985 } 986 987 enum MissingType { 988 plurals, main_exemplars, no_main, collation, index_exemplars, punct_exemplars 989 } 990 TestCoreData()991 public void TestCoreData() { 992 Set<String> availableLanguages = testInfo.getCldrFactory() 993 .getAvailableLanguages(); 994 PluralInfo rootRules = SUPPLEMENTAL_DATA_INFO.getPlurals( 995 PluralType.cardinal, "root"); 996 Multimap<MissingType, Comparable> errors = TreeMultimap.create(); 997 errors.put(MissingType.collation, "?"); 998 999 Multimap<MissingType, Comparable> warnings = TreeMultimap.create(); 1000 warnings.put(MissingType.collation, "?"); 1001 warnings.put(MissingType.index_exemplars, "?"); 1002 warnings.put(MissingType.punct_exemplars, "?"); 1003 1004 Set<String> collations = new HashSet<String>(); 1005 1006 // collect collation info 1007 Factory collationFactory = Factory.make(CLDRPaths.COLLATION_DIRECTORY, 1008 ".*", DraftStatus.contributed); 1009 for (String localeID : collationFactory.getAvailable()) { 1010 if (isTopLevel(localeID)) { 1011 collations.add(localeID); 1012 } 1013 } 1014 logln(collations.toString()); 1015 1016 Set<String> allLanguages = Builder.with(new TreeSet<String>()) 1017 .addAll(collations).addAll(availableLanguages).freeze(); 1018 1019 for (String localeID : allLanguages) { 1020 if (localeID.equals("root")) { 1021 continue; // skip script locales 1022 } 1023 if (!isTopLevel(localeID)) { 1024 continue; 1025 } 1026 1027 errors.clear(); 1028 warnings.clear(); 1029 1030 String name = "Locale:" + localeID + " (" 1031 + testInfo.getEnglish().getName(localeID) + ")"; 1032 1033 if (!collations.contains(localeID)) { 1034 warnings.put(MissingType.collation, "missing"); 1035 logln(name + " is missing " + MissingType.collation.toString()); 1036 } 1037 1038 try { 1039 CLDRFile cldrFile = testInfo.getCldrFactory().make(localeID, 1040 false, DraftStatus.contributed); 1041 1042 String wholeFileAlias = cldrFile.getStringValue("//ldml/alias"); 1043 if (wholeFileAlias != null) { 1044 logln("Whole-file alias:" + name); 1045 continue; 1046 } 1047 1048 PluralInfo pluralInfo = SUPPLEMENTAL_DATA_INFO.getPlurals( 1049 PluralType.cardinal, localeID); 1050 if (pluralInfo == rootRules) { 1051 logln(name + " is missing " 1052 + MissingType.plurals.toString()); 1053 warnings.put(MissingType.plurals, "missing"); 1054 } 1055 UnicodeSet main = cldrFile.getExemplarSet("", 1056 WinningChoice.WINNING); 1057 if (main == null || main.isEmpty()) { 1058 errln(" " + name + " is missing " 1059 + MissingType.main_exemplars.toString()); 1060 errors.put(MissingType.main_exemplars, "missing"); 1061 } 1062 UnicodeSet index = cldrFile.getExemplarSet("index", 1063 WinningChoice.WINNING); 1064 if (index == null || index.isEmpty()) { 1065 logln(name + " is missing " 1066 + MissingType.index_exemplars.toString()); 1067 warnings.put(MissingType.index_exemplars, "missing"); 1068 } 1069 UnicodeSet punctuation = cldrFile.getExemplarSet("punctuation", 1070 WinningChoice.WINNING); 1071 if (punctuation == null || punctuation.isEmpty()) { 1072 logln(name + " is missing " 1073 + MissingType.punct_exemplars.toString()); 1074 warnings.put(MissingType.punct_exemplars, "missing"); 1075 } 1076 } catch (Exception e) { 1077 StringWriter x = new StringWriter(); 1078 PrintWriter pw = new PrintWriter(x); 1079 e.printStackTrace(pw); 1080 pw.flush(); 1081 errln(" " + name + " is missing main locale data." + x); 1082 errors.put(MissingType.no_main, x.toString()); 1083 } 1084 1085 // report errors 1086 1087 if (errors.isEmpty() && warnings.isEmpty()) { 1088 logln(name + ": No problems..."); 1089 } 1090 } 1091 } 1092 isTopLevel(String localeID)1093 private boolean isTopLevel(String localeID) { 1094 return "root".equals(LocaleIDParser.getParent(localeID)); 1095 } 1096 1097 /** 1098 * Tests that every dtd item is connected from root 1099 */ TestDtdCompleteness()1100 public void TestDtdCompleteness() { 1101 for (DtdType type : DtdType.values()) { 1102 DtdData dtdData = DtdData.getInstance(type); 1103 Set<Element> descendents = new LinkedHashSet<Element>(); 1104 dtdData.getDescendents(dtdData.ROOT, descendents); 1105 Set<Element> elements = dtdData.getElements(); 1106 if (!elements.equals(descendents)) { 1107 for (Element e : elements) { 1108 if (!descendents.contains(e) && !e.equals(dtdData.PCDATA) 1109 && !e.equals(dtdData.ANY)) { 1110 errln(type + ": Element " + e 1111 + " not contained in descendents of ROOT."); 1112 } 1113 } 1114 for (Element e : descendents) { 1115 if (!elements.contains(e)) { 1116 errln(type + ": Element " + e 1117 + ", descendent of ROOT, not in elements."); 1118 } 1119 } 1120 } 1121 LinkedHashSet<Element> all = new LinkedHashSet<Element>(descendents); 1122 all.addAll(elements); 1123 Set<Attribute> attributes = dtdData.getAttributes(); 1124 for (Attribute a : attributes) { 1125 if (!elements.contains(a.element)) { 1126 errln(type + ": Attribute " + a + " isn't for any element."); 1127 } 1128 } 1129 } 1130 } 1131 TestBasicDTDCompatibility()1132 public void TestBasicDTDCompatibility() { 1133 1134 if (logKnownIssue("cldrbug:11583", "Comment out test until last release data is available for unit tests")) { 1135 return; 1136 } 1137 1138 final String oldCommon = CldrVersion.v22_1.getBaseDirectory() + "/common"; 1139 1140 // set up exceptions 1141 Set<String> changedToEmpty = new HashSet<String>( 1142 Arrays.asList(new String[] { "version", "languageCoverage", 1143 "scriptCoverage", "territoryCoverage", 1144 "currencyCoverage", "timezoneCoverage", 1145 "skipDefaultLocale" })); 1146 Set<String> PCDATA = new HashSet<String>(); 1147 PCDATA.add("PCDATA"); 1148 Set<String> EMPTY = new HashSet<String>(); 1149 EMPTY.add("EMPTY"); 1150 Set<String> VERSION = new HashSet<String>(); 1151 VERSION.add("version"); 1152 1153 // test all DTDs 1154 for (DtdType dtd : DtdType.values()) { 1155 try { 1156 ElementAttributeInfo oldDtd = ElementAttributeInfo.getInstance( 1157 oldCommon, dtd); 1158 ElementAttributeInfo newDtd = ElementAttributeInfo 1159 .getInstance(dtd); 1160 1161 if (oldDtd == newDtd) { 1162 continue; 1163 } 1164 Relation<String, String> oldElement2Children = oldDtd 1165 .getElement2Children(); 1166 Relation<String, String> newElement2Children = newDtd 1167 .getElement2Children(); 1168 1169 Relation<String, String> oldElement2Attributes = oldDtd 1170 .getElement2Attributes(); 1171 Relation<String, String> newElement2Attributes = newDtd 1172 .getElement2Attributes(); 1173 1174 for (String element : oldElement2Children.keySet()) { 1175 Set<String> oldChildren = oldElement2Children 1176 .getAll(element); 1177 Set<String> newChildren = newElement2Children 1178 .getAll(element); 1179 if (newChildren == null) { 1180 if (!knownElementExceptions.contains(Pair.of(dtd.toString(), element))) { 1181 errln("Old " + dtd + " contains element not in new: <" 1182 + element + ">"); 1183 } 1184 continue; 1185 } 1186 Set<String> funny = containsInOrder(newChildren, 1187 oldChildren); 1188 if (funny != null) { 1189 if (changedToEmpty.contains(element) 1190 && oldChildren.equals(PCDATA) 1191 && newChildren.equals(EMPTY)) { 1192 // ok, skip 1193 } else { 1194 errln("Old " + dtd + " element <" + element 1195 + "> has children Missing/Misordered:\t" 1196 + funny + "\n\t\tOld:\t" + oldChildren 1197 + "\n\t\tNew:\t" + newChildren); 1198 } 1199 } 1200 1201 Set<String> oldAttributes = oldElement2Attributes 1202 .getAll(element); 1203 if (oldAttributes == null) { 1204 oldAttributes = Collections.emptySet(); 1205 } 1206 Set<String> newAttributes = newElement2Attributes 1207 .getAll(element); 1208 if (newAttributes == null) { 1209 newAttributes = Collections.emptySet(); 1210 } 1211 if (!newAttributes.containsAll(oldAttributes)) { 1212 LinkedHashSet<String> missing = new LinkedHashSet<String>( 1213 oldAttributes); 1214 missing.removeAll(newAttributes); 1215 if (element.equals(dtd.toString()) 1216 && missing.equals(VERSION)) { 1217 // ok, skip 1218 } else { 1219 errln("Old " + dtd + " element <" + element 1220 + "> has attributes Missing:\t" + missing 1221 + "\n\t\tOld:\t" + oldAttributes 1222 + "\n\t\tNew:\t" + newAttributes); 1223 } 1224 } 1225 } 1226 } catch (Exception e) { 1227 e.printStackTrace(); 1228 errln("Failure with " + dtd); 1229 } 1230 } 1231 } 1232 containsInOrder(Set<T> superset, Set<T> subset)1233 private <T> Set<T> containsInOrder(Set<T> superset, Set<T> subset) { 1234 if (!superset.containsAll(subset)) { 1235 LinkedHashSet<T> missing = new LinkedHashSet<T>(subset); 1236 missing.removeAll(superset); 1237 return missing; 1238 } 1239 // ok, we know that they are subsets, try order 1240 Set<T> result = null; 1241 DiscreteComparator<T> comp = new DiscreteComparator.Builder<T>( 1242 Ordering.ARBITRARY).add(superset).get(); 1243 T last = null; 1244 for (T item : subset) { 1245 if (last != null) { 1246 int order = comp.compare(last, item); 1247 if (order != -1) { 1248 if (result == null) { 1249 result = new HashSet<T>(); 1250 result.add(last); 1251 result.add(item); 1252 } 1253 } 1254 } 1255 last = item; 1256 } 1257 return result; 1258 } 1259 TestDtdCompatibility()1260 public void TestDtdCompatibility() { 1261 1262 for (DtdType type : DtdType.values()) { 1263 DtdData dtdData = DtdData.getInstance(type); 1264 Map<String, Element> currentElementFromName = dtdData 1265 .getElementFromName(); 1266 1267 // current has no orphan 1268 Set<Element> orphans = new LinkedHashSet<Element>(dtdData 1269 .getElementFromName().values()); 1270 orphans.remove(dtdData.ROOT); 1271 orphans.remove(dtdData.PCDATA); 1272 orphans.remove(dtdData.ANY); 1273 Set<String> elementsWithoutAlt = new TreeSet<String>(); 1274 Set<String> elementsWithoutDraft = new TreeSet<String>(); 1275 Set<String> elementsWithoutAlias = new TreeSet<String>(); 1276 Set<String> elementsWithoutSpecial = new TreeSet<String>(); 1277 1278 for (Element element : dtdData.getElementFromName().values()) { 1279 Set<Element> children = element.getChildren().keySet(); 1280 orphans.removeAll(children); 1281 if (type == DtdType.ldml 1282 && !SUPPLEMENTAL_DATA_INFO.isDeprecated(type, 1283 element.name, "*", "*")) { 1284 if (element.getType() == ElementType.PCDATA) { 1285 if (element.getAttributeNamed("alt") == null) { 1286 elementsWithoutAlt.add(element.name); 1287 } 1288 if (element.getAttributeNamed("draft") == null) { 1289 elementsWithoutDraft.add(element.name); 1290 } 1291 } else { 1292 if (children.size() != 0 && !"alias".equals(element.name)) { 1293 if (element.getChildNamed("alias") == null) { 1294 elementsWithoutAlias.add(element.name); 1295 } 1296 if (element.getChildNamed("special") == null) { 1297 elementsWithoutSpecial.add(element.name); 1298 } 1299 } 1300 } 1301 } 1302 } 1303 assertEquals(type + " DTD Must not have orphan elements", 1304 Collections.EMPTY_SET, orphans); 1305 assertEquals(type 1306 + " DTD elements with PCDATA must have 'alt' attributes", 1307 Collections.EMPTY_SET, elementsWithoutAlt); 1308 assertEquals(type 1309 + " DTD elements with PCDATA must have 'draft' attributes", 1310 Collections.EMPTY_SET, elementsWithoutDraft); 1311 assertEquals(type 1312 + " DTD elements with children must have 'alias' elements", 1313 Collections.EMPTY_SET, elementsWithoutAlias); 1314 assertEquals( 1315 type 1316 + " DTD elements with children must have 'special' elements", 1317 Collections.EMPTY_SET, elementsWithoutSpecial); 1318 1319 if (logKnownIssue("cldrbug:11583", "Comment out test until last release data is available for unit tests")) { 1320 return; 1321 } 1322 1323 for (CldrVersion version : CldrVersion.CLDR_VERSIONS_DESCENDING) { 1324 if (version == CldrVersion.unknown || version == CldrVersion.baseline) { 1325 continue; 1326 } 1327 DtdData dtdDataOld; 1328 try { 1329 dtdDataOld = DtdData.getInstance(type, version.toString()); 1330 } catch (IllegalArgumentException e) { 1331 boolean tooOld = false; 1332 switch (type) { 1333 case ldmlBCP47: 1334 case ldmlICU: 1335 tooOld = version.isOlderThan(CldrVersion.v1_7_2); 1336 break; 1337 case keyboard: 1338 case platform: 1339 tooOld = version.isOlderThan(CldrVersion.v22_1); 1340 break; 1341 default: 1342 break; 1343 } 1344 if (tooOld) { 1345 continue; 1346 } else { 1347 errln(version + ": " + e.getClass().getSimpleName() + ", " + e.getMessage()); 1348 continue; 1349 } 1350 } 1351 // verify that if E is in dtdDataOld, then it is in dtdData, and 1352 // has at least the same children and attributes 1353 for (Entry<String, Element> entry : dtdDataOld 1354 .getElementFromName().entrySet()) { 1355 Element oldElement = entry.getValue(); 1356 Element newElement = currentElementFromName.get(entry 1357 .getKey()); 1358 if (knownElementExceptions.contains(Pair.of(type.toString(), oldElement.getName()))) { 1359 continue; 1360 } 1361 if (assertNotNull(type 1362 + " DTD for trunk must be superset of v" + version 1363 + ", and must contain «" + oldElement.getName() 1364 + "»", newElement)) { 1365 // TODO Check order also 1366 for (Element oldChild : oldElement.getChildren() 1367 .keySet()) { 1368 if (oldChild == null) { 1369 continue; 1370 } 1371 Element newChild = newElement 1372 .getChildNamed(oldChild.getName()); 1373 1374 if (knownChildExceptions.contains(Pair.of(newElement.getName(), oldChild.getName()))) { 1375 continue; 1376 } 1377 assertNotNull( 1378 type + " DTD - Trunk children of «" 1379 + newElement.getName() 1380 + "» must be superset of v" 1381 + version + ", and must contain «" 1382 + oldChild.getName() + "»", 1383 newChild); 1384 } 1385 for (Attribute oldAttribute : oldElement 1386 .getAttributes().keySet()) { 1387 Attribute newAttribute = newElement 1388 .getAttributeNamed(oldAttribute.getName()); 1389 1390 if (knownAttributeExceptions.contains(Pair.of(newElement.getName(), oldAttribute.getName()))) { 1391 continue; 1392 } 1393 assertNotNull( 1394 type + " DTD - Trunk attributes of «" 1395 + newElement.getName() 1396 + "» must be superset of v" 1397 + version + ", and must contain «" 1398 + oldAttribute.getName() + "»", 1399 newAttribute); 1400 } 1401 } 1402 } 1403 } 1404 } 1405 } 1406 1407 /** 1408 * Compare each path to each other path for every single file in CLDR 1409 */ TestDtdComparison()1410 public void TestDtdComparison() { 1411 // try some simple paths for regression 1412 1413 sortPaths( 1414 DtdData.getInstance(DtdType.ldml).getDtdComparator(null), 1415 "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/dateTimeFormatLength[@type=\"full\"]/dateTimeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]", 1416 "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats"); 1417 1418 sortPaths( 1419 DtdData.getInstance(DtdType.supplementalData).getDtdComparator( 1420 null), 1421 "//supplementalData/territoryContainment/group[@type=\"419\"][@contains=\"013 029 005\"][@grouping=\"true\"]", 1422 "//supplementalData/territoryContainment/group[@type=\"003\"][@contains=\"021 013 029\"][@grouping=\"true\"]"); 1423 1424 } 1425 TestDtdComparisonsAll()1426 public void TestDtdComparisonsAll() { 1427 if (getInclusion() <= 5) { // Only run this test in exhaustive mode. 1428 return; 1429 } 1430 for (File file : CLDRConfig.getInstance().getAllCLDRFilesEndingWith(".xml")) { 1431 checkDtdComparatorFor(file, null); 1432 } 1433 } 1434 checkDtdComparatorForResource(String fileToRead, DtdType overrideDtdType)1435 public void checkDtdComparatorForResource(String fileToRead, 1436 DtdType overrideDtdType) { 1437 MyHandler myHandler = new MyHandler(overrideDtdType); 1438 XMLFileReader xfr = new XMLFileReader().setHandler(myHandler); 1439 try { 1440 myHandler.fileName = fileToRead; 1441 xfr.read(myHandler.fileName, TestBasic.class, -1, true); 1442 logln(myHandler.fileName); 1443 } catch (Exception e) { 1444 Throwable t = e; 1445 StringBuilder b = new StringBuilder(); 1446 String indent = ""; 1447 while (t != null) { 1448 b.append(indent).append(t.getMessage()); 1449 indent = indent.isEmpty() ? "\n\t\t" : indent + "\t"; 1450 t = t.getCause(); 1451 } 1452 errln(b.toString()); 1453 return; 1454 } 1455 DtdData dtdData = DtdData.getInstance(myHandler.dtdType); 1456 sortPaths(dtdData.getDtdComparator(null), myHandler.data); 1457 } 1458 checkDtdComparatorFor(File fileToRead, DtdType overrideDtdType)1459 public void checkDtdComparatorFor(File fileToRead, DtdType overrideDtdType) { 1460 MyHandler myHandler = new MyHandler(overrideDtdType); 1461 XMLFileReader xfr = new XMLFileReader().setHandler(myHandler); 1462 try { 1463 myHandler.fileName = PathUtilities.getNormalizedPathString(fileToRead); 1464 xfr.read(myHandler.fileName, -1, true); 1465 logln(myHandler.fileName); 1466 } catch (Exception e) { 1467 Throwable t = e; 1468 StringBuilder b = new StringBuilder(); 1469 String indent = ""; 1470 while (t != null) { 1471 b.append(indent).append(t.getMessage()); 1472 indent = indent.isEmpty() ? "\n\t\t" : indent + "\t"; 1473 t = t.getCause(); 1474 } 1475 errln(b.toString()); 1476 return; 1477 } 1478 DtdData dtdData = DtdData.getInstance(myHandler.dtdType); 1479 sortPaths(dtdData.getDtdComparator(null), myHandler.data); 1480 } 1481 1482 static class MyHandler extends XMLFileReader.SimpleHandler { 1483 private String fileName; 1484 private DtdType dtdType; 1485 private final Set<String> data = new LinkedHashSet<>(); 1486 MyHandler(DtdType overrideDtdType)1487 public MyHandler(DtdType overrideDtdType) { 1488 dtdType = overrideDtdType; 1489 } 1490 1491 @Override handlePathValue(String path, @SuppressWarnings("unused") String value)1492 public void handlePathValue(String path, @SuppressWarnings("unused") String value) { 1493 if (dtdType == null) { 1494 try { 1495 dtdType = DtdType.fromPath(path); 1496 } catch (Exception e) { 1497 throw new IllegalArgumentException( 1498 "Can't read " + fileName, e); 1499 } 1500 } 1501 data.add(path); 1502 } 1503 } 1504 sortPaths(Comparator<String> dc, Collection<String> paths)1505 public void sortPaths(Comparator<String> dc, Collection<String> paths) { 1506 String[] array = paths.toArray(new String[paths.size()]); 1507 sortPaths(dc, array); 1508 } 1509 sortPaths(Comparator<String> dc, String... array)1510 public void sortPaths(Comparator<String> dc, String... array) { 1511 Arrays.sort(array, 0, array.length, dc); 1512 } 1513 // public void TestNewDtdData() moved to TestDtdData 1514 } 1515