1 package org.unicode.cldr.tool; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.PrintWriter; 6 import java.util.ArrayList; 7 import java.util.Arrays; 8 import java.util.Collection; 9 import java.util.Collections; 10 import java.util.HashSet; 11 import java.util.List; 12 import java.util.Map; 13 import java.util.Map.Entry; 14 import java.util.Objects; 15 import java.util.Set; 16 import java.util.TreeMap; 17 import java.util.TreeSet; 18 import java.util.regex.Matcher; 19 import java.util.regex.Pattern; 20 21 import org.unicode.cldr.draft.FileUtilities; 22 import org.unicode.cldr.tool.FormattedFileWriter.Anchors; 23 import org.unicode.cldr.tool.Option.Options; 24 import org.unicode.cldr.tool.Option.Params; 25 import org.unicode.cldr.util.CLDRConfig; 26 import org.unicode.cldr.util.CLDRFile; 27 import org.unicode.cldr.util.CLDRFile.Status; 28 import org.unicode.cldr.util.CLDRPaths; 29 import org.unicode.cldr.util.CldrUtility; 30 import org.unicode.cldr.util.Counter; 31 import org.unicode.cldr.util.DtdData; 32 import org.unicode.cldr.util.DtdType; 33 import org.unicode.cldr.util.Factory; 34 import org.unicode.cldr.util.LanguageTagParser; 35 import org.unicode.cldr.util.Level; 36 import org.unicode.cldr.util.LocaleIDParser; 37 import org.unicode.cldr.util.Pair; 38 import org.unicode.cldr.util.PathHeader; 39 import org.unicode.cldr.util.PathHeader.PageId; 40 import org.unicode.cldr.util.PathStarrer; 41 import org.unicode.cldr.util.PatternCache; 42 import org.unicode.cldr.util.SimpleXMLSource; 43 import org.unicode.cldr.util.SupplementalDataInfo; 44 import org.unicode.cldr.util.TransliteratorUtilities; 45 import org.unicode.cldr.util.XMLFileReader; 46 import org.unicode.cldr.util.XPathParts; 47 48 import com.google.common.base.Splitter; 49 import com.google.common.collect.Multimap; 50 import com.google.common.collect.TreeMultimap; 51 import com.ibm.icu.dev.util.CollectionUtilities; 52 import com.ibm.icu.impl.Relation; 53 import com.ibm.icu.impl.Row.R2; 54 import com.ibm.icu.impl.Row.R3; 55 import com.ibm.icu.impl.Row.R4; 56 import com.ibm.icu.text.NumberFormat; 57 import com.ibm.icu.util.ICUUncheckedIOException; 58 import com.ibm.icu.util.Output; 59 60 public class ChartDelta extends Chart { 61 private static final String DIR_NAME = "delta"; 62 63 private static final boolean SKIP_REFORMAT_ANNOTATIONS = ToolConstants.PREVIOUS_CHART_VERSION.compareTo("30") >= 0; 64 65 private static final PageId DEBUG_PAGE_ID = PageId.DayPeriod; 66 67 private static final SupplementalDataInfo SUPPLEMENTAL_DATA_INFO = CLDRConfig.getInstance().getSupplementalDataInfo(); 68 69 private static final String LAST_ARCHIVE_DIRECTORY = CLDRPaths.LAST_DIRECTORY; 70 private static final String CURRENT_DIRECTORY = CLDRPaths.ARCHIVE_DIRECTORY + "cldr-" + 71 ToolConstants.LAST_CHART_VERSION + "/"; 72 private static final String LOG_DIR = CLDRPaths.GEN_DIRECTORY + "charts/"; 73 74 enum MyOptions { 75 fileFilter(new Params().setHelp("filter by dir/locale, eg: ^main/en$ or .*/en").setDefault(".*").setMatch(".*")), verbose( 76 new Params().setHelp("verbose debugging messages")), 77 ; 78 79 // BOILERPLATE TO COPY 80 final Option option; 81 MyOptions(Params params)82 private MyOptions(Params params) { 83 option = new Option(this, params); 84 } 85 86 private static Options myOptions = new Options(); 87 static { 88 for (MyOptions option : MyOptions.values()) { myOptions.add(option, option.option)89 myOptions.add(option, option.option); 90 } 91 } 92 parse(String[] args, boolean showArguments)93 private static Set<String> parse(String[] args, boolean showArguments) { 94 return myOptions.parse(MyOptions.values()[0], args, true); 95 } 96 } 97 98 private Matcher fileFilter; 99 private boolean verbose; 100 ChartDelta(Matcher fileFilter, boolean verbose)101 public ChartDelta(Matcher fileFilter, boolean verbose) { 102 this.fileFilter = fileFilter; 103 this.verbose = verbose; 104 } 105 main(String[] args)106 public static void main(String[] args) { 107 System.out.println("use -DCHART_VERSION=28 to generate version 28."); 108 MyOptions.parse(args, true); 109 Matcher fileFilter = !MyOptions.fileFilter.option.doesOccur() ? null : PatternCache.get(MyOptions.fileFilter.option.getValue()).matcher(""); 110 boolean verbose = MyOptions.verbose.option.doesOccur(); 111 ChartDelta temp = new ChartDelta(fileFilter, verbose); 112 temp.writeChart(null); 113 temp.showTotals(); 114 } 115 116 private static final String SEP = "\u0001"; 117 private static final boolean DEBUG = false; 118 private static final String DEBUG_FILE = null; // "windowsZones.xml"; 119 static Pattern fileMatcher = PatternCache.get(".*"); 120 121 static String DIR = CLDRPaths.CHART_DIRECTORY + "/" + DIR_NAME + "/"; 122 static PathHeader.Factory phf = PathHeader.getFactory(ENGLISH); 123 static final Set<String> DONT_CARE = new HashSet<>(Arrays.asList("draft", "standard", "reference")); 124 125 @Override getDirectory()126 public String getDirectory() { 127 return DIR; 128 } 129 130 @Override getTitle()131 public String getTitle() { 132 return "Delta Charts"; 133 } 134 135 @Override getFileName()136 public String getFileName() { 137 return "index"; 138 } 139 140 @Override getExplanation()141 public String getExplanation() { 142 return "<p>Charts showing the differences from the last version. " 143 + "Titles prefixed by ¤ are special: either the locale data summary or supplemental data. " 144 + "Not all changed data is charted yet. For details see each chart.</p>"; 145 } 146 147 @Override writeContents(FormattedFileWriter pw)148 public void writeContents(FormattedFileWriter pw) throws IOException { 149 FormattedFileWriter.Anchors anchors = new FormattedFileWriter.Anchors(); 150 FileUtilities.copyFile(ChartDelta.class, "index.css", getDirectory()); 151 counter.clear(); 152 fileCounters.clear(); 153 writeNonLdmlPlain(anchors); 154 writeLdml(anchors); 155 pw.setIndex("Main Chart Index", "../index.html"); 156 pw.write(anchors.toString()); 157 } 158 159 static class PathHeaderSegment extends R3<PathHeader, Integer, String> { PathHeaderSegment(PathHeader b, int elementIndex, String attribute)160 public PathHeaderSegment(PathHeader b, int elementIndex, String attribute) { 161 super(b, elementIndex, attribute); 162 } 163 } 164 165 static class PathDiff extends R4<PathHeaderSegment, String, String, String> { PathDiff(String locale, PathHeaderSegment pathHeaderSegment, String oldValue, String newValue)166 public PathDiff(String locale, PathHeaderSegment pathHeaderSegment, String oldValue, String newValue) { 167 super(pathHeaderSegment, locale, oldValue, newValue); 168 } 169 } 170 171 static final CLDRFile EMPTY_CLDR = new CLDRFile(new SimpleXMLSource("und").freeze()); 172 173 enum ChangeType { 174 added, deleted, changed, same; get(String oldValue, String currentValue)175 public static ChangeType get(String oldValue, String currentValue) { 176 return oldValue == null ? added 177 : currentValue == null ? deleted 178 : oldValue.equals(currentValue) ? same 179 : changed; 180 } 181 } 182 183 Counter<ChangeType> counter = new Counter<>(); 184 Map<String, Counter<ChangeType>> fileCounters = new TreeMap<>(); 185 Set<String> badHeaders = new TreeSet<>(); 186 addChange(String file, ChangeType changeType, int count)187 private void addChange(String file, ChangeType changeType, int count) { 188 counter.add(changeType, count); // unified add 189 Counter<ChangeType> fileCounter = fileCounters.get(file); 190 if (fileCounter == null) { 191 fileCounters.put(file, fileCounter = new Counter<>()); 192 } 193 fileCounter.add(changeType, count); 194 } 195 showTotals()196 private void showTotals() { 197 try (PrintWriter pw = FileUtilities.openUTF8Writer(getTsvDir(DIR, DIR_NAME), DIR_NAME + "_summary.tsv")) { 198 pw.println("# percentages are of *new* total"); 199 pw.print("# dir\tfile"); 200 for (ChangeType item : ChangeType.values()) { 201 pw.print("\t" + (item == ChangeType.same ? "total" : item.toString())); 202 } 203 pw.println(); 204 showTotal(pw, "TOTAL/", counter); 205 206 for (Entry<String, Counter<ChangeType>> entry : fileCounters.entrySet()) { 207 showTotal(pw, entry.getKey(), entry.getValue()); 208 } 209 for (String s : badHeaders) { 210 pw.println(s); 211 } 212 pw.println("# EOF"); 213 } catch (IOException e) { 214 throw new ICUUncheckedIOException(e); 215 } 216 } 217 showTotal(PrintWriter pw, String title2, Counter<ChangeType> counter2)218 private void showTotal(PrintWriter pw, String title2, Counter<ChangeType> counter2) { 219 long total = counter2.getTotal(); 220 NumberFormat pf = NumberFormat.getPercentInstance(); 221 pf.setMinimumFractionDigits(2); 222 NumberFormat nf = NumberFormat.getIntegerInstance(); 223 pw.print(title2.replace("/", "\t")); 224 for (ChangeType item : ChangeType.values()) { 225 if (item == ChangeType.same) { 226 pw.print("\t" + nf.format(total)); 227 } else { 228 final long current = counter2.getCount(item); 229 pw.print("\t" + nf.format(current)); 230 } 231 } 232 pw.println(); 233 } 234 writeSubcharts(Anchors anchors)235 public void writeSubcharts(Anchors anchors) throws IOException { 236 FileUtilities.copyFile(ChartDelta.class, "index.css", getDirectory()); 237 counter.clear(); 238 fileCounters.clear(); 239 writeNonLdmlPlain(anchors); 240 writeLdml(anchors); 241 } 242 writeLdml(Anchors anchors)243 private void writeLdml(Anchors anchors) throws IOException { 244 245 try (PrintWriter tsvFile = FileUtilities.openUTF8Writer(getTsvDir(DIR, DIR_NAME), DIR_NAME + ".tsv"); 246 PrintWriter tsvCountFile = FileUtilities.openUTF8Writer(getTsvDir(DIR, DIR_NAME), DIR_NAME + "_count.tsv"); 247 ) { 248 tsvFile.println("# Section\tPage\tHeader\tCode\tLocale\tOld\tNew\tLevel"); 249 250 // set up factories 251 List<Factory> factories = new ArrayList<>(); 252 List<Factory> oldFactories = new ArrayList<>(); 253 // factories.add(Factory.make(CLDRPaths.BASE_DIRECTORY + "common/" + "main", ".*")); 254 // oldFactories.add(Factory.make(LAST_ARCHIVE_DIRECTORY + "common/" + "main", ".*")); 255 256 Counter<PathHeader> counts = new Counter<>(); 257 258 for (String dir : DtdType.ldml.directories) { 259 if (dir.equals("annotationsDerived") || dir.equals("casing")) { 260 continue; 261 } 262 String current = (ToolConstants.CLDR_VERSIONS.contains(ToolConstants.LAST_CHART_VERSION) 263 ? CURRENT_DIRECTORY : CLDRPaths.BASE_DIRECTORY) + "common/" + dir; 264 String past = LAST_ARCHIVE_DIRECTORY + "common/" + dir; 265 try { 266 factories.add(Factory.make(current, ".*")); 267 } catch (Exception e1) { 268 System.out.println("Skipping: " + dir + "\t" + e1.getMessage()); 269 continue; // skip where the directories don't exist in old versions 270 } 271 try { 272 oldFactories.add(Factory.make(past, ".*")); 273 } catch (Exception e) { 274 System.out.println("Couldn't open factory: " + past); 275 past = null; 276 oldFactories.add(null); 277 } 278 System.out.println("Will examine: " + dir + "\t\t" + current + "\t\t" + past); 279 } 280 if (factories.isEmpty()) { 281 throw new IllegalArgumentException("No factories found"); 282 } 283 // get a list of all the locales to cycle over 284 285 Relation<String, String> baseToLocales = Relation.of(new TreeMap<String, Set<String>>(), HashSet.class); 286 Matcher m = fileMatcher.matcher(""); 287 Set<String> defaultContents = SDI.getDefaultContentLocales(); 288 LanguageTagParser ltp = new LanguageTagParser(); 289 LikelySubtags ls = new LikelySubtags(SDI); 290 for (String file : factories.get(0).getAvailable()) { 291 if (defaultContents.contains(file)) { 292 continue; 293 } 294 if (!m.reset(file).matches()) { 295 continue; 296 } 297 String base = file.equals("root") ? "root" : ltp.set(ls.minimize(file)).getLanguageScript(); 298 baseToLocales.put(base, file); 299 } 300 301 // do keyboards later 302 303 Status currentStatus = new Status(); 304 Status oldStatus = new Status(); 305 Set<PathDiff> diff = new TreeSet<>(); 306 Set<String> paths = new HashSet<>(); 307 308 Relation<PathHeader, String> diffAll = Relation.of(new TreeMap<PathHeader, Set<String>>(), TreeSet.class); 309 // XPathParts pathPlain = new XPathParts(); 310 for (Entry<String, Set<String>> baseNLocale : baseToLocales.keyValuesSet()) { 311 String base = baseNLocale.getKey(); 312 // int qCount = 0; 313 for (int i = 0; i < factories.size(); ++i) { 314 Factory factory = factories.get(i); 315 Factory oldFactory = oldFactories.get(i); 316 List<File> sourceDirs = Arrays.asList(factory.getSourceDirectories()); 317 if (sourceDirs.size() != 1) { 318 throw new IllegalArgumentException("Internal error: expect single source dir"); 319 } 320 File sourceDir = sourceDirs.get(0); 321 String sourceDirLeaf = sourceDir.getName(); 322 //System.out.println(sourceDirLeaf); 323 boolean resolving = !sourceDirLeaf.contains("subdivisions") 324 && !sourceDirLeaf.contains("transforms"); 325 for (String locale : baseNLocale.getValue()) { 326 //System.out.println("\t" + locale); 327 String nameAndLocale = sourceDirLeaf + "/" + locale; 328 if (fileFilter != null && !fileFilter.reset(nameAndLocale).find()) { 329 if (verbose) { 330 System.out.println("SKIPPING: " + nameAndLocale); 331 } 332 continue; 333 } 334 // boolean isBase = locale.equals(base); 335 if (verbose) { 336 System.out.println(nameAndLocale); 337 } 338 CLDRFile current = makeWithFallback(factory, locale, resolving); 339 CLDRFile old = makeWithFallback(oldFactory, locale, resolving); 340 if (!locale.equals("root") && current.getLocaleID().equals("root") && old.getLocaleID().equals("root")) { 341 continue; 342 } 343 if (old == EMPTY_CLDR && current == EMPTY_CLDR) { 344 continue; 345 } 346 paths.clear(); 347 for (String path : current.fullIterable()) { 348 paths.add(path); 349 } 350 for (String path : old.fullIterable()) { 351 paths.add(path); 352 } 353 354 Output<String> reformattedValue = new Output<String>(); 355 Output<Boolean> hasReformattedValue = new Output<Boolean>(); 356 357 for (String path : paths) { 358 if (path.startsWith("//ldml/identity") 359 || path.endsWith("/alias") 360 || path.startsWith("//ldml/segmentations") // do later 361 || path.startsWith("//ldml/rbnf") // do later 362 ) { 363 continue; 364 } 365 if (path.contains("/tRule")) { 366 int debug = 0; 367 } 368 PathHeader ph = getPathHeader(path); 369 if (ph == null) { 370 continue; 371 } 372 373 String oldValue = null; 374 String currentValue = null; 375 376 { 377 String sourceLocaleCurrent = current.getSourceLocaleID(path, currentStatus); 378 String sourceLocaleOld = getReformattedPath(oldStatus, old, path, reformattedValue, hasReformattedValue); 379 380 // filter out stuff that differs at a higher level 381 if (!sourceLocaleCurrent.equals(locale) 382 && !sourceLocaleOld.equals(locale)) { 383 continue; 384 } 385 if (!path.equals(currentStatus.pathWhereFound) 386 && !path.equals(oldStatus.pathWhereFound)) { 387 continue; 388 } 389 // fix some incorrect cases? 390 391 currentValue = current.getStringValue(path); 392 oldValue = hasReformattedValue.value ? reformattedValue.value : old.getStringValue(path); 393 } 394 // handle non-distinguishing attributes 395 addPathDiff(sourceDir, old, current, locale, ph, diff); 396 397 addValueDiff(sourceDir, oldValue, currentValue, locale, ph, diff, diffAll); 398 } 399 } 400 } 401 writeDiffs(anchors, base, diff, tsvFile, counts); 402 diff.clear(); 403 } 404 writeDiffs(anchors, diffAll); 405 406 writeCounter(tsvCountFile, "Count", counts); 407 tsvFile.println("# EOF"); 408 tsvCountFile.println("# EOF"); 409 } 410 411 } 412 getReformattedPath(Status oldStatus, CLDRFile old, String path, Output<String> value, Output<Boolean> hasReformattedValue)413 private String getReformattedPath(Status oldStatus, CLDRFile old, String path, Output<String> value, Output<Boolean> hasReformattedValue) { 414 if (SKIP_REFORMAT_ANNOTATIONS || !path.startsWith("//ldml/annotations/")) { 415 hasReformattedValue.value = Boolean.FALSE; 416 return old.getSourceLocaleID(path, oldStatus); 417 } 418 // OLD: <annotation cp='[��]' tts='grinning face'>face; grin</annotation> 419 // NEW: <annotation cp="��">face | grin</annotation> 420 // <annotation cp="��" type="tts">grinning face</annotation> 421 // from the NEW paths, get the OLD values 422 XPathParts parts = XPathParts.getInstance(path); 423 boolean isTts = parts.getAttributeValue(-1, "type") != null; 424 if (isTts) { 425 parts.removeAttribute(-1, "type"); 426 } 427 String cp = parts.getAttributeValue(-1, "cp"); 428 parts.setAttribute(-1, "cp", "[" + cp + "]"); 429 430 String oldStylePath = parts.toString(); 431 String temp = old.getStringValue(oldStylePath); 432 if (temp == null) { 433 hasReformattedValue.value = Boolean.FALSE; 434 } else if (isTts) { 435 String temp2 = old.getFullXPath(oldStylePath); 436 value.value = XPathParts.getInstance(temp2).getAttributeValue(-1, "tts"); 437 hasReformattedValue.value = Boolean.TRUE; 438 } else { 439 value.value = temp.replaceAll("\\s*;\\s*", " | "); 440 hasReformattedValue.value = Boolean.TRUE; 441 } 442 return old.getSourceLocaleID(oldStylePath, oldStatus); 443 } 444 445 PathStarrer starrer = new PathStarrer().setSubstitutionPattern("%A"); 446 getPathHeader(String path)447 private PathHeader getPathHeader(String path) { 448 try { 449 PathHeader ph = phf.fromPath(path); 450 if (ph.getPageId() == PageId.Unknown) { 451 String star = starrer.set(path); 452 badHeaders.add(star); 453 return null; 454 } 455 return ph; 456 } catch (Exception e) { 457 String star = starrer.set(path); 458 badHeaders.add(star); 459 // System.err.println("Skipping path with bad PathHeader: " + path); 460 return null; 461 } 462 } 463 makeWithFallback(Factory oldFactory, String locale, boolean resolving)464 private CLDRFile makeWithFallback(Factory oldFactory, String locale, boolean resolving) { 465 if (oldFactory == null) { 466 return EMPTY_CLDR; 467 } 468 CLDRFile old; 469 String oldLocale = locale; 470 while (true) { // fall back for old, maybe to root 471 try { 472 old = oldFactory.make(oldLocale, resolving); 473 break; 474 } catch (Exception e) { 475 oldLocale = LocaleIDParser.getParent(oldLocale); 476 if (oldLocale == null) { 477 return EMPTY_CLDR; 478 } 479 } 480 } 481 return old; 482 } 483 addPathDiff(File sourceDir, CLDRFile old, CLDRFile current, String locale, PathHeader ph, Set<PathDiff> diff2)484 private void addPathDiff(File sourceDir, CLDRFile old, CLDRFile current, String locale, PathHeader ph, Set<PathDiff> diff2) { 485 String path = ph.getOriginalPath(); 486 String fullPathCurrent = current.getFullXPath(path); 487 String fullPathOld = old.getFullXPath(path); 488 if (Objects.equals(fullPathCurrent, fullPathOld)) { 489 return; 490 } 491 XPathParts pathPlain = new XPathParts().set(path); 492 XPathParts pathCurrent = fullPathCurrent == null ? pathPlain : new XPathParts().set(fullPathCurrent); 493 XPathParts pathOld = fullPathOld == null ? pathPlain : new XPathParts().set(fullPathOld); 494 TreeSet<String> fullAttributes = null; 495 int size = pathCurrent.size(); 496 String parentAndName = parentAndName(sourceDir, locale); 497 for (int elementIndex = 0; elementIndex < size; ++elementIndex) { // will have same size 498 Collection<String> distinguishing = pathPlain.getAttributeKeys(elementIndex); 499 Collection<String> attributesCurrent = pathCurrent.getAttributeKeys(elementIndex); 500 Collection<String> attributesOld = pathCurrent.getAttributeKeys(elementIndex); 501 if (attributesCurrent.isEmpty() && attributesOld.isEmpty()) { 502 continue; 503 } 504 if (fullAttributes == null) { 505 fullAttributes = new TreeSet<String>(); 506 } else { 507 fullAttributes.clear(); 508 } 509 fullAttributes.addAll(attributesCurrent); 510 fullAttributes.addAll(attributesOld); 511 fullAttributes.removeAll(distinguishing); 512 fullAttributes.removeAll(DONT_CARE); 513 514 // at this point we only have non-distinguishing 515 for (String attribute : fullAttributes) { 516 String attributeValueOld = pathOld.getAttributeValue(elementIndex, attribute); 517 String attributeValueCurrent = pathCurrent.getAttributeValue(elementIndex, attribute); 518 if (Objects.equals(attributeValueOld, attributeValueCurrent)) { 519 addChange(parentAndName, ChangeType.same, 1); 520 continue; 521 } 522 addChange(parentAndName, ChangeType.get(attributeValueOld, attributeValueCurrent), 1); 523 524 PathDiff row = new PathDiff( 525 locale, 526 new PathHeaderSegment(ph, size - elementIndex - 1, attribute), 527 attributeValueOld, 528 attributeValueCurrent); 529 if (DEBUG) { 530 System.out.println(row); 531 } 532 diff2.add(row); 533 } 534 } 535 } 536 parentAndName(File sourceDir, String locale)537 private String parentAndName(File sourceDir, String locale) { 538 return sourceDir.getName() + "/" + locale + ".xml"; 539 } 540 addValueDiff(File sourceDir, String valueOld, String valueCurrent, String locale, PathHeader ph, Set<PathDiff> diff, Relation<PathHeader, String> diffAll)541 private void addValueDiff(File sourceDir, String valueOld, String valueCurrent, String locale, PathHeader ph, Set<PathDiff> diff, 542 Relation<PathHeader, String> diffAll) { 543 // handle stuff that can be split specially 544 Splitter splitter = getSplitter(ph.getOriginalPath(), valueOld, valueCurrent); 545 int count = 1; 546 String parentAndName = parentAndName(sourceDir, locale); 547 if (Objects.equals(valueCurrent, valueOld)) { 548 if (splitter != null && valueCurrent != null) { 549 count = splitHandlingNull(splitter, valueCurrent).size(); 550 } 551 addChange(parentAndName, ChangeType.same, count); 552 } else { 553 if (splitter != null) { 554 List<String> setOld = splitHandlingNull(splitter, valueOld); 555 List<String> setNew = splitHandlingNull(splitter, valueCurrent); 556 int[] sameAndNotInSecond = new int[2]; 557 valueOld = getFilteredValue(setOld, setNew, sameAndNotInSecond); 558 addChange(parentAndName, ChangeType.same, sameAndNotInSecond[0]); 559 addChange(parentAndName, ChangeType.deleted, sameAndNotInSecond[1]); 560 sameAndNotInSecond[0] = sameAndNotInSecond[1] = 0; 561 valueCurrent = getFilteredValue(setNew, setOld, sameAndNotInSecond); 562 addChange(parentAndName, ChangeType.added, sameAndNotInSecond[1]); 563 } else { 564 addChange(parentAndName, ChangeType.get(valueOld, valueCurrent), count); 565 } 566 PathDiff row = new PathDiff(locale, new PathHeaderSegment(ph, -1, ""), valueOld, valueCurrent); 567 diff.add(row); 568 diffAll.put(ph, locale); 569 } 570 } 571 splitHandlingNull(Splitter splitter, String value)572 private List<String> splitHandlingNull(Splitter splitter, String value) { 573 return value == null ? null : splitter.splitToList(value); 574 } 575 getSplitter(String path, String valueOld, String valueCurrent)576 private Splitter getSplitter(String path, String valueOld, String valueCurrent) { 577 if (path.contains("/annotation") && !path.contains("tts")) { 578 return DtdData.BAR_SPLITTER; 579 } else if (valueOld != null && valueOld.contains("\n") || valueCurrent != null && valueCurrent.contains("\n")) { 580 return DtdData.CR_SPLITTER; 581 } else { 582 return null; 583 } 584 } 585 586 /** 587 * Return string with all lines from linesToRemove removed 588 * @param toGetStringFor 589 * @param linesToRemove 590 * @return 591 */ getFilteredValue(Collection<String> toGetStringFor, Collection<String> linesToRemove, int[] sameAndDiff)592 private String getFilteredValue(Collection<String> toGetStringFor, Collection<String> linesToRemove, 593 int[] sameAndDiff) { 594 if (toGetStringFor == null) { 595 return null; 596 } 597 StringBuilder buf = new StringBuilder(); 598 Set<String> toRemove = linesToRemove == null ? Collections.emptySet() : new HashSet<>(linesToRemove); 599 boolean removed = false; 600 for (String old : toGetStringFor) { 601 if (toRemove.contains(old)) { 602 removed = true; 603 sameAndDiff[0]++; 604 } else { 605 sameAndDiff[1]++; 606 if (removed) { 607 buf.append("…\n"); 608 removed = false; 609 } 610 buf.append(old).append('\n'); 611 } 612 } 613 if (removed) { 614 buf.append("…"); 615 } else if (buf.length() > 0) { 616 buf.setLength(buf.length() - 1); // remove final \n 617 } 618 return buf.toString(); 619 } 620 writeDiffs(Anchors anchors, String file, String title, Multimap<PathHeader, String> bcp, PrintWriter tsvFile)621 private void writeDiffs(Anchors anchors, String file, String title, Multimap<PathHeader, String> bcp, PrintWriter tsvFile) { 622 TablePrinter tablePrinter = new TablePrinter() 623 .addColumn("Section", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true) 624 .addColumn("Page", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)//.setRepeatDivider(true) 625 .addColumn("Header", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true) 626 .addColumn("Code", "class='source'", null, "class='source'", false) 627 .addColumn("Old", "class='target'", null, "class='target'", false) // width='20%' 628 .addColumn("New", "class='target'", null, "class='target'", false); // width='20%' 629 PathHeader ph1 = phf.fromPath("//supplementalData/metadata/alias/subdivisionAlias[@type=\"TW-TXQ\"]/_reason"); 630 PathHeader ph2 = phf.fromPath("//supplementalData/metadata/alias/subdivisionAlias[@type=\"LA-XN\"]/_replacement"); 631 ph1.compareTo(ph2); 632 for (Entry<PathHeader, Collection<String>> entry : bcp.asMap().entrySet()) { 633 PathHeader ph = entry.getKey(); 634 if (ph.getPageId() == DEBUG_PAGE_ID) { 635 System.out.println(ph + "\t" + ph.getOriginalPath()); 636 } 637 for (String value : entry.getValue()) { 638 String[] oldNew = value.split(SEP); 639 tablePrinter.addRow() 640 .addCell(ph.getSectionId()) 641 .addCell(ph.getPageId()) 642 .addCell(ph.getHeader()) 643 .addCell(ph.getCode()) 644 .addCell(oldNew[0]) 645 .addCell(oldNew[1]) 646 .finishRow(); 647 } 648 } 649 writeTable(anchors, file, tablePrinter, title, tsvFile); 650 } 651 writeDiffs(Anchors anchors, Relation<PathHeader, String> diffAll)652 private void writeDiffs(Anchors anchors, Relation<PathHeader, String> diffAll) { 653 TablePrinter tablePrinter = new TablePrinter() 654 .addColumn("Section", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true) 655 .addColumn("Page", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true) 656 .addColumn("Header", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true) 657 .addColumn("Code", "class='source'", null, "class='source'", true) 658 .addColumn("Locales where different", "class='target'", null, "class='target'", true); 659 for (Entry<PathHeader, Set<String>> row : diffAll.keyValuesSet()) { 660 PathHeader ph = row.getKey(); 661 Set<String> locales = row.getValue(); 662 tablePrinter.addRow() 663 .addCell(ph.getSectionId()) 664 .addCell(ph.getPageId()) 665 .addCell(ph.getHeader()) 666 .addCell(ph.getCode()) 667 .addCell(CollectionUtilities.join(locales, " ")) 668 .finishRow(); 669 } 670 } 671 writeDiffs(Anchors anchors, String file, Set<PathDiff> diff, PrintWriter tsvFile, Counter<PathHeader> counts)672 private void writeDiffs(Anchors anchors, String file, Set<PathDiff> diff, PrintWriter tsvFile, Counter<PathHeader> counts) { 673 if (diff.isEmpty()) { 674 return; 675 } 676 TablePrinter tablePrinter = new TablePrinter() 677 .addColumn("Section", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true) 678 .addColumn("Page", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true) 679 .addColumn("Header", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true) 680 .addColumn("Code", "class='source'", null, "class='source'", true) 681 .addColumn("Locale", "class='source'", null, "class='source'", true) 682 .addColumn("Old", "class='target'", null, "class='target'", true) // width='20%' 683 .addColumn("New", "class='target'", null, "class='target'", true) // width='20%' 684 .addColumn("Level", "class='target'", null, "class='target'", true); 685 686 for (PathDiff row : diff) { 687 PathHeaderSegment phs = row.get0(); 688 counts.add(phs.get0(), 1); 689 String locale = row.get1(); 690 String oldValue = row.get2(); 691 String currentValue = row.get3(); 692 693 PathHeader ph = phs.get0(); 694 Integer pathIndex = phs.get1(); 695 String attribute = phs.get2(); 696 String specialCode = ph.getCode(); 697 698 if (!attribute.isEmpty()) { 699 specialCode += "_" + attribute; 700 if (pathIndex != 0) { 701 specialCode += "|" + pathIndex; 702 } 703 } 704 Level coverageLevel = SUPPLEMENTAL_DATA_INFO.getCoverageLevel(ph.getOriginalPath(), locale); 705 String fixedOldValue = oldValue == null ? "▷missing◁" : TransliteratorUtilities.toHTML.transform(oldValue); 706 String fixedNewValue = currentValue == null ? "▷removed◁" : TransliteratorUtilities.toHTML.transform(currentValue); 707 708 tablePrinter.addRow() 709 .addCell(ph.getSectionId()) 710 .addCell(ph.getPageId()) 711 .addCell(ph.getHeader()) 712 .addCell(specialCode) 713 .addCell(locale) 714 .addCell(fixedOldValue) 715 .addCell(fixedNewValue) 716 .addCell(coverageLevel) 717 .finishRow(); 718 719 } 720 writeTable(anchors, file, tablePrinter, ENGLISH.getName(file) + " Delta", tsvFile); 721 722 diff.clear(); 723 } 724 725 private class ChartDeltaSub extends Chart { 726 String title; 727 String file; 728 private TablePrinter tablePrinter; 729 private PrintWriter tsvFile; 730 ChartDeltaSub(String title, String file, TablePrinter tablePrinter, PrintWriter tsvFile)731 public ChartDeltaSub(String title, String file, TablePrinter tablePrinter, PrintWriter tsvFile) { 732 super(); 733 this.title = title; 734 this.file = file; 735 this.tablePrinter = tablePrinter; 736 this.tsvFile = tsvFile; 737 } 738 739 @Override getDirectory()740 public String getDirectory() { 741 return DIR; 742 } 743 744 @Override getShowDate()745 public boolean getShowDate() { 746 return false; 747 } 748 749 @Override getTitle()750 public String getTitle() { 751 return title; 752 } 753 754 @Override getFileName()755 public String getFileName() { 756 return file; 757 } 758 759 @Override getExplanation()760 public String getExplanation() { 761 return "<p>Lists data fields that differ from the last version." 762 + " Inherited differences in locales are suppressed, except where the source locales are different. " 763 + " The collations and metadata still have a raw format." 764 + " The rbnf, segmentations, and annotations are not yet included.<p>"; 765 } 766 767 @Override writeContents(FormattedFileWriter pw)768 public void writeContents(FormattedFileWriter pw) throws IOException { 769 pw.write(tablePrinter.toTable()); 770 tablePrinter.toTsv(tsvFile); 771 } 772 } 773 writeTable(Anchors anchors, String file, TablePrinter tablePrinter, String title, PrintWriter tsvFile)774 private void writeTable(Anchors anchors, String file, TablePrinter tablePrinter, String title, PrintWriter tsvFile) { 775 ChartDeltaSub chartDeltaSub = new ChartDeltaSub(title, file, tablePrinter, tsvFile); 776 chartDeltaSub.writeChart(anchors); 777 } 778 writeNonLdmlPlain(Anchors anchors)779 public void writeNonLdmlPlain(Anchors anchors) throws IOException { 780 try (PrintWriter tsvFile = FileUtilities.openUTF8Writer(getTsvDir(DIR, DIR_NAME), DIR_NAME + "_supp.tsv"); 781 PrintWriter tsvCountFile = FileUtilities.openUTF8Writer(getTsvDir(DIR, DIR_NAME), DIR_NAME + "_supp_count.tsv"); 782 ) { 783 tsvFile.println("# Section\tPage\tHeader\tCode\tOld\tNew"); 784 785 Multimap<PathHeader, String> bcp = TreeMultimap.create(); 786 Multimap<PathHeader, String> supplemental = TreeMultimap.create(); 787 Multimap<PathHeader, String> transforms = TreeMultimap.create(); 788 789 Counter<PathHeader> countSame = new Counter<>(); 790 Counter<PathHeader> countAdded = new Counter<>(); 791 Counter<PathHeader> countDeleted = new Counter<>(); 792 793 for (String dir : new File(CLDRPaths.BASE_DIRECTORY + "common/").list()) { 794 if (DtdType.ldml.directories.contains(dir) 795 || dir.equals(".DS_Store") 796 || dir.equals("dtd") // TODO as flat files 797 || dir.equals("properties") // TODO as flat files 798 || dir.equals("uca") // TODO as flat files 799 ) { 800 continue; 801 } 802 File dir1 = new File(LAST_ARCHIVE_DIRECTORY + "common/" + dir); 803 File dir2 = new File(CLDRPaths.BASE_DIRECTORY + "common/" + dir); 804 805 for (String file : dir2.list()) { 806 if (!file.endsWith(".xml")) { 807 continue; 808 } 809 String parentAndFile = dir + "/" + file; 810 String base = file.substring(0, file.length() - 4); 811 if (fileFilter != null && !fileFilter.reset(dir + "/" + base).find()) { 812 if (verbose) { 813 System.out.println("SKIPPING: " + dir + "/" + base); 814 } 815 continue; 816 } 817 818 if (verbose) { 819 System.out.println(file); 820 } 821 Relation<PathHeader, String> contents1 = fillData(dir1.toString() + "/", file); 822 Relation<PathHeader, String> contents2 = fillData(dir2.toString() + "/", file); 823 824 Set<PathHeader> keys = new TreeSet<PathHeader>(CldrUtility.ifNull(contents1.keySet(), Collections.<PathHeader> emptySet())); 825 keys.addAll(CldrUtility.ifNull(contents2.keySet(), Collections.<PathHeader> emptySet())); 826 DtdType dtdType = null; 827 for (PathHeader key : keys) { 828 String originalPath = key.getOriginalPath(); 829 if (originalPath.contains("/paradigmLocales")) { 830 int debug = 0; 831 } 832 boolean isTransform = originalPath.contains("/tRule"); 833 if (dtdType == null) { 834 dtdType = DtdType.fromPath(originalPath); 835 } 836 Multimap<PathHeader, String> target = dtdType == DtdType.ldmlBCP47 ? bcp 837 : isTransform ? transforms 838 : supplemental; 839 Set<String> set1 = contents1.get(key); 840 Set<String> set2 = contents2.get(key); 841 842 if (Objects.equals(set1, set2)) { 843 if (file.equals(DEBUG_FILE)) { // for debugging 844 System.out.println("**Same: " + key + "\t" + set1); 845 } 846 addChange(parentAndFile, ChangeType.same, set1.size()); 847 countSame.add(key, 1); 848 continue; 849 } 850 if (set1 == null) { 851 addChange(parentAndFile, ChangeType.added, set2.size()); 852 for (String s : set2) { 853 addRow(target, key, "▷missing◁", s); 854 countAdded.add(key, 1); 855 } 856 } else if (set2 == null) { 857 addChange(parentAndFile, ChangeType.deleted, set1.size()); 858 for (String s : set1) { 859 addRow(target, key, s, "▷removed◁"); 860 countDeleted.add(key, 1); 861 } 862 } else { 863 Set<String> s1M2 = set1; 864 Set<String> s2M1 = set2; 865 // Set<String> s1M2 = new LinkedHashSet<>(set1); 866 // s1M2.removeAll(set2); 867 // Set<String> s2M1 = new LinkedHashSet<>(set2); 868 // s2M1.removeAll(set1); 869 if (s1M2.isEmpty()) { 870 addRow(target, key, "▷missing◁", CollectionUtilities.join(s2M1, ", ")); 871 addChange(parentAndFile, ChangeType.added, s2M1.size()); 872 countAdded.add(key, 1); 873 } else if (s2M1.isEmpty()) { 874 addRow(target, key, CollectionUtilities.join(s1M2, ", "), "▷removed◁"); 875 addChange(parentAndFile, ChangeType.deleted, s1M2.size()); 876 countDeleted.add(key, 1); 877 } else { 878 String valueOld; 879 String valueCurrent; 880 881 int[] sameAndNotInSecond = new int[2]; 882 valueOld = getFilteredValue(s1M2, s1M2, sameAndNotInSecond); 883 addChange(parentAndFile, ChangeType.same, sameAndNotInSecond[0]); 884 countSame.add(key, 1); 885 addChange(parentAndFile, ChangeType.deleted, sameAndNotInSecond[1]); 886 sameAndNotInSecond[1] = 0; 887 countDeleted.add(key, 1); 888 valueCurrent = getFilteredValue(s2M1, s1M2, sameAndNotInSecond); 889 addChange(parentAndFile, ChangeType.added, sameAndNotInSecond[1]); 890 addRow(target, key, valueOld, valueCurrent); 891 countAdded.add(key, 1); 892 } 893 } 894 } 895 } 896 } 897 // private void writeDiffs(Anchors anchors, String file, String title, Relation<PathHeader, String> bcp) { 898 899 writeDiffs(anchors, "bcp47", "¤¤BCP47 Delta", bcp, tsvFile); 900 writeDiffs(anchors, "supplemental-data", "¤¤Supplemental Delta", supplemental, tsvFile); 901 writeDiffs(anchors, "transforms", "¤¤Transforms Delta", transforms, tsvFile); 902 903 writeCounter(tsvCountFile, "CountSame", countSame); 904 tsvCountFile.println(); 905 writeCounter(tsvCountFile, "CountAdded", countAdded); 906 tsvCountFile.println(); 907 writeCounter(tsvCountFile, "CountDeleted", countDeleted); 908 909 tsvFile.println("# EOF"); 910 tsvCountFile.println("# EOF"); 911 } 912 } 913 writeCounter(PrintWriter tsvFile, String title, Counter<PathHeader> countDeleted)914 private void writeCounter(PrintWriter tsvFile, String title, Counter<PathHeader> countDeleted) { 915 tsvFile.append("# " 916 + title 917 + "\tPathHeader\n\n"); 918 for (R2<Long, PathHeader> entry : countDeleted.getEntrySetSortedByCount(false, null)) { 919 tsvFile.println(entry.get0() + "\t" + entry.get1()); 920 } 921 } 922 addRow(Multimap<PathHeader, String> target, PathHeader key, String oldItem, String newItem)923 private void addRow(Multimap<PathHeader, String> target, PathHeader key, String oldItem, String newItem) { 924 if (oldItem.isEmpty() || newItem.isEmpty()) { 925 throw new IllegalArgumentException(); 926 } 927 target.put(key, oldItem + SEP + newItem); 928 } 929 930 // private static final Splitter ONHYPHEN = Splitter.on('-'); 931 // private final LanguageTagParser lparser = new LanguageTagParser(); 932 // private final Map<LstrType, Map<Validity.Status, Set<String>>> validity = Validity.getInstance().getData(); 933 // private final Set<String> regularLanguage = validity.get(LstrType.language).get(Validity.Status.regular); 934 // 935 // private String name(String key) { 936 // // eo-eo_FONIPA 937 // // Latin-ASCII 938 // int i = 0; 939 // try { 940 // StringBuilder sb = new StringBuilder(); 941 // for (String part : ONHYPHEN.split(key)) { 942 // lparser.set(part); 943 // String base = lparser.getLanguage(); 944 // int script = UScript.getCodeFromName(base); 945 // if (script != UScript.INVALID_CODE) { 946 // part = UScript.getName(script); 947 // } else if (regularLanguage.contains(base)) { 948 // part = ENGLISH.getName(part); 949 // } 950 // if (i != 0) { 951 // sb.append('-'); 952 // } 953 // sb.append(part); 954 // ++i; 955 // } 956 // return sb.toString(); 957 // } catch (Exception e) { 958 // // TODO fix this to handle all cases 959 // return key; 960 // } 961 // } 962 // 963 // private String removeStart(String key, String... string) { 964 // for (String start : string) { 965 // if (key.startsWith(start)) { 966 // return "…" + key.substring(start.length()); 967 // } 968 // } 969 // return key; 970 // } 971 // fillData(String directory, String file)972 public Relation<PathHeader, String> fillData(String directory, String file) { 973 Relation<PathHeader, String> results = Relation.of(new TreeMap<PathHeader, Set<String>>(), TreeSet.class); 974 975 List<Pair<String, String>> contents1; 976 try { 977 contents1 = XMLFileReader.loadPathValues(directory + file, new ArrayList<Pair<String, String>>(), true); 978 } catch (Exception e) { 979 return results; 980 } 981 DtdType dtdType = null; 982 DtdData dtdData = null; 983 Multimap<String, String> extras = TreeMultimap.create(); 984 985 for (Pair<String, String> s : contents1) { 986 String path = s.getFirst(); 987 if (path.contains("/collat")) { 988 int debug = 0; 989 } 990 991 String value = s.getSecond(); 992 if (dtdType == null) { 993 dtdType = DtdType.fromPath(path); 994 dtdData = DtdData.getInstance(dtdType); 995 } 996 XPathParts pathPlain = XPathParts.getFrozenInstance(path); 997 if (dtdData.isMetadata(pathPlain)) { 998 continue; 999 } 1000 Set<String> pathForValues = dtdData.getRegularizedPaths(pathPlain, extras); 1001 if (pathForValues != null) { 1002 for (String pathForValue : pathForValues) { 1003 PathHeader pathHeader = phf.fromPath(pathForValue); 1004 Splitter splitter = getValueSplitter(pathPlain); 1005 for (String line : splitter.split(value)) { 1006 // special case # in transforms 1007 if (isComment(pathPlain, line)) { 1008 continue; 1009 } 1010 results.put(pathHeader, line); 1011 } 1012 } 1013 } 1014 for (Entry<String, Collection<String>> entry : extras.asMap().entrySet()) { 1015 final String extraPath = entry.getKey(); 1016 final PathHeader pathHeaderExtra = phf.fromPath(extraPath); 1017 final Collection<String> extraValue = entry.getValue(); 1018 if (isExtraSplit(extraPath)) { 1019 for (String items : extraValue) { 1020 results.putAll(pathHeaderExtra, DtdData.SPACE_SPLITTER.splitToList(items)); 1021 } 1022 } else { 1023 results.putAll(pathHeaderExtra, extraValue); 1024 } 1025 } 1026 if (pathForValues == null && !value.isEmpty()) { 1027 System.err.println("Shouldn't happen"); 1028 } 1029 } 1030 return results; 1031 } 1032 isExtraSplit(String extraPath)1033 private boolean isExtraSplit(String extraPath) { 1034 if (extraPath.endsWith("/_type") && extraPath.startsWith("//supplementalData/metaZones/mapTimezones")) { 1035 return true; 1036 } 1037 return false; 1038 } 1039 getValueSplitter(XPathParts pathPlain)1040 public Splitter getValueSplitter(XPathParts pathPlain) { 1041 return DtdData.getValueSplitter(pathPlain); 1042 } 1043 isComment(XPathParts pathPlain, String line)1044 public static boolean isComment(XPathParts pathPlain, String line) { 1045 if (pathPlain.contains("transform")) { 1046 if (line.startsWith("#")) { 1047 return true; 1048 } 1049 } 1050 return false; 1051 } 1052 1053 // private static final Set<String> SKIP_ATTRIBUTE = new HashSet<String>(Arrays.asList("references")); 1054 // 1055 // private static void checkedPut(Map<String, String> contentsA, String path2, String value) { 1056 // String old = contentsA.get(path2); 1057 // if (old != null) { 1058 // if (old.equals(value)) { 1059 // return; 1060 // } 1061 // value = old + "\n" + value; 1062 // } 1063 // contentsA.put(path2, value); 1064 // } 1065 // 1066 // static final ChainedMap.M4<DtdType, String, String, Boolean> FIXED_DISTINGUISHING = ChainedMap.of( 1067 // new HashMap<DtdType, Object>(), 1068 // new HashMap<String, Object>(), 1069 // new HashMap<String, Object>(), 1070 // Boolean.class); 1071 // 1072 // static final ChainedMap.M3<DtdType, String, Boolean> FIXED_ORDERING = ChainedMap.of( 1073 // new HashMap<DtdType, Object>(), 1074 // new HashMap<String, Object>(), 1075 // Boolean.class); 1076 // 1077 // static { 1078 // //<key name="ca" description="Calendar algorithm key" alias="calendar"> 1079 // //<type name="buddhist" description="Thai Buddhist calendar"/> 1080 // FIXED_DISTINGUISHING.put(DtdType.ldmlBCP47, "key", "alias", false); 1081 // 1082 // FIXED_ORDERING.put(DtdType.supplementalData, "substitute", true); 1083 // 1084 // //<approvalRequirement votes="20" locales="Cldr:modern" paths="//ldml/numbers/symbols[^/]++/(decimal|group|(plus|minus)Sign)"/> 1085 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "approvalRequirement", "locales", true); 1086 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "approvalRequirement", "paths", true); 1087 // // approvalRequirement should be ordered, but for our comparison, simpler to override 1088 // 1089 // //<coverageVariable key="%acctPattern" value="[@type='accounting']/pattern[@type='standard']"/> 1090 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "coverageVariable", "key", true); 1091 // FIXED_ORDERING.put(DtdType.supplementalData, "coverageVariable", false); 1092 // 1093 // //<coverageLevel inLanguage="%phonebookCollationLanguages" value="minimal" match="localeDisplayNames/types/type[@type='phonebook'][@key='collation']"/> 1094 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "coverageLevel", "inLanguage", true); 1095 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "coverageLevel", "inScript", true); 1096 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "coverageLevel", "inTerritory", true); 1097 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "coverageLevel", "match", true); 1098 // FIXED_ORDERING.put(DtdType.supplementalData, "coverageLevel", false); // this should be true, but for our comparison, simpler to override 1099 // 1100 //// <!ATTLIST dayPeriodRule type NMTOKEN #REQUIRED > 1101 //// <!ATTLIST dayPeriodRule at NMTOKEN #IMPLIED > 1102 //// <!ATTLIST dayPeriodRule after NMTOKEN #IMPLIED > 1103 //// <!ATTLIST dayPeriodRule before NMTOKEN #IMPLIED > 1104 //// <!ATTLIST dayPeriodRule from NMTOKEN #IMPLIED > 1105 //// <!ATTLIST dayPeriodRule to NMTOKEN #IMPLIED > 1106 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "dayPeriodRule", "type", false); 1107 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "dayPeriodRule", "at", true); 1108 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "dayPeriodRule", "after", true); 1109 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "dayPeriodRule", "before", true); 1110 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "dayPeriodRule", "from", true); 1111 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "dayPeriodRule", "to", true); 1112 // 1113 //// <personList type="neutral" locales="af bn bg da de en et eu fa fil fi gu hu id ja kn ko ml ms no sv sw ta te th tr vi zu" /> 1114 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "personList", "type", false); 1115 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "personList", "locales", true); 1116 // 1117 // // <languageMatch desired="am_*_*" supported="en_GB" percent="90" oneway="true" /> <!-- fix ICU for en_GB --> 1118 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "languageMatch", "desired", true); 1119 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "languageMatch", "supported", true); 1120 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "languageMatch", "oneway", true); 1121 // FIXED_ORDERING.put(DtdType.supplementalData, "languageMatch", false); // this should be true, but for our comparison, simpler to override 1122 // 1123 // // <usesMetazone to="1979-10-25 23:00" from="1977-10-20 23:00" mzone="Europe_Central"/> 1124 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "usesMetazone", "from", true); 1125 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "usesMetazone", "to", true); 1126 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "usesMetazone", "mzone", false); 1127 // 1128 //// <!ATTLIST numberingSystem type ( numeric | algorithmic ) #REQUIRED > 1129 //// <!ATTLIST numberingSystem id NMTOKEN #REQUIRED > 1130 //// <!ATTLIST numberingSystem radix NMTOKEN #IMPLIED > 1131 //// <!ATTLIST numberingSystem digits CDATA #IMPLIED > 1132 //// <!ATTLIST numberingSystem rules CDATA #IMPLIED > 1133 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "numberingSystem", "type", false); 1134 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "numberingSystem", "radix", false); 1135 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "numberingSystem", "digits", false); 1136 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "numberingSystem", "rules", false); 1137 // 1138 // FIXED_ORDERING.put(DtdType.supplementalData, "pluralRule", false); // this should be true, but for our comparison, simpler to override 1139 // 1140 // //pluralRanges locales 1141 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "pluralRanges", "locales", true); 1142 // 1143 // // <pluralRange start="one" end="one" result="one"/> 1144 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "pluralRange", "start", true); 1145 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "pluralRange", "end", true); 1146 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "pluralRange", "result", false); 1147 // 1148 //// <territory type="AC" gdp="35200000" literacyPercent="99" population="940"> <!--Ascension Island--> 1149 //// <languagePopulation type="en" populationPercent="99" references="R1020"/> <!--English--> 1150 // 1151 // FIXED_DISTINGUISHING.put(DtdType.ldml, "rbnfrule", "value", false); 1152 // FIXED_ORDERING.put(DtdType.ldml, "ruleset", false); // this should be true, but for our comparison, simpler to override 1153 // FIXED_ORDERING.put(DtdType.ldml, "rbnfrule", false); // this should be true, but for our comparison, simpler to override 1154 // 1155 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "transform", "source", true); 1156 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "transform", "target", true); 1157 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "transform", "direction", true); 1158 // FIXED_DISTINGUISHING.put(DtdType.supplementalData, "transform", "variant", true); 1159 // FIXED_ORDERING.put(DtdType.supplementalData, "tRule", false); 1160 // 1161 // System.out.println("Current Overrides for Distinguishing"); 1162 // for (R4<DtdType, String, String, Boolean> entry : FIXED_DISTINGUISHING.rows()){ 1163 // DtdData dtdData = DtdData.getInstance(entry.get0()); 1164 // boolean actual = dtdData.isDistinguishing(entry.get1(), entry.get2()); 1165 // boolean fixed = entry.get3(); 1166 // if (actual != fixed) { 1167 // System.out.println("\tOverride to\t" + entry); 1168 // } else { 1169 // System.out.println("\tUnnecessary\t" + entry); 1170 // } 1171 // } 1172 // System.out.println("Current Overrides for Ordering"); 1173 // for (R3<DtdType, String, Boolean> entry : FIXED_ORDERING.rows()){ 1174 // DtdData dtdData = DtdData.getInstance(entry.get0()); 1175 // boolean actual = dtdData.isOrdered(entry.get1()); 1176 // boolean fixed = entry.get2(); 1177 // if (actual != fixed) { 1178 // System.out.println("\tOverride to\t" + entry); 1179 // } else { 1180 // System.out.println("\tUnnecessary\t" + entry); 1181 // } 1182 // } 1183 // 1184 // // FIXED_ORDERING.freeze(); Add to ChainedMap? 1185 // } 1186 1187 // private static boolean isFixedDistinguishing(DtdType dtdType, String element, String key) { 1188 // Boolean override = FIXED_DISTINGUISHING.get(dtdType, element, key); 1189 // if (override != null) { 1190 // return override; 1191 // } 1192 // return CLDRFile.isDistinguishing(dtdType, element, key); 1193 // } 1194 // 1195 // private static boolean shouldBeOrdered(DtdType dtdType, String element) { 1196 // Boolean override = FIXED_ORDERING.get(dtdType, element); 1197 // if (override != null) { 1198 // return override; 1199 // } 1200 // return CLDRFile.isOrdered(element, dtdType); 1201 // } 1202 1203 } 1204