1 package org.unicode.cldr.util; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.PrintWriter; 6 import java.io.StringWriter; 7 import java.io.Writer; 8 import java.util.ArrayList; 9 import java.util.Arrays; 10 import java.util.Collections; 11 import java.util.EnumSet; 12 import java.util.HashMap; 13 import java.util.HashSet; 14 import java.util.LinkedHashSet; 15 import java.util.List; 16 import java.util.Map; 17 import java.util.Map.Entry; 18 import java.util.Objects; 19 import java.util.Set; 20 import java.util.TreeMap; 21 import java.util.TreeSet; 22 import java.util.regex.Matcher; 23 import java.util.regex.Pattern; 24 25 import org.unicode.cldr.draft.FileUtilities; 26 import org.unicode.cldr.test.CheckCLDR; 27 import org.unicode.cldr.test.CheckCLDR.CheckStatus; 28 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 29 import org.unicode.cldr.test.CheckCoverage; 30 import org.unicode.cldr.test.CheckNew; 31 import org.unicode.cldr.test.CoverageLevel2; 32 import org.unicode.cldr.test.OutdatedPaths; 33 import org.unicode.cldr.tool.Option; 34 import org.unicode.cldr.tool.Option.Options; 35 import org.unicode.cldr.tool.ToolConstants; 36 import org.unicode.cldr.util.CLDRFile.Status; 37 import org.unicode.cldr.util.PathHeader.PageId; 38 import org.unicode.cldr.util.PathHeader.SectionId; 39 import org.unicode.cldr.util.StandardCodes.LocaleCoverageType; 40 41 import com.ibm.icu.dev.util.CollectionUtilities; 42 import com.ibm.icu.impl.Relation; 43 import com.ibm.icu.impl.Row; 44 import com.ibm.icu.impl.Row.R2; 45 import com.ibm.icu.text.Collator; 46 import com.ibm.icu.text.NumberFormat; 47 import com.ibm.icu.text.UnicodeSet; 48 import com.ibm.icu.util.ICUUncheckedIOException; 49 import com.ibm.icu.util.Output; 50 import com.ibm.icu.util.ULocale; 51 52 /** 53 * Provides a HTML tables showing the important issues for vetters to review for 54 * a given locale. See the main for an example. Most elements have CSS styles, 55 * allowing for customization of the display. 56 * 57 * @author markdavis 58 */ 59 public class VettingViewer<T> { 60 61 private static boolean SHOW_SUBTYPES = true; // CldrUtility.getProperty("SHOW_SUBTYPES", "false").equals("true"); 62 63 private static final String CONNECT_PREFIX = "₍_"; 64 private static final String CONNECT_SUFFIX = "₎"; 65 66 private static final String TH_AND_STYLES = "<th class='tv-th' style='text-align:left'>"; 67 68 private static final String SPLIT_CHAR = "\uFFFE"; 69 70 private static final boolean SUPPRESS = true; 71 72 private static final String TEST_PATH = "//ldml/localeDisplayNames/territories/territory[@type=\"SX\"]"; 73 private static final double NANOSECS = 1000000000.0; 74 private static final boolean TESTING = CldrUtility.getProperty("TEST", false); 75 private static final boolean SHOW_ALL = CldrUtility.getProperty("SHOW", true); 76 77 public static final Pattern ALT_PROPOSED = PatternCache.get("\\[@alt=\"[^\"]*proposed"); 78 79 public static Set<CheckCLDR.CheckStatus.Subtype> OK_IF_VOTED = EnumSet.of(Subtype.sameAsEnglishOrCode, 80 Subtype.sameAsEnglishOrCode); 81 82 public enum Choice { 83 /** 84 * There is a console-check error 85 */ 86 error('E', "Error", "The Survey Tool detected an error in the winning value.", 1), 87 /** 88 * My choice is not the winning item 89 */ 90 weLost( 91 'L', 92 "Losing", 93 "The value that your organization chose (overall) is either not the winning value, or doesn’t have enough votes to be approved. " 94 + "This might be due to a dispute between members of your organization.", 95 2), 96 /** 97 * There is a dispute. 98 */ 99 notApproved('P', "Provisional", "There are not enough votes for this item to be approved (and used).", 3), 100 /** 101 * There is a dispute. 102 */ 103 hasDispute('D', "Disputed", "Different organizations are choosing different values. " 104 + "Please review to approve or reach consensus.", 4), 105 /** 106 * There is a console-check warning 107 */ 108 warning('W', "Warning", "The Survey Tool detected a warning about the winning value.", 5), 109 /** 110 * The English value for the path changed AFTER the current value for 111 * the locale. 112 */ 113 englishChanged('U', "English Changed", 114 "The English value has changed in CLDR, but the corresponding value for your language has not. Check if any changes are needed in your language.", 115 6), 116 /** 117 * The value changed from the last version of CLDR 118 */ 119 changedOldValue('N', "New", "The winning value was altered from the last-released CLDR value. (Informational)", 7), 120 /** 121 * Given the users' coverage, some items are missing. 122 */ 123 missingCoverage( 124 'M', 125 "Missing", 126 "Your current coverage level requires the item to be present. (During the vetting phase, this is informational: you can’t add new values.)", 8), 127 // /** 128 // * There is a console-check error 129 // */ 130 // other('O', "Other", "Everything else."), 131 ; 132 133 public final char abbreviation; 134 public final String buttonLabel; 135 public final String description; 136 public final int order; 137 Choice(char abbreviation, String buttonLabel, String description, int order)138 Choice(char abbreviation, String buttonLabel, String description, int order) { 139 this.abbreviation = abbreviation; 140 this.buttonLabel = TransliteratorUtilities.toHTML.transform(buttonLabel); 141 this.description = TransliteratorUtilities.toHTML.transform(description); 142 this.order = order; 143 144 } 145 appendDisplay(Set<Choice> choices, String htmlMessage, T target)146 public static <T extends Appendable> T appendDisplay(Set<Choice> choices, String htmlMessage, T target) { 147 try { 148 boolean first = true; 149 for (Choice item : choices) { 150 if (first) { 151 first = false; 152 } else { 153 target.append(", "); 154 } 155 item.appendDisplay(htmlMessage, target); 156 } 157 return target; 158 } catch (IOException e) { 159 throw new ICUUncheckedIOException(e); // damn'd checked 160 // exceptions 161 } 162 } 163 appendDisplay(String htmlMessage, T target)164 private <T extends Appendable> void appendDisplay(String htmlMessage, T target) throws IOException { 165 target.append("<span title='") 166 .append(description); 167 if (!htmlMessage.isEmpty()) { 168 target.append(": ") 169 .append(htmlMessage); 170 } 171 target.append("'>") 172 .append(buttonLabel) 173 .append("*</span>"); 174 } 175 fromString(String i)176 public static Choice fromString(String i) { 177 try { 178 return valueOf(i); 179 } catch (NullPointerException e) { 180 throw e; 181 } catch (RuntimeException e) { 182 if (i.isEmpty()) { 183 throw e; 184 } 185 int cp = i.codePointAt(0); 186 for (Choice choice : Choice.values()) { 187 if (cp == choice.abbreviation) { 188 return choice; 189 } 190 } 191 throw e; 192 } 193 } 194 appendRowStyles(Set<Choice> choices, Appendable target)195 public static Appendable appendRowStyles(Set<Choice> choices, Appendable target) { 196 try { 197 if (choices.contains(Choice.changedOldValue)) { 198 int x = 0; // debugging 199 } 200 target.append("hide"); 201 for (Choice item : choices) { 202 target.append(' ').append("vv").append(Character.toLowerCase(item.abbreviation)); 203 } 204 return target; 205 } catch (IOException e) { 206 throw new ICUUncheckedIOException(e); // damn'd checked 207 // exceptions 208 } 209 } 210 } 211 getOutdatedPaths()212 public static OutdatedPaths getOutdatedPaths() { 213 return outdatedPaths; 214 } 215 216 static private PathHeader.Factory pathTransform; 217 static final Pattern breaks = PatternCache.get("\\|"); 218 static final OutdatedPaths outdatedPaths = new OutdatedPaths(); 219 220 // private static final UnicodeSet NEEDS_PERCENT_ESCAPED = new UnicodeSet("[[\\u0000-\\u009F]-[a-zA-z0-9]]"); 221 // private static final Transform<String, String> percentEscape = new Transform<String, String>() { 222 // @Override 223 // public String transform(String source) { 224 // StringBuilder buffer = new StringBuilder(); 225 // buffer.setLength(0); 226 // for (int cp : CharSequences.codePoints(source)) { 227 // if (NEEDS_PERCENT_ESCAPED.contains(cp)) { 228 // buffer.append('%').append(Utility.hex(cp, 2)); 229 // } else { 230 // buffer.appendCodePoint(cp); 231 // } 232 // } 233 // return buffer.toString(); 234 // } 235 // }; 236 237 /** 238 * See VoteResolver getStatusForOrganization to see how this is computed. 239 */ 240 public enum VoteStatus { 241 /** 242 * The value for the path is either contributed or approved, and 243 * the user's organization didn't vote. (see class def for null user) 244 */ 245 ok_novotes, 246 247 /** 248 * The value for the path is either contributed or approved, and 249 * the user's organization chose the winning value. (see class def for null user) 250 */ 251 ok, 252 253 /** 254 * The user's organization chose the winning value for the path, but 255 * that value is neither contributed nor approved. (see class def for null user) 256 */ 257 provisionalOrWorse, 258 259 /** 260 * The user's organization's choice is not winning. There may be 261 * insufficient votes to overcome a previously approved value, or other 262 * organizations may be voting against it. (see class def for null user) 263 */ 264 losing, 265 266 /** 267 * There is a dispute, meaning more than one item with votes, or the item with votes didn't win. 268 */ 269 disputed 270 } 271 272 /** 273 * @author markdavis 274 * 275 * @param <T> 276 */ 277 public static interface UsersChoice<T> { 278 /** 279 * Return the value that the user's organization (as a whole) voted for, 280 * or null if none of the users in the organization voted for the path. <br> 281 * NOTE: Would be easier if this were a method on CLDRFile. 282 * NOTE: if user = null, then it must return the absolute winning value. 283 * 284 * @param locale 285 */ getWinningValueForUsersOrganization(CLDRFile cldrFile, String path, T user)286 public String getWinningValueForUsersOrganization(CLDRFile cldrFile, String path, T user); 287 288 /** 289 * 290 * Return the vote status 291 * NOTE: if user = null, then it must disregard the user and never return losing. See VoteStatus. 292 * 293 * @param locale 294 */ getStatusForUsersOrganization(CLDRFile cldrFile, String path, T user)295 public VoteStatus getStatusForUsersOrganization(CLDRFile cldrFile, String path, T user); 296 } 297 298 public static interface ErrorChecker { 299 enum Status { 300 ok, error, warning 301 } 302 303 /** 304 * Initialize an error checker with a cldrFile. MUST be called before 305 * any getErrorStatus. 306 */ initErrorStatus(CLDRFile cldrFile)307 public Status initErrorStatus(CLDRFile cldrFile); 308 309 /** 310 * Return the detailed CheckStatus information. 311 */ getErrorCheckStatus(String path, String value)312 public List<CheckStatus> getErrorCheckStatus(String path, String value); 313 314 /** 315 * Return the status, and append the error message to the status 316 * message. If there are any errors, then the warnings are not included. 317 */ getErrorStatus(String path, String value, StringBuilder statusMessage)318 public Status getErrorStatus(String path, String value, StringBuilder statusMessage); 319 320 /** 321 * Return the status, and append the error message to the status 322 * message, and get the subtypes. If there are any errors, then the warnings are not included. 323 */ getErrorStatus(String path, String value, StringBuilder statusMessage, EnumSet<Subtype> outputSubtypes)324 public Status getErrorStatus(String path, String value, StringBuilder statusMessage, 325 EnumSet<Subtype> outputSubtypes); 326 } 327 328 public static class NoErrorStatus implements ErrorChecker { 329 @Override initErrorStatus(CLDRFile cldrFile)330 public Status initErrorStatus(CLDRFile cldrFile) { 331 return Status.ok; 332 } 333 334 @Override getErrorCheckStatus(String path, String value)335 public List<CheckStatus> getErrorCheckStatus(String path, String value) { 336 return Collections.emptyList(); 337 } 338 339 @Override getErrorStatus(String path, String value, StringBuilder statusMessage)340 public Status getErrorStatus(String path, String value, StringBuilder statusMessage) { 341 return Status.ok; 342 } 343 344 @Override getErrorStatus(String path, String value, StringBuilder statusMessage, EnumSet<Subtype> outputSubtypes)345 public Status getErrorStatus(String path, String value, StringBuilder statusMessage, 346 EnumSet<Subtype> outputSubtypes) { 347 return Status.ok; 348 } 349 350 } 351 352 public static class DefaultErrorStatus implements ErrorChecker { 353 354 private CheckCLDR checkCldr; 355 private HashMap<String, String> options = new HashMap<String, String>(); 356 private ArrayList<CheckStatus> result = new ArrayList<CheckStatus>(); 357 private CLDRFile cldrFile; 358 private Factory factory; 359 DefaultErrorStatus(Factory cldrFactory)360 public DefaultErrorStatus(Factory cldrFactory) { 361 this.factory = cldrFactory; 362 } 363 364 @Override initErrorStatus(CLDRFile cldrFile)365 public Status initErrorStatus(CLDRFile cldrFile) { 366 this.cldrFile = cldrFile; 367 options = new HashMap<String, String>(); 368 result = new ArrayList<CheckStatus>(); 369 checkCldr = CheckCLDR.getCheckAll(factory, ".*"); 370 checkCldr.setCldrFileToCheck(cldrFile, options, result); 371 return Status.ok; 372 } 373 374 @Override getErrorCheckStatus(String path, String value)375 public List<CheckStatus> getErrorCheckStatus(String path, String value) { 376 String fullPath = cldrFile.getFullXPath(path); 377 ArrayList<CheckStatus> result2 = new ArrayList<CheckStatus>(); 378 checkCldr.check(path, fullPath, value, options, result2); 379 return result2; 380 } 381 382 @Override getErrorStatus(String path, String value, StringBuilder statusMessage)383 public Status getErrorStatus(String path, String value, StringBuilder statusMessage) { 384 return getErrorStatus(path, value, statusMessage, null); 385 } 386 387 @Override getErrorStatus(String path, String value, StringBuilder statusMessage, EnumSet<Subtype> outputSubtypes)388 public Status getErrorStatus(String path, String value, StringBuilder statusMessage, 389 EnumSet<Subtype> outputSubtypes) { 390 Status result0 = Status.ok; 391 StringBuilder errorMessage = new StringBuilder(); 392 String fullPath = cldrFile.getFullXPath(path); 393 checkCldr.check(path, fullPath, value, options, result); 394 for (CheckStatus checkStatus : result) { 395 final CheckCLDR cause = checkStatus.getCause(); 396 if (cause instanceof CheckCoverage || cause instanceof CheckNew) { 397 continue; 398 } 399 CheckStatus.Type statusType = checkStatus.getType(); 400 if (statusType.equals(CheckStatus.errorType)) { 401 // throw away any accumulated warning messages 402 if (result0 == Status.warning) { 403 errorMessage.setLength(0); 404 if (outputSubtypes != null) { 405 outputSubtypes.clear(); 406 } 407 } 408 result0 = Status.error; 409 if (outputSubtypes != null) { 410 outputSubtypes.add(checkStatus.getSubtype()); 411 } 412 appendToMessage(checkStatus.getMessage(), checkStatus.getSubtype(), errorMessage); 413 } else if (result0 != Status.error && statusType.equals(CheckStatus.warningType)) { 414 result0 = Status.warning; 415 // accumulate all the warning messages 416 if (outputSubtypes != null) { 417 outputSubtypes.add(checkStatus.getSubtype()); 418 } 419 appendToMessage(checkStatus.getMessage(), checkStatus.getSubtype(), errorMessage); 420 } 421 } 422 if (result0 != Status.ok) { 423 appendToMessage(errorMessage, statusMessage); 424 } 425 return result0; 426 } 427 } 428 429 private final Factory cldrFactory; 430 private final Factory cldrFactoryOld; 431 private final CLDRFile englishFile; 432 //private final CLDRFile oldEnglishFile; 433 private final UsersChoice<T> userVoteStatus; 434 private final SupplementalDataInfo supplementalDataInfo; 435 private final String lastVersionTitle; 436 private final String currentWinningTitle; 437 //private final PathDescription pathDescription; 438 private ErrorChecker errorChecker; // new 439 440 private final Set<String> defaultContentLocales; 441 442 // NoErrorStatus(); 443 // // 444 // for 445 // testing 446 447 /** 448 * Create the Vetting Viewer. 449 * 450 * @param supplementalDataInfo 451 * @param cldrFactory 452 * @param cldrFactoryOld 453 * @param lastVersionTitle 454 * The title of the last released version of CLDR. 455 * @param currentWinningTitle 456 * The title of the next version of CLDR to be released. 457 */ VettingViewer(SupplementalDataInfo supplementalDataInfo, Factory cldrFactory, Factory cldrFactoryOld, UsersChoice<T> userVoteStatus, String lastVersionTitle, String currentWinningTitle)458 public VettingViewer(SupplementalDataInfo supplementalDataInfo, Factory cldrFactory, Factory cldrFactoryOld, 459 UsersChoice<T> userVoteStatus, 460 String lastVersionTitle, String currentWinningTitle) { 461 super(); 462 this.cldrFactory = cldrFactory; 463 this.cldrFactoryOld = cldrFactoryOld; 464 englishFile = cldrFactory.make("en", true); 465 if (pathTransform == null) { 466 pathTransform = PathHeader.getFactory(englishFile); 467 } 468 //oldEnglishFile = cldrFactoryOld.make("en", true); 469 this.userVoteStatus = userVoteStatus; 470 this.supplementalDataInfo = supplementalDataInfo; 471 this.defaultContentLocales = supplementalDataInfo.getDefaultContentLocales(); 472 473 this.lastVersionTitle = lastVersionTitle; 474 this.currentWinningTitle = currentWinningTitle; 475 //Map<String, List<Set<String>>> starredPaths = new HashMap<String, List<Set<String>>>(); 476 //Map<String, String> extras = new HashMap<String, String>(); 477 reasonsToPaths = Relation.of(new HashMap<String, Set<String>>(), HashSet.class); 478 //this.pathDescription = new PathDescription(supplementalDataInfo, englishFile, extras, starredPaths, 479 // PathDescription.ErrorHandling.CONTINUE); 480 errorChecker = new DefaultErrorStatus(cldrFactory); 481 } 482 483 public class WritingInfo implements Comparable<WritingInfo> { 484 public final PathHeader codeOutput; 485 public final Set<Choice> problems; 486 public final String htmlMessage; 487 WritingInfo(PathHeader pretty, EnumSet<Choice> problems, CharSequence htmlMessage)488 public WritingInfo(PathHeader pretty, EnumSet<Choice> problems, CharSequence htmlMessage) { 489 super(); 490 this.codeOutput = pretty; 491 this.problems = Collections.unmodifiableSet(problems.clone()); 492 this.htmlMessage = htmlMessage.toString(); 493 } 494 495 @Override compareTo(WritingInfo other)496 public int compareTo(WritingInfo other) { 497 return codeOutput.compareTo(other.codeOutput); 498 } 499 getUrl(CLDRLocale locale)500 public String getUrl(CLDRLocale locale) { 501 return urls.forPathHeader(locale, codeOutput); 502 } 503 } 504 505 // public void generateHtmlErrorTablesOld(Appendable output, EnumSet<Choice> choices, String localeID, T user, Level 506 // usersLevel) { 507 // generateHtmlErrorTablesOld(output, choices, localeID, user, usersLevel, false); 508 // } 509 510 // private void generateHtmlErrorTablesOld(Appendable output, EnumSet<Choice> choices, String localeID, T user, 511 // Level usersLevel, boolean showAll) { 512 // 513 // // first gather the relevant paths 514 // // each one will be marked with the choice that it triggered. 515 // 516 // CLDRFile sourceFile = cldrFactory.make(localeID, true); 517 // Matcher altProposed = PatternCache.get("\\[@alt=\"[^\"]*proposed").matcher(""); 518 // EnumSet<Choice> problems = EnumSet.noneOf(Choice.class); 519 // 520 // // Initialize 521 // CoverageLevel2 coverage = CoverageLevel2.getInstance(supplementalDataInfo, localeID); 522 // CLDRFile lastSourceFile = null; 523 // try { 524 // lastSourceFile = cldrFactoryOld.make(localeID, true); 525 // } catch (Exception e) { 526 // } 527 // 528 // // set the following only where needed. 529 // Status status = null; 530 // 531 // Map<String, String> options = null; 532 // List<CheckStatus> result = null; 533 // 534 // for (Choice choice : choices) { 535 // switch (choice) { 536 // case changedOldValue: 537 // break; 538 // case missingCoverage: 539 // status = new Status(); 540 // break; 541 // case englishChanged: 542 // break; 543 // case error: 544 // case warning: 545 // errorChecker.initErrorStatus(sourceFile); 546 // break; 547 // case weLost: 548 // case hasDispute: 549 // //case other: 550 // break; 551 // default: 552 // System.out.println(choice + " not implemented yet"); 553 // } 554 // } 555 // 556 // // now look through the paths 557 // 558 // Relation<R2<SectionId, PageId>, WritingInfo> sorted = Relation.of(new TreeMap<R2<SectionId, PageId>, 559 // Set<WritingInfo>>(), TreeSet.class); 560 // 561 // Counter<Choice> problemCounter = new Counter<Choice>(); 562 // StringBuilder htmlMessage = new StringBuilder(); 563 // StringBuilder statusMessage = new StringBuilder(); 564 // 565 // for (String path : sourceFile) { 566 // progressCallback.nudge(); // Let the user know we're moving along. 567 // 568 // // note that the value might be missing! 569 // 570 // // make sure we only look at the real values 571 // if (altProposed.reset(path).find()) { 572 // continue; 573 // } 574 // 575 // if (path.contains("/exemplarCharacters") || path.contains("/references")) { 576 // continue; 577 // } 578 // 579 // Level level = coverage.getLevel(path); 580 // 581 // // skip anything above the requested level 582 // if (level.compareTo(usersLevel) > 0) { 583 // continue; 584 // } 585 // 586 // String value = sourceFile.getWinningValue(path); 587 // 588 // problems.clear(); 589 // htmlMessage.setLength(0); 590 // boolean haveError = false; 591 // VoteStatus voteStatus = null; 592 // 593 // for (Choice choice : choices) { 594 // switch (choice) { 595 // case changedOldValue: 596 // String oldValue = lastSourceFile == null ? null : lastSourceFile.getWinningValue(path); 597 // if (oldValue != null && !oldValue.equals(value)) { 598 // problems.add(choice); 599 // problemCounter.increment(choice); 600 // } 601 // break; 602 // case missingCoverage: 603 // if (showAll && !localeID.equals("root")) { 604 // if (isMissing(sourceFile, path, status)) { 605 // problems.add(choice); 606 // problemCounter.increment(choice); 607 // } 608 // } 609 // break; 610 // case englishChanged: 611 // if (outdatedPaths.isOutdated(localeID, path) 612 // // || 613 // // !CharSequences.equals(englishFile.getWinningValue(path), 614 // // oldEnglishFile.getWinningValue(path)) 615 // ) { 616 // // the outdated paths compares the base value, before 617 // // data submission, 618 // // so see if the value changed. 619 // String lastValue = lastSourceFile == null ? null : lastSourceFile.getWinningValue(path); 620 // if (CharSequences.equals(value, lastValue)) { 621 // problems.add(choice); 622 // problemCounter.increment(choice); 623 // } 624 // } 625 // break; 626 // case error: 627 // case warning: 628 // if (haveError) { 629 // break; 630 // } 631 // statusMessage.setLength(0); 632 // ErrorChecker.Status errorStatus = errorChecker.getErrorStatus(path, value, statusMessage); 633 // if ((choice == Choice.error && errorStatus == ErrorChecker.Status.error) 634 // || (choice == Choice.warning && errorStatus == ErrorChecker.Status.warning)) { 635 // if (choice == Choice.warning) { 636 // // for now, suppress cases where the English changed 637 // if (outdatedPaths.isOutdated(localeID, path)) { 638 // break; 639 // } 640 // } 641 // problems.add(choice); 642 // appendToMessage(statusMessage, htmlMessage); 643 // problemCounter.increment(choice); 644 // haveError = true; 645 // break; 646 // } 647 // break; 648 // case weLost: 649 // if (voteStatus == null) { 650 // voteStatus = userVoteStatus.getStatusForUsersOrganization(sourceFile, path, user); 651 // } 652 // switch (voteStatus) { 653 // case provisionalOrWorse: 654 // case losing: 655 // if (choice == Choice.weLost) { 656 // problems.add(choice); 657 // problemCounter.increment(choice); 658 // String usersValue = userVoteStatus.getWinningValueForUsersOrganization(sourceFile, path, user); 659 // // appendToMessage(usersValue, testMessage); 660 // } 661 // break; 662 // } 663 // break; 664 // case hasDispute: 665 // if (voteStatus == null) { 666 // voteStatus = userVoteStatus.getStatusForUsersOrganization(sourceFile, path, user); 667 // } 668 // if (voteStatus == VoteStatus.disputed) { 669 // problems.add(choice); 670 // problemCounter.increment(choice); 671 // String usersValue = userVoteStatus.getWinningValueForUsersOrganization(sourceFile, path, user); 672 // if (usersValue != null) { 673 // // appendToMessage(usersValue, testMessage); 674 // } 675 // } 676 // break; 677 // } 678 // } 679 // if (!problems.isEmpty()) { // showAll || 680 // // if (showAll && problems.isEmpty()) { 681 // // problems.add(Choice.other); 682 // // problemCounter.increment(Choice.other); 683 // // } 684 // reasonsToPaths.clear(); 685 // // appendToMessage("level:" + level.toString(), testMessage); 686 // // final String description = 687 // // pathDescription.getDescription(path, value, level, null); 688 // // if (!reasonsToPaths.isEmpty()) { 689 // // appendToMessage(level + " " + 690 // // TransliteratorUtilities.toHTML.transform(reasonsToPaths.toString()), 691 // // testMessage); 692 // // } 693 // // if (description != null && !description.equals("SKIP")) { 694 // // appendToMessage(TransliteratorUtilities.toHTML.transform(description), 695 // // testMessage); 696 // // } 697 // //final String prettyPath = pathTransform.getPrettyPath(path); 698 // // String[] pathParts = breaks.split(prettyPath); 699 // // String section = pathParts.length == 3 ? pathParts[0] : 700 // // "Unknown"; 701 // // String subsection = pathParts.length == 3 ? pathParts[1] : 702 // // "Unknown"; 703 // // String code = pathParts.length == 3 ? pathParts[2] : pretty; 704 // 705 // PathHeader pretty = pathTransform.fromPath(path); 706 // //String[] pathParts = breaks.split(pretty); 707 // // String sectionOutput = pathParts.length == 3 ? pathParts[0] : "Unknown"; 708 // // String subsectionOutput = pathParts.length == 3 ? pathParts[1] : "Unknown"; 709 // // String codeOutput = pathParts.length == 3 ? pathParts[2] : pretty; 710 // 711 // R2<SectionId, PageId> group = Row.of(pretty.getSectionId(), pretty.getPageId()); 712 // 713 // sorted.put(group, new WritingInfo(pretty, problems, htmlMessage)); 714 // } 715 // } 716 // 717 // // now write the results out 718 // writeTables(output, sourceFile, lastSourceFile, sorted, problemCounter, choices, localeID, showAll); 719 // } 720 721 /** 722 * Show a table of values, filtering according to the choices here and in 723 * the constructor. 724 * 725 * @param output 726 * @param choices 727 * See the class description for more information. 728 * @param localeId 729 * @param user 730 * @param usersLevel 731 * @param nonVettingPhase 732 */ generateHtmlErrorTables(Appendable output, EnumSet<Choice> choices, String localeID, T user, Level usersLevel, boolean nonVettingPhase, boolean quick)733 public void generateHtmlErrorTables(Appendable output, EnumSet<Choice> choices, String localeID, T user, 734 Level usersLevel, boolean nonVettingPhase, boolean quick) { 735 736 // Gather the relevant paths 737 // each one will be marked with the choice that it triggered. 738 Relation<R2<SectionId, PageId>, WritingInfo> sorted = Relation.of( 739 new TreeMap<R2<SectionId, PageId>, Set<WritingInfo>>(), TreeSet.class); 740 741 CLDRFile sourceFile = cldrFactory.make(localeID, true); 742 743 // Initialize 744 CLDRFile lastSourceFile = null; 745 if (!quick) { 746 try { 747 lastSourceFile = cldrFactoryOld.make(localeID, true); 748 } catch (Exception e) { 749 } 750 } 751 752 FileInfo fileInfo = new FileInfo().getFileInfo(sourceFile, lastSourceFile, sorted, choices, localeID, nonVettingPhase, user, 753 usersLevel, quick); 754 755 // now write the results out 756 writeTables(output, sourceFile, lastSourceFile, sorted, choices, localeID, nonVettingPhase, fileInfo, quick); 757 } 758 759 /** 760 * Give the list of errors 761 * 762 * @param output 763 * @param choices 764 * See the class description for more information. 765 * @param localeId 766 * @param user 767 * @param usersLevel 768 * @param nonVettingPhase 769 */ generateFileInfoReview(Appendable output, EnumSet<Choice> choices, String localeID, T user, Level usersLevel, boolean nonVettingPhase, boolean quick)770 public Relation<R2<SectionId, PageId>, WritingInfo> generateFileInfoReview(Appendable output, EnumSet<Choice> choices, String localeID, T user, 771 Level usersLevel, boolean nonVettingPhase, boolean quick) { 772 773 // Gather the relevant paths 774 // each one will be marked with the choice that it triggered. 775 Relation<R2<SectionId, PageId>, WritingInfo> sorted = Relation.of( 776 new TreeMap<R2<SectionId, PageId>, Set<WritingInfo>>(), TreeSet.class); 777 778 CLDRFile sourceFile = cldrFactory.make(localeID, true); 779 780 // Initialize 781 CLDRFile lastSourceFile = null; 782 if (!quick) { 783 try { 784 lastSourceFile = cldrFactoryOld.make(localeID, true); 785 } catch (Exception e) { 786 } 787 } 788 789 FileInfo fileInfo = new FileInfo().getFileInfo(sourceFile, lastSourceFile, sorted, choices, localeID, nonVettingPhase, user, 790 usersLevel, quick); 791 792 // now write the results out 793 794 return sorted; 795 } 796 797 class FileInfo { 798 Counter<Choice> problemCounter = new Counter<Choice>(); 799 Counter<Subtype> errorSubtypeCounter = new Counter<Subtype>(); 800 Counter<Subtype> warningSubtypeCounter = new Counter<Subtype>(); 801 EnumSet<Choice> problems = EnumSet.noneOf(Choice.class); 802 addAll(FileInfo other)803 public void addAll(FileInfo other) { 804 problemCounter.addAll(other.problemCounter); 805 errorSubtypeCounter.addAll(other.errorSubtypeCounter); 806 warningSubtypeCounter.addAll(other.warningSubtypeCounter); 807 } 808 getFileInfo(CLDRFile sourceFile, CLDRFile lastSourceFile, Relation<R2<SectionId, PageId>, WritingInfo> sorted, EnumSet<Choice> choices, String localeID, boolean nonVettingPhase, T user, Level usersLevel, boolean quick)809 private FileInfo getFileInfo(CLDRFile sourceFile, CLDRFile lastSourceFile, 810 Relation<R2<SectionId, PageId>, WritingInfo> sorted, 811 EnumSet<Choice> choices, String localeID, boolean nonVettingPhase, 812 T user, Level usersLevel, boolean quick) { 813 return this.getFileInfo(sourceFile, lastSourceFile, sorted, 814 choices, localeID, nonVettingPhase, 815 user, usersLevel, quick, null); 816 } 817 getFileInfo(CLDRFile sourceFile, CLDRFile lastSourceFile, Relation<R2<SectionId, PageId>, WritingInfo> sorted, EnumSet<Choice> choices, String localeID, boolean nonVettingPhase, T user, Level usersLevel, boolean quick, String xpath)818 private FileInfo getFileInfo(CLDRFile sourceFile, CLDRFile lastSourceFile, 819 Relation<R2<SectionId, PageId>, WritingInfo> sorted, 820 EnumSet<Choice> choices, String localeID, boolean nonVettingPhase, 821 T user, Level usersLevel, boolean quick, String xpath) { 822 823 Status status = new Status(); 824 errorChecker.initErrorStatus(sourceFile); 825 Matcher altProposed = ALT_PROPOSED.matcher(""); 826 problems = EnumSet.noneOf(Choice.class); 827 828 // now look through the paths 829 830 StringBuilder htmlMessage = new StringBuilder(); 831 StringBuilder statusMessage = new StringBuilder(); 832 EnumSet<Subtype> subtypes = EnumSet.noneOf(Subtype.class); 833 Set<String> seenSoFar = new HashSet<String>(); 834 boolean latin = VettingViewer.isLatinScriptLocale(sourceFile); 835 for (String path : sourceFile.fullIterable()) { 836 if (xpath != null && !xpath.equals(path)) 837 continue; 838 String value = sourceFile.getWinningValue(path); 839 statusMessage.setLength(0); 840 subtypes.clear(); 841 ErrorChecker.Status errorStatus = errorChecker.getErrorStatus(path, value, statusMessage, subtypes); 842 843 if (quick && errorStatus != ErrorChecker.Status.error && errorStatus != ErrorChecker.Status.warning) { //skip all values but errors and warnings if in "quick" mode 844 continue; 845 } 846 847 if (seenSoFar.contains(path)) { 848 continue; 849 } 850 seenSoFar.add(path); 851 progressCallback.nudge(); // Let the user know we're moving along. 852 853 PathHeader pretty = pathTransform.fromPath(path); 854 if (pretty.getSurveyToolStatus() == PathHeader.SurveyToolStatus.HIDE) { 855 continue; 856 } 857 858 // note that the value might be missing! 859 860 // make sure we only look at the real values 861 if (altProposed.reset(path).find()) { 862 continue; 863 } 864 865 if (path.contains("/references")) { 866 continue; 867 } 868 869 Level level = supplementalDataInfo.getCoverageLevel(path, sourceFile.getLocaleID()); 870 871 // skip anything above the requested level 872 if (level.compareTo(usersLevel) > 0) { 873 continue; 874 } 875 876 problems.clear(); 877 htmlMessage.setLength(0); 878 String oldValue = lastSourceFile == null ? null : lastSourceFile.getWinningValue(path); 879 880 if (choices.contains(Choice.changedOldValue)) { 881 if (oldValue != null && !oldValue.equals(value)) { 882 problems.add(Choice.changedOldValue); 883 problemCounter.increment(Choice.changedOldValue); 884 } 885 } 886 VoteStatus voteStatus = userVoteStatus.getStatusForUsersOrganization(sourceFile, path, user); 887 888 MissingStatus missingStatus = getMissingStatus(sourceFile, path, status, latin); 889 if (choices.contains(Choice.missingCoverage) && missingStatus == MissingStatus.ABSENT) { 890 problems.add(Choice.missingCoverage); 891 problemCounter.increment(Choice.missingCoverage); 892 } 893 boolean itemsOkIfVoted = SUPPRESS 894 && voteStatus == VoteStatus.ok; 895 896 if (!itemsOkIfVoted 897 && outdatedPaths.isOutdated(localeID, path)) { 898 // the outdated paths compares the base value, before 899 // data submission, 900 // so see if the value changed. 901 // String lastValue = lastSourceFile == null ? null : lastSourceFile.getWinningValue(path); 902 if (Objects.equals(value, oldValue) && choices.contains(Choice.englishChanged)) { 903 // check to see if we voted 904 problems.add(Choice.englishChanged); 905 problemCounter.increment(Choice.englishChanged); 906 } 907 } 908 909 Choice choice = errorStatus == ErrorChecker.Status.error ? Choice.error 910 : errorStatus == ErrorChecker.Status.warning ? Choice.warning 911 : null; 912 if (choice == Choice.error && choices.contains(Choice.error) 913 && (!itemsOkIfVoted 914 || !OK_IF_VOTED.containsAll(subtypes))) { 915 problems.add(choice); 916 appendToMessage(statusMessage, htmlMessage); 917 problemCounter.increment(choice); 918 for (Subtype subtype : subtypes) { 919 errorSubtypeCounter.increment(subtype); 920 } 921 } else if (choice == Choice.warning && choices.contains(Choice.warning) 922 && (!itemsOkIfVoted 923 || !OK_IF_VOTED.containsAll(subtypes))) { 924 problems.add(choice); 925 appendToMessage(statusMessage, htmlMessage); 926 problemCounter.increment(choice); 927 for (Subtype subtype : subtypes) { 928 warningSubtypeCounter.increment(subtype); 929 } 930 } 931 932 switch (voteStatus) { 933 case losing: 934 if (choices.contains(Choice.weLost)) { 935 problems.add(Choice.weLost); 936 problemCounter.increment(Choice.weLost); 937 } 938 String usersValue = userVoteStatus.getWinningValueForUsersOrganization(sourceFile, path, user); 939 if (usersValue != null) { 940 usersValue = "Losing value: <" + TransliteratorUtilities.toHTML.transform(usersValue) + ">"; 941 appendToMessage(usersValue, htmlMessage); 942 } 943 break; 944 case disputed: 945 if (choices.contains(Choice.hasDispute)) { 946 problems.add(Choice.hasDispute); 947 problemCounter.increment(Choice.hasDispute); 948 } 949 break; 950 case provisionalOrWorse: 951 if (missingStatus == MissingStatus.PRESENT && choices.contains(Choice.notApproved)) { 952 problems.add(Choice.notApproved); 953 problemCounter.increment(Choice.notApproved); 954 } 955 break; 956 default: 957 } 958 959 if (xpath != null) 960 return this; 961 962 if (!problems.isEmpty()) { 963 // showAll || 964 // if (showAll && problems.isEmpty()) { 965 // problems.add(Choice.other); 966 // problemCounter.increment(Choice.other); 967 // } 968 if (sorted != null) { 969 reasonsToPaths.clear(); 970 // final String prettyPath = pathTransform.getPrettyPath(path); 971 972 // String[] pathParts = breaks.split(pretty); 973 // String sectionOutput = pathParts.length == 3 ? pathParts[0] : "Unknown"; 974 // String subsectionOutput = pathParts.length == 3 ? pathParts[1] : "Unknown"; 975 // String codeOutput = pathParts.length == 3 ? pathParts[2] : pretty; 976 977 R2<SectionId, PageId> group = Row.of(pretty.getSectionId(), pretty.getPageId()); 978 979 sorted.put(group, new WritingInfo(pretty, problems, htmlMessage)); 980 } 981 } 982 983 } 984 return this; 985 } 986 } 987 988 public static final class LocalesWithExplicitLevel implements Predicate<String> { 989 private final Organization org; 990 private final Level desiredLevel; 991 LocalesWithExplicitLevel(Organization org, Level level)992 public LocalesWithExplicitLevel(Organization org, Level level) { 993 this.org = org; 994 this.desiredLevel = level; 995 } 996 997 @Override is(String localeId)998 public boolean is(String localeId) { 999 Output<LocaleCoverageType> output = new Output<LocaleCoverageType>(); 1000 // For admin - return true if SOME organization has explicit coverage for the locale 1001 // TODO: Make admin pick up any locale that has a vote 1002 if (org.equals(Organization.surveytool)) { 1003 for (Organization checkorg : Organization.values()) { 1004 StandardCodes.make().getLocaleCoverageLevel(checkorg, localeId, output); 1005 if (output.value == StandardCodes.LocaleCoverageType.explicit) { 1006 return true; 1007 } 1008 } 1009 return false; 1010 } else { 1011 Level level = StandardCodes.make().getLocaleCoverageLevel(org, localeId, output); 1012 return desiredLevel == level && output.value == StandardCodes.LocaleCoverageType.explicit; 1013 } 1014 } 1015 }; 1016 generateSummaryHtmlErrorTables(Appendable output, EnumSet<Choice> choices, Predicate<String> includeLocale, T organization)1017 public void generateSummaryHtmlErrorTables(Appendable output, EnumSet<Choice> choices, 1018 Predicate<String> includeLocale, T organization) { 1019 try { 1020 1021 output 1022 .append("<p>The following summarizes the Priority Items across locales, " + 1023 "using the default coverage levels for your organization for each locale. " + 1024 "Before using, please read the instructions at " + 1025 "<a target='CLDR_ST_DOCS' href='http://cldr.unicode.org/translation/vetting-summary'>Priority " + 1026 "Items Summary</a>.</p>\n"); 1027 1028 StringBuilder headerRow = new StringBuilder(); 1029 headerRow 1030 .append("<tr class='tvs-tr'>") 1031 .append(TH_AND_STYLES) 1032 .append("Locale</th>") 1033 .append(TH_AND_STYLES) 1034 .append("Codes</th>"); 1035 for (Choice choice : choices) { 1036 headerRow.append("<th class='tv-th'>"); 1037 choice.appendDisplay("", headerRow); 1038 headerRow.append("</th>"); 1039 } 1040 headerRow.append("</tr>\n"); 1041 String header = headerRow.toString(); 1042 1043 if (organization.equals(Organization.surveytool)) { 1044 writeSummaryTable(output, header, Level.COMPREHENSIVE, choices, organization); 1045 } else { 1046 for (Level level : Level.values()) { 1047 writeSummaryTable(output, header, level, choices, organization); 1048 } 1049 } 1050 } catch (IOException e) { 1051 throw new ICUUncheckedIOException(e); // dang'ed checked exceptions 1052 } 1053 1054 } 1055 writeSummaryTable(Appendable output, String header, Level desiredLevel, EnumSet<Choice> choices, T organization)1056 private void writeSummaryTable(Appendable output, String header, Level desiredLevel, 1057 EnumSet<Choice> choices, T organization) throws IOException { 1058 1059 Map<String, String> sortedNames = new TreeMap<String, String>(Collator.getInstance()); 1060 1061 // Gather the relevant paths 1062 // Each one will be marked with the choice that it triggered. 1063 1064 // TODO Fix HACK 1065 // We are going to ignore the predicate for now, just using the locales that have explicit coverage. 1066 // in that locale, or allow all locales for admin@ 1067 LocalesWithExplicitLevel includeLocale = new LocalesWithExplicitLevel((Organization) organization, desiredLevel); 1068 1069 for (String localeID : cldrFactory.getAvailable()) { 1070 if (defaultContentLocales.contains(localeID) 1071 || localeID.equals("en") 1072 || !includeLocale.is(localeID)) { 1073 continue; 1074 } 1075 1076 sortedNames.put(getName(localeID), localeID); 1077 } 1078 if (sortedNames.isEmpty()) { 1079 return; 1080 } 1081 1082 EnumSet<Choice> thingsThatRequireOldFile = EnumSet.of(Choice.englishChanged, Choice.missingCoverage, Choice.changedOldValue); 1083 EnumSet<Choice> ourChoicesThatRequireOldFile = choices.clone(); 1084 ourChoicesThatRequireOldFile.retainAll(thingsThatRequireOldFile); 1085 output.append("<h2>Level: ").append(desiredLevel.toString()).append("</h2>"); 1086 output.append("<table class='tvs-table'>\n"); 1087 char lastChar = ' '; 1088 Map<String, FileInfo> localeNameToFileInfo = new TreeMap(); 1089 FileInfo totals = new FileInfo(); 1090 1091 for (Entry<String, String> entry : sortedNames.entrySet()) { 1092 String name = entry.getKey(); 1093 String localeID = entry.getValue(); 1094 // Initialize 1095 1096 CLDRFile sourceFile = cldrFactory.make(localeID, true); 1097 1098 CLDRFile lastSourceFile = null; 1099 if (!ourChoicesThatRequireOldFile.isEmpty()) { 1100 try { 1101 lastSourceFile = cldrFactoryOld.make(localeID, true); 1102 } catch (Exception e) { 1103 } 1104 } 1105 Level level = Level.MODERN; 1106 if (organization != null) { 1107 level = StandardCodes.make().getLocaleCoverageLevel(organization.toString(), localeID); 1108 } 1109 FileInfo fileInfo = new FileInfo().getFileInfo(sourceFile, lastSourceFile, null, choices, localeID, true, organization, level, false); 1110 localeNameToFileInfo.put(name, fileInfo); 1111 totals.addAll(fileInfo); 1112 1113 char nextChar = name.charAt(0); 1114 if (lastChar != nextChar) { 1115 output.append(header); 1116 lastChar = nextChar; 1117 } 1118 1119 writeSummaryRow(output, choices, fileInfo.problemCounter, name, localeID); 1120 1121 if (output instanceof Writer) { 1122 ((Writer) output).flush(); 1123 } 1124 } 1125 output.append(header); 1126 writeSummaryRow(output, choices, totals.problemCounter, "Total", null); 1127 output.append("</table>"); 1128 if (SHOW_SUBTYPES) { 1129 showSubtypes(output, sortedNames, localeNameToFileInfo, totals, true); 1130 showSubtypes(output, sortedNames, localeNameToFileInfo, totals, false); 1131 } 1132 } 1133 showSubtypes(Appendable output, Map<String, String> sortedNames, Map<String, FileInfo> localeNameToFileInfo, FileInfo totals, boolean errors)1134 private void showSubtypes(Appendable output, Map<String, String> sortedNames, 1135 Map<String, FileInfo> localeNameToFileInfo, 1136 FileInfo totals, 1137 boolean errors) throws IOException { 1138 output.append("<h3>Details: ").append(errors ? "Error Types" : "Warning Types").append("</h3>"); 1139 output.append("<table class='tvs-table'>"); 1140 Counter<Subtype> subtypeCounterTotals = errors ? totals.errorSubtypeCounter : totals.warningSubtypeCounter; 1141 Set<Subtype> sortedBySize = subtypeCounterTotals.getKeysetSortedByCount(false); 1142 1143 // header 1144 writeDetailHeader(subtypeCounterTotals, sortedBySize, output); 1145 1146 // items 1147 for (Entry<String, FileInfo> entry : localeNameToFileInfo.entrySet()) { 1148 Counter<Subtype> counter = errors ? entry.getValue().errorSubtypeCounter : entry.getValue().warningSubtypeCounter; 1149 if (counter.getTotal() == 0) { 1150 continue; 1151 } 1152 String name = entry.getKey(); 1153 //String[] names = name.split(SPLIT_CHAR); 1154 String localeID = sortedNames.get(name); 1155 output.append("<tr>").append(TH_AND_STYLES); 1156 appendNameAndCode(name, localeID, output); 1157 output.append("</th>"); 1158 for (Subtype subtype : sortedBySize) { 1159 long count = counter.get(subtype); 1160 output.append("<td class='tvs-count'>"); 1161 if (count != 0) { 1162 output.append(nf.format(count)); 1163 } 1164 output.append("</td>"); 1165 } 1166 } 1167 1168 // subtotals 1169 writeDetailHeader(subtypeCounterTotals, sortedBySize, output); 1170 output.append("<tr>").append(TH_AND_STYLES).append("<i>Total</i>").append("</th>").append(TH_AND_STYLES).append("</th>"); 1171 for (Subtype subtype : sortedBySize) { 1172 long count = subtypeCounterTotals.get(subtype); 1173 output.append("<td class='tvs-count'>"); 1174 if (count != 0) { 1175 output.append("<b>").append(nf.format(count)).append("</b>"); 1176 } 1177 output.append("</td>"); 1178 } 1179 output.append("</table>"); 1180 } 1181 writeDetailHeader(Counter<Subtype> subtypeCounterTotals, Set<Subtype> sortedBySize, Appendable output)1182 private void writeDetailHeader(Counter<Subtype> subtypeCounterTotals, Set<Subtype> sortedBySize, Appendable output) throws IOException { 1183 output.append("<tr>") 1184 .append(TH_AND_STYLES).append("Name").append("</th>") 1185 .append(TH_AND_STYLES).append("ID").append("</th>"); 1186 for (Subtype subtype : sortedBySize) { 1187 output.append(TH_AND_STYLES).append(subtype.toString()).append("</th>"); 1188 } 1189 } 1190 writeSummaryRow(Appendable output, EnumSet<Choice> choices, Counter<Choice> problemCounter, String name, String localeID)1191 private void writeSummaryRow(Appendable output, EnumSet<Choice> choices, Counter<Choice> problemCounter, 1192 String name, String localeID) throws IOException { 1193 output 1194 .append("<tr>") 1195 .append(TH_AND_STYLES); 1196 if (localeID == null) { 1197 output 1198 .append("<i>") 1199 .append(name) 1200 .append("</i>") 1201 .append("</th>") 1202 .append(TH_AND_STYLES); 1203 } else { 1204 appendNameAndCode(name, localeID, output); 1205 } 1206 output.append("</th>\n"); 1207 for (Choice choice : choices) { 1208 long count = problemCounter.get(choice); 1209 output.append("<td class='tvs-count'>"); 1210 if (localeID == null) { 1211 output.append("<b>"); 1212 } 1213 output.append(nf.format(count)); 1214 if (localeID == null) { 1215 output.append("</b>"); 1216 } 1217 output.append("</td>\n"); 1218 } 1219 output.append("</tr>\n"); 1220 } 1221 appendNameAndCode(String name, String localeID, Appendable output)1222 private void appendNameAndCode(String name, String localeID, Appendable output) throws IOException { 1223 String[] names = name.split(SPLIT_CHAR); 1224 output 1225 .append("<a href='" + urls.forSpecial(CLDRURLS.Special.Vetting, CLDRLocale.getInstance(localeID))) 1226 .append("'>") 1227 .append(TransliteratorUtilities.toHTML.transform(names[0])) 1228 .append("</a>") 1229 .append("</th>") 1230 .append(TH_AND_STYLES) 1231 .append("<code>") 1232 .append(names[1]) 1233 .append("</code>"); 1234 } 1235 1236 LanguageTagParser ltp = new LanguageTagParser(); 1237 getName(String localeID)1238 private String getName(String localeID) { 1239 Set<String> contents = supplementalDataInfo.getEquivalentsForLocale(localeID); 1240 // put in special character that can be split on later 1241 String name = englishFile.getName(localeID, true, CLDRFile.SHORT_ALTS) + SPLIT_CHAR + gatherCodes(contents); 1242 return name; 1243 } 1244 1245 /** 1246 * Collapse the names 1247 {en_Cyrl, en_Cyrl_US} => en_Cyrl(_US) 1248 {en_GB, en_Latn_GB} => en(_Latn)_GB 1249 {en, en_US, en_Latn, en_Latn_US} => en(_Latn)(_US) 1250 {az_IR, az_Arab, az_Arab_IR} => az_IR, az_Arab(_IR) 1251 */ gatherCodes(Set<String> contents)1252 public static String gatherCodes(Set<String> contents) { 1253 Set<Set<String>> source = new LinkedHashSet<Set<String>>(); 1254 for (String s : contents) { 1255 source.add(new LinkedHashSet<String>(Arrays.asList(s.split("_")))); 1256 } 1257 Set<Set<String>> oldSource = new LinkedHashSet<Set<String>>(); 1258 1259 do { 1260 // exchange source/target 1261 oldSource.clear(); 1262 oldSource.addAll(source); 1263 source.clear(); 1264 Set<String> last = null; 1265 for (Set<String> ss : oldSource) { 1266 if (last == null) { 1267 last = ss; 1268 } else { 1269 if (ss.containsAll(last)) { 1270 last = combine(last, ss); 1271 } else { 1272 source.add(last); 1273 last = ss; 1274 } 1275 } 1276 } 1277 source.add(last); 1278 } while (oldSource.size() != source.size()); 1279 1280 StringBuilder b = new StringBuilder(); 1281 for (Set<String> stringSet : source) { 1282 if (b.length() != 0) { 1283 b.append(", "); 1284 } 1285 String sep = ""; 1286 for (String string : stringSet) { 1287 if (string.startsWith(CONNECT_PREFIX)) { 1288 b.append(string + CONNECT_SUFFIX); 1289 } else { 1290 b.append(sep + string); 1291 } 1292 sep = "_"; 1293 } 1294 } 1295 return b.toString(); 1296 } 1297 combine(Set<String> last, Set<String> ss)1298 private static Set<String> combine(Set<String> last, Set<String> ss) { 1299 LinkedHashSet<String> result = new LinkedHashSet<String>(); 1300 for (String s : ss) { 1301 if (last.contains(s)) { 1302 result.add(s); 1303 } else { 1304 result.add(CONNECT_PREFIX + s); 1305 } 1306 } 1307 return result; 1308 } 1309 1310 public enum MissingStatus { 1311 PRESENT, ALIASED, MISSING_OK, ROOT_OK, ABSENT 1312 } 1313 getMissingStatus(CLDRFile sourceFile, String path, Status status, boolean latin)1314 public static MissingStatus getMissingStatus(CLDRFile sourceFile, String path, Status status, boolean latin) { 1315 if (sourceFile == null) { 1316 return MissingStatus.ABSENT; 1317 } 1318 if ("root".equals(sourceFile.getLocaleID()) || path.startsWith("//ldml/layout/orientation/")) { 1319 return MissingStatus.MISSING_OK; 1320 } 1321 if (path.equals(TEST_PATH)) { 1322 int debug = 1; 1323 } 1324 MissingStatus result; 1325 1326 String value = sourceFile.getStringValue(path); 1327 boolean isAliased = path.equals(status.pathWhereFound); 1328 1329 if (value == null) { 1330 result = ValuePathStatus.isMissingOk(sourceFile, path, latin, isAliased) ? MissingStatus.MISSING_OK : MissingStatus.ABSENT; 1331 } else { 1332 String localeFound = sourceFile.getSourceLocaleID(path, status); 1333 1334 // only count it as missing IF the (localeFound is root or codeFallback) 1335 // AND the aliasing didn't change the path 1336 if (localeFound.equals("root") 1337 || localeFound.equals(XMLSource.CODE_FALLBACK_ID) 1338 // || voteStatus == VoteStatus.provisionalOrWorse 1339 ) { 1340 result = ValuePathStatus.isMissingOk(sourceFile, path, latin, isAliased) 1341 || sourceFile.getLocaleID().equals("en") ? MissingStatus.ROOT_OK : MissingStatus.ABSENT; 1342 } else if (isAliased) { 1343 result = MissingStatus.PRESENT; 1344 // } else if (path.contains("decimalFormatLength[@type=\"long\"]") && 1345 // path.contains("pattern[@type=\"1")) { // aliased 1346 // // special case compact numbers 1347 // // 1348 // ldml/numbers/decimalFormats[@numberSystem="latn"]/decimalFormatLength[@type="long"]/decimalFormat[@type="standard"]/pattern[@type="10000000"] 1349 // result = MissingStatus.ABSENT; 1350 } else { 1351 result = MissingStatus.ALIASED; 1352 } 1353 } 1354 return result; 1355 } 1356 1357 public static final UnicodeSet LATIN = ValuePathStatus.LATIN; 1358 isLatinScriptLocale(CLDRFile sourceFile)1359 public static boolean isLatinScriptLocale(CLDRFile sourceFile) { 1360 return ValuePathStatus.isLatinScriptLocale(sourceFile); 1361 } 1362 appendToMessage(CharSequence usersValue, EnumSet<Subtype> subtypes, StringBuilder testMessage)1363 private static StringBuilder appendToMessage(CharSequence usersValue, EnumSet<Subtype> subtypes, StringBuilder testMessage) { 1364 if (subtypes != null) { 1365 usersValue = "<" + CollectionUtilities.join(subtypes, ", ") + "> " + usersValue; 1366 } 1367 return appendToMessage(usersValue, testMessage); 1368 } 1369 appendToMessage(CharSequence usersValue, Subtype subtype, StringBuilder testMessage)1370 private static StringBuilder appendToMessage(CharSequence usersValue, Subtype subtype, StringBuilder testMessage) { 1371 if (subtype != null) { 1372 usersValue = "<" + subtype + "> " + usersValue; 1373 } 1374 return appendToMessage(usersValue, testMessage); 1375 } 1376 appendToMessage(CharSequence usersValue, StringBuilder testMessage)1377 private static StringBuilder appendToMessage(CharSequence usersValue, StringBuilder testMessage) { 1378 if (usersValue.length() == 0) { 1379 return testMessage; 1380 } 1381 if (testMessage.length() != 0) { 1382 testMessage.append("<br>"); 1383 } 1384 return testMessage.append(usersValue); 1385 } 1386 1387 static final NumberFormat nf = NumberFormat.getIntegerInstance(ULocale.ENGLISH); 1388 private Relation<String, String> reasonsToPaths; 1389 private CLDRURLS urls = CLDRConfig.getInstance().urls(); 1390 1391 static { 1392 nf.setGroupingUsed(true); 1393 } 1394 1395 /** 1396 * Class that allows the relaying of progress information 1397 * 1398 * @author srl 1399 * 1400 */ 1401 public static class ProgressCallback { 1402 /** 1403 * Note any progress. This will be called before any output is printed. 1404 * It will be called approximately once per xpath. 1405 */ nudge()1406 public void nudge() { 1407 } 1408 1409 /** 1410 * Called when all operations are complete. 1411 */ done()1412 public void done() { 1413 } 1414 } 1415 1416 private ProgressCallback progressCallback = new ProgressCallback(); // null 1417 1418 // instance 1419 // by 1420 // default 1421 1422 /** 1423 * Select a new callback. Must be set before running. 1424 * 1425 * @return 1426 * 1427 */ setProgressCallback(ProgressCallback newCallback)1428 public VettingViewer<T> setProgressCallback(ProgressCallback newCallback) { 1429 progressCallback = newCallback; 1430 return this; 1431 } 1432 getErrorChecker()1433 public ErrorChecker getErrorChecker() { 1434 return errorChecker; 1435 } 1436 1437 /** 1438 * Select a new error checker. Must be set before running. 1439 * 1440 * @return 1441 * 1442 */ setErrorChecker(ErrorChecker errorChecker)1443 public VettingViewer<T> setErrorChecker(ErrorChecker errorChecker) { 1444 this.errorChecker = errorChecker; 1445 return this; 1446 } 1447 1448 /** 1449 * Provide the styles for inclusion into the ST <head> element. 1450 * 1451 * @return 1452 */ getHeaderStyles()1453 public static String getHeaderStyles() { 1454 return "<style type='text/css'>\n" 1455 + ".hide {display:none}\n" 1456 + ".vve {}\n" 1457 + ".vvn {}\n" 1458 + ".vvp {}\n" 1459 + ".vvl {}\n" 1460 + ".vvm {}\n" 1461 + ".vvu {}\n" 1462 + ".vvw {}\n" 1463 + ".vvd {}\n" 1464 + ".vvo {}\n" 1465 + "</style>"; 1466 } 1467 writeTables(Appendable output, CLDRFile sourceFile, CLDRFile lastSourceFile, Relation<R2<SectionId, PageId>, WritingInfo> sorted, EnumSet<Choice> choices, String localeID, boolean nonVettingPhase, FileInfo outputFileInfo, boolean quick)1468 private void writeTables(Appendable output, CLDRFile sourceFile, CLDRFile lastSourceFile, 1469 Relation<R2<SectionId, PageId>, WritingInfo> sorted, 1470 EnumSet<Choice> choices, 1471 String localeID, 1472 boolean nonVettingPhase, 1473 FileInfo outputFileInfo, 1474 boolean quick) { 1475 try { 1476 1477 boolean latin = VettingViewer.isLatinScriptLocale(sourceFile); 1478 1479 Status status = new Status(); 1480 1481 output.append("<h2>Summary</h2>\n") 1482 .append("<p><i>It is important that you read " + 1483 "<a target='CLDR-ST-DOCS' href='http://cldr.unicode.org/translation/vetting-view'>" + 1484 "Priority Items</a> before starting!</i></p>") 1485 .append("<form name='checkboxes' action='#'>\n") 1486 .append("<table class='tvs-table'>\n") 1487 .append("<tr class='tvs-tr'>" + 1488 "<th class='tv-th'>Count</th>" + 1489 "<th class='tv-th'>Issue</th>" + 1490 "<th class='tv-th'>Description</th>" + 1491 "</tr>\n"); 1492 1493 // find the choice to check 1494 // OLD if !vetting and missing != 0, use missing. Otherwise pick first. 1495 Choice checkedItem = null; 1496 // if (nonVettingPhase && problemCounter.get(Choice.missingCoverage) != 0) { 1497 // checkedItem = Choice.missingCoverage; 1498 // } 1499 1500 for (Choice choice : choices) { 1501 if (quick && choice != Choice.error && choice != Choice.warning) { //if "quick" mode, only show errors and warnings 1502 continue; 1503 } 1504 long count = outputFileInfo.problemCounter.get(choice); 1505 output.append("<tr><td class='tvs-count'>") 1506 .append(nf.format(count)) 1507 .append("</td>\n\t<td nowrap class='tvs-abb'>") 1508 .append("<input type='checkbox' name='") 1509 .append(Character.toLowerCase(choice.abbreviation)) 1510 .append("' onclick='setStyles()'"); 1511 if (checkedItem == choice || checkedItem == null && count != 0) { 1512 output.append(" checked"); 1513 checkedItem = choice; 1514 } 1515 output.append(">"); 1516 choice.appendDisplay("", output); 1517 output.append("</td>\n\t<td class='tvs-desc'>") 1518 .append(choice.description) 1519 .append("</td></tr>\n"); 1520 } 1521 output.append("</table>\n</form>\n" 1522 + "<script type='text/javascript'>\n" + 1523 "<!-- \n" + 1524 "setStyles()\n" + 1525 "-->\n" 1526 + "</script>"); 1527 1528 // gather information on choices on each page 1529 1530 Relation<Row.R3<SectionId, PageId, String>, Choice> choicesForHeader = Relation.of( 1531 new HashMap<Row.R3<SectionId, PageId, String>, Set<Choice>>(), HashSet.class); 1532 1533 Relation<Row.R2<SectionId, PageId>, Choice> choicesForSection = Relation.of( 1534 new HashMap<R2<SectionId, PageId>, Set<Choice>>(), HashSet.class); 1535 1536 for (Entry<R2<SectionId, PageId>, Set<WritingInfo>> entry0 : sorted.keyValuesSet()) { 1537 SectionId section = entry0.getKey().get0(); 1538 PageId subsection = entry0.getKey().get1(); 1539 final Set<WritingInfo> rows = entry0.getValue(); 1540 for (WritingInfo pathInfo : rows) { 1541 String header = pathInfo.codeOutput.getHeader(); 1542 Set<Choice> choicesForPath = pathInfo.problems; 1543 choicesForSection.putAll(Row.of(section, subsection), choicesForPath); 1544 choicesForHeader.putAll(Row.of(section, subsection, header), choicesForPath); 1545 } 1546 } 1547 1548 final String localeId = sourceFile.getLocaleID(); 1549 final CLDRLocale locale = CLDRLocale.getInstance(localeId); 1550 int count = 0; 1551 for (Entry<R2<SectionId, PageId>, Set<WritingInfo>> entry0 : sorted.keyValuesSet()) { 1552 SectionId section = entry0.getKey().get0(); 1553 PageId subsection = entry0.getKey().get1(); 1554 final Set<WritingInfo> rows = entry0.getValue(); 1555 1556 rows.iterator().next(); // getUrl(localeId); (no side effect?) 1557 // http://kwanyin.unicode.org/cldr-apps/survey?_=ur&x=scripts 1558 // http://unicode.org/cldr-apps/survey?_=ur&x=scripts 1559 1560 output.append("\n<h2 class='tv-s'>Section: ") 1561 .append(section.toString()) 1562 .append(" — <i><a " + /*target='CLDR_ST-SECTION' */"href='") 1563 .append(urls.forPage(locale, subsection)) 1564 .append("'>Page: ") 1565 .append(subsection.toString()) 1566 .append("</a></i> (" + rows.size() + ")</h2>\n"); 1567 startTable(choicesForSection.get(Row.of(section, subsection)), output); 1568 1569 String oldHeader = ""; 1570 for (WritingInfo pathInfo : rows) { 1571 String header = pathInfo.codeOutput.getHeader(); 1572 String code = pathInfo.codeOutput.getCode(); 1573 String path = pathInfo.codeOutput.getOriginalPath(); 1574 Set<Choice> choicesForPath = pathInfo.problems; 1575 1576 if (!header.equals(oldHeader)) { 1577 Set<Choice> headerChoices = choicesForHeader.get(Row.of(section, subsection, header)); 1578 output.append("<tr class='"); 1579 Choice.appendRowStyles(headerChoices, output); 1580 output.append("'>\n"); 1581 output.append(" <th class='partsection' colSpan='6'>"); 1582 output.append(header); 1583 output.append("</th>\n</tr>\n"); 1584 oldHeader = header; 1585 } 1586 1587 output.append("<tr class='"); 1588 Choice.appendRowStyles(choicesForPath, output); 1589 output.append("'>\n"); 1590 addCell(output, nf.format(++count), null, "tv-num", HTMLType.plain); 1591 // path 1592 addCell(output, code, null, "tv-code", HTMLType.plain); 1593 // English value 1594 if (choicesForPath.contains(Choice.englishChanged)) { 1595 String winning = englishFile.getWinningValue(path); 1596 String cellValue = winning == null ? "<i>missing</i>" : TransliteratorUtilities.toHTML 1597 .transform(winning); 1598 String previous = outdatedPaths.getPreviousEnglish(path); 1599 if (previous != null) { 1600 cellValue += "<br><span style='color:#900'><b>OLD: </b>" 1601 + TransliteratorUtilities.toHTML.transform(previous) + "</span>"; 1602 } else { 1603 cellValue += "<br><b><i>missing</i></b>"; 1604 } 1605 addCell(output, cellValue, null, "tv-eng", HTMLType.markup); 1606 } else { 1607 addCell(output, englishFile.getWinningValue(path), null, "tv-eng", HTMLType.plain); 1608 } 1609 // value for last version 1610 final String oldStringValue = lastSourceFile == null ? null : lastSourceFile.getWinningValue(path); 1611 MissingStatus oldValueMissing = getMissingStatus(lastSourceFile, path, status, latin); 1612 1613 addCell(output, oldStringValue, null, oldValueMissing != MissingStatus.PRESENT ? "tv-miss" 1614 : "tv-last", HTMLType.plain); 1615 // value for last version 1616 String newWinningValue = sourceFile.getWinningValue(path); 1617 if (Objects.equals(newWinningValue, oldStringValue)) { 1618 newWinningValue = "="; 1619 } 1620 addCell(output, newWinningValue, null, choicesForPath.contains(Choice.missingCoverage) ? "tv-miss" 1621 : "tv-win", HTMLType.plain); 1622 // Fix? 1623 // http://unicode.org/cldr/apps/survey?_=az&xpath=%2F%2Fldml%2FlocaleDisplayNames%2Flanguages%2Flanguage%5B%40type%3D%22az%22%5D 1624 output.append(" <td class='tv-fix'><a target='_blank' href='") 1625 .append(pathInfo.getUrl(locale)) // .append(c)baseUrl + "?_=") 1626 // .append(localeID) 1627 // .append("&xpath=") 1628 // .append(percentEscape.transform(path)) 1629 .append("'>"); 1630 Choice.appendDisplay(choicesForPath, "", output); 1631 // String otherUrl = pathInfo.getUrl(sourceFile.getLocaleID()); 1632 output.append("</a></td>"); 1633 // if (!otherUrl.equals(url)) { 1634 // output.append("<td class='tv-test'><a "+/*target='CLDR_ST-SECTION' */"href='") 1635 // .append(otherUrl) 1636 // .append("'><i>Section*</i></a></td>"); 1637 // } 1638 if (!pathInfo.htmlMessage.isEmpty()) { 1639 addCell(output, pathInfo.htmlMessage, null, "tv-test", HTMLType.markup); 1640 } 1641 output.append("</tr>\n"); 1642 } 1643 output.append("</table>\n"); 1644 } 1645 } catch (IOException e) { 1646 throw new ICUUncheckedIOException(e); // damn'ed checked exceptions 1647 } 1648 } 1649 1650 /** 1651 * 1652 * @param output 1653 * @param choices 1654 * See the class description for more information. 1655 * @param localeId 1656 * @param user 1657 * @param usersLevel 1658 * @param nonVettingPhase 1659 */ getErrorOnPath(EnumSet<Choice> choices, String localeID, T user, Level usersLevel, boolean nonVettingPhase, String path)1660 public ArrayList<String> getErrorOnPath(EnumSet<Choice> choices, String localeID, T user, 1661 Level usersLevel, boolean nonVettingPhase, String path) { 1662 1663 // Gather the relevant paths 1664 // each one will be marked with the choice that it triggered. 1665 Relation<R2<SectionId, PageId>, WritingInfo> sorted = Relation.of( 1666 new TreeMap<R2<SectionId, PageId>, Set<WritingInfo>>(), TreeSet.class); 1667 1668 CLDRFile sourceFile = cldrFactory.make(localeID, true); 1669 1670 // Initialize 1671 CLDRFile lastSourceFile = null; 1672 try { 1673 lastSourceFile = cldrFactoryOld.make(localeID, true); 1674 } catch (Exception e) { 1675 } 1676 1677 EnumSet<Choice> errors = new FileInfo().getFileInfo(sourceFile, lastSourceFile, sorted, choices, localeID, nonVettingPhase, user, usersLevel, 1678 false, path).problems; 1679 1680 ArrayList<String> out = new ArrayList<String>(); 1681 for (Object error : errors.toArray()) { 1682 out.add(((Choice) error).buttonLabel); 1683 } 1684 1685 return out; 1686 } 1687 1688 /*private void getJSONReview(Appendable output, CLDRFile sourceFile, CLDRFile lastSourceFile, 1689 Relation<R2<SectionId, PageId>, WritingInfo> sorted, 1690 EnumSet<Choice> choices, 1691 String localeID, 1692 boolean nonVettingPhase, 1693 FileInfo outputFileInfo, 1694 boolean quick 1695 ) { 1696 1697 try { 1698 boolean latin = VettingViewer.isLatinScriptLocale(sourceFile); 1699 JSONObject reviewInfo = new JSONObject(); 1700 JSONArray notificationsCount = new JSONArray(); 1701 List<String> notifications = new ArrayList<String>(); 1702 Status status = new Status(); 1703 1704 1705 1706 for (Choice choice : choices) { 1707 notificationsCount.put(new JSONObject().put("name",choice.buttonLabel.replace(' ', '_')).put("description", choice.description).put("count", outputFileInfo.problemCounter.get(choice))); 1708 notifications.add(choice.buttonLabel); 1709 } 1710 1711 reviewInfo.put("notification", notificationsCount); 1712 // gather information on choices on each page 1713 //output.append(reviewInfo.toString()); 1714 1715 1716 Relation<Row.R3<SectionId, PageId, String>, Choice> choicesForHeader = Relation.of( 1717 new HashMap<Row.R3<SectionId, PageId, String>, Set<Choice>>(), HashSet.class); 1718 1719 Relation<Row.R2<SectionId, PageId>, Choice> choicesForSection = Relation.of( 1720 new HashMap<R2<SectionId, PageId>, Set<Choice>>(), HashSet.class); 1721 1722 Comparator<? super R4<Choice, SectionId, PageId, String>> comparator = new Comparator<Row.R4<Choice,SectionId, PageId, String>>() { 1723 1724 1725 @Override 1726 public int compare(R4<Choice, SectionId, PageId, String> o1, R4<Choice, SectionId, PageId, String> o2) { 1727 int compChoice = o2.get0().order - o1.get0().order; 1728 if(compChoice == 0) { 1729 int compSection = o1.get1().compareTo(o2.get1()); 1730 if(compSection == 0) { 1731 int compPage = o1.get2().compareTo(o2.get2()); 1732 if(compPage == 0) 1733 return o1.get3().compareTo(o2.get3()); 1734 else 1735 return 0; 1736 } 1737 else 1738 return compSection; 1739 } 1740 else 1741 return compChoice; 1742 } 1743 }; 1744 1745 Relation<Row.R4<Choice,SectionId, PageId, String>, WritingInfo> notificationsList = Relation.of( 1746 new TreeMap<Row.R4<Choice,SectionId, PageId, String>, Set<WritingInfo>>(comparator), TreeSet.class); 1747 1748 1749 //TODO we can prob do it in only one loop, but with that we can sort 1750 for (Entry<R2<SectionId, PageId>, Set<WritingInfo>> entry0 : sorted.keyValuesSet()) { 1751 final Set<WritingInfo> rows = entry0.getValue(); 1752 for (WritingInfo pathInfo : rows) { 1753 Set<Choice> choicesForPath = pathInfo.problems; 1754 SectionId section = entry0.getKey().get0(); 1755 PageId subsection = entry0.getKey().get1(); 1756 for(Choice choice : choicesForPath) { 1757 //reviewInfo 1758 notificationsList.put(Row.of(choice, section, subsection, pathInfo.codeOutput.getHeader()), pathInfo); 1759 } 1760 } 1761 1762 } 1763 1764 JSONArray allNotifications = new JSONArray(); 1765 for(Entry<R4<Choice, SectionId, PageId, String>, Set<WritingInfo>> entry : notificationsList.keyValuesSet()) { 1766 1767 String notificationName = entry.getKey().get0().buttonLabel.replace(' ', '_'); 1768 int notificationNumber = entry.getKey().get0().order; 1769 1770 String sectionName = entry.getKey().get1().name(); 1771 String pageName = entry.getKey().get2().name(); 1772 String headerName = entry.getKey().get3(); 1773 1774 if(allNotifications.optJSONObject(notificationNumber) == null) { 1775 allNotifications.put(notificationNumber,new JSONObject().put(notificationName, new JSONObject())); 1776 } 1777 1778 JSONObject sections = allNotifications.getJSONObject(notificationNumber).getJSONObject(notificationName); 1779 1780 if(sections.optJSONObject(sectionName) == null) { 1781 sections.accumulate(sectionName, new JSONObject()); 1782 } 1783 JSONObject pages = sections.getJSONObject(sectionName); 1784 1785 if(pages.optJSONObject(pageName) == null) { 1786 pages.accumulate(pageName, new JSONObject()); 1787 } 1788 JSONObject header = pages.getJSONObject(pageName); 1789 1790 JSONArray allContent = new JSONArray(); 1791 //real info 1792 for(WritingInfo info : entry.getValue()) { 1793 JSONObject content = new JSONObject(); 1794 String code = info.codeOutput.getCode(); 1795 String path = info.codeOutput.getOriginalPath(); 1796 Set<Choice> choicesForPath = info.problems; 1797 1798 //code 1799 content.put("code",code); 1800 content.put("path", ctx.sm.xpt.getByXpath(path)); 1801 1802 //english 1803 if (choicesForPath.contains(Choice.englishChanged)) { 1804 String winning = englishFile.getWinningValue(path); 1805 String cellValue = winning == null ? "<i>missing</i>" : TransliteratorUtilities.toHTML 1806 .transform(winning); 1807 String previous = outdatedPaths.getPreviousEnglish(path); 1808 if (previous != null) { 1809 cellValue += "<br><span style='color:#900'><b>OLD: </b>" 1810 + TransliteratorUtilities.toHTML.transform(previous) + "</span>"; 1811 } else { 1812 cellValue += "<br><b><i>missing</i></b>"; 1813 } 1814 content.put("english", cellValue); 1815 } else { 1816 content.put("english",englishFile.getWinningValue(path)); 1817 } 1818 1819 //old release 1820 final String oldStringValue = lastSourceFile == null ? null : lastSourceFile.getWinningValue(path); 1821 content.put("old", oldStringValue); 1822 1823 // 1824 1825 //winning value 1826 String newWinningValue = sourceFile.getWinningValue(path); 1827 if (CharSequences.equals(newWinningValue, oldStringValue)) { 1828 newWinningValue = "="; 1829 } 1830 content.put("winning",newWinningValue); 1831 1832 //comment 1833 String comment = ""; 1834 if (!info.htmlMessage.isEmpty()) { 1835 comment = info.htmlMessage; 1836 } 1837 content.put("comment", comment.replace("\"", """)); 1838 1839 content.put("id", StringId.getHexId(info.codeOutput.getOriginalPath())); 1840 allContent.put(content); 1841 } 1842 header.put(headerName, allContent); 1843 1844 } 1845 reviewInfo.put("allNotifications", allNotifications); 1846 1847 //hidden info 1848 ReviewHide review = new ReviewHide(); 1849 reviewInfo.put("hidden", review.getHiddenField(ctx.userId(), ctx.getLocale().toString())); 1850 reviewInfo.put("direction", ctx.getDirectionForLocale()); 1851 output.append(reviewInfo.toString()); 1852 } 1853 catch (JSONException | IOException e) { 1854 e.printStackTrace(); 1855 } 1856 } 1857 */ startTable(Set<Choice> choices, Appendable output)1858 private void startTable(Set<Choice> choices, Appendable output) throws IOException { 1859 output.append("<table class='tv-table'>\n"); 1860 output.append("<tr class='"); 1861 Choice.appendRowStyles(choices, output); 1862 output.append("'>" + 1863 "<th class='tv-th'>No.</th>" + 1864 "<th class='tv-th'>Code</th>" + 1865 "<th class='tv-th'>English</th>" + 1866 "<th class='tv-th'>" + lastVersionTitle + "</th>" + 1867 "<th class='tv-th'>" + currentWinningTitle + "</th>" + 1868 "<th class='tv-th'>Fix?</th>" + 1869 "<th class='tv-th'>Comment</th>" + 1870 "</tr>\n"); 1871 } 1872 1873 enum HTMLType { 1874 plain, markup 1875 } 1876 addCell(Appendable output, String value, String title, String classValue, HTMLType htmlType)1877 private void addCell(Appendable output, String value, String title, String classValue, HTMLType htmlType) 1878 throws IOException { 1879 output.append(" <td class='") 1880 .append(classValue); 1881 if (value == null) { 1882 output.append(" tv-null'><i>missing</i></td>"); 1883 } else { 1884 if (title != null && !title.equals(value)) { 1885 output.append("title='").append(TransliteratorUtilities.toHTML.transform(title)).append('\''); 1886 } 1887 output 1888 .append("'>") 1889 .append(htmlType == HTMLType.markup ? value : TransliteratorUtilities.toHTML.transform(value)) 1890 .append("</td>\n"); 1891 } 1892 } 1893 1894 /** 1895 * Find the status of the items in the file. 1896 * @param file the source. Must be a resolved file, made with minimalDraftStatus = unconfirmed 1897 * @param pathHeaderFactory PathHeaderFactory. 1898 * @param foundCounter output counter of the number of paths with values having contributed or approved status 1899 * @param unconfirmedCounter output counter of the number of paths with values, but neither contributed nor approved status 1900 * @param missingCounter output counter of the number of paths without values 1901 * @param missingPaths output if not null, the specific paths that are missing. 1902 * @param unconfirmedPaths TODO 1903 */ getStatus(CLDRFile file, PathHeader.Factory pathHeaderFactory, Counter<Level> foundCounter, Counter<Level> unconfirmedCounter, Counter<Level> missingCounter, Relation<MissingStatus, String> missingPaths, Set<String> unconfirmedPaths)1904 public static void getStatus(CLDRFile file, PathHeader.Factory pathHeaderFactory, 1905 Counter<Level> foundCounter, Counter<Level> unconfirmedCounter, 1906 Counter<Level> missingCounter, 1907 Relation<MissingStatus, String> missingPaths, 1908 Set<String> unconfirmedPaths) { 1909 getStatus(file.fullIterable(), file, pathHeaderFactory, foundCounter, unconfirmedCounter, missingCounter, missingPaths, unconfirmedPaths); 1910 } 1911 1912 /** 1913 * Find the status of the items in the file. 1914 * @param allPaths manual list of paths 1915 * @param file the source. Must be a resolved file, made with minimalDraftStatus = unconfirmed 1916 * @param pathHeaderFactory PathHeaderFactory. 1917 * @param foundCounter output counter of the number of paths with values having contributed or approved status 1918 * @param unconfirmedCounter output counter of the number of paths with values, but neither contributed nor approved status 1919 * @param missingCounter output counter of the number of paths without values 1920 * @param missingPaths output if not null, the specific paths that are missing. 1921 * @param unconfirmedPaths TODO 1922 */ getStatus(Iterable<String> allPaths, CLDRFile file, PathHeader.Factory pathHeaderFactory, Counter<Level> foundCounter, Counter<Level> unconfirmedCounter, Counter<Level> missingCounter, Relation<MissingStatus, String> missingPaths, Set<String> unconfirmedPaths)1923 public static void getStatus(Iterable<String> allPaths, CLDRFile file, 1924 PathHeader.Factory pathHeaderFactory, Counter<Level> foundCounter, 1925 Counter<Level> unconfirmedCounter, 1926 Counter<Level> missingCounter, 1927 Relation<MissingStatus, String> missingPaths, Set<String> unconfirmedPaths) { 1928 1929 if (!file.isResolved()) { 1930 throw new IllegalArgumentException("File must be resolved, no minimal draft status"); 1931 } 1932 foundCounter.clear(); 1933 unconfirmedCounter.clear(); 1934 missingCounter.clear(); 1935 1936 Status status = new Status(); 1937 boolean latin = VettingViewer.isLatinScriptLocale(file); 1938 CoverageLevel2 coverageLevel2 = CoverageLevel2.getInstance(file.getLocaleID()); 1939 1940 for (String path : allPaths) { 1941 1942 PathHeader ph = pathHeaderFactory.fromPath(path); 1943 if (ph.getSectionId() == SectionId.Special) { 1944 continue; 1945 } 1946 1947 Level level = coverageLevel2.getLevel(path); 1948 // String localeFound = file.getSourceLocaleID(path, status); 1949 // String value = file.getSourceLocaleID(path, status); 1950 MissingStatus missingStatus = VettingViewer.getMissingStatus(file, path, status, latin); 1951 1952 switch (missingStatus) { 1953 case ABSENT: 1954 missingCounter.add(level, 1); 1955 if (missingPaths != null && level.compareTo(Level.MODERN) <= 0) { 1956 missingPaths.put(missingStatus, path); 1957 } 1958 break; 1959 case ALIASED: 1960 case PRESENT: 1961 String fullPath = file.getFullXPath(path); 1962 if (fullPath.contains("unconfirmed") 1963 || fullPath.contains("provisional")) { 1964 unconfirmedCounter.add(level, 1); 1965 if (unconfirmedPaths != null && level.compareTo(Level.MODERN) <= 0) { 1966 unconfirmedPaths.add(path); 1967 } 1968 } else { 1969 foundCounter.add(level, 1); 1970 } 1971 break; 1972 case MISSING_OK: 1973 case ROOT_OK: 1974 break; 1975 default: 1976 throw new IllegalArgumentException(); 1977 } 1978 } 1979 } 1980 1981 /** 1982 * Simple example of usage 1983 * 1984 * @param args 1985 * @throws IOException 1986 */ 1987 final static Options myOptions = new Options(); 1988 1989 enum MyOptions { 1990 repeat(null, null, "Repeat indefinitely"), filter(".*", ".*", "Filter files"), locale(".*", "af", "Single locale for testing"), source(".*", 1991 CLDRPaths.MAIN_DIRECTORY, // CldrUtility.TMP2_DIRECTORY + "/vxml/common/main" 1992 "if summary, creates filtered version (eg -d main): does a find in the name, which is of the form dir/file"), verbose(null, null, 1993 "verbose debugging messages"), output(".*", CLDRPaths.GEN_DIRECTORY + "vetting/", "filter the raw files (non-summary, mostly for debugging)"),; 1994 // boilerplate 1995 final Option option; 1996 MyOptions(String argumentPattern, String defaultArgument, String helpText)1997 MyOptions(String argumentPattern, String defaultArgument, String helpText) { 1998 option = myOptions.add(this, argumentPattern, defaultArgument, helpText); 1999 } 2000 } 2001 main(String[] args)2002 public static void main(String[] args) throws IOException { 2003 SHOW_SUBTYPES = true; 2004 myOptions.parse(MyOptions.source, args, true); 2005 boolean repeat = MyOptions.repeat.option.doesOccur(); 2006 String fileFilter = MyOptions.filter.option.getValue(); 2007 String myOutputDir = repeat ? null : MyOptions.output.option.getValue(); 2008 String LOCALE = MyOptions.locale.option.getValue(); 2009 String CURRENT_MAIN = MyOptions.source.option.getValue(); 2010 final String version = ToolConstants.PREVIOUS_CHART_VERSION; 2011 final String lastMain = CLDRPaths.ARCHIVE_DIRECTORY + "/cldr-" + version + "/common/main"; 2012 //final String lastMain = CLDRPaths.ARCHIVE_DIRECTORY + "/common/main"; 2013 do { 2014 Timer timer = new Timer(); 2015 timer.start(); 2016 2017 Factory cldrFactory = Factory.make(CURRENT_MAIN, fileFilter); 2018 cldrFactory.setSupplementalDirectory(new File(CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 2019 Factory cldrFactoryOld = Factory.make(lastMain, fileFilter); 2020 SupplementalDataInfo supplementalDataInfo = SupplementalDataInfo 2021 .getInstance(CLDRPaths.SUPPLEMENTAL_DIRECTORY); 2022 CheckCLDR.setDisplayInformation(cldrFactory.make("en", true)); 2023 2024 // FAKE this, because we don't have access to ST data 2025 2026 UsersChoice<Organization> usersChoice = new UsersChoice<Organization>() { 2027 // Fake values for now 2028 public String getWinningValueForUsersOrganization(CLDRFile cldrFile, String path, Organization user) { 2029 if (path.contains("USD")) { 2030 return "&dummy ‘losing’ value"; 2031 } 2032 return null; // assume we didn't vote on anything else. 2033 } 2034 2035 // Fake values for now 2036 public VoteStatus getStatusForUsersOrganization(CLDRFile cldrFile, String path, Organization user) { 2037 String usersValue = getWinningValueForUsersOrganization(cldrFile, path, user); 2038 String winningValue = cldrFile.getWinningValue(path); 2039 if (usersValue != null && !Objects.equals(usersValue, winningValue)) { 2040 return VoteStatus.losing; 2041 } 2042 String fullPath = cldrFile.getFullXPath(path); 2043 if (fullPath.contains("AMD") || fullPath.contains("unconfirmed") || fullPath.contains("provisional")) { 2044 return VoteStatus.provisionalOrWorse; 2045 } else if (fullPath.contains("AED")) { 2046 return VoteStatus.disputed; 2047 } else if (fullPath.contains("AED")) { 2048 return VoteStatus.ok_novotes; 2049 } 2050 return VoteStatus.ok; 2051 } 2052 }; 2053 2054 // create the tableView and set the options desired. 2055 // The Options should come from a GUI; from each you can get a long 2056 // description and a button label. 2057 // Assuming user can be identified by an int 2058 VettingViewer<Organization> tableView = new VettingViewer<Organization>(supplementalDataInfo, cldrFactory, 2059 cldrFactoryOld, usersChoice, "CLDR " + version, 2060 "Winning Proposed"); 2061 2062 // here are per-view parameters 2063 2064 final EnumSet<Choice> choiceSet = EnumSet.allOf(Choice.class); 2065 String localeStringID = LOCALE; 2066 int userNumericID = 666; 2067 Level usersLevel = Level.MODERN; 2068 // http: // unicode.org/cldr-apps/survey?_=ur 2069 2070 if (!repeat) { 2071 FileCopier.ensureDirectoryExists(myOutputDir); 2072 FileCopier.copy(VettingViewer.class, "vettingView.css", myOutputDir); 2073 FileCopier.copy(VettingViewer.class, "vettingView.js", myOutputDir); 2074 } 2075 System.out.println("Creation: " + timer.getDuration() / NANOSECS + " secs"); 2076 2077 // timer.start(); 2078 // writeFile(tableView, choiceSet, "", localeStringID, userNumericID, usersLevel, CodeChoice.oldCode); 2079 // System.out.println(timer.getDuration() / NANOSECS + " secs"); 2080 2081 timer.start(); 2082 writeFile(myOutputDir, tableView, choiceSet, "", localeStringID, userNumericID, usersLevel, CodeChoice.newCode, null); 2083 System.out.println("Code: " + timer.getDuration() / NANOSECS + " secs"); 2084 2085 timer.start(); 2086 writeFile(myOutputDir, tableView, choiceSet, "", localeStringID, userNumericID, usersLevel, CodeChoice.summary, 2087 Organization.google); 2088 System.out.println("Summary: " + timer.getDuration() / NANOSECS + " secs"); 2089 2090 // timer.start(); 2091 // writeFile(tableView, choiceSet, "", localeStringID, userNumericID, usersLevel, CodeChoice.summary, 2092 // Organization.ibm); 2093 // System.out.println(timer.getDuration() / NANOSECS + " secs"); 2094 2095 // // check that the choices work. 2096 // for (Choice choice : choiceSet) { 2097 // timer.start(); 2098 // writeFile(tableView, EnumSet.of(choice), "-" + choice.abbreviation, localeStringID, userNumericID, 2099 // usersLevel); 2100 // System.out.println(timer.getDuration() / NANOSECS + " secs"); 2101 // } 2102 } while (repeat); 2103 } 2104 2105 public enum CodeChoice { 2106 /** For the normal (locale) view of data **/ 2107 newCode, 2108 // /** @deprecated **/ 2109 // oldCode, 2110 /** For a summary view of data **/ 2111 summary 2112 } 2113 writeFile(String myOutputDir, VettingViewer<Organization> tableView, final EnumSet<Choice> choiceSet, String name, String localeStringID, int userNumericID, Level usersLevel, CodeChoice newCode, Organization organization)2114 public static void writeFile(String myOutputDir, VettingViewer<Organization> tableView, final EnumSet<Choice> choiceSet, 2115 String name, String localeStringID, int userNumericID, 2116 Level usersLevel, 2117 CodeChoice newCode, Organization organization) 2118 throws IOException { 2119 // open up a file, and output some of the styles to control the table 2120 // appearance 2121 PrintWriter out = myOutputDir == null ? new PrintWriter(new StringWriter()) 2122 : FileUtilities.openUTF8Writer(myOutputDir, "vettingView" 2123 + name 2124 + (newCode == CodeChoice.newCode ? "" : newCode == CodeChoice.summary ? "-summary" : "") 2125 + (organization == null ? "" : "-" + organization.toString()) 2126 + ".html"); 2127 // FileUtilities.appendFile(VettingViewer.class, "vettingViewerHead.txt", out); 2128 FileCopier.copy(VettingViewer.class, "vettingViewerHead.txt", out); 2129 out.append(getHeaderStyles()); 2130 out.append("</head><body>\n"); 2131 2132 out.println( 2133 "<p>Note: this is just a sample run. The user, locale, user's coverage level, and choices of tests will change the output. In a real ST page using these, the first three would " 2134 + "come from context, and the choices of tests would be set with radio buttons. Demo settings are: </p>\n<ol>" 2135 + "<li>choices: " 2136 + choiceSet 2137 + "</li><li>localeStringID: " 2138 + localeStringID 2139 + "</li><li>userNumericID: " 2140 + userNumericID 2141 + "</li><li>usersLevel: " 2142 + usersLevel 2143 + "</ol>" 2144 + "<p>Notes: This is a static version, using old values and faked values (L) just for testing." 2145 + (TESTING ? "Also, the white cell after the Fix column is just for testing." : "") 2146 + "</p><hr>\n"); 2147 2148 // now generate the table with the desired options 2149 // The options should come from a GUI; from each you can get a long 2150 // description and a button label. 2151 // Assuming user can be identified by an int 2152 2153 switch (newCode) { 2154 case newCode: 2155 tableView.generateHtmlErrorTables(out, choiceSet, localeStringID, organization, usersLevel, SHOW_ALL, false); 2156 break; 2157 // case oldCode: 2158 // tableView.generateHtmlErrorTablesOld(out, choiceSet, localeStringID, userNumericID, usersLevel, SHOW_ALL); 2159 // break; 2160 case summary: 2161 //System.out.println(tableView.getName("zh_Hant_HK")); 2162 tableView.generateSummaryHtmlErrorTables(out, choiceSet, null, organization); 2163 break; 2164 } 2165 out.println("</body>\n</html>\n"); 2166 out.close(); 2167 } 2168 } 2169